From b4c264329145c3779afe80d0533ada2f5e80052f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 22 Oct 2021 09:21:34 +0800 Subject: [PATCH 001/363] fix(api-clients): escape the searching string --- apps/emqx_management/src/emqx_mgmt_api_clients.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 2fe6a5ccb..1ddd87a3d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -334,7 +334,7 @@ query({Qs, Fuzzy}, Start, Limit) -> match_fun(Ms, Fuzzy) -> MsC = ets:match_spec_compile(Ms), REFuzzy = lists:map(fun({K, like, S}) -> - {ok, RE} = re:compile(S), + {ok, RE} = re:compile(escape(S)), {K, like, RE} end, Fuzzy), fun(Rows) -> @@ -347,6 +347,9 @@ match_fun(Ms, Fuzzy) -> end end. +escape(B) when is_binary(B) -> + re:replace(B, <<"\\\\">>, <<"\\\\\\\\">>, [{return, binary}, global]). + run_fuzzy_match(_, []) -> true; run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> @@ -450,4 +453,9 @@ params2qs_test() -> [{{'$1', #{}, '_'}, [], ['$_']}] = qs2ms([]). +escape_test() -> + Str = <<"\\n">>, + {ok, Re} = re:compile(escape(Str)), + {match, _} = re:run(<<"\\name">>, Re). + -endif. From 791caba2ede8710bb9600e9b66b5bbb8153d1123 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Fri, 29 Oct 2021 09:55:19 +0200 Subject: [PATCH 002/363] fix(broker): Fix out-of-order message delivery in a cluster Fixes: #4658 --- etc/emqx.conf | 2 +- priv/emqx.schema | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 6043dc361..f24d7ada2 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -344,7 +344,7 @@ rpc.port_discovery = stateless ## ## Value: Interger [0-256] ## Default = 1 -#rpc.tcp_client_num = 1 +#rpc.tcp_client_num = 0 ## RCP Client connect timeout. ## diff --git a/priv/emqx.schema b/priv/emqx.schema index 86a2a2892..4b33cf65c 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -376,7 +376,7 @@ end}. {translation, "gen_rpc.tcp_client_num", fun(Conf) -> case cuttlefish:conf_get("rpc.tcp_client_num", Conf) of - 0 -> 1; %% keep allowing 0 for backward compatibility + 0 -> max(1, erlang:system_info(schedulers) div 2); V -> V end end}. diff --git a/rebar.config b/rebar.config index f56e8e2c2..bd39f9816 100644 --- a/rebar.config +++ b/rebar.config @@ -44,7 +44,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.3"}}} - , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.6.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} From f8acb31f89baff71424a3ac878f09b675ffb78a2 Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 27 Oct 2021 11:43:05 +0800 Subject: [PATCH 003/363] feat: add slow topics statistics plugin --- .../etc/emqx_st_statistics.conf | 38 ++ .../priv/emqx_st_statistics.schema | 58 +++ apps/emqx_st_statistics/rebar.config | 23 ++ .../src/emqx_st_statistics.app.src | 12 + .../src/emqx_st_statistics.appup.src | 9 + .../src/emqx_st_statistics.erl | 331 ++++++++++++++++++ .../src/emqx_st_statistics_app.erl | 33 ++ .../src/emqx_st_statistics_sup.erl | 35 ++ .../test/emqx_st_statistics_SUITE.erl | 110 ++++++ rebar.config.erl | 1 + scripts/apps-version-check.sh | 90 +++-- src/emqx_session.erl | 9 +- 12 files changed, 714 insertions(+), 35 deletions(-) create mode 100644 apps/emqx_st_statistics/etc/emqx_st_statistics.conf create mode 100644 apps/emqx_st_statistics/priv/emqx_st_statistics.schema create mode 100644 apps/emqx_st_statistics/rebar.config create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics.app.src create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics.appup.src create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics.erl create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics_app.erl create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl create mode 100644 apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl diff --git a/apps/emqx_st_statistics/etc/emqx_st_statistics.conf b/apps/emqx_st_statistics/etc/emqx_st_statistics.conf new file mode 100644 index 000000000..88ca0c37f --- /dev/null +++ b/apps/emqx_st_statistics/etc/emqx_st_statistics.conf @@ -0,0 +1,38 @@ +##-------------------------------------------------------------------- +## EMQ X Slow Topics Statistics +##-------------------------------------------------------------------- + +## Threshold time of slow topics statistics +## +## Default: 10 seconds +st_statistics.threshold_time = 10S + +## Time window of slow topics statistics +## +## Value: 5 minutes +st_statistics.time_window = 5M + +## Maximum of slow topics log, log will clear when enter new time window +## +## Value: 500 +st_statistics.max_log_num = 500 + +## Top-K record for slow topics, update from logs +## +## Value: 500 +st_statistics.top_k_num = 500 + +## Topic of notification +## +## Defaut: $slow_topics +st_statistics.notice_topic = $slow_topics + +## QoS of notification message in notice topic +## +## Defaut: 0 +st_statistics.notice_qos = 0 + +## Maximum information number in one notification +## +## Default: 500 +st_statistics.notice_batch_size = 500 diff --git a/apps/emqx_st_statistics/priv/emqx_st_statistics.schema b/apps/emqx_st_statistics/priv/emqx_st_statistics.schema new file mode 100644 index 000000000..da9a2f810 --- /dev/null +++ b/apps/emqx_st_statistics/priv/emqx_st_statistics.schema @@ -0,0 +1,58 @@ +%%-*- mode: erlang -*- +%% st_statistics config mapping + +%% Threshold time of slow topics statistics +%% {$configurable} +{mapping, "st_statistics.threshold_time", "emqx_st_statistics.threshold_time", + [ + {default, "10S"}, + {datatype, [integer, {duration, ms}]} + ]}. + +%% Time window of slow topics statistics +%% {$configurable} +{mapping, "st_statistics.time_window", "emqx_st_statistics.time_window", + [ + {default, "5M"}, + {datatype, [integer, {duration, ms}]} + ]}. + +%% Maximum of slow topics log +%% {$configurable} +{mapping, "st_statistics.max_log_num", "emqx_st_statistics.max_log_num", + [ + {default, 500}, + {datatype, integer} + ]}. + +%% Top-K record for slow topics, update from logs +%% {$configurable} +{mapping, "st_statistics.top_k_num", "emqx_st_statistics.top_k_num", + [ + {default, 500}, + {datatype, integer} + ]}. + +%% Topic of notification +%% {$configurable} +{mapping, "st_statistics.notice_topic", "emqx_st_statistics.notice_topic", + [ + {default, <<"slow_topics">>}, + {datatype, string} + ]}. + +%% QoS of notification message in notice topic +%% {$configurable} +{mapping, "st_statistics.notice_qos", "emqx_st_statistics.notice_qos", + [ + {default, 0}, + {datatype, integer} + ]}. + +%% Maximum entities per notification message +%% {$configurable} +{mapping, "st_statistics.notice_batch_size", "emqx_st_statistics.notice_batch_size", + [ + {default, 500}, + {datatype, integer} + ]}. diff --git a/apps/emqx_st_statistics/rebar.config b/apps/emqx_st_statistics/rebar.config new file mode 100644 index 000000000..6433d92d6 --- /dev/null +++ b/apps/emqx_st_statistics/rebar.config @@ -0,0 +1,23 @@ +{deps, []}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard + ]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{profiles, + [{test, + [{deps, + [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}}]} + ]} + ]}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.app.src b/apps/emqx_st_statistics/src/emqx_st_statistics.app.src new file mode 100644 index 000000000..b1eb1612a --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics.app.src @@ -0,0 +1,12 @@ +{application, emqx_st_statistics, + [{description, "EMQ X Slow Topics Statistics"}, + {vsn, "1.0.0"}, % strict semver, bump manually! + {modules, []}, + {registered, [emqx_st_statistics_sup]}, + {applications, [kernel,stdlib]}, + {mod, {emqx_st_statistics_app,[]}}, + {env, []}, + {licenses, ["Apache-2.0"]}, + {maintainers, ["EMQ X Team "]}, + {links, []} + ]}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src b/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src new file mode 100644 index 000000000..dcf0d8cdd --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src @@ -0,0 +1,9 @@ +%% -*-: erlang -*- +{VSN, + [ + {<<".*">>, []} + ], + [ + {<<".*">>, []} + ] +}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.erl b/apps/emqx_st_statistics/src/emqx_st_statistics.erl new file mode 100644 index 000000000..8a6c1535f --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics.erl @@ -0,0 +1,331 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics). + +-behaviour(gen_server). + +-include_lib("include/emqx.hrl"). +-include_lib("include/logger.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). + +-logger_header("[SLOW TOPICS]"). + +-export([ start_link/1, on_publish_done/3, enable/0 + , disable/0 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-compile(nowarn_unused_type). + +-type state() :: #{ config := proplist:proplist() + , index := index_map() + , begin_time := pos_integer() + , counter := counters:counter_ref() + , enable := boolean() + }. + +-type log() :: #{ topic := emqx_types:topic() + , times := pos_integer() + , average := float() + }. + +-type window_log() :: #{ begin_time := pos_integer() + , logs := [log()] + }. + +-record(slow_log, { topic :: emqx_types:topic() + , times :: non_neg_integer() + , elapsed :: non_neg_integer() + }). + +-record(top_k, { key :: any() + , average :: float()}). + +-type message() :: #message{}. + +-import(proplists, [get_value/2]). + +-define(LOG_TAB, emqx_st_statistics_log). +-define(TOPK_TAB, emqx_st_statistics_topk). +-define(NOW, erlang:system_time(millisecond)). +-define(TOP_KEY(Times, Topic), {Times, Topic}). +-define(QUOTA_IDX, 1). + +-type top_key() :: ?TOP_KEY(pos_integer(), emqx_types:topic()). +-type index_map() :: #{emqx_types:topic() => pos_integer()}. + +%% erlang term order +%% number < atom < reference < fun < port < pid < tuple < list < bit string + +%% ets ordered_set is ascending by term order + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +%% @doc Start the st_statistics +-spec(start_link(Env :: list()) -> emqx_types:startlink_ret()). +start_link(Env) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). + +-spec on_publish_done(message(), pos_integer(), counters:counters_ref()) -> ok. +on_publish_done(#message{timestamp = Timestamp} = Msg, Threshold, Counter) -> + case ?NOW - Timestamp of + Elapsed when Elapsed > Threshold -> + case get_log_quota(Counter) of + true -> + update_log(Msg, Elapsed); + _ -> + ok + end; + _ -> + ok + end. + +enable() -> + gen_server:call(?MODULE, {enable, true}). + +disable() -> + gen_server:call(?MODULE, {enable, false}). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Env]) -> + erlang:process_flag(trap_exit, true), + init_log_tab(Env), + init_topk_tab(Env), + notification_tick(Env), + Counter = counters:new(1, [write_concurrency]), + set_log_quota(Env, Counter), + Threshold = get_value(threshold_time, Env), + load(Threshold, Counter), + {ok, #{config => Env, + index => #{}, + begin_time => ?NOW, + counter => Counter, + enable => true}}. + +handle_call({enable, Enable}, _From, + #{config := Cfg, counter := Counter, enable := IsEnable} = State) -> + State2 = case Enable of + IsEnable -> + State; + true -> + Threshold = get_value(threshold_time, Cfg), + load(Threshold, Counter), + State#{enable := true}; + _ -> + unload(), + State#{enable := false} + end, + {reply, ok, State2}; + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(notification_tick, #{config := Cfg} = State) -> + notification_tick(Cfg), + Index2 = do_notification(State), + {noreply, State#{index := Index2, + begin_time := ?NOW}}; + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _) -> + unload(), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +notification_tick(Env) -> + TimeWindow = get_value(time_window, Env), + erlang:send_after(TimeWindow, self(), ?FUNCTION_NAME). + +init_log_tab(_) -> + ?LOG_TAB = ets:new(?LOG_TAB, [ set, public, named_table + , {keypos, #slow_log.topic}, {write_concurrency, true} + , {read_concurrency, true} + ]). + +init_topk_tab(_) -> + ?TOPK_TAB = ets:new(?TOPK_TAB, [ ordered_set, protected, named_table + , {keypos, #top_k.key}, {write_concurrency, true} + , {read_concurrency, false} + ]). + +-spec get_log_quota(counter:counter_ref()) -> boolean(). +get_log_quota(Counter) -> + case counters:get(Counter, ?QUOTA_IDX) of + Quota when Quota > 0 -> + counters:sub(Counter, ?QUOTA_IDX, 1), + true; + _ -> + false + end. + +-spec set_log_quota(proplists:proplist(), counter:counter_ref()) -> ok. +set_log_quota(Cfg, Counter) -> + MaxLogNum = get_value(max_log_num, Cfg), + counters:put(Counter, ?QUOTA_IDX, MaxLogNum). + +-spec update_log(message(), non_neg_integer()) -> ok. +update_log(#message{topic = Topic}, Elapsed) -> + _ = ets:update_counter(?LOG_TAB, + Topic, + [{#slow_log.times, 1}, {#slow_log.elapsed, Elapsed}], + #slow_log{topic = Topic, + times = 1, + elapsed = Elapsed}), + ok. + +-spec do_notification(state()) -> index_map(). +do_notification(#{begin_time := BeginTime, + config := Cfg, + index := IndexMap, + counter := Counter}) -> + Logs = ets:tab2list(?LOG_TAB), + ets:delete_all_objects(?LOG_TAB), + start_publish(Logs, BeginTime, Cfg), + set_log_quota(Cfg, Counter), + MaxRecord = get_value(top_k_num, Cfg), + Size = ets:info(?TOPK_TAB, size), + update_top_k(Logs, erlang:max(0, MaxRecord - Size), IndexMap). + +-spec update_top_k(list(#slow_log{}), non_neg_integer(), index_map()) -> index_map(). +update_top_k([#slow_log{topic = Topic, + times = NewTimes, + elapsed = Elapsed} = Log | T], + Left, + IndexMap) -> + case maps:get(Topic, IndexMap, 0) of + 0 -> + try_insert_new(Log, Left, T, IndexMap); + Times -> + [#top_k{key = Key, average = Average}] = ets:lookup(?TOPK_TAB, ?TOP_KEY(Times, Topic)), + Times2 = Times + NewTimes, + Total = Times * Average + Elapsed, + Average2 = Total / Times2, + ets:delete(?TOPK_TAB, Key), + ets:insert(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times2, Topic), average = Average2}), + update_top_k(T, Left, IndexMap#{Topic := Times2}) + end; + +update_top_k([], _, IndexMap) -> + IndexMap. + +-spec try_insert_new(#slow_log{}, + non_neg_integer(), list(#slow_log{}), index_map()) -> index_map(). +try_insert_new(#slow_log{topic = Topic, + times = Times, + elapsed = Elapsed}, Left, Logs, IndexMap) when Left > 0 -> + Average = Elapsed / Times, + ets:insert_new(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times, Topic), average = Average}), + update_top_k(Logs, Left - 1, IndexMap#{Topic => Times}); + +try_insert_new(#slow_log{topic = Topic, + times = Times, + elapsed = Elapsed}, Left, Logs, IndexMap) -> + ?TOP_KEY(MinTimes, MinTopic) = MinKey = ets:first(?TOPK_TAB), + case MinTimes > Times of + true -> + update_top_k(Logs, Left, IndexMap); + _ -> + Average = Elapsed / Times, + ets:delete(?TOPK_TAB, MinKey), + ets:insert_new(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times, Topic), average = Average}), + update_top_k(Logs, + Left - 1, + maps:put(Topic, Times, maps:remove(MinTopic, IndexMap))) + end. + +start_publish(Logs, BeginTime, Cfg) -> + emqx_pool:async_submit({fun do_publish/3, [Logs, BeginTime, Cfg]}). + +do_publish([], _, _) -> + ok; + +do_publish(Logs, BeginTime, Cfg) -> + BatchSize = get_value(notice_batch_size, Cfg), + do_publish(Logs, BatchSize, BeginTime, Cfg, []). + +do_publish([Log | T], Size, BeginTime, Cfg, Cache) when Size > 0 -> + Cache2 = [convert_to_notice(Log) | Cache], + do_publish(T, Size - 1, BeginTime, Cfg, Cache2); + +do_publish(Logs, Size, BeginTime, Cfg, Cache) when Size =:= 0 -> + publish(BeginTime, Cfg, Cache), + do_publish(Logs, BeginTime, Cfg); + +do_publish([], _, BeginTime, Cfg, Cache) -> + publish(BeginTime, Cfg, Cache), + ok. + +convert_to_notice(#slow_log{topic = Topic, + times = Times, + elapsed = Elapsed}) -> + #{topic => Topic, + times => Times, + average => Elapsed / Times}. + +publish(BeginTime, Cfg, Notices) -> + WindowLog = #{begin_time => BeginTime, + logs => Notices}, + Payload = emqx_json:encode(WindowLog), + _ = emqx:publish(#message{ id = emqx_guid:gen() + , qos = get_value(notice_qos, Cfg) + , from = ?MODULE + , topic = get_topic(Cfg) + , payload = Payload + , timestamp = ?NOW + }), + ok. + +load(Threshold, Counter) -> + _ = emqx:hook('message.publish_done', fun ?MODULE:on_publish_done/3, [Threshold, Counter]), + ok. + +unload() -> + emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/3). + +get_topic(Cfg) -> + case get_value(notice_topic, Cfg) of + Topic when is_binary(Topic) -> + Topic; + Topic -> + erlang:list_to_binary(Topic) + end. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl new file mode 100644 index 000000000..e5b62a00e --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([ start/2 + , stop/1 + ]). + +start(_Type, _Args) -> + Env = application:get_all_env(emqx_st_statistics), + {ok, Sup} = emqx_st_statistics_sup:start_link(Env), + {ok, Sup}. + +stop(_State) -> + ok. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl new file mode 100644 index 000000000..cbff854ef --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics_sup). + +-behaviour(supervisor). + +-export([start_link/1]). + +-export([init/1]). + +start_link(Env) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). + +init([Env]) -> + {ok, {{one_for_one, 10, 3600}, + [#{id => st_statistics, + start => {emqx_st_statistics, start_link, [Env]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_st_statistics]}]}}. diff --git a/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl b/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl new file mode 100644 index 000000000..c08c8d986 --- /dev/null +++ b/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl @@ -0,0 +1,110 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("include/emqx.hrl"). + +%-define(LOGT(Format, Args), ct:pal(Format, Args)). + +-define(LOG_TAB, emqx_st_statistics_log). +-define(TOPK_TAB, emqx_st_statistics_topk). +-define(NOW, erlang:system_time(millisecond)). + +all() -> emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_st_statistics], fun set_special_cfg/1), + Config. + +set_special_cfg(_) -> + application:set_env([{emqx_st_statistics, base_conf()}]), + ok. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_st_statistics]), + Config. + +%%-------------------------------------------------------------------- +%% Test Cases +%%-------------------------------------------------------------------- +t_log_and_pub(_) -> + %% Sub topic first + SubBase = "/test", + emqx:subscribe("$slow_topics"), + Clients = start_client(SubBase), + timer:sleep(1000), + Now = ?NOW, + %% publish + ?assert(ets:info(?LOG_TAB, size) =:= 0), + lists:foreach(fun(I) -> + Topic = list_to_binary(io_lib:format("~s~p", [SubBase, I])), + Msg = emqx_message:make(Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 1000}) + end, + lists:seq(1, 10)), + + ?assert(ets:info(?LOG_TAB, size) =:= 5), + + timer:sleep(2400), + + ?assert(ets:info(?LOG_TAB, size) =:= 0), + ?assert(ets:info(?TOPK_TAB, size) =:= 3), + try_receive(3), + try_receive(2), + [Client ! stop || Client <- Clients], + ok. + +base_conf() -> + [{top_k_num, 3}, + {threshold_time, 10}, + {notice_qos, 0}, + {notice_batch_size, 3}, + {notice_topic,"$slow_topics"}, + {time_window, 2000}, + {max_log_num, 5}]. + +start_client(Base) -> + [spawn(fun() -> + Topic = list_to_binary(io_lib:format("~s~p", [Base, I])), + client(Topic) + end) + || I <- lists:seq(1, 10)]. + +client(Topic) -> + {ok, C} = emqtt:start_link([{host, "localhost"}, + {clientid, Topic}, + {username, <<"plain">>}, + {password, <<"plain">>}]), + {ok, _} = emqtt:connect(C), + {ok, _, _} = emqtt:subscribe(C, Topic), + receive + stop -> + ok + end. + +try_receive(L) -> + receive + {deliver, _, #message{payload = Payload}} -> + #{<<"logs">> := Logs} = emqx_json:decode(Payload, [return_maps]), + ?assertEqual(length(Logs), L) + after 500 -> + ?assert(false) + end. diff --git a/rebar.config.erl b/rebar.config.erl index 1000a2c92..38dcb681b 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -303,6 +303,7 @@ relx_plugin_apps(ReleaseType) -> , emqx_recon , emqx_rule_engine , emqx_sasl + , emqx_st_statistics ] ++ [emqx_telemetry || not is_enterprise()] ++ relx_plugin_apps_per_rel(ReleaseType) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 596f7404a..7c2ea5eb2 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -5,38 +5,64 @@ latest_release=$(git describe --abbrev=0 --tags) bad_app_count=0 -while read -r app; do - if [ "$app" != "emqx" ]; then - app_path="$app" - else - app_path="." - fi - src_file="$app_path/src/$(basename "$app").app.src" - old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"')" - now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') - if [ "$old_app_version" = "$now_app_version" ]; then - changed="$(git diff --name-only "$latest_release"...HEAD \ - -- "$app_path/src" \ - -- "$app_path/priv" \ - -- "$app_path/c_src" | { grep -v -E 'appup\.src' || true; } | wc -l)" - if [ "$changed" -gt 0 ]; then - echo "$src_file needs a vsn bump" - bad_app_count=$(( bad_app_count + 1)) - elif [[ ${app_path} = *emqx_dashboard* ]]; then - ## emqx_dashboard is ensured to be upgraded after all other plugins - ## at the end of its appup instructions, there is the final instruction - ## {apply, {emqx_plugins, load, []} - ## since we don't know which plugins are stopped during the upgrade - ## for safty, we just force a dashboard version bump for each and every release - ## even if there is nothing changed in the app - echo "$src_file needs a vsn bump to ensure plugins loaded after upgrade" - bad_app_count=$(( bad_app_count + 1)) +get_vsn() { + commit="$1" + app_src_file="$2" + if [ "$commit" = 'HEAD' ]; then + if [ -f "$app_src_file" ]; then + grep vsn "$app_src_file" | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true fi + else + git show "$commit":"$app_src_file" 2>/dev/null | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true fi -done < <(./scripts/find-apps.sh) +} -if [ $bad_app_count -gt 0 ]; then - exit 1 -else - echo "apps version check successfully" -fi +check_apps() { + while read -r app_path; do + app=$(basename "$app_path") + src_file="$app_path/src/$app.app.src" + old_app_version="$(get_vsn "$latest_release" "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$old_app_version" = '' ]; then + old_app_version="$(get_vsn "$latest_release" 'src/emqx.app.src')" + fi + now_app_version="$(get_vsn 'HEAD' "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$now_app_version" = '' ]; then + now_app_version="$(get_vsn 'HEAD' 'src/emqx.app.src')" + fi + if [ -z "$now_app_version" ]; then + echo "failed_to_get_new_app_vsn for $app" + exit 1 + fi + if [ -z "${old_app_version:-}" ]; then + echo "skiped checking new app ${app}" + elif [ "$old_app_version" = "$now_app_version" ]; then + lines="$(git diff --name-only "$latest_release"...HEAD \ + -- "$app_path/src" \ + -- "$app_path/priv" \ + -- "$app_path/c_src")" + if [ "$lines" != '' ]; then + echo "$src_file needs a vsn bump (old=$old_app_version)" + echo "changed: $lines" + bad_app_count=$(( bad_app_count + 1)) + fi + fi + done < <(./scripts/find-apps.sh) + + if [ $bad_app_count -gt 0 ]; then + exit 1 + else + echo "apps version check successfully" + fi +} + +_main() { + if echo "${latest_release}" |grep -oE '[0-9]+.[0-9]+.[0-9]+' > /dev/null 2>&1; then + check_apps + else + echo "skiped unstable tag: ${latest_release}" + fi +} + +_main diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 9463345d4..db09bf8c9 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -319,6 +319,7 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, puback(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> + emqx:run_hook('message.publish_done', [Msg]), Inflight1 = emqx_inflight:delete(PacketId, Inflight), return_with(Msg, dequeue(Session#session{inflight = Inflight1})); {value, {_Pubrel, _Ts}} -> @@ -343,6 +344,8 @@ return_with(Msg, {ok, Publishes, Session}) -> pubrec(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> + %% execute hook here, because message record will be replaced by pubrel + emqx:run_hook('message.publish_done', [Msg]), Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; {value, {pubrel, _Ts}} -> @@ -439,11 +442,12 @@ deliver([Msg | More], Acc, Session) -> end. deliver_msg(Msg = #message{qos = ?QOS_0}, Session) -> + emqx:run_hook('message.publish_done', [Msg]), {ok, [{undefined, maybe_ack(Msg)}], Session}; deliver_msg(Msg = #message{qos = QoS}, Session = - #session{next_pkt_id = PacketId, inflight = Inflight}) - when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 -> + #session{next_pkt_id = PacketId, inflight = Inflight}) + when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 -> case emqx_inflight:is_full(Inflight) of true -> Session1 = case maybe_nack(Msg) of @@ -696,4 +700,3 @@ age(Now, Ts) -> Now - Ts. set_field(Name, Value, Session) -> Pos = emqx_misc:index_of(Name, record_info(fields, session)), setelement(Pos+1, Session, Value). - From b9270ad7197fdeb50de646452e8cee136bec57c0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 3 Nov 2021 17:45:48 -0300 Subject: [PATCH 004/363] feat(stats): track live / connected channel count for monitoring In order to correctly display the number of connected clients in our monitor dashboard, we need to track those connections that are actually connected to clients, not considering connections from persistent sessions that are disconnected. Today, the `connections.count` that is displayed in the dashboards considers those disconnected persistent sessions as well. --- src/emqx_channel.erl | 5 +- src/emqx_cm.erl | 39 +++- src/emqx_connection.erl | 6 +- src/emqx_stats.erl | 13 +- test/emqx_broker_SUITE.erl | 291 +++++++++++++++++++++++++++++- test/emqx_channel_SUITE.erl | 3 +- test/emqx_connection_SUITE.erl | 18 +- test/emqx_mqueue_SUITE.erl | 2 +- test/emqx_ws_connection_SUITE.erl | 11 +- 9 files changed, 359 insertions(+), 29 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 7bfef472d..5f164d99e 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1536,6 +1536,8 @@ ensure_connected(Channel = #channel{conninfo = ConnInfo, clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.connected', [ClientInfo, NConnInfo]), + ChanPid = self(), + emqx_cm:mark_channel_connected(ChanPid), Channel#channel{conninfo = NConnInfo, conn_state = connected }. @@ -1624,6 +1626,8 @@ ensure_disconnected(Reason, Channel = #channel{conninfo = ConnInfo, clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.disconnected', [ClientInfo, Reason, NConnInfo]), + ChanPid = self(), + emqx_cm:mark_channel_disconnected(ChanPid), Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. %%-------------------------------------------------------------------- @@ -1725,4 +1729,3 @@ flag(false) -> 0. set_field(Name, Value, Channel) -> Pos = emqx_misc:index_of(Name, record_info(fields, channel)), setelement(Pos+1, Channel, Value). - diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 23f078568..c54089e41 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -22,6 +22,7 @@ -include("emqx.hrl"). -include("logger.hrl"). -include("types.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[CM]"). @@ -72,7 +73,12 @@ ]). %% Internal export --export([stats_fun/0, clean_down/1]). +-export([ stats_fun/0 + , clean_down/1 + , mark_channel_connected/1 + , mark_channel_disconnected/1 + , get_connected_client_count/0 + ]). -type(chan_pid() :: pid()). @@ -80,11 +86,13 @@ -define(CHAN_TAB, emqx_channel). -define(CHAN_CONN_TAB, emqx_channel_conn). -define(CHAN_INFO_TAB, emqx_channel_info). +-define(CHAN_LIVE_TAB, emqx_channel_live). -define(CHAN_STATS, [{?CHAN_TAB, 'channels.count', 'channels.max'}, {?CHAN_TAB, 'sessions.count', 'sessions.max'}, - {?CHAN_CONN_TAB, 'connections.count', 'connections.max'} + {?CHAN_CONN_TAB, 'connections.count', 'connections.max'}, + {?CHAN_LIVE_TAB, 'live_connections.count', 'live_connections.max'} ]). %% Batch drain @@ -437,8 +445,10 @@ init([]) -> ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true}|TabOpts]), ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]), ok = emqx_tables:new(?CHAN_INFO_TAB, [set, compressed | TabOpts]), + ok = emqx_tables:new(?CHAN_LIVE_TAB, [set, {write_concurrency, true} | TabOpts]), ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0), - {ok, #{chan_pmon => emqx_pmon:new()}}. + State = #{chan_pmon => emqx_pmon:new()}, + {ok, State}. handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), @@ -447,17 +457,21 @@ handle_call(Req, _From, State) -> handle_cast({registered, {ClientId, ChanPid}}, State = #{chan_pmon := PMon}) -> PMon1 = emqx_pmon:monitor(ChanPid, ClientId, PMon), {noreply, State#{chan_pmon := PMon1}}; - handle_cast(Msg, State) -> ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> + ?tp(emqx_cm_process_down, #{pid => Pid, reason => _Reason}), ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), + lists:foreach( + fun({ChanPid, _ClientID}) -> + mark_channel_disconnected(ChanPid) + end, + Items), ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]), {noreply, State#{chan_pmon := PMon1}}; - handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), {noreply, State}. @@ -493,3 +507,18 @@ get_chann_conn_mod(ClientId, ChanPid) when node(ChanPid) == node() -> get_chann_conn_mod(ClientId, ChanPid) -> rpc_call(node(ChanPid), get_chann_conn_mod, [ClientId, ChanPid], ?T_GET_INFO). +mark_channel_connected(ChanPid) -> + ?tp(emqx_cm_connected_client_count_inc, #{}), + ets:insert_new(?CHAN_LIVE_TAB, {ChanPid, true}), + ok. + +mark_channel_disconnected(ChanPid) -> + ?tp(emqx_cm_connected_client_count_dec, #{}), + ets:delete(?CHAN_LIVE_TAB, ChanPid), + ok. + +get_connected_client_count() -> + case ets:info(?CHAN_LIVE_TAB, size) of + undefined -> 0; + Size -> Size + end. diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index ab91c02b4..84086015d 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -518,7 +518,7 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport, ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S}) end, ?tp(info, terminate, #{reason => Reason}), - maybe_raise_excption(Reason). + maybe_raise_exception(Reason). %% close socket, discard new state, always return ok. close_socket_ok(State) -> @@ -526,12 +526,12 @@ close_socket_ok(State) -> ok. %% tell truth about the original exception -maybe_raise_excption(#{exception := Exception, +maybe_raise_exception(#{exception := Exception, context := Context, stacktrace := Stacktrace }) -> erlang:raise(Exception, Context, Stacktrace); -maybe_raise_excption(Reason) -> +maybe_raise_exception(Reason) -> exit(Reason). %%-------------------------------------------------------------------- diff --git a/src/emqx_stats.erl b/src/emqx_stats.erl index f53549e65..ba61143ac 100644 --- a/src/emqx_stats.erl +++ b/src/emqx_stats.erl @@ -21,6 +21,7 @@ -include("emqx.hrl"). -include("logger.hrl"). -include("types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[Stats]"). @@ -67,8 +68,10 @@ %% Connection stats -define(CONNECTION_STATS, - ['connections.count', %% Count of Concurrent Connections - 'connections.max' %% Maximum Number of Concurrent Connections + [ 'connections.count' %% Count of Concurrent Connections + , 'connections.max' %% Maximum Number of Concurrent Connections + , 'live_connections.count' %% Count of connected clients + , 'live_connections.max' %% Maximum number of connected clients ]). %% Channel stats @@ -216,6 +219,11 @@ handle_cast({setstat, Stat, MaxStat, Val}, State) -> ets:insert(?TAB, {MaxStat, Val}) end, safe_update_element(Stat, Val), + ?tp(emqx_stats_setstat, + #{ count_stat => Stat + , max_stat => MaxStat + , value => Val + }), {noreply, State}; handle_cast({update_interval, Update = #update{name = Name}}, @@ -274,4 +282,3 @@ safe_update_element(Key, Val) -> error:badarg -> ?LOG(warning, "Failed to update ~0p to ~0p", [Key, Val]) end. - diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl index 4cef68660..57f0d6acf 100644 --- a/test/emqx_broker_SUITE.erl +++ b/test/emqx_broker_SUITE.erl @@ -23,20 +23,63 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> + [ {group, all_cases} + , {group, connected_client_count_group} + ]. -init_per_suite(Config) -> +groups() -> + TCs = emqx_ct:all(?MODULE), + ConnClientTCs = [ t_connected_client_count_persistent + , t_connected_client_count_anonymous + , t_connected_client_stats + ], + OtherTCs = TCs -- ConnClientTCs, + [ {all_cases, [], OtherTCs} + , {connected_client_count_group, [ {group, tcp} + , {group, ws} + ]} + , {tcp, [], ConnClientTCs} + , {ws, [], ConnClientTCs} + ]. + +init_per_group(connected_client_count_group, Config) -> + Config; +init_per_group(tcp, Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([]), + [{conn_fun, connect} | Config]; +init_per_group(ws, Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([]), + [ {ssl, false} + , {enable_websocket, true} + , {conn_fun, ws_connect} + , {port, 8083} + , {host, "localhost"} + | Config + ]; +init_per_group(_Group, Config) -> emqx_ct_helpers:boot_modules(all), emqx_ct_helpers:start_apps([]), Config. -end_per_suite(_Config) -> +end_per_group(connected_client_count_group, _Config) -> + ok; +end_per_group(_Group, _Config) -> emqx_ct_helpers:stop_apps([]). +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + init_per_testcase(Case, Config) -> ?MODULE:Case({init, Config}). @@ -277,6 +320,248 @@ t_stats_fun({'end', _Config}) -> ok = emqx_broker:unsubscribe(<<"topic">>), ok = emqx_broker:unsubscribe(<<"topic2">>). +%% persistent sessions, when gone, do not contribute to connected +%% client count +t_connected_client_count_persistent({init, Config}) -> + ok = snabbkaffe:start_trace(), + process_flag(trap_exit, true), + Config; +t_connected_client_count_persistent(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ClientID = <<"clientid">>, + ?assertEqual(0, emqx_cm:get_connected_client_count()), + {ok, ConnPid0} = emqtt:start_link([ {clean_start, false} + , {clientid, ClientID} + | Config]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid0) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + {ok, {ok, [_]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid0) end, + [emqx_cm_connected_client_count_dec] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% reconnecting + {ok, ConnPid1} = emqtt:start_link([ {clean_start, false} + , {clientid, ClientID} + | Config + ]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid1) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% taking over + {ok, ConnPid2} = emqtt:start_link([ {clean_start, false} + , {clientid, ClientID} + | Config + ]), + {{ok, _}, {ok, [_, _]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid2) end, + [ emqx_cm_connected_client_count_inc + , emqx_cm_connected_client_count_dec + ], + 500 + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% abnormal exit of channel process + ChanPids = emqx_cm:all_channels(), + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> + lists:foreach( + fun(ChanPid) -> exit(ChanPid, kill) end, + ChanPids) + end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ok; +t_connected_client_count_persistent({'end', _Config}) -> + snabbkaffe:stop(), + ok. + +%% connections without client_id also contribute to connected client +%% count +t_connected_client_count_anonymous({init, Config}) -> + ok = snabbkaffe:start_trace(), + process_flag(trap_exit, true), + Config; +t_connected_client_count_anonymous(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% first client + {ok, ConnPid0} = emqtt:start_link([ {clean_start, true} + | Config]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid0) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% second client + {ok, ConnPid1} = emqtt:start_link([ {clean_start, true} + | Config]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid1) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(2, emqx_cm:get_connected_client_count()), + %% when first client disconnects, shouldn't affect the second + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid0) end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% reconnecting + {ok, ConnPid2} = emqtt:start_link([ {clean_start, true} + | Config + ]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid2) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(2, emqx_cm:get_connected_client_count()), + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid1) end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% abnormal exit of channel process + Chans = emqx_cm:all_channels(), + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> + lists:foreach( + fun(ChanPid) -> exit(ChanPid, kill) end, + Chans) + end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ok; +t_connected_client_count_anonymous({'end', _Config}) -> + snabbkaffe:stop(), + ok. + +t_connected_client_stats({init, Config}) -> + ok = supervisor:terminate_child(emqx_kernel_sup, emqx_stats), + {ok, _} = supervisor:restart_child(emqx_kernel_sup, emqx_stats), + ok = snabbkaffe:start_trace(), + Config; +t_connected_client_stats(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ?assertEqual(0, emqx_stats:getstat('live_connections.count')), + ?assertEqual(0, emqx_stats:getstat('live_connections.max')), + {ok, ConnPid} = emqtt:start_link([ {clean_start, true} + , {clientid, <<"clientid">>} + | Config + ]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid) end, + [emqx_cm_connected_client_count_inc] + ), + %% ensure stats are synchronized + {_, {ok, [_]}} = wait_for_stats( + fun emqx_cm:stats_fun/0, + [#{count_stat => 'live_connections.count', + max_stat => 'live_connections.max'}] + ), + ?assertEqual(1, emqx_stats:getstat('live_connections.count')), + ?assertEqual(1, emqx_stats:getstat('live_connections.max')), + {ok, {ok, [_]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid) end, + [emqx_cm_connected_client_count_dec] + ), + %% ensure stats are synchronized + {_, {ok, [_]}} = wait_for_stats( + fun emqx_cm:stats_fun/0, + [#{count_stat => 'live_connections.count', + max_stat => 'live_connections.max'}] + ), + ?assertEqual(0, emqx_stats:getstat('live_connections.count')), + ?assertEqual(1, emqx_stats:getstat('live_connections.max')), + ok; +t_connected_client_stats({'end', _Config}) -> + ok = snabbkaffe:stop(), + ok = supervisor:terminate_child(emqx_kernel_sup, emqx_stats), + {ok, _} = supervisor:restart_child(emqx_kernel_sup, emqx_stats), + ok. + +%% the count must be always non negative +t_connect_client_never_negative({init, Config}) -> + Config; +t_connect_client_never_negative(Config) when is_list(Config) -> + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% would go to -1 + ChanPid = list_to_pid("<0.0.1>"), + emqx_cm:mark_channel_disconnected(ChanPid), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% would be 0, if really went to -1 + emqx_cm:mark_channel_connected(ChanPid), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + ok; +t_connect_client_never_negative({'end', _Config}) -> + ok. + +wait_for_events(Action, Kinds) -> + wait_for_events(Action, Kinds, 500). + +wait_for_events(Action, Kinds, Timeout) -> + Predicate = fun(#{?snk_kind := K}) -> + lists:member(K, Kinds) + end, + N = length(Kinds), + {ok, Sub} = snabbkaffe_collector:subscribe(Predicate, N, Timeout, 0), + Res = Action(), + case snabbkaffe_collector:receive_events(Sub) of + {timeout, _} -> + {Res, timeout}; + {ok, Events} -> + {Res, {ok, Events}} + end. + +wait_for_stats(Action, Stats) -> + Predicate = fun(Event = #{?snk_kind := emqx_stats_setstat}) -> + Stat = maps:with( + [ count_stat + , max_stat + ], Event), + lists:member(Stat, Stats); + (_) -> + false + end, + N = length(Stats), + Timeout = 500, + {ok, Sub} = snabbkaffe_collector:subscribe(Predicate, N, Timeout, 0), + Res = Action(), + case snabbkaffe_collector:receive_events(Sub) of + {timeout, _} -> + {Res, timeout}; + {ok, Events} -> + {Res, {ok, Events}} + end. + +insert_fake_channels() -> + %% Insert copies to simulate missed counts + Tab = emqx_channel_info, + Key = ets:first(Tab), + [{_Chan, ChanInfo = #{conn_state := connected}, Stats}] = ets:lookup(Tab, Key), + ets:insert(Tab, [ {{"fake" ++ integer_to_list(N), undefined}, ChanInfo, Stats} + || N <- lists:seq(1, 9)]), + %% these should not be counted + ets:insert(Tab, [ { {"fake" ++ integer_to_list(N), undefined} + , ChanInfo#{conn_state := disconnected}, Stats} + || N <- lists:seq(10, 20)]). + recv_msgs(Count) -> recv_msgs(Count, []). diff --git a/test/emqx_channel_SUITE.erl b/test/emqx_channel_SUITE.erl index 9558dfd28..60e2541aa 100644 --- a/test/emqx_channel_SUITE.erl +++ b/test/emqx_channel_SUITE.erl @@ -33,6 +33,8 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> %% CM Meck ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), + ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Access Control Meck ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), ok = meck:expect(emqx_access_control, authenticate, @@ -835,4 +837,3 @@ session(InitFields) when is_map(InitFields) -> quota() -> emqx_limiter:init(zone, [{conn_messages_routing, {5, 1}}, {overall_messages_routing, {10, 1}}]). - diff --git a/test/emqx_connection_SUITE.erl b/test/emqx_connection_SUITE.erl index a6b2b614a..3dcc280d4 100644 --- a/test/emqx_connection_SUITE.erl +++ b/test/emqx_connection_SUITE.erl @@ -36,6 +36,8 @@ init_per_suite(Config) -> ok = meck:new(emqx_channel, [passthrough, no_history, no_link]), %% Meck Cm ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), + ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Meck Limiter ok = meck:new(emqx_limiter, [passthrough, no_history, no_link]), %% Meck Pd @@ -112,7 +114,7 @@ t_ws_pingreq_before_connected(_) -> t_info(_) -> CPid = spawn(fun() -> - receive + receive {'$gen_call', From, info} -> gen_server:reply(From, emqx_connection:info(st())) after @@ -132,7 +134,7 @@ t_info_limiter(_) -> t_stats(_) -> CPid = spawn(fun() -> - receive + receive {'$gen_call', From, stats} -> gen_server:reply(From, emqx_connection:stats(st())) after @@ -147,10 +149,10 @@ t_stats(_) -> {send_pend,0}| _] , Stats). t_process_msg(_) -> - with_conn(fun(CPid) -> - ok = meck:expect(emqx_channel, handle_in, - fun(_Packet, Channel) -> - {ok, Channel} + with_conn(fun(CPid) -> + ok = meck:expect(emqx_channel, handle_in, + fun(_Packet, Channel) -> + {ok, Channel} end), CPid ! {incoming, ?PACKET(?PINGREQ)}, CPid ! {incoming, undefined}, @@ -318,7 +320,7 @@ t_with_channel(_) -> t_handle_outgoing(_) -> ?assertEqual(ok, emqx_connection:handle_outgoing(?PACKET(?PINGRESP), st())), ?assertEqual(ok, emqx_connection:handle_outgoing([?PACKET(?PINGRESP)], st())). - + t_handle_info(_) -> ?assertMatch({ok, {event,running}, _NState}, emqx_connection:handle_info(activate_socket, st())), @@ -345,7 +347,7 @@ t_activate_socket(_) -> State = st(), {ok, NStats} = emqx_connection:activate_socket(State), ?assertEqual(running, emqx_connection:info(sockstate, NStats)), - + State1 = st(#{sockstate => blocked}), ?assertEqual({ok, State1}, emqx_connection:activate_socket(State1)), diff --git a/test/emqx_mqueue_SUITE.erl b/test/emqx_mqueue_SUITE.erl index 599e68da1..52b93dceb 100644 --- a/test/emqx_mqueue_SUITE.erl +++ b/test/emqx_mqueue_SUITE.erl @@ -121,7 +121,7 @@ t_priority_mqueue(_) -> ?assertEqual(5, ?Q:len(Q5)), {_, Q6} = ?Q:in(#message{qos = 1, topic = <<"t2">>}, Q5), ?assertEqual(5, ?Q:len(Q6)), - {{value, Msg}, Q7} = ?Q:out(Q6), + {{value, _Msg}, Q7} = ?Q:out(Q6), ?assertEqual(4, ?Q:len(Q7)). t_priority_mqueue_conservation(_) -> diff --git a/test/emqx_ws_connection_SUITE.erl b/test/emqx_ws_connection_SUITE.erl index 56d038c23..3c1aad3de 100644 --- a/test/emqx_ws_connection_SUITE.erl +++ b/test/emqx_ws_connection_SUITE.erl @@ -48,6 +48,10 @@ init_per_testcase(TestCase, Config) when TestCase =/= t_ws_pingreq_before_connected, TestCase =/= t_ws_non_check_origin -> + %% Meck Cm + ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), + ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Mock cowboy_req ok = meck:new(cowboy_req, [passthrough, no_history, no_link]), ok = meck:expect(cowboy_req, header, fun(_, _, _) -> <<>> end), @@ -95,7 +99,8 @@ end_per_testcase(TestCase, _Config) when TestCase =/= t_ws_pingreq_before_connected -> lists:foreach(fun meck:unload/1, - [cowboy_req, + [emqx_cm, + cowboy_req, emqx_zone, emqx_access_control, emqx_broker, @@ -389,14 +394,12 @@ t_handle_info_close(_) -> {[{close, _}], _St} = ?ws_conn:handle_info({close, protocol_error}, st()). t_handle_info_event(_) -> - ok = meck:new(emqx_cm, [passthrough, no_history]), ok = meck:expect(emqx_cm, register_channel, fun(_,_,_) -> ok end), ok = meck:expect(emqx_cm, insert_channel_info, fun(_,_,_) -> ok end), ok = meck:expect(emqx_cm, connection_closed, fun(_) -> true end), {ok, _} = ?ws_conn:handle_info({event, connected}, st()), {ok, _} = ?ws_conn:handle_info({event, disconnected}, st()), - {ok, _} = ?ws_conn:handle_info({event, updated}, st()), - ok = meck:unload(emqx_cm). + {ok, _} = ?ws_conn:handle_info({event, updated}, st()). t_handle_timeout_idle_timeout(_) -> TRef = make_ref(), From fec83590a6aa5d984bdc50a6e5dc2d862e96213e Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 8 Nov 2021 21:38:18 +0100 Subject: [PATCH 005/363] chore: bump release version to 4.4.0 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index c89dde010..24f6feb57 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.9"}). +-define(EMQX_RELEASE, {opensource, "4.4.0"}). -else. From 9ea5c5e58df3bcd8376c81cbbaf89adad7927439 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 8 Nov 2021 23:11:32 +0100 Subject: [PATCH 006/363] fix(emqx_mgmt): support v4.4 data export --- apps/emqx_management/include/emqx_mgmt.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index 6d510ed0c..26c566f50 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -32,4 +32,4 @@ -define(ERROR14, 114). %% OldPassword error -define(ERROR15, 115). %% bad topic --define(VERSIONS, ["4.0", "4.1", "4.2", "4.3"]). \ No newline at end of file +-define(VERSIONS, ["4.0", "4.1", "4.2", "4.3", "4.4"]). From 00ba4d32f3351235c2f1a6a933da2fd22dc18246 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 3 Nov 2021 11:19:37 +0800 Subject: [PATCH 007/363] fix(syntax): allow single quotes in the FROM clause --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 16f7193bb..a38d74e88 100644 --- a/rebar.config +++ b/rebar.config @@ -51,7 +51,7 @@ , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}} - , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.2"}}} + , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} From 2242bb93763bcd9c32b98db9e99b22529f946b89 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 9 Nov 2021 08:30:57 +0800 Subject: [PATCH 008/363] fix(rule): force max speed to 2 decimal digits of precision --- apps/emqx_rule_engine/src/emqx_rule_metrics.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index bc2b04c07..c88533f6f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -410,7 +410,7 @@ calculate_speed(CurrVal, #rule_speed{max = MaxSpeed0, last_v = LastVal, last5m_smpl = Last5MinSamples, tick = Tick + 1}. format_rule_speed(#rule_speed{max = Max, current = Current, last5m = Last5Min}) -> - #{max => Max, current => precision(Current, 2), last5m => precision(Last5Min, 2)}. + #{max => precision(Max, 2), current => precision(Current, 2), last5m => precision(Last5Min, 2)}. precision(Float, N) -> Base = math:pow(10, N), From 77c82cf1898ff790d359e34775ae6d91bcafd28b Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 9 Nov 2021 08:39:36 +0800 Subject: [PATCH 009/363] fix(code_style): some elvis complaints --- apps/emqx_rule_engine/src/emqx_rule_metrics.erl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index c88533f6f..57ba6e3da 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -73,6 +73,8 @@ , terminate/2 ]). +-elvis([{elvis_style, god_modules, disable}]). + -ifndef(TEST). -define(SECS_5M, 300). -define(SAMPLING, 10). @@ -235,7 +237,7 @@ start_link() -> init([]) -> erlang:process_flag(trap_exit, true), %% the overall counters - [ok = emqx_metrics:ensure(Metric)|| Metric <- overall_metrics()], + [ok = emqx_metrics:ensure(Metric) || Metric <- overall_metrics()], %% the speed metrics erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), {ok, #state{overall_rule_speed = #rule_speed{}}}. @@ -388,17 +390,19 @@ calculate_speed(CurrVal, #rule_speed{max = MaxSpeed0, last_v = LastVal, %% calculate the max speed since the emqx startup MaxSpeed = - if MaxSpeed0 >= CurrSpeed -> MaxSpeed0; - true -> CurrSpeed + case MaxSpeed0 >= CurrSpeed of + true -> MaxSpeed0; + false -> CurrSpeed end, %% calculate the average speed in last 5 mins {Last5MinSamples, Acc5Min, Last5Min} = - if Tick =< ?SAMPCOUNT_5M -> + case Tick =< ?SAMPCOUNT_5M of + true -> Acc = AccSpeed5Min0 + CurrSpeed, {lists:reverse([CurrSpeed | lists:reverse(Last5MinSamples0)]), Acc, Acc / Tick}; - true -> + false -> [FirstSpeed | Speeds] = Last5MinSamples0, Acc = AccSpeed5Min0 + CurrSpeed - FirstSpeed, {lists:reverse([CurrSpeed | lists:reverse(Speeds)]), From 354b0bc08e24edf99b7867eadf07eea29f55f00a Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 5 Nov 2021 16:11:48 +0800 Subject: [PATCH 010/363] refactor(emqx_st_statistics): optimize the implementation of topk --- .../src/emqx_st_statistics.appup.src | 9 - .../src/emqx_st_statistics.erl | 199 ++++++++++-------- .../test/emqx_st_statistics_SUITE.erl | 9 +- 3 files changed, 118 insertions(+), 99 deletions(-) delete mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics.appup.src diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src b/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src deleted file mode 100644 index dcf0d8cdd..000000000 --- a/apps/emqx_st_statistics/src/emqx_st_statistics.appup.src +++ /dev/null @@ -1,9 +0,0 @@ -%% -*-: erlang -*- -{VSN, - [ - {<<".*">>, []} - ], - [ - {<<".*">>, []} - ] -}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.erl b/apps/emqx_st_statistics/src/emqx_st_statistics.erl index 8a6c1535f..801a21c96 100644 --- a/apps/emqx_st_statistics/src/emqx_st_statistics.erl +++ b/apps/emqx_st_statistics/src/emqx_st_statistics.erl @@ -40,28 +40,30 @@ -compile(nowarn_unused_type). -type state() :: #{ config := proplist:proplist() - , index := index_map() - , begin_time := pos_integer() + , period := pos_integer() + , last_tick_at := pos_integer() , counter := counters:counter_ref() , enable := boolean() }. -type log() :: #{ topic := emqx_types:topic() - , times := pos_integer() + , count := pos_integer() , average := float() }. --type window_log() :: #{ begin_time := pos_integer() +-type window_log() :: #{ last_tick_at := pos_integer() , logs := [log()] }. -record(slow_log, { topic :: emqx_types:topic() - , times :: non_neg_integer() - , elapsed :: non_neg_integer() + , count :: pos_integer() + , elapsed :: pos_integer() }). --record(top_k, { key :: any() - , average :: float()}). +-record(top_k, { rank :: pos_integer() + , topic :: emqx_types:topic() + , average_count :: number() + , average_elapsed :: number()}). -type message() :: #message{}. @@ -70,11 +72,11 @@ -define(LOG_TAB, emqx_st_statistics_log). -define(TOPK_TAB, emqx_st_statistics_topk). -define(NOW, erlang:system_time(millisecond)). --define(TOP_KEY(Times, Topic), {Times, Topic}). -define(QUOTA_IDX, 1). --type top_key() :: ?TOP_KEY(pos_integer(), emqx_types:topic()). --type index_map() :: #{emqx_types:topic() => pos_integer()}. +-type slow_log() :: #slow_log{}. +-type top_k() :: #top_k{}. +-type top_k_map() :: #{emqx_types:topic() => top_k()}. %% erlang term order %% number < atom < reference < fun < port < pid < tuple < list < bit string @@ -124,8 +126,8 @@ init([Env]) -> Threshold = get_value(threshold_time, Env), load(Threshold, Counter), {ok, #{config => Env, - index => #{}, - begin_time => ?NOW, + period => 1, + last_tick_at => ?NOW, counter => Counter, enable => true}}. @@ -152,11 +154,11 @@ handle_cast(Msg, State) -> ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. -handle_info(notification_tick, #{config := Cfg} = State) -> +handle_info(notification_tick, #{config := Cfg, period := Period} = State) -> notification_tick(Cfg), - Index2 = do_notification(State), - {noreply, State#{index := Index2, - begin_time := ?NOW}}; + do_notification(State), + {noreply, State#{last_tick_at := ?NOW, + period := Period + 1}}; handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), @@ -183,9 +185,9 @@ init_log_tab(_) -> ]). init_topk_tab(_) -> - ?TOPK_TAB = ets:new(?TOPK_TAB, [ ordered_set, protected, named_table - , {keypos, #top_k.key}, {write_concurrency, true} - , {read_concurrency, false} + ?TOPK_TAB = ets:new(?TOPK_TAB, [ set, protected, named_table + , {keypos, #top_k.rank}, {write_concurrency, false} + , {read_concurrency, true} ]). -spec get_log_quota(counter:counter_ref()) -> boolean(). @@ -203,107 +205,104 @@ set_log_quota(Cfg, Counter) -> MaxLogNum = get_value(max_log_num, Cfg), counters:put(Counter, ?QUOTA_IDX, MaxLogNum). --spec update_log(message(), non_neg_integer()) -> ok. +-spec update_log(message(), pos_integer()) -> ok. update_log(#message{topic = Topic}, Elapsed) -> _ = ets:update_counter(?LOG_TAB, Topic, - [{#slow_log.times, 1}, {#slow_log.elapsed, Elapsed}], + [{#slow_log.count, 1}, {#slow_log.elapsed, Elapsed}], #slow_log{topic = Topic, - times = 1, - elapsed = Elapsed}), + count = 1, + elapsed = Elapsed}), ok. --spec do_notification(state()) -> index_map(). -do_notification(#{begin_time := BeginTime, +-spec do_notification(state()) -> true. +do_notification(#{last_tick_at := TickTime, config := Cfg, - index := IndexMap, + period := Period, counter := Counter}) -> Logs = ets:tab2list(?LOG_TAB), ets:delete_all_objects(?LOG_TAB), - start_publish(Logs, BeginTime, Cfg), + start_publish(Logs, TickTime, Cfg), set_log_quota(Cfg, Counter), MaxRecord = get_value(top_k_num, Cfg), - Size = ets:info(?TOPK_TAB, size), - update_top_k(Logs, erlang:max(0, MaxRecord - Size), IndexMap). + update_topk(Logs, MaxRecord, Period). --spec update_top_k(list(#slow_log{}), non_neg_integer(), index_map()) -> index_map(). -update_top_k([#slow_log{topic = Topic, - times = NewTimes, - elapsed = Elapsed} = Log | T], - Left, - IndexMap) -> - case maps:get(Topic, IndexMap, 0) of - 0 -> - try_insert_new(Log, Left, T, IndexMap); - Times -> - [#top_k{key = Key, average = Average}] = ets:lookup(?TOPK_TAB, ?TOP_KEY(Times, Topic)), - Times2 = Times + NewTimes, - Total = Times * Average + Elapsed, - Average2 = Total / Times2, - ets:delete(?TOPK_TAB, Key), - ets:insert(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times2, Topic), average = Average2}), - update_top_k(T, Left, IndexMap#{Topic := Times2}) +-spec update_topk(list(slow_log()), pos_integer(), pos_integer()) -> true. +update_topk(Logs, MaxRecord, Period) -> + TopkMap = get_topk_map(Period), + TopkMap2 = update_topk_map(Logs, Period, TopkMap), + SortFun = fun(A, B) -> + A#top_k.average_count > B#top_k.average_count + end, + TopkL = lists:sort(SortFun, maps:values(TopkMap2)), + TopkL2 = lists:sublist(TopkL, 1, MaxRecord), + update_topk_tab(TopkL2). + +-spec update_topk_map(list(slow_log()), pos_integer(), top_k_map()) -> top_k_map(). +update_topk_map([#slow_log{topic = Topic, + count = LogTimes, + elapsed = LogElapsed} | T], Period, TopkMap) -> + case maps:get(Topic, TopkMap, undefined) of + undefined -> + Record = #top_k{rank = 1, + topic = Topic, + average_count = LogTimes, + average_elapsed = LogElapsed}, + TopkMap2 = TopkMap#{Topic => Record}, + update_topk_map(T, Period, TopkMap2); + #top_k{average_count = AvgCount, + average_elapsed = AvgElapsed} = Record -> + NewPeriod = Period + 1, + %% (a + b) / c = a / c + b / c + %% average_count(elapsed) dived NewPeriod in function get_topk_maps + AvgCount2 = AvgCount + LogTimes / NewPeriod, + AvgElapsed2 = AvgElapsed + LogElapsed / NewPeriod, + Record2 = Record#top_k{average_count = AvgCount2, + average_elapsed = AvgElapsed2}, + update_topk_map(T, Period, TopkMap#{Topic := Record2}) end; -update_top_k([], _, IndexMap) -> - IndexMap. +update_topk_map([], _, TopkMap) -> + TopkMap. --spec try_insert_new(#slow_log{}, - non_neg_integer(), list(#slow_log{}), index_map()) -> index_map(). -try_insert_new(#slow_log{topic = Topic, - times = Times, - elapsed = Elapsed}, Left, Logs, IndexMap) when Left > 0 -> - Average = Elapsed / Times, - ets:insert_new(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times, Topic), average = Average}), - update_top_k(Logs, Left - 1, IndexMap#{Topic => Times}); +-spec update_topk_tab(list(top_k())) -> true. +update_topk_tab(Records) -> + Zip = fun(Rank, Item) -> Item#top_k{rank = Rank} end, + Len = erlang:length(Records), + RankedTopics = lists:zipwith(Zip, lists:seq(1, Len), Records), + ets:insert(?TOPK_TAB, RankedTopics). -try_insert_new(#slow_log{topic = Topic, - times = Times, - elapsed = Elapsed}, Left, Logs, IndexMap) -> - ?TOP_KEY(MinTimes, MinTopic) = MinKey = ets:first(?TOPK_TAB), - case MinTimes > Times of - true -> - update_top_k(Logs, Left, IndexMap); - _ -> - Average = Elapsed / Times, - ets:delete(?TOPK_TAB, MinKey), - ets:insert_new(?TOPK_TAB, #top_k{key = ?TOP_KEY(Times, Topic), average = Average}), - update_top_k(Logs, - Left - 1, - maps:put(Topic, Times, maps:remove(MinTopic, IndexMap))) - end. - -start_publish(Logs, BeginTime, Cfg) -> - emqx_pool:async_submit({fun do_publish/3, [Logs, BeginTime, Cfg]}). +start_publish(Logs, TickTime, Cfg) -> + emqx_pool:async_submit({fun do_publish/3, [Logs, TickTime, Cfg]}). do_publish([], _, _) -> ok; -do_publish(Logs, BeginTime, Cfg) -> +do_publish(Logs, TickTime, Cfg) -> BatchSize = get_value(notice_batch_size, Cfg), - do_publish(Logs, BatchSize, BeginTime, Cfg, []). + do_publish(Logs, BatchSize, TickTime, Cfg, []). -do_publish([Log | T], Size, BeginTime, Cfg, Cache) when Size > 0 -> +do_publish([Log | T], Size, TickTime, Cfg, Cache) when Size > 0 -> Cache2 = [convert_to_notice(Log) | Cache], - do_publish(T, Size - 1, BeginTime, Cfg, Cache2); + do_publish(T, Size - 1, TickTime, Cfg, Cache2); -do_publish(Logs, Size, BeginTime, Cfg, Cache) when Size =:= 0 -> - publish(BeginTime, Cfg, Cache), - do_publish(Logs, BeginTime, Cfg); +do_publish(Logs, Size, TickTime, Cfg, Cache) when Size =:= 0 -> + publish(TickTime, Cfg, Cache), + do_publish(Logs, TickTime, Cfg); -do_publish([], _, BeginTime, Cfg, Cache) -> - publish(BeginTime, Cfg, Cache), +do_publish([], _, TickTime, Cfg, Cache) -> + publish(TickTime, Cfg, Cache), ok. convert_to_notice(#slow_log{topic = Topic, - times = Times, + count = Count, elapsed = Elapsed}) -> #{topic => Topic, - times => Times, - average => Elapsed / Times}. + count => Count, + average => Elapsed / Count}. -publish(BeginTime, Cfg, Notices) -> - WindowLog = #{begin_time => BeginTime, +publish(TickTime, Cfg, Notices) -> + WindowLog = #{last_tick_at => TickTime, logs => Notices}, Payload = emqx_json:encode(WindowLog), _ = emqx:publish(#message{ id = emqx_guid:gen() @@ -322,6 +321,7 @@ load(Threshold, Counter) -> unload() -> emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/3). +-spec get_topic(proplists:proplist()) -> binary(). get_topic(Cfg) -> case get_value(notice_topic, Cfg) of Topic when is_binary(Topic) -> @@ -329,3 +329,24 @@ get_topic(Cfg) -> Topic -> erlang:list_to_binary(Topic) end. + +-spec get_topk_map(pos_integer()) -> top_k_map(). +get_topk_map(Period) -> + Size = ets:info(?TOPK_TAB, size), + get_topk_map(1, Size, Period, #{}). + +-spec get_topk_map(pos_integer(), + non_neg_integer(), pos_integer(), top_k_map()) -> top_k_map(). +get_topk_map(Index, Size, _, TopkMap) when Index > Size -> + TopkMap; +get_topk_map(Index, Size, Period, TopkMap) -> + [#top_k{topic = Topic, + average_count = AvgCount, + average_elapsed = AvgElapsed} = R] = ets:lookup(?TOPK_TAB, Index), + NewPeriod = Period + 1, + TotalTimes = AvgCount * Period, + AvgCount2 = TotalTimes / NewPeriod, + AvgElapsed2 = TotalTimes * AvgElapsed / NewPeriod, + TopkMap2 = TopkMap#{Topic => R#top_k{average_count = AvgCount2, + average_elapsed = AvgElapsed2}}, + get_topk_map(Index + 1, Size, Period, TopkMap2). diff --git a/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl b/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl index c08c8d986..e8bdd129f 100644 --- a/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl +++ b/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl @@ -61,7 +61,14 @@ t_log_and_pub(_) -> end, lists:seq(1, 10)), - ?assert(ets:info(?LOG_TAB, size) =:= 5), + timer:sleep(100), + + case ets:info(?LOG_TAB, size) of + 5 -> + ok; + _ -> + ?assert(ets:info(?TOPK_TAB, size) =/= 0) + end, timer:sleep(2400), From 388c29344a6ad7133f0a55480de9d6618a054962 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 10 Nov 2021 09:51:49 +0800 Subject: [PATCH 011/363] fix(relup): configs for plugins are missing after relup --- bin/install_upgrade.escript | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 0658bdef2..97548cba8 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -248,15 +248,20 @@ parse_version(V) when is_list(V) -> hd(string:tokens(V,"/")). check_and_install(TargetNode, Vsn) -> - {ok, [[CurrAppConf]]} = rpc:call(TargetNode, init, get_argument, [config], ?TIMEOUT), + %% Backup the vm.args. VM args should be unchanged during hot upgrade + %% but we still backup it here {ok, [[CurrVmArgs]]} = rpc:call(TargetNode, init, get_argument, [vm_args], ?TIMEOUT), - case filename:extension(CurrAppConf) of - ".config" -> - {ok, _} = file:copy(CurrAppConf, filename:join(["releases", Vsn, "sys.config"])); - _ -> - {ok, _} = file:copy(CurrAppConf++".config", filename:join(["releases", Vsn, "sys.config"])) - end, {ok, _} = file:copy(CurrVmArgs, filename:join(["releases", Vsn, "vm.args"])), + %% Backup the sys.config, this will be used when we check and install release + %% NOTE: We cannot backup the old sys.config directly, because the + %% configs for plugins are only in app-envs, not in the old sys.config + Configs0 = + [{AppName, rpc:call(TargetNode, application, get_all_env, [AppName], ?TIMEOUT)} + || {AppName, _, _} <- rpc:call(TargetNode, application, which_applications, [], ?TIMEOUT)], + Configs1 = [{AppName, Conf} || {AppName, Conf} <- Configs0, Conf =/= []], + ok = file:write_file(filename:join(["releases", Vsn, "sys.config"]), io_lib:format("~p.", [Configs1])), + + %% check and install release case rpc:call(TargetNode, release_handler, check_install_release, [Vsn], ?TIMEOUT) of {ok, _OtherVsn, _Desc} -> From fae815b35c283434121cbb3eba3c5be83f0e587c Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 10 Nov 2021 11:08:06 +0800 Subject: [PATCH 012/363] Feat/slow topic api (#6101) * feat(emqx_st_statistics): add api --- .../include/emqx_st_statistics.hrl | 25 +++ .../src/emqx_st_statistics.erl | 17 +- .../src/emqx_st_statistics_api.erl | 75 +++++++++ .../src/emqx_st_statistics_app.erl | 8 +- .../test/emqx_st_statistics_api_SUITE.erl | 152 ++++++++++++++++++ .../src/emqx_mod_st_statistics.erl | 47 ++++++ lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- .../emqx_modules/src/emqx_modules.appup.src | 6 + priv/emqx.schema | 1 + 9 files changed, 328 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_st_statistics/include/emqx_st_statistics.hrl create mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics_api.erl create mode 100644 apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl create mode 100644 lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl diff --git a/apps/emqx_st_statistics/include/emqx_st_statistics.hrl b/apps/emqx_st_statistics/include/emqx_st_statistics.hrl new file mode 100644 index 000000000..9184c3194 --- /dev/null +++ b/apps/emqx_st_statistics/include/emqx_st_statistics.hrl @@ -0,0 +1,25 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(LOG_TAB, emqx_st_statistics_log). +-define(TOPK_TAB, emqx_st_statistics_topk). + +-record(top_k, { rank :: pos_integer() + , topic :: emqx_types:topic() + , average_count :: number() + , average_elapsed :: number()}). + +-type top_k() :: #top_k{}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.erl b/apps/emqx_st_statistics/src/emqx_st_statistics.erl index 801a21c96..2aa5c9fb4 100644 --- a/apps/emqx_st_statistics/src/emqx_st_statistics.erl +++ b/apps/emqx_st_statistics/src/emqx_st_statistics.erl @@ -25,7 +25,7 @@ -logger_header("[SLOW TOPICS]"). -export([ start_link/1, on_publish_done/3, enable/0 - , disable/0 + , disable/0, clear_history/0 ]). %% gen_server callbacks @@ -78,6 +78,12 @@ -type top_k() :: #top_k{}. -type top_k_map() :: #{emqx_types:topic() => top_k()}. +-ifdef(TEST). +-define(TOPK_ACCESS, public). +-else. +-define(TOPK_ACCESS, protected). +-endif. + %% erlang term order %% number < atom < reference < fun < port < pid < tuple < list < bit string @@ -106,6 +112,9 @@ on_publish_done(#message{timestamp = Timestamp} = Msg, Threshold, Counter) -> ok end. +clear_history() -> + gen_server:call(?MODULE, ?FUNCTION_NAME). + enable() -> gen_server:call(?MODULE, {enable, true}). @@ -146,6 +155,10 @@ handle_call({enable, Enable}, _From, end, {reply, ok, State2}; +handle_call(clear_history, _, State) -> + ets:delete_all_objects(?TOPK_TAB), + {reply, ok, State}; + handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, State}. @@ -185,7 +198,7 @@ init_log_tab(_) -> ]). init_topk_tab(_) -> - ?TOPK_TAB = ets:new(?TOPK_TAB, [ set, protected, named_table + ?TOPK_TAB = ets:new(?TOPK_TAB, [ set, ?TOPK_ACCESS, named_table , {keypos, #top_k.rank}, {write_concurrency, false} , {read_concurrency, true} ]). diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_api.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_api.erl new file mode 100644 index 000000000..3980caccd --- /dev/null +++ b/apps/emqx_st_statistics/src/emqx_st_statistics_api.erl @@ -0,0 +1,75 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics_api). + +-rest_api(#{name => clear_history, + method => 'DELETE', + path => "/slow_topic", + func => clear_history, + descr => "Clear current data and re count slow topic"}). + +-rest_api(#{name => get_history, + method => 'GET', + path => "/slow_topic", + func => get_history, + descr => "Get slow topics statistics record data"}). + +-export([ clear_history/2 + , get_history/2 + ]). + +-include_lib("emqx_st_statistics/include/emqx_st_statistics.hrl"). + +-import(minirest, [return/1]). + +%%-------------------------------------------------------------------- +%% HTTP API +%%-------------------------------------------------------------------- + +clear_history(_Bindings, _Params) -> + ok = emqx_st_statistics:clear_history(), + return(ok). + +get_history(_Bindings, Params) -> + PageT = proplists:get_value(<<"_page">>, Params), + LimitT = proplists:get_value(<<"_limit">>, Params), + Page = erlang:binary_to_integer(PageT), + Limit = erlang:binary_to_integer(LimitT), + Start = (Page - 1) * Limit + 1, + Size = ets:info(?TOPK_TAB, size), + EndT = Start + Limit - 1, + End = erlang:min(EndT, Size), + Infos = lists:foldl(fun(Rank, Acc) -> + [#top_k{topic = Topic + , average_count = Count + , average_elapsed = Elapsed}] = ets:lookup(?TOPK_TAB, Rank), + + Info =[ {rank, Rank} + , {topic, Topic} + , {count, Count} + , {elapsed, Elapsed}], + + [Info | Acc] + end, + [], + lists:seq(Start, End)), + + return({ok, #{meta => #{page => Page, + limit => Limit, + hasnext => End < Size, + count => End - Start + 1}, + data => Infos}}). diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl index e5b62a00e..b8e5987db 100644 --- a/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl +++ b/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl @@ -22,11 +22,15 @@ -export([ start/2 , stop/1 + , start/0 ]). start(_Type, _Args) -> - Env = application:get_all_env(emqx_st_statistics), - {ok, Sup} = emqx_st_statistics_sup:start_link(Env), + start(). + +start() -> + Conf = application:get_all_env(emqx_st_statistics), + {ok, Sup} = emqx_st_statistics_sup:start_link(Conf), {ok, Sup}. stop(_State) -> diff --git a/apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl b/apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl new file mode 100644 index 000000000..e6b99e70a --- /dev/null +++ b/apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl @@ -0,0 +1,152 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_st_statistics_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx_management/include/emqx_mgmt.hrl"). +-include_lib("emqx_st_statistics/include/emqx_st_statistics.hrl"). + +-define(CONTENT_TYPE, "application/x-www-form-urlencoded"). + +-define(HOST, "http://127.0.0.1:8081/"). + +-define(API_VERSION, "v4"). + +-define(BASE_PATH, "api"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_modules), + emqx_ct_helpers:start_apps([emqx_st_statistics, emqx_management]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_st_statistics, emqx_management]), + Config. + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, Config) -> + Config. + +get(Key, ResponseBody) -> + maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). + +lookup_alarm(Name, [#{<<"name">> := Name} | _More]) -> + true; +lookup_alarm(Name, [_Alarm | More]) -> + lookup_alarm(Name, More); +lookup_alarm(_Name, []) -> + false. + +is_existing(Name, [#{name := Name} | _More]) -> + true; +is_existing(Name, [_Alarm | More]) -> + is_existing(Name, More); +is_existing(_Name, []) -> + false. + +t_get_history(_) -> + ets:insert(?TOPK_TAB, #top_k{rank = 1, + topic = <<"test">>, + average_count = 12, + average_elapsed = 1500}), + + {ok, Data} = request_api(get, api_path(["slow_topic"]), "_page=1&_limit=10", + auth_header_()), + + Return = #{meta => #{page => 1, + limit => 10, + hasnext => false, + count => 1}, + data => [#{topic => <<"test">>, + rank => 1, + elapsed => 1500, + count => 12}], + code => 0}, + + ShouldBe = emqx_json:encode(Return), + + ?assertEqual(ShouldBe, erlang:list_to_binary(Data)). + +t_clear(_) -> + ets:insert(?TOPK_TAB, #top_k{rank = 1, + topic = <<"test">>, + average_count = 12, + average_elapsed = 1500}), + + {ok, _} = request_api(delete, api_path(["slow_topic"]), [], + auth_header_()), + + ?assertEqual(0, ets:info(?TOPK_TAB, size)). + +request_api(Method, Url, Auth) -> + request_api(Method, Url, [], Auth, []). + +request_api(Method, Url, QueryParams, Auth) -> + request_api(Method, Url, QueryParams, Auth, []). + +request_api(Method, Url, QueryParams, Auth, []) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth]}); +request_api(Method, Url, QueryParams, Auth, Body) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + +do_request_api(Method, Request)-> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +auth_header_() -> + AppId = <<"admin">>, + AppSecret = <<"public">>, + auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User,":",Pass])), + {"Authorization","Basic " ++ Encoded}. + +api_path(Parts)-> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). + +filter(List, Key, Value) -> + lists:filter(fun(Item) -> + maps:get(Key, Item) == Value + end, List). diff --git a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl b/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl new file mode 100644 index 000000000..01a1444bd --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_st_statistics). + +-behaviour(emqx_gen_mod). + +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[st_statistics]"). + +%% emqx_gen_mod callbacks +-export([ load/1 + , unload/1 + , description/0 + ]). + +%%-------------------------------------------------------------------- +%% Load/Unload +%%-------------------------------------------------------------------- + +-spec(load(list()) -> ok). +load(_Env) -> + _ = emqx_st_statistics_app:start(), + ok. + +-spec(unload(list()) -> ok). +unload(_Env) -> + emqx_st_statistics_app:stop(undefined). + +description() -> + "EMQ X Slow Topic Statistics Module". + +%%-------------------------------------------------------------------- diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index bcb05fe31..47a3d8888 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.3"}, + {vsn, "4.3.4"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index f52ba1a61..aed228213 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,6 +1,9 @@ %% -*-: erlang -*- {VSN, [ + {"4.3.3", [ + {load_module, emqx_mod_st_statistics, brutal_purge, soft_purge, []} + ]}, {"4.3.2", [ {load_module, emqx_mod_presence, brutal_purge, soft_purge, []} ]}, @@ -16,6 +19,9 @@ {<<".*">>, []} ], [ + {"4.3.3", [ + {load_module, emqx_mod_st_statistics, brutal_purge, soft_purge, []} + ]}, {"4.3.2", [ {load_module, emqx_mod_presence, brutal_purge, soft_purge, []} ]}, diff --git a/priv/emqx.schema b/priv/emqx.schema index 4b33cf65c..6418501e3 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2217,6 +2217,7 @@ end}. [{emqx_mod_rewrite, Rewrites()}], [{emqx_mod_topic_metrics, []}], [{emqx_mod_delayed, []}], + [{emqx_mod_st_statistics, []}], [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] ]) end}. From 06a1b37992494fa0b6c22ac82ab7479993186c8b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 10 Nov 2021 15:41:31 +0800 Subject: [PATCH 013/363] fix(test): flaky mqtt expiry test case. (#6111) --- test/emqx_mqtt_SUITE.erl | 112 +++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 39 deletions(-) diff --git a/test/emqx_mqtt_SUITE.erl b/test/emqx_mqtt_SUITE.erl index ae7e7382c..3a77134b8 100644 --- a/test/emqx_mqtt_SUITE.erl +++ b/test/emqx_mqtt_SUITE.erl @@ -62,79 +62,104 @@ t_conn_stats(_) -> t_tcp_sock_passive(_) -> with_client(fun(CPid) -> CPid ! {tcp_passive, sock} end, []). -t_message_expiry_interval_1(_) -> - ClientA = message_expiry_interval_init(), - [message_expiry_interval_exipred(ClientA, QoS) || QoS <- [0,1,2]], - emqtt:stop(ClientA). +t_message_expiry_interval(_) -> + {CPublish, CControl} = message_expiry_interval_init(), + [message_expiry_interval_exipred(CPublish, CControl, QoS) || QoS <- [0,1,2]], + emqtt:stop(CPublish), + emqtt:stop(CControl). -t_message_expiry_interval_2(_) -> - ClientA = message_expiry_interval_init(), - [message_expiry_interval_not_exipred(ClientA, QoS) || QoS <- [0,1,2]], - emqtt:stop(ClientA). +t_message_not_expiry_interval(_) -> + {CPublish, CControl} = message_expiry_interval_init(), + [message_expiry_interval_not_exipred(CPublish, CControl, QoS) || QoS <- [0,1,2]], + emqtt:stop(CPublish), + emqtt:stop(CControl). message_expiry_interval_init() -> - {ok, ClientA} = emqtt:start_link([{proto_ver,v5}, - {clientid, <<"client-a">>}, + {ok, CPublish} = emqtt:start_link([{proto_ver,v5}, + {clientid, <<"Client-Publish">>}, {clean_start, false}, {properties, #{'Session-Expiry-Interval' => 360}}]), - {ok, ClientB} = emqtt:start_link([{proto_ver,v5}, - {clientid, <<"client-b">>}, + {ok, CVerify} = emqtt:start_link([{proto_ver,v5}, + {clientid, <<"Client-Verify">>}, {clean_start, false}, {properties, #{'Session-Expiry-Interval' => 360}}]), - {ok, _} = emqtt:connect(ClientA), - {ok, _} = emqtt:connect(ClientB), - %% subscribe and disconnect client-b - emqtt:subscribe(ClientB, <<"t/a">>, 1), - emqtt:stop(ClientB), - ClientA. + {ok, CControl} = emqtt:start_link([{proto_ver,v5}, + {clientid, <<"Client-Control">>}, + {clean_start, false}, + {properties, #{'Session-Expiry-Interval' => 360}}]), + {ok, _} = emqtt:connect(CPublish), + {ok, _} = emqtt:connect(CVerify), + {ok, _} = emqtt:connect(CControl), + %% subscribe and disconnect Client-verify + emqtt:subscribe(CControl, <<"t/a">>, 1), + emqtt:subscribe(CVerify, <<"t/a">>, 1), + emqtt:stop(CVerify), + {CPublish, CControl}. -message_expiry_interval_exipred(ClientA, QoS) -> +message_expiry_interval_exipred(CPublish, CControl, QoS) -> ct:pal("~p ~p", [?FUNCTION_NAME, QoS]), %% publish to t/a and waiting for the message expired - emqtt:publish(ClientA, <<"t/a">>, #{'Message-Expiry-Interval' => 1}, <<"this will be purged in 1s">>, [{qos, QoS}]), - ct:sleep(2000), + emqtt:publish(CPublish, <<"t/a">>, #{'Message-Expiry-Interval' => 1}, + <<"this will be purged in 1s">>, [{qos, QoS}]), + %% CControl make sure publish already store in broker. + receive + {publish,#{client_pid := CControl, topic := <<"t/a">>}} -> + ok + after 1000 -> + ct:fail(should_receive_publish) + end, + ct:sleep(1100), - %% resume the session for client-b - {ok, ClientB1} = emqtt:start_link([{proto_ver,v5}, - {clientid, <<"client-b">>}, + %% resume the session for Client-Verify + {ok, CVerify} = emqtt:start_link([{proto_ver,v5}, + {clientid, <<"Client-Verify">>}, {clean_start, false}, {properties, #{'Session-Expiry-Interval' => 360}}]), - {ok, _} = emqtt:connect(ClientB1), + {ok, _} = emqtt:connect(CVerify), - %% verify client-b could not receive the publish message + %% verify Client-Verify could not receive the publish message receive - {publish,#{client_pid := ClientB1, topic := <<"t/a">>}} -> + {publish,#{client_pid := CVerify, topic := <<"t/a">>}} -> ct:fail(should_have_expired) after 300 -> ok end, - emqtt:stop(ClientB1). + emqtt:stop(CVerify). -message_expiry_interval_not_exipred(ClientA, QoS) -> +message_expiry_interval_not_exipred(CPublish, CControl, QoS) -> ct:pal("~p ~p", [?FUNCTION_NAME, QoS]), %% publish to t/a - emqtt:publish(ClientA, <<"t/a">>, #{'Message-Expiry-Interval' => 20}, <<"this will be purged in 1s">>, [{qos, QoS}]), + emqtt:publish(CPublish, <<"t/a">>, #{'Message-Expiry-Interval' => 20}, + <<"this will be purged in 20s">>, [{qos, QoS}]), - %% wait for 1s and then resume the session for client-b, the message should not expires + %% CControl make sure publish already store in broker. + receive + {publish,#{client_pid := CControl, topic := <<"t/a">>}} -> + ok + after 1000 -> + ct:fail(should_receive_publish) + end, + + %% wait for 1.2s and then resume the session for Client-Verify, the message should not expires %% as Message-Expiry-Interval = 20s - ct:sleep(1000), - {ok, ClientB1} = emqtt:start_link([{proto_ver,v5}, - {clientid, <<"client-b">>}, + ct:sleep(1200), + {ok, CVerify} = emqtt:start_link([{proto_ver,v5}, + {clientid, <<"Client-Verify">>}, {clean_start, false}, {properties, #{'Session-Expiry-Interval' => 360}}]), - {ok, _} = emqtt:connect(ClientB1), + {ok, _} = emqtt:connect(CVerify), - %% verify client-b could receive the publish message and the Message-Expiry-Interval is set + %% verify Client-Verify could receive the publish message and the Message-Expiry-Interval is set receive - {publish,#{client_pid := ClientB1, topic := <<"t/a">>, + {publish,#{client_pid := CVerify, topic := <<"t/a">>, properties := #{'Message-Expiry-Interval' := MsgExpItvl}}} - when MsgExpItvl < 20 -> ok; + when MsgExpItvl =< 20 -> ok; {publish, _} = Msg -> ct:fail({incorrect_publish, Msg}) after 300 -> ct:fail(no_publish_received) end, - emqtt:stop(ClientB1). + emqtt:stop(CVerify). with_client(TestFun, _Options) -> ClientId = <<"t_conn">>, @@ -156,6 +181,15 @@ t_async_set_keepalive('end', _Config) -> ok. t_async_set_keepalive(_) -> + case os:type() of + {unix, darwin} -> + %% Mac OSX don't support the feature + ok; + _ -> + do_async_set_keepalive() + end. + +do_async_set_keepalive() -> ClientID = <<"client-tcp-keepalive">>, {ok, Client} = emqtt:start_link([{host, "localhost"}, {proto_ver,v5}, From 1dd18aa07a2168ed3fd706c70aa14a5b07290833 Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 11 Nov 2021 10:16:06 +0800 Subject: [PATCH 014/363] =?UTF-8?q?fix(emqx=5Fst=5Fstatistics):=20change?= =?UTF-8?q?=20emqx=5Fst=5Fstatistics=20implementation=20fro=E2=80=A6=20(#6?= =?UTF-8?q?115)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(emqx_st_statistics): change emqx_st_statistics implementation from plugin to module --- .../etc/emqx_st_statistics.conf | 38 ------------ .../priv/emqx_st_statistics.schema | 58 ------------------- apps/emqx_st_statistics/rebar.config | 23 -------- .../src/emqx_st_statistics.app.src | 12 ---- .../src/emqx_st_statistics_app.erl | 37 ------------ .../src/emqx_st_statistics_sup.erl | 35 ----------- etc/emqx.conf | 38 ++++++++++++ .../include/emqx_st_statistics.hrl | 0 .../src/emqx_mod_st_statistics.erl | 47 --------------- lib-ce/emqx_modules/src/emqx_mod_sup.erl | 21 ++++--- .../emqx_mod_st_statistics.erl | 40 +++++++++---- .../emqx_st_statistics_api.erl | 4 +- .../test/emqx_st_statistics_SUITE.erl | 16 +++-- .../test/emqx_st_statistics_api_SUITE.erl | 8 ++- priv/emqx.schema | 44 +++++++++++++- rebar.config.erl | 1 - 16 files changed, 140 insertions(+), 282 deletions(-) delete mode 100644 apps/emqx_st_statistics/etc/emqx_st_statistics.conf delete mode 100644 apps/emqx_st_statistics/priv/emqx_st_statistics.schema delete mode 100644 apps/emqx_st_statistics/rebar.config delete mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics.app.src delete mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics_app.erl delete mode 100644 apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl rename {apps/emqx_st_statistics => lib-ce/emqx_modules}/include/emqx_st_statistics.hrl (100%) delete mode 100644 lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl rename apps/emqx_st_statistics/src/emqx_st_statistics.erl => lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl (94%) rename {apps/emqx_st_statistics/src => lib-ce/emqx_modules/src/emqx_st_statistics}/emqx_st_statistics_api.erl (96%) rename {apps/emqx_st_statistics => lib-ce/emqx_modules}/test/emqx_st_statistics_SUITE.erl (93%) rename {apps/emqx_st_statistics => lib-ce/emqx_modules}/test/emqx_st_statistics_api_SUITE.erl (94%) diff --git a/apps/emqx_st_statistics/etc/emqx_st_statistics.conf b/apps/emqx_st_statistics/etc/emqx_st_statistics.conf deleted file mode 100644 index 88ca0c37f..000000000 --- a/apps/emqx_st_statistics/etc/emqx_st_statistics.conf +++ /dev/null @@ -1,38 +0,0 @@ -##-------------------------------------------------------------------- -## EMQ X Slow Topics Statistics -##-------------------------------------------------------------------- - -## Threshold time of slow topics statistics -## -## Default: 10 seconds -st_statistics.threshold_time = 10S - -## Time window of slow topics statistics -## -## Value: 5 minutes -st_statistics.time_window = 5M - -## Maximum of slow topics log, log will clear when enter new time window -## -## Value: 500 -st_statistics.max_log_num = 500 - -## Top-K record for slow topics, update from logs -## -## Value: 500 -st_statistics.top_k_num = 500 - -## Topic of notification -## -## Defaut: $slow_topics -st_statistics.notice_topic = $slow_topics - -## QoS of notification message in notice topic -## -## Defaut: 0 -st_statistics.notice_qos = 0 - -## Maximum information number in one notification -## -## Default: 500 -st_statistics.notice_batch_size = 500 diff --git a/apps/emqx_st_statistics/priv/emqx_st_statistics.schema b/apps/emqx_st_statistics/priv/emqx_st_statistics.schema deleted file mode 100644 index da9a2f810..000000000 --- a/apps/emqx_st_statistics/priv/emqx_st_statistics.schema +++ /dev/null @@ -1,58 +0,0 @@ -%%-*- mode: erlang -*- -%% st_statistics config mapping - -%% Threshold time of slow topics statistics -%% {$configurable} -{mapping, "st_statistics.threshold_time", "emqx_st_statistics.threshold_time", - [ - {default, "10S"}, - {datatype, [integer, {duration, ms}]} - ]}. - -%% Time window of slow topics statistics -%% {$configurable} -{mapping, "st_statistics.time_window", "emqx_st_statistics.time_window", - [ - {default, "5M"}, - {datatype, [integer, {duration, ms}]} - ]}. - -%% Maximum of slow topics log -%% {$configurable} -{mapping, "st_statistics.max_log_num", "emqx_st_statistics.max_log_num", - [ - {default, 500}, - {datatype, integer} - ]}. - -%% Top-K record for slow topics, update from logs -%% {$configurable} -{mapping, "st_statistics.top_k_num", "emqx_st_statistics.top_k_num", - [ - {default, 500}, - {datatype, integer} - ]}. - -%% Topic of notification -%% {$configurable} -{mapping, "st_statistics.notice_topic", "emqx_st_statistics.notice_topic", - [ - {default, <<"slow_topics">>}, - {datatype, string} - ]}. - -%% QoS of notification message in notice topic -%% {$configurable} -{mapping, "st_statistics.notice_qos", "emqx_st_statistics.notice_qos", - [ - {default, 0}, - {datatype, integer} - ]}. - -%% Maximum entities per notification message -%% {$configurable} -{mapping, "st_statistics.notice_batch_size", "emqx_st_statistics.notice_batch_size", - [ - {default, 500}, - {datatype, integer} - ]}. diff --git a/apps/emqx_st_statistics/rebar.config b/apps/emqx_st_statistics/rebar.config deleted file mode 100644 index 6433d92d6..000000000 --- a/apps/emqx_st_statistics/rebar.config +++ /dev/null @@ -1,23 +0,0 @@ -{deps, []}. - -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard - ]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. - -{profiles, - [{test, - [{deps, - [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}}]} - ]} - ]}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.app.src b/apps/emqx_st_statistics/src/emqx_st_statistics.app.src deleted file mode 100644 index b1eb1612a..000000000 --- a/apps/emqx_st_statistics/src/emqx_st_statistics.app.src +++ /dev/null @@ -1,12 +0,0 @@ -{application, emqx_st_statistics, - [{description, "EMQ X Slow Topics Statistics"}, - {vsn, "1.0.0"}, % strict semver, bump manually! - {modules, []}, - {registered, [emqx_st_statistics_sup]}, - {applications, [kernel,stdlib]}, - {mod, {emqx_st_statistics_app,[]}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, []} - ]}. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl deleted file mode 100644 index b8e5987db..000000000 --- a/apps/emqx_st_statistics/src/emqx_st_statistics_app.erl +++ /dev/null @@ -1,37 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_st_statistics_app). - --behaviour(application). - --emqx_plugin(?MODULE). - --export([ start/2 - , stop/1 - , start/0 - ]). - -start(_Type, _Args) -> - start(). - -start() -> - Conf = application:get_all_env(emqx_st_statistics), - {ok, Sup} = emqx_st_statistics_sup:start_link(Conf), - {ok, Sup}. - -stop(_State) -> - ok. diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl b/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl deleted file mode 100644 index cbff854ef..000000000 --- a/apps/emqx_st_statistics/src/emqx_st_statistics_sup.erl +++ /dev/null @@ -1,35 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_st_statistics_sup). - --behaviour(supervisor). - --export([start_link/1]). - --export([init/1]). - -start_link(Env) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]). - -init([Env]) -> - {ok, {{one_for_one, 10, 3600}, - [#{id => st_statistics, - start => {emqx_st_statistics, start_link, [Env]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_st_statistics]}]}}. diff --git a/etc/emqx.conf b/etc/emqx.conf index cf1279de1..a94afd961 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2211,6 +2211,44 @@ module.presence.qos = 1 ## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1 ## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 +##-------------------------------------------------------------------- +## Slow Topic Statistics Module + +## Threshold time of slow topics statistics +## +## Default: 10 seconds +#module.st_statistics.threshold_time = 10S + +## Time window of slow topics statistics +## +## Value: 5 minutes +#module.st_statistics.time_window = 5M + +## Maximum of slow topics log, log will clear when enter new time window +## +## Value: 500 +#module.st_statistics.max_log_num = 500 + +## Top-K record for slow topics, update from logs +## +## Value: 500 +#module.st_statistics.top_k_num = 500 + +## Topic of notification +## +## Defaut: $slow_topics +#module.st_statistics.notice_topic = $slow_topics + +## QoS of notification message in notice topic +## +## Defaut: 0 +#module.st_statistics.notice_qos = 0 + +## Maximum information number in one notification +## +## Default: 500 +#module.st_statistics.notice_batch_size = 500 + ## CONFIG_SECTION_END=modules ================================================== ##------------------------------------------------------------------- diff --git a/apps/emqx_st_statistics/include/emqx_st_statistics.hrl b/lib-ce/emqx_modules/include/emqx_st_statistics.hrl similarity index 100% rename from apps/emqx_st_statistics/include/emqx_st_statistics.hrl rename to lib-ce/emqx_modules/include/emqx_st_statistics.hrl diff --git a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl b/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl deleted file mode 100644 index 01a1444bd..000000000 --- a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl +++ /dev/null @@ -1,47 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_mod_st_statistics). - --behaviour(emqx_gen_mod). - --include_lib("emqx/include/logger.hrl"). - --logger_header("[st_statistics]"). - -%% emqx_gen_mod callbacks --export([ load/1 - , unload/1 - , description/0 - ]). - -%%-------------------------------------------------------------------- -%% Load/Unload -%%-------------------------------------------------------------------- - --spec(load(list()) -> ok). -load(_Env) -> - _ = emqx_st_statistics_app:start(), - ok. - --spec(unload(list()) -> ok). -unload(_Env) -> - emqx_st_statistics_app:stop(undefined). - -description() -> - "EMQ X Slow Topic Statistics Module". - -%%-------------------------------------------------------------------- diff --git a/lib-ce/emqx_modules/src/emqx_mod_sup.erl b/lib-ce/emqx_modules/src/emqx_mod_sup.erl index 755e52a60..60907f580 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_sup.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_sup.erl @@ -23,18 +23,22 @@ -export([ start_link/0 , start_child/1 , start_child/2 + , start_child/3 , stop_child/1 ]). -export([init/1]). %% Helper macro for declaring children of supervisor --define(CHILD(Mod, Type), #{id => Mod, - start => {Mod, start_link, []}, - restart => permanent, - shutdown => 5000, - type => Type, - modules => [Mod]}). +-define(CHILD(Mod, Type, Args), + #{id => Mod, + start => {Mod, start_link, Args}, + restart => permanent, + shutdown => 5000, + type => Type, + modules => [Mod]}). + +-define(CHILD(MOD, Type), ?CHILD(MOD, Type, [])). -spec(start_link() -> startlink_ret()). start_link() -> @@ -48,6 +52,10 @@ start_child(ChildSpec) when is_map(ChildSpec) -> start_child(Mod, Type) when is_atom(Mod) andalso is_atom(Type) -> assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Type))). +-spec start_child(atom(), atom(), list(any())) -> ok. +start_child(Mod, Type, Args) when is_atom(Mod) andalso is_atom(Type) -> + assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Type, Args))). + -spec(stop_child(any()) -> ok | {error, term()}). stop_child(ChildId) -> case supervisor:terminate_child(?MODULE, ChildId) of @@ -71,4 +79,3 @@ assert_started({ok, _Pid}) -> ok; assert_started({ok, _Pid, _Info}) -> ok; assert_started({error, {already_tarted, _Pid}}) -> ok; assert_started({error, Reason}) -> erlang:error(Reason). - diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics.erl b/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl similarity index 94% rename from apps/emqx_st_statistics/src/emqx_st_statistics.erl rename to lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl index 2aa5c9fb4..a66975767 100644 --- a/apps/emqx_st_statistics/src/emqx_st_statistics.erl +++ b/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_st_statistics). +-module(emqx_mod_st_statistics). +-behaviour(emqx_gen_mod). -behaviour(gen_server). -include_lib("include/emqx.hrl"). -include_lib("include/logger.hrl"). --include_lib("stdlib/include/ms_transform.hrl"). +-include("include/emqx_st_statistics.hrl"). -logger_header("[SLOW TOPICS]"). @@ -28,6 +29,12 @@ , disable/0, clear_history/0 ]). +%% emqx_gen_mod callbacks +-export([ load/1 + , unload/1 + , description/0 + ]). + %% gen_server callbacks -export([ init/1 , handle_call/3 @@ -60,22 +67,14 @@ , elapsed :: pos_integer() }). --record(top_k, { rank :: pos_integer() - , topic :: emqx_types:topic() - , average_count :: number() - , average_elapsed :: number()}). - -type message() :: #message{}. -import(proplists, [get_value/2]). --define(LOG_TAB, emqx_st_statistics_log). --define(TOPK_TAB, emqx_st_statistics_topk). -define(NOW, erlang:system_time(millisecond)). -define(QUOTA_IDX, 1). -type slow_log() :: #slow_log{}. --type top_k() :: #top_k{}. -type top_k_map() :: #{emqx_types:topic() => top_k()}. -ifdef(TEST). @@ -90,9 +89,26 @@ %% ets ordered_set is ascending by term order %%-------------------------------------------------------------------- -%% APIs +%% Load/Unload %%-------------------------------------------------------------------- +-spec(load(list()) -> ok). +load(Env) -> + emqx_mod_sup:start_child(?MODULE, worker, [Env]), + ok. + +-spec(unload(list()) -> ok). +unload(_Env) -> + _ = emqx_mod_sup:stop_child(?MODULE), + ok. + +description() -> + "EMQ X Slow Topic Statistics Module". + +%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- %% @doc Start the st_statistics -spec(start_link(Env :: list()) -> emqx_types:startlink_ret()). start_link(Env) -> diff --git a/apps/emqx_st_statistics/src/emqx_st_statistics_api.erl b/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl similarity index 96% rename from apps/emqx_st_statistics/src/emqx_st_statistics_api.erl rename to lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl index 3980caccd..328fddf28 100644 --- a/apps/emqx_st_statistics/src/emqx_st_statistics_api.erl +++ b/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl @@ -32,7 +32,7 @@ , get_history/2 ]). --include_lib("emqx_st_statistics/include/emqx_st_statistics.hrl"). +-include("include/emqx_st_statistics.hrl"). -import(minirest, [return/1]). @@ -41,7 +41,7 @@ %%-------------------------------------------------------------------- clear_history(_Bindings, _Params) -> - ok = emqx_st_statistics:clear_history(), + ok = emqx_mod_st_statistics:clear_history(), return(ok). get_history(_Bindings, Params) -> diff --git a/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl b/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl similarity index 93% rename from apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl rename to lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl index e8bdd129f..36da813f2 100644 --- a/apps/emqx_st_statistics/test/emqx_st_statistics_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl @@ -31,17 +31,21 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_st_statistics], fun set_special_cfg/1), + emqx_ct_helpers:start_apps([emqx]), Config. -set_special_cfg(_) -> - application:set_env([{emqx_st_statistics, base_conf()}]), - ok. - end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_st_statistics]), + emqx_ct_helpers:stop_apps([emqx]), Config. +init_per_testcase(_, Config) -> + emqx_mod_st_statistics:load(base_conf()), + Config. + +end_per_testcase(_, _) -> + emqx_mod_st_statistics:unload(undefined), + ok. + %%-------------------------------------------------------------------- %% Test Cases %%-------------------------------------------------------------------- diff --git a/apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl similarity index 94% rename from apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl rename to lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl index e6b99e70a..8255cdd5f 100644 --- a/apps/emqx_st_statistics/test/emqx_st_statistics_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_management/include/emqx_mgmt.hrl"). --include_lib("emqx_st_statistics/include/emqx_st_statistics.hrl"). +-include_lib("emqx_modules/include/emqx_st_statistics.hrl"). -define(CONTENT_TYPE, "application/x-www-form-urlencoded"). @@ -39,17 +39,19 @@ all() -> init_per_suite(Config) -> application:load(emqx_modules), - emqx_ct_helpers:start_apps([emqx_st_statistics, emqx_management]), + emqx_ct_helpers:start_apps([emqx_management]), Config. end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_st_statistics, emqx_management]), + emqx_ct_helpers:stop_apps([emqx_management]), Config. init_per_testcase(_, Config) -> + emqx_mod_st_statistics:load(emqx_st_statistics_SUITE:base_conf()), Config. end_per_testcase(_, Config) -> + emqx_mod_st_statistics:unload(undefined), Config. get(Key, ResponseBody) -> diff --git a/priv/emqx.schema b/priv/emqx.schema index 6418501e3..febd4cb9e 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2188,6 +2188,42 @@ end}. {datatype, string} ]}. +{mapping, "module.st_statistics.threshold_time", "emqx.modules", [ + {default, "10s"}, + {datatype, {duration, ms}} +]}. + +{mapping, "module.st_statistics.time_window", "emqx.modules", [ + {default, "5M"}, + {datatype, {duration, ms}} +]}. + +{mapping, "module.st_statistics.max_log_num", "emqx.modules", [ + {default, 500}, + {datatype, integer} +]}. + +{mapping, "module.st_statistics.top_k_num", "emqx.modules", [ + {default, 500}, + {datatype, integer} +]}. + +{mapping, "module.st_statistics.notice_topic", "emqx.modules", [ + {default, "$slow_topics"}, + {datatype, string} +]}. + +{mapping, "module.st_statistics.notice_qos", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +{mapping, "module.st_statistics.notice_batch_size", "emqx.modules", [ + {default, 500}, + {datatype, integer} +]}. + {translation, "emqx.modules", fun(Conf, _, Conf1) -> Subscriptions = fun() -> List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), @@ -2211,13 +2247,19 @@ end}. {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} end, TotalRules) end, + + SlowTopic = fun() -> + List = cuttlefish_variable:filter_by_prefix("module.st_statistics", Conf), + [{erlang:list_to_atom(Key), Value} || {[_, _, Key], Value} <- List] + end, + lists:append([ [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}], [{emqx_mod_subscription, Subscriptions()}], [{emqx_mod_rewrite, Rewrites()}], [{emqx_mod_topic_metrics, []}], [{emqx_mod_delayed, []}], - [{emqx_mod_st_statistics, []}], + [{emqx_mod_st_statistics, SlowTopic()}], [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] ]) end}. diff --git a/rebar.config.erl b/rebar.config.erl index 38dcb681b..1000a2c92 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -303,7 +303,6 @@ relx_plugin_apps(ReleaseType) -> , emqx_recon , emqx_rule_engine , emqx_sasl - , emqx_st_statistics ] ++ [emqx_telemetry || not is_enterprise()] ++ relx_plugin_apps_per_rel(ReleaseType) From 835ad52498efc79a62d795c183fce12585f47ddb Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 11 Nov 2021 14:54:40 +0800 Subject: [PATCH 015/363] feat(trace): http api for trace (#6052) * feat(trace): add http trace API * feat: sub/unsub trace topic * chore(trace): stream log use _page and _limit * fix: elvis warning * fix: mod_trace test failed * fix: http api meta rename _page/_limit to _page/_limit * fix: clientid string not working * fix: add emqx_mod_trace to module * fix(trace): fixed by review suggestions --- .../src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 122 +++-- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 26 +- deploy/charts/emqx/values.yaml | 2 + include/emqx_mqtt.hrl | 18 + lib-ce/emqx_modules/src/emqx_mod_sup.erl | 2 +- lib-ce/emqx_modules/src/emqx_mod_trace.erl | 508 ++++++++++++++++++ .../emqx_modules/src/emqx_mod_trace_api.erl | 191 +++++++ .../test/emqx_mod_rewrite_SUITE.erl | 2 +- .../emqx_modules/test/emqx_mod_sup_SUITE.erl | 3 +- .../test/emqx_mod_trace_SUITE.erl | 478 ++++++++++++++++ priv/emqx.schema | 1 + src/emqx_broker.erl | 50 +- src/emqx_guid.erl | 4 +- src/emqx_misc.erl | 38 +- src/emqx_packet.erl | 59 +- src/emqx_tracer.erl | 175 ++++-- test/emqx_tracer_SUITE.erl | 119 ++-- 18 files changed, 1580 insertions(+), 220 deletions(-) create mode 100644 lib-ce/emqx_modules/src/emqx_mod_trace.erl create mode 100644 lib-ce/emqx_modules/src/emqx_mod_trace_api.erl create mode 100644 lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 405b4c244..fe68fef44 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.8"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 95f5121cd..97cb182e6 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -21,7 +21,9 @@ -include("emqx_mgmt.hrl"). --define(PRINT_CMD(Cmd, Descr), io:format("~-48s# ~s~n", [Cmd, Descr])). +-elvis([{elvis_style, invalid_dynamic_call, #{ ignore => [emqx_mgmt_cli]}}]). + +-define(PRINT_CMD(Cmd, Desc), io:format("~-48s# ~s~n", [Cmd, Desc])). -export([load/0]). @@ -74,11 +76,8 @@ mgmt(["insert", AppId, Name]) -> mgmt(["lookup", AppId]) -> case emqx_mgmt_auth:lookup_app(list_to_binary(AppId)) of - {AppId1, AppSecret, Name, Desc, Status, Expired} -> - emqx_ctl:print("app_id: ~s~nsecret: ~s~nname: ~s~ndesc: ~s~nstatus: ~s~nexpired: ~p~n", - [AppId1, AppSecret, Name, Desc, Status, Expired]); - undefined -> - emqx_ctl:print("Not Found.~n") + undefined -> emqx_ctl:print("Not Found.~n"); + App -> print_app_info(App) end; mgmt(["update", AppId, Status]) -> @@ -99,10 +98,7 @@ mgmt(["delete", AppId]) -> end; mgmt(["list"]) -> - lists:foreach(fun({AppId, AppSecret, Name, Desc, Status, Expired}) -> - emqx_ctl:print("app_id: ~s, secret: ~s, name: ~s, desc: ~s, status: ~s, expired: ~p~n", - [AppId, AppSecret, Name, Desc, Status, Expired]) - end, emqx_mgmt_auth:list_apps()); + lists:foreach(fun print_app_info/1, emqx_mgmt_auth:list_apps()); mgmt(_) -> emqx_ctl:usage([{"mgmt list", "List Applications"}, @@ -128,10 +124,12 @@ broker([]) -> [emqx_ctl:print("~-10s: ~s~n", [Fun, emqx_sys:Fun()]) || Fun <- Funs]; broker(["stats"]) -> - [emqx_ctl:print("~-30s: ~w~n", [Stat, Val]) || {Stat, Val} <- lists:sort(emqx_stats:getstats())]; + [emqx_ctl:print("~-30s: ~w~n", [Stat, Val]) || + {Stat, Val} <- lists:sort(emqx_stats:getstats())]; broker(["metrics"]) -> - [emqx_ctl:print("~-30s: ~w~n", [Metric, Val]) || {Metric, Val} <- lists:sort(emqx_metrics:all())]; + [emqx_ctl:print("~-30s: ~w~n", [Metric, Val]) || + {Metric, Val} <- lists:sort(emqx_metrics:all())]; broker(_) -> emqx_ctl:usage([{"broker", "Show broker version, uptime and description"}, @@ -256,10 +254,12 @@ subscriptions(["del", ClientId, Topic]) -> end; subscriptions(_) -> - emqx_ctl:usage([{"subscriptions list", "List all subscriptions"}, - {"subscriptions show ", "Show subscriptions of a client"}, - {"subscriptions add ", "Add a static subscription manually"}, - {"subscriptions del ", "Delete a static subscription manually"}]). + emqx_ctl:usage([{"subscriptions list", "List all subscriptions"}, + {"subscriptions show ", "Show subscriptions of a client"}, + {"subscriptions add ", + "Add a static subscription manually"}, + {"subscriptions del ", + "Delete a static subscription manually"}]). if_valid_qos(QoS, Fun) -> try list_to_integer(QoS) of @@ -328,14 +328,20 @@ vm(["memory"]) -> [emqx_ctl:print("memory/~-17s: ~w~n", [Cat, Val]) || {Cat, Val} <- erlang:memory()]; vm(["process"]) -> - [emqx_ctl:print("process/~-16s: ~w~n", [Name, erlang:system_info(Key)]) || {Name, Key} <- [{limit, process_limit}, {count, process_count}]]; + [emqx_ctl:print("process/~-16s: ~w~n", + [Name, erlang:system_info(Key)]) || + {Name, Key} <- [{limit, process_limit}, {count, process_count}]]; vm(["io"]) -> IoInfo = lists:usort(lists:flatten(erlang:system_info(check_io))), - [emqx_ctl:print("io/~-21s: ~w~n", [Key, proplists:get_value(Key, IoInfo)]) || Key <- [max_fds, active_fds]]; + [emqx_ctl:print("io/~-21s: ~w~n", + [Key, proplists:get_value(Key, IoInfo)]) || + Key <- [max_fds, active_fds]]; vm(["ports"]) -> - [emqx_ctl:print("ports/~-16s: ~w~n", [Name, erlang:system_info(Key)]) || {Name, Key} <- [{count, port_count}, {limit, port_limit}]]; + [emqx_ctl:print("ports/~-16s: ~w~n", + [Name, erlang:system_info(Key)]) || + {Name, Key} <- [{count, port_count}, {limit, port_limit}]]; vm(_) -> emqx_ctl:usage([{"vm all", "Show info of Erlang VM"}, @@ -372,8 +378,9 @@ log(["primary-level", Level]) -> emqx_ctl:print("~s~n", [emqx_logger:get_primary_log_level()]); log(["handlers", "list"]) -> - _ = [emqx_ctl:print("LogHandler(id=~s, level=~s, destination=~s, status=~s)~n", [Id, Level, Dst, Status]) - || #{id := Id, level := Level, dst := Dst, status := Status} <- emqx_logger:get_log_handlers()], + _ = [emqx_ctl:print("LogHandler(id=~s, level=~s, destination=~s, status=~s)~n", + [Id, Level, Dst, Status]) || #{id := Id, level := Level, dst := Dst, status := Status} + <- emqx_logger:get_log_handlers()], ok; log(["handlers", "start", HandlerId]) -> @@ -406,15 +413,18 @@ log(_) -> {"log handlers list", "Show log handlers"}, {"log handlers start ", "Start a log handler"}, {"log handlers stop ", "Stop a log handler"}, - {"log handlers set-level ", "Set log level of a log handler"}]). + {"log handlers set-level ", + "Set log level of a log handler"}]). %%-------------------------------------------------------------------- %% @doc Trace Command trace(["list"]) -> - lists:foreach(fun({{Who, Name}, {Level, LogFile}}) -> - emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Who, Name, Level, LogFile]) - end, emqx_tracer:lookup_traces()); + lists:foreach(fun(Trace) -> + #{type := Type, level := Level, dst := Dst} = Trace, + Who = maps:get(Type, Trace), + emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Who, Level, Dst]) + end, emqx_tracer:lookup_traces()); trace(["stop", "client", ClientId]) -> trace_off(clientid, ClientId); @@ -441,8 +451,9 @@ trace(_) -> {"trace start topic [] ", "Traces for a topic"}, {"trace stop topic ", "Stop tracing for a topic"}]). +-dialyzer({nowarn_function, [trace_on/4, trace_off/2]}). trace_on(Who, Name, Level, LogFile) -> - case emqx_tracer:start_trace({Who, iolist_to_binary(Name)}, Level, LogFile) of + case emqx_tracer:start_trace(Who, Name, Level, LogFile) of ok -> emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]); {error, Error} -> @@ -450,7 +461,7 @@ trace_on(Who, Name, Level, LogFile) -> end. trace_off(Who, Name) -> - case emqx_tracer:stop_trace({Who, iolist_to_binary(Name)}) of + case emqx_tracer:stop_trace(Who, Name) of ok -> emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]); {error, Error} -> @@ -472,18 +483,20 @@ listeners([]) -> lists:foreach(fun indent_print/1, Info) end, esockd:listeners()), lists:foreach(fun({Protocol, Opts}) -> - Port = proplists:get_value(port, Opts), - Info = [{listen_on, {string, emqx_listeners:format_listen_on(Port)}}, - {acceptors, maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0)}, - {max_conns, proplists:get_value(max_connections, Opts)}, - {current_conn, proplists:get_value(all_connections, Opts)}, - {shutdown_count, []}], - emqx_ctl:print("~s~n", [listener_identifier(Protocol, Port)]), - lists:foreach(fun indent_print/1, Info) - end, ranch:info()); + Port = proplists:get_value(port, Opts), + Acceptors = maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0), + Info = [{listen_on, {string, emqx_listeners:format_listen_on(Port)}}, + {acceptors, Acceptors}, + {max_conns, proplists:get_value(max_connections, Opts)}, + {current_conn, proplists:get_value(all_connections, Opts)}, + {shutdown_count, []}], + emqx_ctl:print("~s~n", [listener_identifier(Protocol, Port)]), + lists:foreach(fun indent_print/1, Info) + end, ranch:info()); listeners(["stop", Name = "http" ++ _N | _MaybePort]) -> - %% _MaybePort is to be backward compatible, to stop http listener, there is no need for the port number + %% _MaybePort is to be backward compatible, to stop http listener, + %% there is no need for the port number case minirest:stop_http(list_to_atom(Name)) of ok -> emqx_ctl:print("Stop ~s listener successfully.~n", [Name]); @@ -564,7 +577,8 @@ data(["import", Filename, "--env", Env]) -> {error, unsupported_version} -> emqx_ctl:print("The emqx data import failed: Unsupported version.~n"); {error, Reason} -> - emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", [Reason, Filename]) + emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", + [Reason, Filename]) end; data(_) -> @@ -657,19 +671,23 @@ print({client, {ClientId, ChanPid}}) -> maps:with([created_at], Session)]), InfoKeys = [clientid, username, peername, clean_start, keepalive, expiry_interval, - subscriptions_cnt, inflight_cnt, awaiting_rel_cnt, send_msg, mqueue_len, mqueue_dropped, - connected, created_at, connected_at] ++ case maps:is_key(disconnected_at, Info) of - true -> [disconnected_at]; - false -> [] - end, + subscriptions_cnt, inflight_cnt, awaiting_rel_cnt, + send_msg, mqueue_len, mqueue_dropped, + connected, created_at, connected_at] ++ + case maps:is_key(disconnected_at, Info) of + true -> [disconnected_at]; + false -> [] + end, emqx_ctl:print("Client(~s, username=~s, peername=~s, " - "clean_start=~s, keepalive=~w, session_expiry_interval=~w, " - "subscriptions=~w, inflight=~w, awaiting_rel=~w, delivered_msgs=~w, enqueued_msgs=~w, dropped_msgs=~w, " - "connected=~s, created_at=~w, connected_at=~w" ++ case maps:is_key(disconnected_at, Info) of - true -> ", disconnected_at=~w)~n"; - false -> ")~n" - end, - [format(K, maps:get(K, Info)) || K <- InfoKeys]); + "clean_start=~s, keepalive=~w, session_expiry_interval=~w, " + "subscriptions=~w, inflight=~w, awaiting_rel=~w, " + "delivered_msgs=~w, enqueued_msgs=~w, dropped_msgs=~w, " + "connected=~s, created_at=~w, connected_at=~w" ++ + case maps:is_key(disconnected_at, Info) of + true -> ", disconnected_at=~w)~n"; + false -> ")~n" + end, + [format(K, maps:get(K, Info)) || K <- InfoKeys]); print({emqx_route, #route{topic = Topic, dest = {_, Node}}}) -> emqx_ctl:print("~s -> ~s~n", [Topic, Node]); @@ -721,3 +739,7 @@ restart_http_listener(Scheme, AppName) -> http_mod_name(emqx_management) -> emqx_mgmt_http; http_mod_name(Name) -> Name. + +print_app_info({AppId, AppSecret, Name, Desc, Status, Expired}) -> + emqx_ctl:print("app_id: ~s, secret: ~s, name: ~s, desc: ~s, status: ~s, expired: ~p~n", + [AppId, AppSecret, Name, Desc, Status, Expired]). diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index a96ee7a62..698cbf605 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -17,6 +17,9 @@ -module(emqx_rule_funcs). -include("rule_engine.hrl"). +-elvis([{elvis_style, god_modules, disable}]). +-elvis([{elvis_style, function_naming_convention, disable}]). +-elvis([{elvis_style, macro_names, disable}]). %% IoT Funcs -export([ msgid/0 @@ -438,7 +441,8 @@ subbits(Bits, Len) when is_integer(Len), is_bitstring(Bits) -> subbits(Bits, Start, Len) when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> get_subbits(Bits, Start, Len, <<"integer">>, <<"unsigned">>, <<"big">>). -subbits(Bits, Start, Len, Type, Signedness, Endianness) when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> +subbits(Bits, Start, Len, Type, Signedness, Endianness) + when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> get_subbits(Bits, Start, Len, Type, Signedness, Endianness). get_subbits(Bits, Start, Len, Type, Signedness, Endianness) -> @@ -520,7 +524,7 @@ map(Data) -> emqx_rule_utils:map(Data). bin2hexstr(Bin) when is_binary(Bin) -> - emqx_misc:bin2hexstr_A_F(Bin). + emqx_misc:bin2hexstr_a_f_upper(Bin). hexstr2bin(Str) when is_binary(Str) -> emqx_misc:hexstr2bin(Str). @@ -608,7 +612,8 @@ tokens(S, Separators) -> [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators))]. tokens(S, Separators, <<"nocrlf">>) -> - [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators) ++ [$\r,$\n,[$\r,$\n]])]. + [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), + binary_to_list(Separators) ++ [$\r,$\n,[$\r,$\n]])]. concat(S1, S2) when is_binary(S1), is_binary(S2) -> unicode:characters_to_binary([S1, S2], unicode). @@ -646,7 +651,8 @@ replace(SrcStr, P, RepStr) when is_binary(SrcStr), is_binary(P), is_binary(RepSt replace(SrcStr, P, RepStr, <<"all">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> iolist_to_binary(string:replace(SrcStr, P, RepStr, all)); -replace(SrcStr, P, RepStr, <<"trailing">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> +replace(SrcStr, P, RepStr, <<"trailing">>) + when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> iolist_to_binary(string:replace(SrcStr, P, RepStr, trailing)); replace(SrcStr, P, RepStr, <<"leading">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> @@ -662,7 +668,7 @@ regex_replace(SrcStr, RE, RepStr) -> re:replace(SrcStr, RE, RepStr, [global, {return,binary}]). ascii(Char) when is_binary(Char) -> - [FirstC| _] = binary_to_list(Char), + [FirstC | _] = binary_to_list(Char), FirstC. find(S, P) when is_binary(S), is_binary(P) -> @@ -782,7 +788,7 @@ sha256(S) when is_binary(S) -> hash(sha256, S). hash(Type, Data) -> - emqx_misc:bin2hexstr_a_f(crypto:hash(Type, Data)). + emqx_misc:bin2hexstr_a_f_lower(crypto:hash(Type, Data)). %%------------------------------------------------------------------------------ %% Data encode and decode Funcs @@ -875,23 +881,23 @@ time_unit(<<"nanosecond">>) -> nanosecond. %% the function handling to the worker module. %% @end -ifdef(EMQX_ENTERPRISE). -'$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> +'$handle_undefined_function'(schema_decode, [SchemaId, Data | MoreArgs]) -> emqx_schema_parser:decode(SchemaId, Data, MoreArgs); '$handle_undefined_function'(schema_decode, Args) -> error({args_count_error, {schema_decode, Args}}); -'$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> +'$handle_undefined_function'(schema_encode, [SchemaId, Term | MoreArgs]) -> emqx_schema_parser:encode(SchemaId, Term, MoreArgs); '$handle_undefined_function'(schema_encode, Args) -> error({args_count_error, {schema_encode, Args}}); -'$handle_undefined_function'(sprintf, [Format|Args]) -> +'$handle_undefined_function'(sprintf, [Format | Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> error({sql_function_not_supported, function_literal(Fun, Args)}). -else. -'$handle_undefined_function'(sprintf, [Format|Args]) -> +'$handle_undefined_function'(sprintf, [Format | Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index c30e237b7..62cf779ea 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -94,6 +94,8 @@ emqxLoadedPlugins: > emqxLoadedModules: > {emqx_mod_acl_internal, true}. {emqx_mod_presence, true}. + {emqx_mod_trace, false}. + {emqx_mod_st_statistics, false}. {emqx_mod_delayed, false}. {emqx_mod_rewrite, false}. {emqx_mod_subscription, false}. diff --git a/include/emqx_mqtt.hrl b/include/emqx_mqtt.hrl index 5dd9a317c..71c2f25f3 100644 --- a/include/emqx_mqtt.hrl +++ b/include/emqx_mqtt.hrl @@ -542,4 +542,22 @@ -define(SHARE(Group, Topic), emqx_topic:join([<>, Group, Topic])). -define(IS_SHARE(Topic), case Topic of <> -> true; _ -> false end). +-define(TYPE_NAMES, { + 'CONNECT' + , 'CONNACK' + , 'PUBLISH' + , 'PUBACK' + , 'PUBREC' + , 'PUBREL' + , 'PUBCOMP' + , 'SUBSCRIBE' + , 'SUBACK' + , 'UNSUBSCRIBE' + , 'UNSUBACK' + , 'PINGREQ' + , 'PINGRESP' + , 'DISCONNECT' + , 'AUTH' + }). + -endif. diff --git a/lib-ce/emqx_modules/src/emqx_mod_sup.erl b/lib-ce/emqx_modules/src/emqx_mod_sup.erl index 60907f580..109564f65 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_sup.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_sup.erl @@ -77,5 +77,5 @@ init([]) -> assert_started({ok, _Pid}) -> ok; assert_started({ok, _Pid, _Info}) -> ok; -assert_started({error, {already_tarted, _Pid}}) -> ok; +assert_started({error, {already_started, _Pid}}) -> ok; assert_started({error, Reason}) -> erlang:error(Reason). diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace.erl b/lib-ce/emqx_modules/src/emqx_mod_trace.erl new file mode 100644 index 000000000..737ee7e97 --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_trace.erl @@ -0,0 +1,508 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_trace). + +-behaviour(gen_server). +-behaviour(emqx_gen_mod). + + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("kernel/include/file.hrl"). + +-logger_header("[Trace]"). + +%% Mnesia bootstrap +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-export([ load/1 + , unload/1 + , description/0 + ]). + +-export([ start_link/0 + , list/0 + , list/1 + , get_trace_filename/1 + , create/1 + , delete/1 + , clear/0 + , update/2 + ]). + +-export([ format/1 + , zip_dir/0 + , trace_dir/0 + , trace_file/1 + , delete_files_after_send/2 + ]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). + +-define(TRACE, ?MODULE). +-define(PACKETS, tuple_to_list(?TYPE_NAMES)). +-define(MAX_SIZE, 30). + +-ifdef(TEST). +-export([log_file/2]). +-endif. + +-record(?TRACE, + { name :: binary() | undefined | '_' + , type :: clientid | topic | undefined | '_' + , topic :: emqx_types:topic() | undefined | '_' + , clientid :: emqx_types:clientid() | undefined | '_' + , packets = [] :: list() | '_' + , enable = true :: boolean() | '_' + , start_at :: integer() | undefined | binary() | '_' + , end_at :: integer() | undefined | binary() | '_' + , log_size = #{} :: map() | '_' + }). + +mnesia(boot) -> + ok = ekka_mnesia:create_table(?TRACE, [ + {type, set}, + {disc_copies, [node()]}, + {record_name, ?TRACE}, + {attributes, record_info(fields, ?TRACE)}]); +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?TRACE, disc_copies). + +description() -> + "EMQ X Trace Module". + +-spec load(any()) -> ok. +load(_Env) -> + emqx_mod_sup:start_child(?MODULE, worker). + +-spec unload(any()) -> ok. +unload(_Env) -> + _ = emqx_mod_sup:stop_child(?MODULE), + stop_all_trace_handler(). + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec list() -> [tuple()]. +list() -> + ets:match_object(?TRACE, #?TRACE{_ = '_'}). + +-spec list(boolean()) -> [tuple()]. +list(Enable) -> + ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). + +-spec create([{Key :: binary(), Value :: binary()}]) -> + ok | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. +create(Trace) -> + case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of + true -> + case to_trace(Trace) of + {ok, TraceRec} -> create_new_trace(TraceRec); + {error, Reason} -> {error, Reason} + end; + false -> + {error, """The number of traces created has reached the maximum, +please delete the useless ones first"""} + end. + +-spec delete(Name :: binary()) -> ok | {error, not_found}. +delete(Name) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name) of + [_] -> mnesia:delete(?TRACE, Name, write); + [] -> mnesia:abort(not_found) + end + end, + transaction(Tran). + +-spec clear() -> ok | {error, Reason :: term()}. +clear() -> + case mnesia:clear_table(?TRACE) of + {atomic, ok} -> ok; + {aborted, Reason} -> {error, Reason} + end. + +-spec update(Name :: binary(), Enable :: boolean()) -> + ok | {error, not_found | finished}. +update(Name, Enable) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name) of + [] -> mnesia:abort(not_found); + [#?TRACE{enable = Enable}] -> ok; + [Rec] -> + case erlang:system_time(second) >= Rec#?TRACE.end_at of + false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write); + true -> mnesia:abort(finished) + end + end + end, + transaction(Tran). + +-spec get_trace_filename(Name :: binary()) -> + {ok, FileName :: string()} | {error, not_found}. +get_trace_filename(Name) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name, read) of + [] -> mnesia:abort(not_found); + [#?TRACE{start_at = Start}] -> {ok, filename(Name, Start)} + end end, + transaction(Tran). + +-spec trace_file(File :: list()) -> + {ok, Node :: list(), Binary :: binary()} | + {error, Node :: list(), Reason :: term()}. +trace_file(File) -> + FileName = filename:join(trace_dir(), File), + Node = atom_to_list(node()), + case file:read_file(FileName) of + {ok, Bin} -> {ok, Node, Bin}; + {error, Reason} -> {error, Node, Reason} + end. + +delete_files_after_send(TraceLog, Zips) -> + gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}). + +-spec format(list(#?TRACE{})) -> list(map()). +format(Traces) -> + Fields = record_info(fields, ?TRACE), + lists:map(fun(Trace0 = #?TRACE{start_at = StartAt, end_at = EndAt}) -> + Trace = Trace0#?TRACE{ + start_at = list_to_binary(calendar:system_time_to_rfc3339(StartAt)), + end_at = list_to_binary(calendar:system_time_to_rfc3339(EndAt)) + }, + [_ | Values] = tuple_to_list(Trace), + maps:from_list(lists:zip(Fields, Values)) + end, Traces). + +init([]) -> + erlang:process_flag(trap_exit, true), + OriginLogLevel = emqx_logger:get_primary_log_level(), + ok = filelib:ensure_dir(trace_dir()), + ok = filelib:ensure_dir(zip_dir()), + {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), + Traces = get_enable_trace(), + ok = update_log_primary_level(Traces, OriginLogLevel), + TRef = update_trace(Traces), + {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ok, State}. + +handle_cast({delete_tag, Pid, Files}, State = #{monitors := Monitors}) -> + erlang:monitor(process, Pid), + {noreply, State#{monitors => Monitors#{Pid => Files}}}; +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitors}) -> + case maps:take(Pid, Monitors) of + error -> {noreply, State}; + {Files, NewMonitors} -> + lists:foreach(fun(F) -> file:delete(F) end, Files), + {noreply, State#{monitors => NewMonitors}} + end; +handle_info({timeout, TRef, update_trace}, + #{timer := TRef, primary_log_level := OriginLogLevel} = State) -> + Traces = get_enable_trace(), + ok = update_log_primary_level(Traces, OriginLogLevel), + NextTRef = update_trace(Traces), + {noreply, State#{timer => NextTRef}}; + +handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> + emqx_misc:cancel_timer(TRef), + handle_info({timeout, TRef, update_trace}, State); + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> + ok = set_log_primary_level(OriginLogLevel), + _ = mnesia:unsubscribe({table, ?TRACE, simple}), + emqx_misc:cancel_timer(TRef), + stop_all_trace_handler(), + _ = file:del_dir_r(zip_dir()), + ok. + +code_change(_, State, _Extra) -> + {ok, State}. + +create_new_trace(Trace) -> + Tran = fun() -> + case mnesia:read(?TRACE, Trace#?TRACE.name) of + [] -> + #?TRACE{start_at = StartAt, topic = Topic, + clientid = ClientId, packets = Packets} = Trace, + Match = #?TRACE{_ = '_', start_at = StartAt, topic = Topic, + clientid = ClientId, packets = Packets}, + case mnesia:match_object(?TRACE, Match, read) of + [] -> mnesia:write(?TRACE, Trace, write); + [#?TRACE{name = Name}] -> mnesia:abort({duplicate_condition, Name}) + end; + [#?TRACE{name = Name}] -> mnesia:abort({already_existed, Name}) + end + end, + transaction(Tran). + +update_trace(Traces) -> + Now = erlang:system_time(second), + {_Waiting, Running, Finished} = classify_by_time(Traces, Now), + disable_finished(Finished), + Started = already_running(), + {NeedRunning, AllStarted} = start_trace(Running, Started), + NeedStop = AllStarted -- NeedRunning, + ok = stop_trace(NeedStop, Started), + clean_stale_trace_files(), + NextTime = find_closest_time(Traces, Now), + emqx_misc:start_timer(NextTime, update_trace). + +stop_all_trace_handler() -> + lists:foreach(fun(#{type := Type, name := Name} = Trace) -> + _ = emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name) + end + , already_running()). + +already_running() -> + emqx_tracer:lookup_traces(). + +get_enable_trace() -> + {atomic, Traces} = + mnesia:transaction(fun() -> + mnesia:match_object(?TRACE, #?TRACE{enable = true, _ = '_'}, read) + end), + Traces. + +find_closest_time(Traces, Now) -> + Sec = + lists:foldl( + fun(#?TRACE{start_at = Start, end_at = End}, Closest) + when Start >= Now andalso Now < End -> %% running + min(End - Now, Closest); + (#?TRACE{start_at = Start}, Closest) when Start < Now -> %% waiting + min(Now - Start, Closest); + (_, Closest) -> Closest %% finished + end, 60 * 15, Traces), + timer:seconds(Sec). + +disable_finished([]) -> ok; +disable_finished(Traces) -> + NameWithLogSize = + lists:map(fun(#?TRACE{name = Name, start_at = StartAt}) -> + FileSize = filelib:file_size(log_file(Name, StartAt)), + {Name, FileSize} end, Traces), + transaction(fun() -> + lists:map(fun({Name, LogSize}) -> + case mnesia:read(?TRACE, Name, write) of + [] -> ok; + [Trace = #?TRACE{log_size = Logs}] -> + mnesia:write(?TRACE, Trace#?TRACE{enable = false, + log_size = Logs#{node() => LogSize}}, write) + end end, NameWithLogSize) + end). + +start_trace(Traces, Started0) -> + Started = lists:map(fun(#{name := Name}) -> Name end, Started0), + lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> + case lists:member(Name, StartedAcc) of + true -> {[Name | Running], StartedAcc}; + false -> + case start_trace(Trace) of + ok -> {[Name | Running], [Name | StartedAcc]}; + Error -> + ?LOG(error, "(~p)start trace failed by:~p", [Name, Error]), + {[Name | Running], StartedAcc} + end + end + end, {[], Started}, Traces). + +start_trace(Trace) -> + #?TRACE{name = Name + , type = Type + , clientid = ClientId + , topic = Topic + , packets = Packets + , start_at = Start + } = Trace, + Who0 = #{name => Name, labels => Packets}, + Who = + case Type of + topic -> Who0#{type => topic, topic => Topic}; + clientid -> Who0#{type => clientid, clientid => ClientId} + end, + case emqx_tracer:start_trace(Who, debug, log_file(Name, Start)) of + ok -> ok; + {error, {already_exist, _}} -> ok; + {error, Reason} -> {error, Reason} + end. + +stop_trace(Finished, Started) -> + lists:foreach(fun(#{name := Name, type := Type} = Trace) -> + case lists:member(Name, Finished) of + true -> emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name); + false -> ok + end + end, Started). + +clean_stale_trace_files() -> + TraceDir = trace_dir(), + case file:list_dir(TraceDir) of + {ok, AllFiles} when AllFiles =/= ["zip"] -> + FileFun = fun(#?TRACE{name = Name, start_at = StartAt}) -> filename(Name, StartAt) end, + KeepFiles = lists:map(FileFun, list()), + case AllFiles -- ["zip" | KeepFiles] of + [] -> ok; + DeleteFiles -> + DelFun = fun(F) -> file:delete(filename:join(TraceDir, F)) end, + lists:foreach(DelFun, DeleteFiles) + end; + _ -> ok + end. + +classify_by_time(Traces, Now) -> + classify_by_time(Traces, Now, [], [], []). + +classify_by_time([], _Now, Wait, Run, Finish) -> {Wait, Run, Finish}; +classify_by_time([Trace = #?TRACE{start_at = Start} | Traces], + Now, Wait, Run, Finish) when Start > Now -> + classify_by_time(Traces, Now, [Trace | Wait], Run, Finish); +classify_by_time([Trace = #?TRACE{end_at = End} | Traces], + Now, Wait, Run, Finish) when End =< Now -> + classify_by_time(Traces, Now, Wait, Run, [Trace | Finish]); +classify_by_time([Trace | Traces], Now, Wait, Run, Finish) -> + classify_by_time(Traces, Now, Wait, [Trace | Run], Finish). + +to_trace(TraceList) -> + case to_trace(TraceList, #?TRACE{}) of + {error, Reason} -> {error, Reason}; + {ok, #?TRACE{name = undefined}} -> + {error, "name required"}; + {ok, #?TRACE{type = undefined}} -> + {error, "type required"}; + {ok, #?TRACE{topic = undefined, clientid = undefined}} -> + {error, "topic/clientid cannot be both empty"}; + {ok, Trace} -> + case fill_default(Trace) of + #?TRACE{start_at = Start, end_at = End} when End =< Start -> + {error, "failed by start_at >= end_at"}; + Trace1 -> {ok, Trace1} + end + end. + +fill_default(Trace = #?TRACE{start_at = undefined}) -> + fill_default(Trace#?TRACE{start_at = erlang:system_time(second)}); +fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> + fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60}); +fill_default(Trace) -> Trace. + +to_trace([], Rec) -> {ok, Rec}; +to_trace([{<<"name">>, Name} | Trace], Rec) -> + case binary:match(Name, [<<"/">>], []) of + nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); + _ -> {error, "name cannot contain /"} + end; +to_trace([{<<"type">>, Type} | Trace], Rec) -> + case lists:member(Type, [<<"clientid">>, <<"topic">>]) of + true -> to_trace(Trace, Rec#?TRACE{type = binary_to_existing_atom(Type)}); + false -> {error, "incorrect type: only support clientid/topic"} + end; +to_trace([{<<"topic">>, Topic} | Trace], Rec) -> + case validate_topic(Topic) of + ok -> to_trace(Trace, Rec#?TRACE{topic = Topic}); + {error, Reason} -> {error, Reason} + end; +to_trace([{<<"start_at">>, StartAt} | Trace], Rec) -> + case to_system_second(StartAt) of + {ok, Sec} -> to_trace(Trace, Rec#?TRACE{start_at = Sec}); + {error, Reason} -> {error, Reason} + end; +to_trace([{<<"end_at">>, EndAt} | Trace], Rec) -> + Now = erlang:system_time(second), + case to_system_second(EndAt) of + {ok, Sec} when Sec > Now -> + to_trace(Trace, Rec#?TRACE{end_at = Sec}); + {ok, _Sec} -> + {error, "end_at time has already passed"}; + {error, Reason} -> + {error, Reason} + end; +to_trace([{<<"clientid">>, ClientId} | Trace], Rec) -> + to_trace(Trace, Rec#?TRACE{clientid = ClientId}); +to_trace([{<<"packets">>, PacketList} | Trace], Rec) -> + case to_packets(PacketList) of + {ok, Packets} -> to_trace(Trace, Rec#?TRACE{packets = Packets}); + {error, Reason} -> {error, io_lib:format("unsupport packets: ~p", [Reason])} + end; +to_trace([Unknown | _Trace], _Rec) -> {error, io_lib:format("unknown field: ~p", [Unknown])}. + +validate_topic(TopicName) -> + try emqx_topic:validate(name, TopicName) of + true -> ok + catch + error:Error -> + {error, io_lib:format("~s invalid by ~p", [TopicName, Error])} + end. + +to_system_second(At) -> + try + Sec = calendar:rfc3339_to_system_time(binary_to_list(At), [{unit, second}]), + {ok, Sec} + catch error: {badmatch, _} -> + {error, ["The rfc3339 specification not satisfied: ", At]} + end. + +to_packets(Packets) when is_list(Packets) -> + AtomTypes = lists:map(fun(Type) -> binary_to_existing_atom(Type) end, Packets), + case lists:filter(fun(T) -> not lists:member(T, ?PACKETS) end, AtomTypes) of + [] -> {ok, AtomTypes}; + InvalidE -> {error, InvalidE} + end; +to_packets(Packets) -> {error, Packets}. + +zip_dir() -> + trace_dir() ++ "zip/". + +trace_dir() -> + filename:join(emqx:get_env(data_dir), "trace") ++ "/". + +log_file(Name, Start) -> + filename:join(trace_dir(), filename(Name, Start)). + +filename(Name, Start) -> + [Time, _] = string:split(calendar:system_time_to_rfc3339(Start), "T", leading), + lists:flatten(["trace_", binary_to_list(Name), "_", Time, ".log"]). + +transaction(Tran) -> + case mnesia:transaction(Tran) of + {atomic, Res} -> Res; + {aborted, Reason} -> {error, Reason} + end. + +update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel); +update_log_primary_level(_, _) -> set_log_primary_level(debug). + +set_log_primary_level(NewLevel) -> + case NewLevel =/= emqx_logger:get_primary_log_level() of + true -> emqx_logger:set_primary_log_level(NewLevel); + false -> ok + end. diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl new file mode 100644 index 000000000..6d2813b96 --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -0,0 +1,191 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_trace_api). +-include_lib("emqx/include/logger.hrl"). + +%% API +-export([ list_trace/2 + , create_trace/2 + , update_trace/2 + , delete_trace/2 + , clear_traces/2 + , download_zip_log/2 + , stream_log_file/2 +]). +-export([read_trace_file/3]). + +-define(TO_BIN(_B_), iolist_to_binary(_B_)). +-define(RETURN_NOT_FOUND(N), return({error, 'NOT_FOUND', ?TO_BIN([N, "NOT FOUND"])})). + +-import(minirest, [return/1]). + +-rest_api(#{name => list_trace, + method => 'GET', + path => "/trace/", + func => list_trace, + descr => "list all traces"}). + +-rest_api(#{name => create_trace, + method => 'POST', + path => "/trace/", + func => create_trace, + descr => "create trace"}). + +-rest_api(#{name => delete_trace, + method => 'DELETE', + path => "/trace/:bin:name", + func => delete_trace, + descr => "delete trace"}). + +-rest_api(#{name => clear_trace, + method => 'DELETE', + path => "/trace/", + func => clear_traces, + descr => "clear all traces"}). + +-rest_api(#{name => update_trace, + method => 'PUT', + path => "/trace/:bin:name/:atom:operation", + func => update_trace, + descr => "diable/enable trace"}). + +-rest_api(#{name => download_zip_log, + method => 'GET', + path => "/trace/:bin:name/download", + func => download_zip_log, + descr => "download trace's log"}). + +-rest_api(#{name => stream_log_file, + method => 'GET', + path => "/trace/:bin:name/log", + func => stream_log_file, + descr => "download trace's log"}). + +list_trace(_, Params) -> + List = + case Params of + [{<<"enable">>, Enable}] -> emqx_mod_trace:list(Enable); + _ -> emqx_mod_trace:list() + end, + return({ok, emqx_mod_trace:format(List)}). + +create_trace(_, Param) -> + case emqx_mod_trace:create(Param) of + ok -> return(ok); + {error, {already_existed, Name}} -> + return({error, 'ALREADY_EXISTED', ?TO_BIN([Name, "Already Exists"])}); + {error, {duplicate_condition, Name}} -> + return({error, 'DUPLICATE_CONDITION', ?TO_BIN([Name, "Duplication Condition"])}); + {error, Reason} -> + return({error, 'INCORRECT_PARAMS', ?TO_BIN(Reason)}) + end. + +delete_trace(#{name := Name}, _Param) -> + case emqx_mod_trace:delete(Name) of + ok -> return(ok); + {error, not_found} -> ?RETURN_NOT_FOUND(Name) + end. + +clear_traces(_, _) -> + return(emqx_mod_trace:clear()). + +update_trace(#{name := Name, operation := Operation}, _Param) -> + Enable = case Operation of disable -> false; enable -> true end, + case emqx_mod_trace:update(Name, Enable) of + ok -> return({ok, #{enable => Enable, name => Name}}); + {error, not_found} -> ?RETURN_NOT_FOUND(Name) + end. + +%% if HTTP request headers include accept-encoding: gzip and file size > 300 bytes. +%% cowboy_compress_h will auto encode gzip format. +download_zip_log(#{name := Name}, _Param) -> + case emqx_mod_trace:get_trace_filename(Name) of + {ok, TraceLog} -> + TraceFiles = collect_trace_file(TraceLog), + ZipDir = emqx_mod_trace:zip_dir(), + Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), + ZipFileName = ZipDir ++ TraceLog, + {ok, ZipFile} = zip:zip(ZipFileName, Zips), + emqx_mod_trace:delete_files_after_send(ZipFileName, Zips), + {ok, #{}, {sendfile, 0, filelib:file_size(ZipFile), ZipFile}}; + {error, Reason} -> + return({error, Reason}) + end. + +group_trace_file(ZipDir, TraceLog, TraceFiles) -> + lists:foldl(fun(Res, Acc) -> + case Res of + {ok, Node, Bin} -> + ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, + ok = file:write_file(ZipName, Bin), + [ZipName | Acc]; + {error, Node, Reason} -> + ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), + Acc + end + end, [], TraceFiles). + +collect_trace_file(TraceLog) -> + Nodes = ekka_mnesia:running_nodes(), + {Files, BadNodes} = rpc:multicall(Nodes, emqx_mod_trace, trace_file, [TraceLog], 60000), + BadNodes =/= [] andalso ?LOG(error, "download log rpc failed on ~p", [BadNodes]), + Files. + +%% _page as position and _limit as bytes for front-end reusing components +stream_log_file(#{name := Name}, Params) -> + Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), + Position0 = proplists:get_value(<<"_page">>, Params, <<"0">>), + Bytes0 = proplists:get_value(<<"_limit">>, Params, <<"500">>), + Node = binary_to_existing_atom(Node0), + Position = binary_to_integer(Position0), + Bytes = binary_to_integer(Bytes0), + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + {ok, Bin} -> + Meta = #{<<"page">> => Position + byte_size(Bin), <<"limit">> => Bytes}, + return({ok, #{meta => Meta, items => Bin}}); + eof -> + Meta = #{<<"page">> => Position, <<"limit">> => Bytes}, + return({ok, #{meta => Meta, items => <<"">>}}); + {error, Reason} -> + logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), + return({error, Reason}) + end. + +%% this is an rpc call for stream_log_file/2 +read_trace_file(Name, Position, Limit) -> + TraceDir = emqx_mod_trace:trace_dir(), + {ok, AllFiles} = file:list_dir(TraceDir), + TracePrefix = "trace_" ++ binary_to_list(Name) ++ "_", + Filter = fun(FileName) -> nomatch =/= string:prefix(FileName, TracePrefix) end, + case lists:filter(Filter, AllFiles) of + [TraceFile] -> + TracePath = filename:join([TraceDir, TraceFile]), + read_file(TracePath, Position, Limit); + [] -> {error, not_found} + end. + +read_file(Path, Offset, Bytes) -> + {ok, IoDevice} = file:open(Path, [read, raw, binary]), + try + _ = case Offset of + 0 -> ok; + _ -> file:position(IoDevice, {bof, Offset}) + end, + file:read(IoDevice, Bytes) + after + file:close(IoDevice) + end. diff --git a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl index 997eff1c2..e6a9f6c16 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl @@ -62,7 +62,7 @@ t_mod_rewrite(_Config) -> timer:sleep(100), ?assertEqual([], emqx_broker:subscriptions(<<"rewrite_client">>)), %% Pub Rules - {ok, _Props, _} = emqtt:subscribe(C, [{Topic, ?QOS_1} || Topic <- PubDestTopics]), + {ok, _Props1, _} = emqtt:subscribe(C, [{Topic, ?QOS_1} || Topic <- PubDestTopics]), RecvTopics2 = [begin ok = emqtt:publish(C, Topic, <<"payload">>), {ok, #{topic := RecvTopic}} = receive_publish(100), diff --git a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl index 59d0ffde2..7c666ea9a 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl @@ -41,9 +41,8 @@ t_start_child(_) -> modules => [Mod]}, ok = emqx_mod_sup:start_child(Mod, worker), - ?assertError({already_started, _}, emqx_mod_sup:start_child(Spec)), + ?assertEqual(ok, emqx_mod_sup:start_child(Spec)), ok = emqx_mod_sup:stop_child(Mod), {error, not_found} = emqx_mod_sup:stop_child(Mod), ok. - diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl new file mode 100644 index 000000000..c643c7908 --- /dev/null +++ b/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl @@ -0,0 +1,478 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mod_trace_SUITE). + +%% API +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-define(HOST, "http://127.0.0.1:18083/"). +-define(API_VERSION, "v4"). +-define(BASE_PATH, "api"). + +-record(emqx_mod_trace, { + name, + type, + topic, + clientid, + packets = [], + enable = true, + start_at, + end_at, + log_size = #{} + }). + +-define(PACKETS, ['CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL' + , 'PUBCOMP', 'SUBSCRIBE', 'SUBACK', 'UNSUBSCRIBE', 'UNSUBACK' + , 'PINGREQ', 'PINGRESP', 'DISCONNECT', 'AUTH']). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_modules, emqx_dashboard]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules, emqx_dashboard]). + +t_base_create_delete(_Config) -> + ok = emqx_mod_trace:clear(), + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + End = to_rfc3339(Now + 30 * 60), + Name = <<"name1">>, + ClientId = <<"test-device">>, + Packets = [atom_to_binary(E) || E <- ?PACKETS], + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, + {<<"clientid">>, ClientId}, + {<<"packets">>, Packets}, + {<<"start_at">>, Start}, + {<<"end_at">>, End} + ], + AnotherTrace = lists:keyreplace(<<"name">>, 1, Trace, {<<"name">>, <<"AnotherName">>}), + ok = emqx_mod_trace:create(Trace), + ?assertEqual({error, {already_existed, Name}}, emqx_mod_trace:create(Trace)), + ?assertEqual({error, {duplicate_condition, Name}}, emqx_mod_trace:create(AnotherTrace)), + [TraceRec] = emqx_mod_trace:list(), + Expect = #emqx_mod_trace{ + name = Name, + type = clientid, + topic = undefined, + clientid = ClientId, + packets = ?PACKETS, + start_at = Now, + end_at = Now + 30 * 60 + }, + ?assertEqual(Expect, TraceRec), + ExpectFormat = [ + #{ + clientid => <<"test-device">>, + enable => true, + type => clientid, + packets => ?PACKETS, + name => <<"name1">>, + start_at => Start, + end_at => End, + log_size => #{}, + topic => undefined + } + ], + ?assertEqual(ExpectFormat, emqx_mod_trace:format([TraceRec])), + ?assertEqual(ok, emqx_mod_trace:delete(Name)), + ?assertEqual({error, not_found}, emqx_mod_trace:delete(Name)), + ?assertEqual([], emqx_mod_trace:list()), + ok. + +t_create_size_max(_Config) -> + emqx_mod_trace:clear(), + lists:map(fun(Seq) -> + Name = list_to_binary("name" ++ integer_to_list(Seq)), + Trace = [{<<"name">>, Name}, {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], + ok = emqx_mod_trace:create(Trace) + end, lists:seq(1, 30)), + Trace31 = [{<<"name">>, <<"name31">>}, {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/31">>}], + {error, _} = emqx_mod_trace:create(Trace31), + ok = emqx_mod_trace:delete(<<"name30">>), + ok = emqx_mod_trace:create(Trace31), + ?assertEqual(30, erlang:length(emqx_mod_trace:list())), + ok. + +t_create_failed(_Config) -> + ok = emqx_mod_trace:clear(), + UnknownField = [{<<"unknown">>, 12}], + {error, Reason1} = emqx_mod_trace:create(UnknownField), + ?assertEqual(<<"unknown field: {<<\"unknown\">>,12}">>, iolist_to_binary(Reason1)), + + InvalidTopic = [{<<"topic">>, "#/#//"}], + {error, Reason2} = emqx_mod_trace:create(InvalidTopic), + ?assertEqual(<<"#/#// invalid by function_clause">>, iolist_to_binary(Reason2)), + + InvalidStart = [{<<"start_at">>, <<"2021-12-3:12">>}], + {error, Reason3} = emqx_mod_trace:create(InvalidStart), + ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, + iolist_to_binary(Reason3)), + + InvalidEnd = [{<<"end_at">>, <<"2021-12-3:12">>}], + {error, Reason4} = emqx_mod_trace:create(InvalidEnd), + ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, + iolist_to_binary(Reason4)), + + InvalidPackets = [{<<"packets">>, [<<"publish">>]}], + {error, Reason5} = emqx_mod_trace:create(InvalidPackets), + ?assertEqual(<<"unsupport packets: [publish]">>, iolist_to_binary(Reason5)), + + InvalidPackets2 = [{<<"packets">>, <<"publish">>}], + {error, Reason6} = emqx_mod_trace:create(InvalidPackets2), + ?assertEqual(<<"unsupport packets: <<\"publish\">>">>, iolist_to_binary(Reason6)), + + {error, Reason7} = emqx_mod_trace:create([{<<"name">>, <<"test">>}, {<<"type">>, <<"clientid">>}]), + ?assertEqual(<<"topic/clientid cannot be both empty">>, iolist_to_binary(Reason7)), + + InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, {<<"type">>, <<"clientid">>}], + {error, Reason9} = emqx_mod_trace:create(InvalidPackets4), + ?assertEqual(<<"name cannot contain /">>, iolist_to_binary(Reason9)), + + ?assertEqual({error, "type required"}, emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, + {<<"packets">>, []}, {<<"clientid">>, <<"good">>}])), + + ?assertEqual({error, "incorrect type: only support clientid/topic"}, + emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, + {<<"packets">>, []}, {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), + ok. + +t_create_default(_Config) -> + ok = emqx_mod_trace:clear(), + {error, "name required"} = emqx_mod_trace:create([]), + ok = emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, + {<<"type">>, <<"clientid">>}, {<<"packets">>, []}, {<<"clientid">>, <<"good">>}]), + [#emqx_mod_trace{packets = Packets}] = emqx_mod_trace:list(), + ?assertEqual([], Packets), + ok = emqx_mod_trace:clear(), + Trace = [ + {<<"name">>, <<"test-name">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"start_at">>, <<"2021-10-28T10:54:47+08:00">>}, + {<<"end_at">>, <<"2021-10-27T10:54:47+08:00">>} + ], + {error, "end_at time has already passed"} = emqx_mod_trace:create(Trace), + Now = erlang:system_time(second), + Trace2 = [ + {<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"start_at">>, to_rfc3339(Now + 10)}, + {<<"end_at">>, to_rfc3339(Now + 3)} + ], + {error, "failed by start_at >= end_at"} = emqx_mod_trace:create(Trace2), + ok = emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>}]), + [#emqx_mod_trace{start_at = Start, end_at = End}] = emqx_mod_trace:list(), + ?assertEqual(10 * 60, End - Start), + ?assertEqual(true, Start - erlang:system_time(second) < 5), + ok. + +t_update_enable(_Config) -> + ok = emqx_mod_trace:clear(), + Name = <<"test-name">>, + Now = erlang:system_time(second), + End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), + ok = emqx_mod_trace:create([{<<"name">>, Name}, {<<"packets">>, [<<"PUBLISH">>]}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), + [#emqx_mod_trace{enable = Enable}] = emqx_mod_trace:list(), + ?assertEqual(Enable, true), + ok = emqx_mod_trace:update(Name, false), + [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), + ok = emqx_mod_trace:update(Name, false), + [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), + ok = emqx_mod_trace:update(Name, true), + [#emqx_mod_trace{enable = true}] = emqx_mod_trace:list(), + ok = emqx_mod_trace:update(Name, false), + [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), + ?assertEqual({error, not_found}, emqx_mod_trace:update(<<"Name not found">>, true)), + ct:sleep(2100), + ?assertEqual({error, finished}, emqx_mod_trace:update(Name, true)), + ok. + +t_load_state(_Config) -> + emqx_mod_trace:clear(), + ok = emqx_mod_trace:load(test), + Now = erlang:system_time(second), + Running = [{<<"name">>, <<"Running">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/1">>}, {<<"start_at">>, to_rfc3339(Now - 1)}, {<<"end_at">>, to_rfc3339(Now + 2)}], + Waiting = [{<<"name">>, <<"Waiting">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/2">>}, {<<"start_at">>, to_rfc3339(Now + 3)}, {<<"end_at">>, to_rfc3339(Now + 8)}], + Finished = [{<<"name">>, <<"Finished">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/3">>}, {<<"start_at">>, to_rfc3339(Now - 5)}, {<<"end_at">>, to_rfc3339(Now)}], + ok = emqx_mod_trace:create(Running), + ok = emqx_mod_trace:create(Waiting), + {error, "end_at time has already passed"} = emqx_mod_trace:create(Finished), + Traces = emqx_mod_trace:format(emqx_mod_trace:list()), + ?assertEqual(2, erlang:length(Traces)), + Enables = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces), + ExpectEnables = [{<<"Running">>, true}, {<<"Waiting">>, true}], + ?assertEqual(ExpectEnables, lists:sort(Enables)), + ct:sleep(3500), + Traces2 = emqx_mod_trace:format(emqx_mod_trace:list()), + ?assertEqual(2, erlang:length(Traces2)), + Enables2 = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces2), + ExpectEnables2 = [{<<"Running">>, false}, {<<"Waiting">>, true}], + ?assertEqual(ExpectEnables2, lists:sort(Enables2)), + ok = emqx_mod_trace:unload(test), + ok. + +t_client_event(_Config) -> + application:set_env(emqx, allow_anonymous, true), + emqx_mod_trace:clear(), + ClientId = <<"client-test">>, + ok = emqx_mod_trace:load(test), + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + Name = <<"test_client_id_event">>, + ok = emqx_mod_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + emqtt:ping(Client), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]), + ct:sleep(200), + ok = emqx_mod_trace:create([{<<"name">>, <<"test_topic">>}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Bin} = file:read_file(emqx_mod_trace:log_file(Name, Now)), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ct:sleep(200), + {ok, Bin2} = file:read_file(emqx_mod_trace:log_file(Name, Now)), + {ok, Bin3} = file:read_file(emqx_mod_trace:log_file(<<"test_topic">>, Now)), + ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), + ?assert(erlang:byte_size(Bin) > 0), + ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)), + ?assert(erlang:byte_size(Bin3) > 0), + ok = emqx_mod_trace:unload(test), + ok. + +t_get_log_filename(_Config) -> + ok = emqx_mod_trace:clear(), + ok = emqx_mod_trace:load(test), + Now = erlang:system_time(second), + Start = calendar:system_time_to_rfc3339(Now), + End = calendar:system_time_to_rfc3339(Now + 2), + Name = <<"name1">>, + ClientId = <<"test-device">>, + Packets = [atom_to_binary(E) || E <- ?PACKETS], + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, + {<<"clientid">>, ClientId}, + {<<"packets">>, Packets}, + {<<"start_at">>, list_to_binary(Start)}, + {<<"end_at">>, list_to_binary(End)} + ], + ok = emqx_mod_trace:create(Trace), + ?assertEqual({error, not_found}, emqx_mod_trace:get_trace_filename(<<"test">>)), + ?assertEqual(ok, element(1, emqx_mod_trace:get_trace_filename(Name))), + ct:sleep(3000), + ?assertEqual(ok, element(1, emqx_mod_trace:get_trace_filename(Name))), + ok = emqx_mod_trace:unload(test), + ok. + +t_trace_file(_Config) -> + FileName = "test.log", + Content = <<"test \n test">>, + TraceDir = emqx_mod_trace:trace_dir(), + File = filename:join(TraceDir, FileName), + ok = file:write_file(File, Content), + {ok, Node, Bin} = emqx_mod_trace:trace_file(FileName), + ?assertEqual(Node, atom_to_list(node())), + ?assertEqual(Content, Bin), + ok = file:delete(File), + ok. + +t_http_test(_Config) -> + emqx_mod_trace:clear(), + emqx_mod_trace:load(test), + Header = auth_header_(), + %% list + {ok, Empty} = request_api(get, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(Empty)), + %% create + ErrorTrace = #{}, + {ok, Error} = request_api(post, api_path("trace"), Header, ErrorTrace), + ?assertEqual(#{<<"message">> => <<"unknown field: {}">>, <<"code">> => <<"INCORRECT_PARAMS">>}, json(Error)), + + Name = <<"test-name">>, + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, <<"/x/y/z">>} + ], + + {ok, Create} = request_api(post, api_path("trace"), Header, Trace), + ?assertEqual(#{<<"code">> => 0}, json(Create)), + + {ok, List} = request_api(get, api_path("trace"), Header), + #{<<"code">> := 0, <<"data">> := [Data]} = json(List), + ?assertEqual(Name, maps:get(<<"name">>, Data)), + + %% update + {ok, Update} = request_api(put, api_path("trace/test-name/disable"), Header, #{}), + ?assertEqual(#{<<"code">> => 0, + <<"data">> => #{<<"enable">> => false, + <<"name">> => <<"test-name">>}}, json(Update)), + + {ok, List1} = request_api(get, api_path("trace"), Header), + #{<<"code">> := 0, <<"data">> := [Data1]} = json(List1), + ?assertEqual(false, maps:get(<<"enable">>, Data1)), + + %% delete + {ok, Delete} = request_api(delete, api_path("trace/test-name"), Header), + ?assertEqual(#{<<"code">> => 0}, json(Delete)), + + {ok, DeleteNotFound} = request_api(delete, api_path("trace/test-name"), Header), + ?assertEqual(#{<<"code">> => <<"NOT_FOUND">>, + <<"message">> => <<"test-nameNOT FOUND">>}, json(DeleteNotFound)), + + {ok, List2} = request_api(get, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(List2)), + + %% clear + {ok, Create1} = request_api(post, api_path("trace"), Header, Trace), + ?assertEqual(#{<<"code">> => 0}, json(Create1)), + + {ok, Clear} = request_api(delete, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0}, json(Clear)), + + emqx_mod_trace:unload(test), + ok. + +t_download_log(_Config) -> + emqx_mod_trace:clear(), + emqx_mod_trace:load(test), + ClientId = <<"client-test">>, + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + Name = <<"test_client_id">>, + ok = emqx_mod_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], + ct:sleep(100), + {ok, #{}, {sendfile, 0, ZipFileSize, _ZipFile}} = + emqx_mod_trace_api:download_zip_log(#{name => Name}, []), + ?assert(ZipFileSize > 0), + %% download zip file failed by server_closed occasionally? + %Header = auth_header_(), + %{ok, ZipBin} = request_api(get, api_path("trace/test_client_id/download"), Header), + %{ok, ZipHandler} = zip:zip_open(ZipBin), + %{ok, [ZipName]} = zip:zip_get(ZipHandler), + %?assertNotEqual(nomatch, string:find(ZipName, "test@127.0.0.1")), + %{ok, _} = file:read_file(emqx_mod_trace:log_file(<<"test_client_id">>, Now)), + ok = emqtt:disconnect(Client), + emqx_mod_trace:unload(test), + ok. + +t_stream_log(_Config) -> + application:set_env(emqx, allow_anonymous, true), + emqx_mod_trace:clear(), + emqx_mod_trace:load(test), + ClientId = <<"client-stream">>, + Now = erlang:system_time(second), + Name = <<"test_stream_log">>, + Start = to_rfc3339(Now - 10), + ok = emqx_mod_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], + emqtt:publish(Client, <<"/good">>, #{}, <<"ghood1">>, [{qos, 0}]), + emqtt:publish(Client, <<"/good">>, #{}, <<"ghood2">>, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ct:sleep(200), + File = emqx_mod_trace:log_file(Name, Now), + ct:pal("FileName: ~p", [File]), + {ok, FileBin} = file:read_file(File), + ct:pal("FileBin: ~p ~s", [byte_size(FileBin), FileBin]), + Header = auth_header_(), + {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?_limit=10"), Header), + #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta, <<"items">> := Bin}} = json(Binary), + ?assertEqual(10, byte_size(Bin)), + ?assertEqual(#{<<"page">> => 10, <<"limit">> => 10}, Meta), + {ok, Binary1} = request_api(get, api_path("trace/test_stream_log/log?_page=20&_limit=10"), Header), + #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta1, <<"items">> := Bin1}} = json(Binary1), + ?assertEqual(#{<<"page">> => 30, <<"limit">> => 10}, Meta1), + ?assertEqual(10, byte_size(Bin1)), + emqx_mod_trace:unload(test), + ok. + +to_rfc3339(Second) -> + list_to_binary(calendar:system_time_to_rfc3339(Second)). + +auth_header_() -> + auth_header_("admin", "public"). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), + {"Authorization", "Basic " ++ Encoded}. + +request_api(Method, Url, Auth) -> do_request_api(Method, {Url, [Auth]}). + +request_api(Method, Url, Auth, Body) -> + Request = {Url, [Auth], "application/json", emqx_json:encode(Body)}, + do_request_api(Method, Request). + +do_request_api(Method, Request) -> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], [{body_format, binary}]) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {error,{shutdown, server_closed}} -> + {error, server_closed}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +api_path(Path) -> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION, Path]). + +json(Data) -> + {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. diff --git a/priv/emqx.schema b/priv/emqx.schema index febd4cb9e..69f3acc41 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2259,6 +2259,7 @@ end}. [{emqx_mod_rewrite, Rewrites()}], [{emqx_mod_topic_metrics, []}], [{emqx_mod_delayed, []}], + [{emqx_mod_trace, []}], [{emqx_mod_st_statistics, SlowTopic()}], [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] ]) diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index d3ad128bb..14617e2e1 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -82,7 +82,7 @@ -define(SUBSCRIPTION, emqx_subscription). %% Guards --define(is_subid(Id), (is_binary(Id) orelse is_atom(Id))). +-define(IS_SUBID(Id), (is_binary(Id) orelse is_atom(Id))). -spec(start_link(atom(), pos_integer()) -> startlink_ret()). start_link(Pool, Id) -> @@ -118,14 +118,15 @@ subscribe(Topic) when is_binary(Topic) -> subscribe(Topic, undefined). -spec(subscribe(emqx_topic:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok). -subscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> +subscribe(Topic, SubId) when is_binary(Topic), ?IS_SUBID(SubId) -> subscribe(Topic, SubId, ?DEFAULT_SUBOPTS); subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> subscribe(Topic, undefined, SubOpts). -spec(subscribe(emqx_topic:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok). -subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?is_subid(SubId), is_map(SubOpts0) -> +subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) -> SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0), + _ = emqx_tracer:trace_subscribe(Topic, SubId, SubOpts), case ets:member(?SUBOPTION, {SubPid = self(), Topic}) of false -> %% New ok = emqx_broker_helper:register_sub(SubPid, SubId), @@ -171,6 +172,7 @@ unsubscribe(Topic) when is_binary(Topic) -> SubPid = self(), case ets:lookup(?SUBOPTION, {SubPid, Topic}) of [{_, SubOpts}] -> + emqx_tracer:trace_unsubscribe(Topic, SubOpts), _ = emqx_broker_helper:reclaim_seq(Topic), do_unsubscribe(Topic, SubPid, SubOpts); [] -> ok @@ -183,13 +185,7 @@ do_unsubscribe(Topic, SubPid, SubOpts) -> do_unsubscribe(Group, Topic, SubPid, SubOpts). do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> - case maps:get(shard, SubOpts, 0) of - 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - cast(pick(Topic), {unsubscribed, Topic}); - I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end; - + clean_subscribe(SubOpts, Topic, SubPid); do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> emqx_shared_sub:unsubscribe(Group, Topic, SubPid). @@ -199,7 +195,7 @@ do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> -spec(publish(emqx_types:message()) -> emqx_types:publish_result()). publish(Msg) when is_record(Msg, message) -> - _ = emqx_tracer:trace(publish, Msg), + _ = emqx_tracer:trace_publish(Msg), emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of #message{headers = #{allow_publish := false}} -> @@ -231,8 +227,7 @@ delivery(Msg) -> #delivery{sender = self(), message = Msg}. -spec(route([emqx_types:route_entry()], emqx_types:delivery()) -> emqx_types:publish_result()). route([], #delivery{message = Msg}) -> - ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]), - ok = inc_dropped_cnt(Msg), + drop_message(Msg), []; route(Routes, Delivery) -> @@ -240,6 +235,10 @@ route(Routes, Delivery) -> [do_route(Route, Delivery) | Acc] end, [], Routes). +drop_message(Msg) -> + ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]), + ok = inc_dropped_cnt(Msg). + do_route({To, Node}, Delivery) when Node =:= node() -> {Node, To, dispatch(To, Delivery)}; do_route({To, Node}, Delivery) when is_atom(Node) -> @@ -261,7 +260,7 @@ aggre(Routes) -> end, [], Routes). %% @doc Forward message to another node. --spec(forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode::sync|async) +-spec(forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode::sync | async) -> emqx_types:deliver_result()). forward(Node, To, Delivery, async) -> case emqx_rpc:cast(To, Node, ?BROKER, dispatch, [To, Delivery]) of @@ -288,8 +287,7 @@ dispatch(Topic, #delivery{message = Msg}) -> end, 0, subscribers(Topic)), case DispN of 0 -> - ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]), - ok = inc_dropped_cnt(Msg), + drop_message(Msg), {error, no_subscribers}; _ -> {ok, DispN} @@ -336,17 +334,20 @@ subscriber_down(SubPid) -> SubOpts when is_map(SubOpts) -> _ = emqx_broker_helper:reclaim_seq(Topic), true = ets:delete(?SUBOPTION, {SubPid, Topic}), - case maps:get(shard, SubOpts, 0) of - 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end; + clean_subscribe(SubOpts, Topic, SubPid); undefined -> ok end end, lookup_value(?SUBSCRIPTION, SubPid, [])), ets:delete(?SUBSCRIPTION, SubPid). +clean_subscribe(SubOpts, Topic, SubPid) -> + case maps:get(shard, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end. + %%-------------------------------------------------------------------- %% Management APIs %%-------------------------------------------------------------------- @@ -366,14 +367,14 @@ subscriptions(SubId) -> -spec(subscribed(pid() | emqx_types:subid(), emqx_topic:topic()) -> boolean()). subscribed(SubPid, Topic) when is_pid(SubPid) -> ets:member(?SUBOPTION, {SubPid, Topic}); -subscribed(SubId, Topic) when ?is_subid(SubId) -> +subscribed(SubId, Topic) when ?IS_SUBID(SubId) -> SubPid = emqx_broker_helper:lookup_subpid(SubId), ets:member(?SUBOPTION, {SubPid, Topic}). -spec(get_subopts(pid(), emqx_topic:topic()) -> maybe(emqx_types:subopts())). get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> lookup_value(?SUBOPTION, {SubPid, Topic}); -get_subopts(SubId, Topic) when ?is_subid(SubId) -> +get_subopts(SubId, Topic) when ?IS_SUBID(SubId) -> case emqx_broker_helper:lookup_subpid(SubId) of SubPid when is_pid(SubPid) -> get_subopts(SubPid, Topic); @@ -498,4 +499,3 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- - diff --git a/src/emqx_guid.erl b/src/emqx_guid.erl index 3b66f6e92..4963c48d4 100644 --- a/src/emqx_guid.erl +++ b/src/emqx_guid.erl @@ -39,6 +39,8 @@ , from_base62/1 ]). +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + -define(TAG_VERSION, 131). -define(PID_EXT, 103). -define(NEW_PID_EXT, 88). @@ -137,7 +139,7 @@ npid() -> NPid. to_hexstr(I) when byte_size(I) =:= 16 -> - emqx_misc:bin2hexstr_A_F(I). + emqx_misc:bin2hexstr_a_f_upper(I). from_hexstr(S) when byte_size(S) =:= 32 -> emqx_misc:hexstr2bin(S). diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index eb6a25377..2b77d0455 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -21,6 +21,8 @@ -include("types.hrl"). -include("logger.hrl"). +-elvis([{elvis_style, god_modules, disable}]). + -export([ merge_opts/2 , maybe_apply/2 , compose/1 @@ -47,8 +49,8 @@ , ipv6_probe/1 ]). --export([ bin2hexstr_A_F/1 - , bin2hexstr_a_f/1 +-export([ bin2hexstr_a_f_upper/1 + , bin2hexstr_a_f_lower/1 , hexstr2bin/1 ]). @@ -98,9 +100,9 @@ maybe_apply(Fun, Arg) when is_function(Fun) -> -spec(compose(list(F)) -> G when F :: fun((any()) -> any()), G :: fun((any()) -> any())). -compose([F|More]) -> compose(F, More). +compose([F | More]) -> compose(F, More). --spec(compose(F, G|[Gs]) -> C +-spec(compose(F, G | [Gs]) -> C when F :: fun((X1) -> X2), G :: fun((X2) -> X3), Gs :: [fun((Xn) -> Xn1)], @@ -108,19 +110,19 @@ compose([F|More]) -> compose(F, More). X3 :: any(), Xn :: any(), Xn1 :: any(), Xm :: any()). compose(F, G) when is_function(G) -> fun(X) -> G(F(X)) end; compose(F, [G]) -> compose(F, G); -compose(F, [G|More]) -> compose(compose(F, G), More). +compose(F, [G | More]) -> compose(compose(F, G), More). %% @doc RunFold run_fold([], Acc, _State) -> Acc; -run_fold([Fun|More], Acc, State) -> +run_fold([Fun | More], Acc, State) -> run_fold(More, Fun(Acc, State), State). %% @doc Pipeline pipeline([], Input, State) -> {ok, Input, State}; -pipeline([Fun|More], Input, State) -> +pipeline([Fun | More], Input, State) -> case apply_fun(Fun, Input, State) of ok -> pipeline(More, Input, State); {ok, NState} -> @@ -169,7 +171,7 @@ drain_deliver(0, Acc) -> drain_deliver(N, Acc) -> receive Deliver = {deliver, _Topic, _Msg} -> - drain_deliver(N-1, [Deliver|Acc]) + drain_deliver(N-1, [Deliver | Acc]) after 0 -> lists:reverse(Acc) end. @@ -184,7 +186,7 @@ drain_down(0, Acc) -> drain_down(Cnt, Acc) -> receive {'DOWN', _MRef, process, Pid, _Reason} -> - drain_down(Cnt-1, [Pid|Acc]) + drain_down(Cnt-1, [Pid | Acc]) after 0 -> lists:reverse(Acc) end. @@ -210,7 +212,7 @@ check_oom(Pid, #{message_queue_len := MaxQLen, end. do_check_oom([]) -> ok; -do_check_oom([{Val, Max, Reason}|Rest]) -> +do_check_oom([{Val, Max, Reason} | Rest]) -> case is_integer(Max) andalso (0 < Max) andalso (Max < Val) of true -> {shutdown, Reason}; false -> do_check_oom(Rest) @@ -257,8 +259,8 @@ proc_stats(Pid) -> reductions, memory]) of undefined -> []; - [{message_queue_len, Len}|ProcStats] -> - [{mailbox_len, Len}|ProcStats] + [{message_queue_len, Len} | ProcStats] -> + [{mailbox_len, Len} | ProcStats] end. rand_seed() -> @@ -278,17 +280,17 @@ index_of(E, L) -> index_of(_E, _I, []) -> error(badarg); -index_of(E, I, [E|_]) -> +index_of(E, I, [E | _]) -> I; -index_of(E, I, [_|L]) -> +index_of(E, I, [_ | L]) -> index_of(E, I+1, L). --spec(bin2hexstr_A_F(binary()) -> binary()). -bin2hexstr_A_F(B) when is_binary(B) -> +-spec(bin2hexstr_a_f_upper(binary()) -> binary()). +bin2hexstr_a_f_upper(B) when is_binary(B) -> << <<(int2hexchar(H, upper)), (int2hexchar(L, upper))>> || <> <= B>>. --spec(bin2hexstr_a_f(binary()) -> binary()). -bin2hexstr_a_f(B) when is_binary(B) -> +-spec(bin2hexstr_a_f_lower(binary()) -> binary()). +bin2hexstr_a_f_lower(B) when is_binary(B) -> << <<(int2hexchar(H, lower)), (int2hexchar(L, lower))>> || <> <= B>>. int2hexchar(I, _) when I >= 0 andalso I < 10 -> I + $0; diff --git a/src/emqx_packet.erl b/src/emqx_packet.erl index a4d440ba1..2a1137f57 100644 --- a/src/emqx_packet.erl +++ b/src/emqx_packet.erl @@ -46,24 +46,6 @@ -export([format/1]). --define(TYPE_NAMES, - { 'CONNECT' - , 'CONNACK' - , 'PUBLISH' - , 'PUBACK' - , 'PUBREC' - , 'PUBREL' - , 'PUBCOMP' - , 'SUBSCRIBE' - , 'SUBACK' - , 'UNSUBSCRIBE' - , 'UNSUBACK' - , 'PINGREQ' - , 'PINGRESP' - , 'DISCONNECT' - , 'AUTH' - }). - -type(connect() :: #mqtt_packet_connect{}). -type(publish() :: #mqtt_packet_publish{}). -type(subscribe() :: #mqtt_packet_subscribe{}). @@ -107,14 +89,14 @@ retain(#mqtt_packet{header = #mqtt_packet_header{retain = Retain}}) -> %%-------------------------------------------------------------------- %% @doc Protocol name of the CONNECT Packet. --spec(proto_name(emqx_types:packet()|connect()) -> binary()). +-spec(proto_name(emqx_types:packet() | connect()) -> binary()). proto_name(?CONNECT_PACKET(ConnPkt)) -> proto_name(ConnPkt); proto_name(#mqtt_packet_connect{proto_name = Name}) -> Name. %% @doc Protocol version of the CONNECT Packet. --spec(proto_ver(emqx_types:packet()|connect()) -> emqx_types:version()). +-spec(proto_ver(emqx_types:packet() | connect()) -> emqx_types:version()). proto_ver(?CONNECT_PACKET(ConnPkt)) -> proto_ver(ConnPkt); proto_ver(#mqtt_packet_connect{proto_ver = Ver}) -> @@ -249,7 +231,7 @@ set_props(Props, #mqtt_packet_auth{} = Pkt) -> %%-------------------------------------------------------------------- %% @doc Check PubSub Packet. --spec(check(emqx_types:packet()|publish()|subscribe()|unsubscribe()) +-spec(check(emqx_types:packet() | publish() | subscribe() | unsubscribe()) -> ok | {error, emqx_types:reason_code()}). check(#mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH}, variable = PubPkt}) when not is_tuple(PubPkt) -> @@ -318,7 +300,7 @@ check_pub_props(#{'Response-Topic' := ResponseTopic}) -> check_pub_props(_Props) -> ok. %% @doc Check CONNECT Packet. --spec(check(emqx_types:packet()|connect(), Opts :: map()) +-spec(check(emqx_types:packet() | connect(), Opts :: map()) -> ok | {error, emqx_types:reason_code()}). check(?CONNECT_PACKET(ConnPkt), Opts) -> check(ConnPkt, Opts); @@ -357,11 +339,13 @@ check_conn_props(#mqtt_packet_connect{properties = undefined}, _Opts) -> ok; check_conn_props(#mqtt_packet_connect{properties = #{'Receive-Maximum' := 0}}, _Opts) -> {error, ?RC_PROTOCOL_ERROR}; -check_conn_props(#mqtt_packet_connect{properties = #{'Request-Response-Information' := ReqRespInfo}}, _Opts) - when ReqRespInfo =/= 0, ReqRespInfo =/= 1 -> +check_conn_props(#mqtt_packet_connect{properties = + #{'Request-Response-Information' := ReqRespInfo}}, _Opts) + when ReqRespInfo =/= 0, ReqRespInfo =/= 1 -> {error, ?RC_PROTOCOL_ERROR}; -check_conn_props(#mqtt_packet_connect{properties = #{'Request-Problem-Information' := ReqProInfo}}, _Opts) - when ReqProInfo =/= 0, ReqProInfo =/= 1 -> +check_conn_props(#mqtt_packet_connect{properties = + #{'Request-Problem-Information' := ReqProInfo}}, _Opts) + when ReqProInfo =/= 0, ReqProInfo =/= 1 -> {error, ?RC_PROTOCOL_ERROR}; check_conn_props(_ConnPkt, _Opts) -> ok. @@ -382,7 +366,7 @@ check_will_msg(#mqtt_packet_connect{will_topic = WillTopic}, _Opts) -> run_checks([], _Packet, _Options) -> ok; -run_checks([Check|More], Packet, Options) -> +run_checks([Check | More], Packet, Options) -> case Check(Packet, Options) of ok -> run_checks(More, Packet, Options); Error = {error, _Reason} -> Error @@ -419,7 +403,8 @@ to_message(#mqtt_packet{ Msg#message{flags = #{dup => Dup, retain => Retain}, headers = Headers#{properties => Props}}. --spec(will_msg(#mqtt_packet_connect{}) -> emqx_types:message()). +-type(connectPacket() :: #mqtt_packet_connect{}). +-spec(will_msg(connectPacket()) -> emqx_types:message()). will_msg(#mqtt_packet_connect{will_flag = false}) -> undefined; will_msg(#mqtt_packet_connect{clientid = ClientId, @@ -468,13 +453,16 @@ format_variable(#mqtt_packet_connect{ will_payload = WillPayload, username = Username, password = Password}) -> - Format = "ClientId=~s, ProtoName=~s, ProtoVsn=~p, CleanStart=~s, KeepAlive=~p, Username=~s, Password=~s", - Args = [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)], - {Format1, Args1} = if - WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~s, Payload=~0p)", - Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]}; - true -> {Format, Args} - end, + Format = "ClientId=~s, ProtoName=~s, ProtoVsn=~p, CleanStart=~s," + " KeepAlive=~p, Username=~s, Password=~s", + Args = [ClientId, ProtoName, ProtoVer, CleanStart, + KeepAlive, Username, format_password(Password)], + {Format1, Args1} = + case WillFlag of + true -> {Format ++ ", Will(Q~p, R~p, Topic=~s, Payload=~0p)", + Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]}; + false -> {Format, Args} + end, io_lib:format(Format1, Args1); format_variable(#mqtt_packet_disconnect @@ -520,4 +508,3 @@ format_password(_Password) -> '******'. i(true) -> 1; i(false) -> 0; i(I) when is_integer(I) -> I. - diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl index 995712f6c..e563c8eab 100644 --- a/src/emqx_tracer.erl +++ b/src/emqx_tracer.erl @@ -22,13 +22,29 @@ -logger_header("[Tracer]"). %% APIs --export([ trace/2 +-export([ trace_publish/1 + , trace_subscribe/3 + , trace_unsubscribe/2 , start_trace/3 + , start_trace/4 , lookup_traces/0 - , stop_trace/1 + , stop_trace/3 + , stop_trace/2 ]). --type(trace_who() :: {clientid | topic, binary()}). +-ifdef(TEST). +-export([is_match/3]). +-endif. + +-type(label() :: 'CONNECT' | 'CONNACK' | 'PUBLISH' | 'PUBACK' | 'PUBREC' | + 'PUBREL' | 'PUBCOMP' | 'SUBSCRIBE' | 'SUBACK' | 'UNSUBSCRIBE' | + 'UNSUBACK' | 'PINGREQ' | 'PINGRESP' | 'DISCONNECT' | 'AUTH'). + +-type(tracer() :: #{name := binary(), + type := clientid | topic, + clientid => emqx_types:clientid(), + topic => emqx_types:topic(), + labels := [label()]}). -define(TRACER, ?MODULE). -define(FORMAT, {logger_formatter, @@ -44,10 +60,16 @@ msg, "\n"], single_line => false }}). --define(TOPIC_TRACE_ID(T), "trace_topic_"++T). --define(CLIENT_TRACE_ID(C), "trace_clientid_"++C). --define(TOPIC_TRACE(T), {topic, T}). --define(CLIENT_TRACE(C), {clientid, C}). +-define(TOPIC_COMBINATOR, <<"_trace_topic_">>). +-define(CLIENTID_COMBINATOR, <<"_trace_clientid_">>). +-define(TOPIC_TRACE_ID(T, N), + binary_to_atom(<<(N)/binary, ?TOPIC_COMBINATOR/binary, (T)/binary>>)). +-define(CLIENT_TRACE_ID(C, N), + binary_to_atom(<<(N)/binary, ?CLIENTID_COMBINATOR/binary, (C)/binary>>)). +-define(TOPIC_TRACE(T, N, M), {topic, T, N, M}). +-define(CLIENT_TRACE(C, N, M), {clientid, C, N, M}). +-define(TOPIC_TRACE(T, N), {topic, T, N}). +-define(CLIENT_TRACE(C, N), {clientid, C, N}). -define(IS_LOG_LEVEL(L), L =:= emergency orelse @@ -59,28 +81,49 @@ L =:= info orelse L =:= debug). --dialyzer({nowarn_function, [install_trace_handler/3]}). - %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ -trace(publish, #message{topic = <<"$SYS/", _/binary>>}) -> +trace_publish(#message{topic = <<"$SYS/", _/binary>>}) -> %% Do not trace '$SYS' publish ignore; -trace(publish, #message{from = From, topic = Topic, payload = Payload}) +trace_publish(#message{from = From, topic = Topic, payload = Payload}) when is_binary(From); is_atom(From) -> emqx_logger:info(#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} }, "PUBLISH to ~s: ~0p", [Topic, Payload]). +trace_subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore; +trace_subscribe(Topic, SubId, SubOpts) -> + emqx_logger:info(#{topic => Topic, + mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "~ts SUBSCRIBE ~ts: Options: ~0p", [SubId, Topic, SubOpts]). + +trace_unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; +trace_unsubscribe(Topic, SubOpts) -> + emqx_logger:info(#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "~ts UNSUBSCRIBE ~ts: Options: ~0p", + [maps:get(subid, SubOpts, ""), Topic, SubOpts]). + +-spec(start_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic(), + logger:level() | all, string()) -> ok | {error, term()}). +start_trace(clientid, ClientId0, Level, LogFile) -> + ClientId = ensure_bin(ClientId0), + Who = #{type => clientid, clientid => ClientId, name => ClientId, labels => []}, + start_trace(Who, Level, LogFile); +start_trace(topic, Topic0, Level, LogFile) -> + Topic = ensure_bin(Topic0), + Who = #{type => topic, topic => Topic, name => Topic, labels => []}, + start_trace(Who, Level, LogFile). + %% @doc Start to trace clientid or topic. --spec(start_trace(trace_who(), logger:level() | all, string()) -> ok | {error, term()}). +-spec(start_trace(tracer(), logger:level() | all, string()) -> ok | {error, term()}). start_trace(Who, all, LogFile) -> start_trace(Who, debug, LogFile); start_trace(Who, Level, LogFile) -> case ?IS_LOG_LEVEL(Level) of true -> - #{level := PrimaryLevel} = logger:get_primary_config(), + PrimaryLevel = emqx_logger:get_primary_log_level(), try logger:compare_levels(Level, PrimaryLevel) of lt -> {error, @@ -96,13 +139,28 @@ start_trace(Who, Level, LogFile) -> false -> {error, {invalid_log_level, Level}} end. +-spec(stop_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic()) -> + ok | {error, term()}). +stop_trace(Type, ClientIdOrTopic) -> + stop_trace(Type, ClientIdOrTopic, ClientIdOrTopic). + %% @doc Stop tracing clientid or topic. --spec(stop_trace(trace_who()) -> ok | {error, term()}). -stop_trace(Who) -> +-spec(stop_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic(), binary()) -> + ok | {error, term()}). +stop_trace(clientid, ClientId, Name) -> + Who = #{type => clientid, clientid => ensure_bin(ClientId), name => ensure_bin(Name)}, + uninstall_trance_handler(Who); +stop_trace(topic, Topic, Name) -> + Who = #{type => topic, topic => ensure_bin(Topic), name => ensure_bin(Name)}, uninstall_trance_handler(Who). %% @doc Lookup all traces --spec(lookup_traces() -> [{Who :: trace_who(), LogFile :: string()}]). +-spec(lookup_traces() -> [#{ type => topic | clientid, + name => binary(), + topic => emqx_types:topic(), + level => logger:level(), + dst => file:filename() | console | unknown + }]). lookup_traces() -> lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). @@ -116,7 +174,8 @@ install_trace_handler(Who, Level, LogFile) -> {fun filter_by_meta_key/2, Who}}]}) of ok -> - ?LOG(info, "Start trace for ~p", [Who]); + ?LOG(info, "Start trace for ~p", [Who]), + ok; {error, Reason} -> ?LOG(error, "Start trace for ~p failed, error: ~p", [Who, Reason]), {error, Reason} @@ -125,44 +184,78 @@ install_trace_handler(Who, Level, LogFile) -> uninstall_trance_handler(Who) -> case logger:remove_handler(handler_id(Who)) of ok -> - ?LOG(info, "Stop trace for ~p", [Who]); + ?LOG(info, "Stop trace for ~p", [Who]), + ok; {error, Reason} -> ?LOG(error, "Stop trace for ~p failed, error: ~p", [Who, Reason]), {error, Reason} end. filter_traces(#{id := Id, level := Level, dst := Dst}, Acc) -> - case atom_to_list(Id) of - ?TOPIC_TRACE_ID(T)-> - [{?TOPIC_TRACE(T), {Level, Dst}} | Acc]; - ?CLIENT_TRACE_ID(C) -> - [{?CLIENT_TRACE(C), {Level, Dst}} | Acc]; - _ -> Acc + IdStr = atom_to_binary(Id), + case binary:split(IdStr, [?TOPIC_COMBINATOR]) of + [Name, Topic] -> + [#{ type => topic, + name => Name, + topic => Topic, + level => Level, + dst => Dst} | Acc]; + _ -> + case binary:split(IdStr, [?CLIENTID_COMBINATOR]) of + [Name, ClientId] -> + [#{ type => clientid, + name => Name, + clientid => ClientId, + level => Level, + dst => Dst} | Acc]; + _ -> Acc + end end. -handler_id(?TOPIC_TRACE(Topic)) -> - list_to_atom(?TOPIC_TRACE_ID(handler_name(Topic))); -handler_id(?CLIENT_TRACE(ClientId)) -> - list_to_atom(?CLIENT_TRACE_ID(handler_name(ClientId))). +%% Plan to support topic_and_client type, so we have type field. +handler_id(#{type := topic, topic := Topic, name := Name}) -> + ?TOPIC_TRACE_ID(format(Topic), format(Name)); +handler_id(#{type := clientid, clientid := ClientId, name := Name}) -> + ?CLIENT_TRACE_ID(format(ClientId), format(Name)). -filter_by_meta_key(#{meta := Meta} = Log, {Key, Value}) -> - case is_meta_match(Key, Value, Meta) of +filter_by_meta_key(#{meta := Meta, level := Level} = Log, Context) -> + case is_match(Context, Meta, Level) of true -> Log; false -> ignore end. -is_meta_match(clientid, ClientId, #{clientid := ClientIdStr}) -> - ClientId =:= iolist_to_binary(ClientIdStr); -is_meta_match(topic, TopicFilter, #{topic := TopicMeta}) -> - emqx_topic:match(TopicMeta, TopicFilter); -is_meta_match(_, _, _) -> +%% When the log level is higher than debug and clientid/topic is match, +%% it will be logged without judging the content inside the labels. +%% When the log level is debug, in addition to the matched clientid/topic, +%% you also need to determine whether the label is in the labels +is_match(#{type := clientid, clientid := ExpectId, labels := Labels}, + #{clientid := RealId} = Meta, + Level) -> + is_match(ExpectId =:= iolist_to_binary(RealId), Level, Meta, Labels); +is_match(#{type := topic, topic := TopicFilter, labels := Labels}, + #{topic := Topic} = Meta, Level) -> + is_match(emqx_topic:match(Topic, TopicFilter), Level, Meta, Labels); +is_match(_, _, _) -> false. -handler_name(Bin) -> - case byte_size(Bin) of - Size when Size =< 200 -> binary_to_list(Bin); - _ -> hashstr(Bin) +is_match(true, debug, Meta, Labels) -> is_match_labels(Meta, Labels); +is_match(Boolean, _, _Meta, _Labels) -> Boolean. + +is_match_labels(#{trace_label := 'ALL'}, _Context) -> true; +is_match_labels(_, []) -> true; +is_match_labels(#{trace_label := Packet}, Context) -> + lists:member(Packet, Context); +is_match_labels(_, _) -> false. + +format(List)when is_list(List) -> + format(list_to_binary(List)); +format(Atom)when is_atom(Atom) -> + format(atom_to_list(Atom)); +format(Bin0)when is_binary(Bin0) -> + case byte_size(Bin0) of + Size when Size =< 200 -> Bin0; + _ -> emqx_misc:bin2hexstr_a_f_upper(Bin0) end. -hashstr(Bin) -> - binary_to_list(emqx_misc:bin2hexstr_A_F(Bin)). +ensure_bin(List) when is_list(List) -> iolist_to_binary(List); +ensure_bin(Bin) when is_binary(Bin) -> Bin. diff --git a/test/emqx_tracer_SUITE.erl b/test/emqx_tracer_SUITE.erl index f5af65440..5f7105774 100644 --- a/test/emqx_tracer_SUITE.erl +++ b/test/emqx_tracer_SUITE.erl @@ -22,8 +22,13 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-define(CLIENT, [{host, "localhost"}, + {clientid, <<"client">>}, + {username, <<"testuser">>}, + {password, <<"pass">>} + ]). -all() -> [t_trace_clientid, t_trace_topic]. +all() -> [t_trace_clientid, t_trace_topic, t_is_match]. init_per_suite(Config) -> emqx_ct_helpers:boot_modules(all), @@ -34,33 +39,36 @@ end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). t_trace_clientid(_Config) -> - {ok, T} = emqtt:start_link([{host, "localhost"}, - {clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]), + {ok, T} = emqtt:start_link(?CLIENT), emqtt:connect(T), %% Start tracing emqx_logger:set_log_level(error), - {error, _} = emqx_tracer:start_trace({clientid, <<"client">>}, debug, "tmp/client.log"), + {error, _} = emqx_tracer:start_trace(clientid, <<"client">>, debug, "tmp/client.log"), emqx_logger:set_log_level(debug), - ok = emqx_tracer:start_trace({clientid, <<"client">>}, debug, "tmp/client.log"), - ok = emqx_tracer:start_trace({clientid, <<"client2">>}, all, "tmp/client2.log"), - ok = emqx_tracer:start_trace({clientid, <<"client3">>}, all, "tmp/client3.log"), - {error, {invalid_log_level, bad_level}} = emqx_tracer:start_trace({clientid, <<"client4">>}, bad_level, "tmp/client4.log"), - {error, {handler_not_added, {file_error,".",eisdir}}} = emqx_tracer:start_trace({clientid, <<"client5">>}, debug, "."), + %% add list clientid + ok = emqx_tracer:start_trace(clientid, "client", debug, "tmp/client.log"), + ok = emqx_tracer:start_trace(clientid, <<"client2">>, all, "tmp/client2.log"), + ok = emqx_tracer:start_trace(clientid, <<"client3">>, all, "tmp/client3.log"), + {error, {invalid_log_level, bad_level}} = + emqx_tracer:start_trace(clientid, <<"client4">>, bad_level, "tmp/client4.log"), + {error, {handler_not_added, {file_error, ".", eisdir}}} = + emqx_tracer:start_trace(clientid, <<"client5">>, debug, "."), ct:sleep(100), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/client.log")), ?assert(filelib:is_regular("tmp/client2.log")), + ?assert(filelib:is_regular("tmp/client3.log")), %% Get current traces - ?assertEqual([{{clientid,"client"},{debug,"tmp/client.log"}}, - {{clientid,"client2"},{debug,"tmp/client2.log"}}, - {{clientid,"client3"},{debug,"tmp/client3.log"}} - ], emqx_tracer:lookup_traces()), + ?assertEqual([#{type => clientid, clientid => <<"client">>, + name => <<"client">>, level => debug, dst => "tmp/client.log"}, + #{type => clientid, clientid => <<"client2">>, + name => <<"client2">>, level => debug, dst => "tmp/client2.log"}, + #{type => clientid, clientid => <<"client3">>, + name => <<"client3">>, level => debug, dst => "tmp/client3.log"} + ], emqx_tracer:lookup_traces()), %% set the overall log level to debug emqx_logger:set_log_level(debug), @@ -74,47 +82,90 @@ t_trace_clientid(_Config) -> ?assert(filelib:file_size("tmp/client2.log") == 0), %% Stop tracing - ok = emqx_tracer:stop_trace({clientid, <<"client">>}), - ok = emqx_tracer:stop_trace({clientid, <<"client2">>}), - ok = emqx_tracer:stop_trace({clientid, <<"client3">>}), + ok = emqx_tracer:stop_trace(clientid, <<"client">>), + ok = emqx_tracer:stop_trace(clientid, <<"client2">>), + ok = emqx_tracer:stop_trace(clientid, <<"client3">>), emqtt:disconnect(T), emqx_logger:set_log_level(warning). t_trace_topic(_Config) -> - {ok, T} = emqtt:start_link([{host, "localhost"}, - {clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]), + {ok, T} = emqtt:start_link(?CLIENT), emqtt:connect(T), %% Start tracing emqx_logger:set_log_level(debug), - ok = emqx_tracer:start_trace({topic, <<"x/#">>}, all, "tmp/topic_trace.log"), - ok = emqx_tracer:start_trace({topic, <<"y/#">>}, all, "tmp/topic_trace.log"), + ok = emqx_tracer:start_trace(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), + ok = emqx_tracer:start_trace(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), ct:sleep(100), %% Verify the tracing file exits - ?assert(filelib:is_regular("tmp/topic_trace.log")), + ?assert(filelib:is_regular("tmp/topic_trace_x.log")), + ?assert(filelib:is_regular("tmp/topic_trace_y.log")), %% Get current traces - ?assertEqual([{{topic,"x/#"},{debug,"tmp/topic_trace.log"}}, - {{topic,"y/#"},{debug,"tmp/topic_trace.log"}}], emqx_tracer:lookup_traces()), + ?assertEqual([#{type => topic, topic => <<"x/#">>, name => <<"x/#">>, + level => debug, dst => "tmp/topic_trace_x.log"}, + #{type => topic, topic => <<"y/#">>, name => <<"y/#">>, + level => debug, dst => "tmp/topic_trace_y.log"}], + emqx_tracer:lookup_traces()), %% set the overall log level to debug emqx_logger:set_log_level(debug), %% Client with clientid = "client" publishes a "hi" message to "x/y/z". - emqtt:publish(T, <<"x/y/z">>, <<"hi">>), + emqtt:publish(T, <<"x/y/z">>, <<"hi1">>), + emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), ct:sleep(200), - ?assert(filelib:file_size("tmp/topic_trace.log") > 0), + ?assert(filelib:file_size("tmp/topic_trace_x.log") > 0), + ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0), %% Stop tracing - ok = emqx_tracer:stop_trace({topic, <<"x/#">>}), - ok = emqx_tracer:stop_trace({topic, <<"y/#">>}), - {error, _Reason} = emqx_tracer:stop_trace({topic, <<"z/#">>}), + ok = emqx_tracer:stop_trace(topic, <<"x/#">>), + ok = emqx_tracer:stop_trace(topic, <<"y/#">>), + {error, _Reason} = emqx_tracer:stop_trace(topic, <<"z/#">>), emqtt:disconnect(T), emqx_logger:set_log_level(warning). + +t_is_match(_Config) -> + ClientId = <<"test">>, + ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, + #{clientid => ClientId}, warning)), + ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, + #{clientid => ClientId}, debug)), + ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, + labels => ['PUBLISH']}, #{clientid => ClientId}, debug)), + ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, + #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), + ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['PUBLISH']}, + #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), + ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['SUBACK']}, + #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), + ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['SUBACK']}, + #{clientid => ClientId, trace_label => 'ALL'}, debug)), + ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, + #{clientid => <<"Bad">>}, warning)), + ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, + #{clientid => <<"Bad">>, trace_label => 'PUBLISH'}, debug)), + + Topic = <<"/test/#">>, + ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, + #{topic => <<"/test/1">>}, warning)), + ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, + #{topic => <<"/test/1/2">>}, debug)), + ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['SUBSCRIBE']}, + #{topic => <<"/test/1/2">>}, debug)), + ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, + #{topic => <<"/test/3">>, trace_label => 'PUBLISH'}, debug)), + ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['PUBLISH']}, + #{topic => <<"/test/398/">>, trace_label => 'PUBLISH'}, debug)), + ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['SUBACK']}, + #{topic => <<"/test/1/xy/y">>, trace_label => 'PUBLISH'}, debug)), + + ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['PUBLISH']}, + #{topic => <<"/t1est/398/">>, trace_label => 'PUBLISH'}, debug)), + ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, + #{topic => <<"/t1est/1/xy/y">>, trace_label => 'PUBLISH'}, debug)), + ok. From 9d4f2916c2445e1a015a2dc6ba81f576353173ef Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 11 Nov 2021 23:49:54 +0800 Subject: [PATCH 016/363] refactor(emqx_st_statistics): optimize the code directory structure (#6128) --- .../include/emqx_st_statistics.hrl | 0 .../src/emqx_plugin_libs.app.src | 2 +- .../emqx_st_statistics/emqx_st_statistics.erl | 26 +--------- .../emqx_st_statistics_api.erl | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.erl | 18 +++---- .../test/emqx_dashboard_SUITE.erl | 1 + .../src/emqx_mod_st_statistics.erl | 49 +++++++++++++++++++ .../test/emqx_st_statistics_SUITE.erl | 9 ---- .../test/emqx_st_statistics_api_SUITE.erl | 11 +++-- 9 files changed, 68 insertions(+), 50 deletions(-) rename {lib-ce/emqx_modules => apps/emqx_plugin_libs}/include/emqx_st_statistics.hrl (100%) rename lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl => apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl (95%) rename {lib-ce/emqx_modules => apps/emqx_plugin_libs}/src/emqx_st_statistics/emqx_st_statistics_api.erl (98%) create mode 100644 lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl diff --git a/lib-ce/emqx_modules/include/emqx_st_statistics.hrl b/apps/emqx_plugin_libs/include/emqx_st_statistics.hrl similarity index 100% rename from lib-ce/emqx_modules/include/emqx_st_statistics.hrl rename to apps/emqx_plugin_libs/include/emqx_st_statistics.hrl diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 82937d033..f72ffc229 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.3.1"}, + {vsn, "4.3.2"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl similarity index 95% rename from lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl rename to apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl index a66975767..ac1604c9b 100644 --- a/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_mod_st_statistics.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl @@ -14,9 +14,8 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mod_st_statistics). +-module(emqx_st_statistics). --behaviour(emqx_gen_mod). -behaviour(gen_server). -include_lib("include/emqx.hrl"). @@ -29,12 +28,6 @@ , disable/0, clear_history/0 ]). -%% emqx_gen_mod callbacks --export([ load/1 - , unload/1 - , description/0 - ]). - %% gen_server callbacks -export([ init/1 , handle_call/3 @@ -88,23 +81,6 @@ %% ets ordered_set is ascending by term order -%%-------------------------------------------------------------------- -%% Load/Unload -%%-------------------------------------------------------------------- - --spec(load(list()) -> ok). -load(Env) -> - emqx_mod_sup:start_child(?MODULE, worker, [Env]), - ok. - --spec(unload(list()) -> ok). -unload(_Env) -> - _ = emqx_mod_sup:stop_child(?MODULE), - ok. - -description() -> - "EMQ X Slow Topic Statistics Module". - %%-------------------------------------------------------------------- %%-------------------------------------------------------------------- %% APIs diff --git a/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl similarity index 98% rename from lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl rename to apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl index 328fddf28..ba13d70e2 100644 --- a/lib-ce/emqx_modules/src/emqx_st_statistics/emqx_st_statistics_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl @@ -41,7 +41,7 @@ %%-------------------------------------------------------------------- clear_history(_Bindings, _Params) -> - ok = emqx_mod_st_statistics:clear_history(), + ok = emqx_st_statistics:clear_history(), return(ok). get_history(_Bindings, Params) -> diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl index 0390339d3..40a978a37 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl @@ -42,17 +42,16 @@ start_listeners() -> lists:foreach(fun(Listener) -> start_listener(Listener) end, listeners()). %% Start HTTP Listener -start_listener({Proto, Port, Options}) when Proto == http -> +start_listener({Proto, Port, Options}) -> Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_http(listener_name(Proto), ranch_opts(Port, Options), Dispatch); - -start_listener({Proto, Port, Options}) when Proto == https -> - Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, - {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_https(listener_name(Proto), ranch_opts(Port, Options), Dispatch). + Server = listener_name(Proto), + RanchOpts = ranch_opts(Port, Options), + case Proto of + http -> minirest:start_http(Server, RanchOpts, Dispatch); + https -> minirest:start_https(Server, RanchOpts, Dispatch) + end. ranch_opts(Port, Options0) -> NumAcceptors = get_value(num_acceptors, Options0, 4), @@ -89,7 +88,7 @@ listener_name(Proto) -> http_handlers() -> Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), [{"/api/v4/", - minirest:handler(#{apps => Plugins ++ [emqx_modules], + minirest:handler(#{apps => Plugins ++ [emqx_modules, emqx_plugin_libs], filter => fun ?MODULE:filter/1}), [{authorization, fun ?MODULE:is_authorized/1}]}]. @@ -116,6 +115,7 @@ is_authorized(_Path, Req) -> _ -> false end. +filter(#{app := emqx_plugin_libs}) -> true; filter(#{app := emqx_modules}) -> true; filter(#{app := App}) -> case emqx_plugins:find_plugin(App) of diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl index ef2e747fa..550f1e9de 100644 --- a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -54,6 +54,7 @@ groups() -> ]. init_per_suite(Config) -> + application:load(emqx_plugin_libs), emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_dashboard]), Config. diff --git a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl b/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl new file mode 100644 index 000000000..d6796122c --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_st_statistics). + +-behaviour(emqx_gen_mod). + +-include_lib("include/emqx.hrl"). +-include_lib("include/logger.hrl"). + +-logger_header("[SLOW TOPICS]"). + +%% emqx_gen_mod callbacks +-export([ load/1 + , unload/1 + , description/0 + ]). + +-define(LIB, emqx_st_statistics). + +%%-------------------------------------------------------------------- +%% Load/Unload +%%-------------------------------------------------------------------- + +-spec(load(list()) -> ok). +load(Env) -> + emqx_mod_sup:start_child(?LIB, worker, [Env]), + ok. + +-spec(unload(list()) -> ok). +unload(_Env) -> + _ = emqx_mod_sup:stop_child(?LIB), + ok. + +description() -> + "EMQ X Slow Topic Statistics Module". diff --git a/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl b/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl index 36da813f2..8df07f8b8 100644 --- a/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl @@ -65,15 +65,6 @@ t_log_and_pub(_) -> end, lists:seq(1, 10)), - timer:sleep(100), - - case ets:info(?LOG_TAB, size) of - 5 -> - ok; - _ -> - ?assert(ets:info(?TOPK_TAB, size) =/= 0) - end, - timer:sleep(2400), ?assert(ets:info(?LOG_TAB, size) =:= 0), diff --git a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl index 8255cdd5f..3cf4dafd8 100644 --- a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl @@ -24,11 +24,11 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_management/include/emqx_mgmt.hrl"). --include_lib("emqx_modules/include/emqx_st_statistics.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_st_statistics.hrl"). -define(CONTENT_TYPE, "application/x-www-form-urlencoded"). --define(HOST, "http://127.0.0.1:8081/"). +-define(HOST, "http://127.0.0.1:18083/"). -define(API_VERSION, "v4"). @@ -38,8 +38,9 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - application:load(emqx_modules), - emqx_ct_helpers:start_apps([emqx_management]), + emqx_ct_helpers:boot_modules(all), + application:load(emqx_plugin_libs), + emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_dashboard]), Config. end_per_suite(Config) -> @@ -55,7 +56,7 @@ end_per_testcase(_, Config) -> Config. get(Key, ResponseBody) -> - maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). + maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). lookup_alarm(Name, [#{<<"name">> := Name} | _More]) -> true; From 7193cd4275dd4e24871d6c244c8d55dae32a0457 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 12 Nov 2021 07:33:36 +0800 Subject: [PATCH 017/363] feat(trace): move common trace module to plugin_libs (#6127) * feat(trace): move common mod to plugin_libs * fix: elvis warning --- .../src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_trace/emqx_trace.erl | 487 ++++++++++++++++++ .../src/emqx_trace/emqx_trace_api.erl | 147 ++++++ .../test/emqx_trace_SUITE.erl | 353 +++++++++++++ lib-ce/emqx_dashboard/src/emqx_dashboard.erl | 5 +- .../test/emqx_dashboard_SUITE.erl | 1 - lib-ce/emqx_modules/src/emqx_mod_trace.erl | 475 +---------------- .../emqx_modules/src/emqx_mod_trace_api.erl | 136 +---- .../test/emqx_mod_trace_SUITE.erl | 478 ----------------- .../test/emqx_mod_trace_api_SUITE.erl | 179 +++++++ 10 files changed, 1193 insertions(+), 1070 deletions(-) create mode 100644 apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl create mode 100644 apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl create mode 100644 apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl delete mode 100644 lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl create mode 100644 lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index f72ffc229..7b7dcbc07 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.3.2"}, + {vsn, "4.4.0"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl new file mode 100644 index 000000000..eb2ed7276 --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -0,0 +1,487 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_trace). + +-behaviour(gen_server). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[Trace]"). + +%% Mnesia bootstrap +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-export([ start_link/0 + , list/0 + , list/1 + , get_trace_filename/1 + , create/1 + , delete/1 + , clear/0 + , update/2 + ]). + +-export([ format/1 + , zip_dir/0 + , trace_dir/0 + , trace_file/1 + , delete_files_after_send/2 + ]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). + +-define(TRACE, ?MODULE). +-define(PACKETS, tuple_to_list(?TYPE_NAMES)). +-define(MAX_SIZE, 30). + +-ifdef(TEST). +-export([log_file/2]). +-endif. + +-record(?TRACE, + { name :: binary() | undefined | '_' + , type :: clientid | topic | undefined | '_' + , topic :: emqx_types:topic() | undefined | '_' + , clientid :: emqx_types:clientid() | undefined | '_' + , packets = [] :: list() | '_' + , enable = true :: boolean() | '_' + , start_at :: integer() | undefined | binary() | '_' + , end_at :: integer() | undefined | binary() | '_' + , log_size = #{} :: map() | '_' + }). + +mnesia(boot) -> + ok = ekka_mnesia:create_table(?TRACE, [ + {type, set}, + {disc_copies, [node()]}, + {record_name, ?TRACE}, + {attributes, record_info(fields, ?TRACE)}]); +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?TRACE, disc_copies). + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec list() -> [tuple()]. +list() -> + ets:match_object(?TRACE, #?TRACE{_ = '_'}). + +-spec list(boolean()) -> [tuple()]. +list(Enable) -> + ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). + +-spec create([{Key :: binary(), Value :: binary()}]) -> + ok | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. +create(Trace) -> + case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of + true -> + case to_trace(Trace) of + {ok, TraceRec} -> create_new_trace(TraceRec); + {error, Reason} -> {error, Reason} + end; + false -> + {error, """The number of traces created has reached the maximum, +please delete the useless ones first"""} + end. + +-spec delete(Name :: binary()) -> ok | {error, not_found}. +delete(Name) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name) of + [_] -> mnesia:delete(?TRACE, Name, write); + [] -> mnesia:abort(not_found) + end + end, + transaction(Tran). + +-spec clear() -> ok | {error, Reason :: term()}. +clear() -> + case mnesia:clear_table(?TRACE) of + {atomic, ok} -> ok; + {aborted, Reason} -> {error, Reason} + end. + +-spec update(Name :: binary(), Enable :: boolean()) -> + ok | {error, not_found | finished}. +update(Name, Enable) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name) of + [] -> mnesia:abort(not_found); + [#?TRACE{enable = Enable}] -> ok; + [Rec] -> + case erlang:system_time(second) >= Rec#?TRACE.end_at of + false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write); + true -> mnesia:abort(finished) + end + end + end, + transaction(Tran). + +-spec get_trace_filename(Name :: binary()) -> + {ok, FileName :: string()} | {error, not_found}. +get_trace_filename(Name) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name, read) of + [] -> mnesia:abort(not_found); + [#?TRACE{start_at = Start}] -> {ok, filename(Name, Start)} + end end, + transaction(Tran). + +-spec trace_file(File :: list()) -> + {ok, Node :: list(), Binary :: binary()} | + {error, Node :: list(), Reason :: term()}. +trace_file(File) -> + FileName = filename:join(trace_dir(), File), + Node = atom_to_list(node()), + case file:read_file(FileName) of + {ok, Bin} -> {ok, Node, Bin}; + {error, Reason} -> {error, Node, Reason} + end. + +delete_files_after_send(TraceLog, Zips) -> + gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}). + +-spec format(list(#?TRACE{})) -> list(map()). +format(Traces) -> + Fields = record_info(fields, ?TRACE), + lists:map(fun(Trace0 = #?TRACE{start_at = StartAt, end_at = EndAt}) -> + Trace = Trace0#?TRACE{ + start_at = list_to_binary(calendar:system_time_to_rfc3339(StartAt)), + end_at = list_to_binary(calendar:system_time_to_rfc3339(EndAt)) + }, + [_ | Values] = tuple_to_list(Trace), + maps:from_list(lists:zip(Fields, Values)) + end, Traces). + +init([]) -> + erlang:process_flag(trap_exit, true), + OriginLogLevel = emqx_logger:get_primary_log_level(), + ok = filelib:ensure_dir(trace_dir()), + ok = filelib:ensure_dir(zip_dir()), + {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), + Traces = get_enable_trace(), + ok = update_log_primary_level(Traces, OriginLogLevel), + TRef = update_trace(Traces), + {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ok, State}. + +handle_cast({delete_tag, Pid, Files}, State = #{monitors := Monitors}) -> + erlang:monitor(process, Pid), + {noreply, State#{monitors => Monitors#{Pid => Files}}}; +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitors}) -> + case maps:take(Pid, Monitors) of + error -> {noreply, State}; + {Files, NewMonitors} -> + lists:foreach(fun(F) -> file:delete(F) end, Files), + {noreply, State#{monitors => NewMonitors}} + end; +handle_info({timeout, TRef, update_trace}, + #{timer := TRef, primary_log_level := OriginLogLevel} = State) -> + Traces = get_enable_trace(), + ok = update_log_primary_level(Traces, OriginLogLevel), + NextTRef = update_trace(Traces), + {noreply, State#{timer => NextTRef}}; + +handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> + emqx_misc:cancel_timer(TRef), + handle_info({timeout, TRef, update_trace}, State); + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> + ok = set_log_primary_level(OriginLogLevel), + _ = mnesia:unsubscribe({table, ?TRACE, simple}), + emqx_misc:cancel_timer(TRef), + stop_all_trace_handler(), + _ = file:del_dir_r(zip_dir()), + ok. + +code_change(_, State, _Extra) -> + {ok, State}. + +create_new_trace(Trace) -> + Tran = fun() -> + case mnesia:read(?TRACE, Trace#?TRACE.name) of + [] -> + #?TRACE{start_at = StartAt, topic = Topic, + clientid = ClientId, packets = Packets} = Trace, + Match = #?TRACE{_ = '_', start_at = StartAt, topic = Topic, + clientid = ClientId, packets = Packets}, + case mnesia:match_object(?TRACE, Match, read) of + [] -> mnesia:write(?TRACE, Trace, write); + [#?TRACE{name = Name}] -> mnesia:abort({duplicate_condition, Name}) + end; + [#?TRACE{name = Name}] -> mnesia:abort({already_existed, Name}) + end + end, + transaction(Tran). + +update_trace(Traces) -> + Now = erlang:system_time(second), + {_Waiting, Running, Finished} = classify_by_time(Traces, Now), + disable_finished(Finished), + Started = already_running(), + {NeedRunning, AllStarted} = start_trace(Running, Started), + NeedStop = AllStarted -- NeedRunning, + ok = stop_trace(NeedStop, Started), + clean_stale_trace_files(), + NextTime = find_closest_time(Traces, Now), + emqx_misc:start_timer(NextTime, update_trace). + +stop_all_trace_handler() -> + lists:foreach(fun(#{type := Type, name := Name} = Trace) -> + _ = emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name) + end + , already_running()). + +already_running() -> + emqx_tracer:lookup_traces(). + +get_enable_trace() -> + {atomic, Traces} = + mnesia:transaction(fun() -> + mnesia:match_object(?TRACE, #?TRACE{enable = true, _ = '_'}, read) + end), + Traces. + +find_closest_time(Traces, Now) -> + Sec = + lists:foldl( + fun(#?TRACE{start_at = Start, end_at = End}, Closest) + when Start >= Now andalso Now < End -> %% running + min(End - Now, Closest); + (#?TRACE{start_at = Start}, Closest) when Start < Now -> %% waiting + min(Now - Start, Closest); + (_, Closest) -> Closest %% finished + end, 60 * 15, Traces), + timer:seconds(Sec). + +disable_finished([]) -> ok; +disable_finished(Traces) -> + NameWithLogSize = + lists:map(fun(#?TRACE{name = Name, start_at = StartAt}) -> + FileSize = filelib:file_size(log_file(Name, StartAt)), + {Name, FileSize} end, Traces), + transaction(fun() -> + lists:map(fun({Name, LogSize}) -> + case mnesia:read(?TRACE, Name, write) of + [] -> ok; + [Trace = #?TRACE{log_size = Logs}] -> + mnesia:write(?TRACE, Trace#?TRACE{enable = false, + log_size = Logs#{node() => LogSize}}, write) + end end, NameWithLogSize) + end). + +start_trace(Traces, Started0) -> + Started = lists:map(fun(#{name := Name}) -> Name end, Started0), + lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> + case lists:member(Name, StartedAcc) of + true -> {[Name | Running], StartedAcc}; + false -> + case start_trace(Trace) of + ok -> {[Name | Running], [Name | StartedAcc]}; + Error -> + ?LOG(error, "(~p)start trace failed by:~p", [Name, Error]), + {[Name | Running], StartedAcc} + end + end + end, {[], Started}, Traces). + +start_trace(Trace) -> + #?TRACE{name = Name + , type = Type + , clientid = ClientId + , topic = Topic + , packets = Packets + , start_at = Start + } = Trace, + Who0 = #{name => Name, labels => Packets}, + Who = + case Type of + topic -> Who0#{type => topic, topic => Topic}; + clientid -> Who0#{type => clientid, clientid => ClientId} + end, + case emqx_tracer:start_trace(Who, debug, log_file(Name, Start)) of + ok -> ok; + {error, {already_exist, _}} -> ok; + {error, Reason} -> {error, Reason} + end. + +stop_trace(Finished, Started) -> + lists:foreach(fun(#{name := Name, type := Type} = Trace) -> + case lists:member(Name, Finished) of + true -> emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name); + false -> ok + end + end, Started). + +clean_stale_trace_files() -> + TraceDir = trace_dir(), + case file:list_dir(TraceDir) of + {ok, AllFiles} when AllFiles =/= ["zip"] -> + FileFun = fun(#?TRACE{name = Name, start_at = StartAt}) -> filename(Name, StartAt) end, + KeepFiles = lists:map(FileFun, list()), + case AllFiles -- ["zip" | KeepFiles] of + [] -> ok; + DeleteFiles -> + DelFun = fun(F) -> file:delete(filename:join(TraceDir, F)) end, + lists:foreach(DelFun, DeleteFiles) + end; + _ -> ok + end. + +classify_by_time(Traces, Now) -> + classify_by_time(Traces, Now, [], [], []). + +classify_by_time([], _Now, Wait, Run, Finish) -> {Wait, Run, Finish}; +classify_by_time([Trace = #?TRACE{start_at = Start} | Traces], + Now, Wait, Run, Finish) when Start > Now -> + classify_by_time(Traces, Now, [Trace | Wait], Run, Finish); +classify_by_time([Trace = #?TRACE{end_at = End} | Traces], + Now, Wait, Run, Finish) when End =< Now -> + classify_by_time(Traces, Now, Wait, Run, [Trace | Finish]); +classify_by_time([Trace | Traces], Now, Wait, Run, Finish) -> + classify_by_time(Traces, Now, Wait, [Trace | Run], Finish). + +to_trace(TraceList) -> + case to_trace(TraceList, #?TRACE{}) of + {error, Reason} -> {error, Reason}; + {ok, #?TRACE{name = undefined}} -> + {error, "name required"}; + {ok, #?TRACE{type = undefined}} -> + {error, "type required"}; + {ok, #?TRACE{topic = undefined, clientid = undefined}} -> + {error, "topic/clientid cannot be both empty"}; + {ok, Trace} -> + case fill_default(Trace) of + #?TRACE{start_at = Start, end_at = End} when End =< Start -> + {error, "failed by start_at >= end_at"}; + Trace1 -> {ok, Trace1} + end + end. + +fill_default(Trace = #?TRACE{start_at = undefined}) -> + fill_default(Trace#?TRACE{start_at = erlang:system_time(second)}); +fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> + fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60}); +fill_default(Trace) -> Trace. + +to_trace([], Rec) -> {ok, Rec}; +to_trace([{<<"name">>, Name} | Trace], Rec) -> + case binary:match(Name, [<<"/">>], []) of + nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); + _ -> {error, "name cannot contain /"} + end; +to_trace([{<<"type">>, Type} | Trace], Rec) -> + case lists:member(Type, [<<"clientid">>, <<"topic">>]) of + true -> to_trace(Trace, Rec#?TRACE{type = binary_to_existing_atom(Type)}); + false -> {error, "incorrect type: only support clientid/topic"} + end; +to_trace([{<<"topic">>, Topic} | Trace], Rec) -> + case validate_topic(Topic) of + ok -> to_trace(Trace, Rec#?TRACE{topic = Topic}); + {error, Reason} -> {error, Reason} + end; +to_trace([{<<"start_at">>, StartAt} | Trace], Rec) -> + case to_system_second(StartAt) of + {ok, Sec} -> to_trace(Trace, Rec#?TRACE{start_at = Sec}); + {error, Reason} -> {error, Reason} + end; +to_trace([{<<"end_at">>, EndAt} | Trace], Rec) -> + Now = erlang:system_time(second), + case to_system_second(EndAt) of + {ok, Sec} when Sec > Now -> + to_trace(Trace, Rec#?TRACE{end_at = Sec}); + {ok, _Sec} -> + {error, "end_at time has already passed"}; + {error, Reason} -> + {error, Reason} + end; +to_trace([{<<"clientid">>, ClientId} | Trace], Rec) -> + to_trace(Trace, Rec#?TRACE{clientid = ClientId}); +to_trace([{<<"packets">>, PacketList} | Trace], Rec) -> + case to_packets(PacketList) of + {ok, Packets} -> to_trace(Trace, Rec#?TRACE{packets = Packets}); + {error, Reason} -> {error, io_lib:format("unsupport packets: ~p", [Reason])} + end; +to_trace([Unknown | _Trace], _Rec) -> {error, io_lib:format("unknown field: ~p", [Unknown])}. + +validate_topic(TopicName) -> + try emqx_topic:validate(name, TopicName) of + true -> ok + catch + error:Error -> + {error, io_lib:format("~s invalid by ~p", [TopicName, Error])} + end. + +to_system_second(At) -> + try + Sec = calendar:rfc3339_to_system_time(binary_to_list(At), [{unit, second}]), + {ok, Sec} + catch error: {badmatch, _} -> + {error, ["The rfc3339 specification not satisfied: ", At]} + end. + +to_packets(Packets) when is_list(Packets) -> + AtomTypes = lists:map(fun(Type) -> binary_to_existing_atom(Type) end, Packets), + case lists:filter(fun(T) -> not lists:member(T, ?PACKETS) end, AtomTypes) of + [] -> {ok, AtomTypes}; + InvalidE -> {error, InvalidE} + end; +to_packets(Packets) -> {error, Packets}. + +zip_dir() -> + trace_dir() ++ "zip/". + +trace_dir() -> + filename:join(emqx:get_env(data_dir), "trace") ++ "/". + +log_file(Name, Start) -> + filename:join(trace_dir(), filename(Name, Start)). + +filename(Name, Start) -> + [Time, _] = string:split(calendar:system_time_to_rfc3339(Start), "T", leading), + lists:flatten(["trace_", binary_to_list(Name), "_", Time, ".log"]). + +transaction(Tran) -> + case mnesia:transaction(Tran) of + {atomic, Res} -> Res; + {aborted, Reason} -> {error, Reason} + end. + +update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel); +update_log_primary_level(_, _) -> set_log_primary_level(debug). + +set_log_primary_level(NewLevel) -> + case NewLevel =/= emqx_logger:get_primary_log_level() of + true -> emqx_logger:set_primary_log_level(NewLevel); + false -> ok + end. diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl new file mode 100644 index 000000000..1bc71654e --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -0,0 +1,147 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_api). +-include_lib("emqx/include/logger.hrl"). + +%% API +-export([ list_trace/2 + , create_trace/2 + , update_trace/2 + , delete_trace/2 + , clear_traces/2 + , download_zip_log/2 + , stream_log_file/2 +]). +-export([read_trace_file/3]). + +-define(TO_BIN(_B_), iolist_to_binary(_B_)). +-define(NOT_FOUND(N), {error, 'NOT_FOUND', ?TO_BIN([N, "NOT FOUND"])}). + +list_trace(_, Params) -> + List = + case Params of + [{<<"enable">>, Enable}] -> emqx_trace:list(Enable); + _ -> emqx_trace:list() + end, + {ok, emqx_trace:format(List)}. + +create_trace(_, Param) -> + case emqx_trace:create(Param) of + ok -> ok; + {error, {already_existed, Name}} -> + {error, 'ALREADY_EXISTED', ?TO_BIN([Name, "Already Exists"])}; + {error, {duplicate_condition, Name}} -> + {error, 'DUPLICATE_CONDITION', ?TO_BIN([Name, "Duplication Condition"])}; + {error, Reason} -> + {error, 'INCORRECT_PARAMS', ?TO_BIN(Reason)} + end. + +delete_trace(#{name := Name}, _Param) -> + case emqx_trace:delete(Name) of + ok -> ok; + {error, not_found} -> ?NOT_FOUND(Name) + end. + +clear_traces(_, _) -> + emqx_trace:clear(). + +update_trace(#{name := Name, operation := Operation}, _Param) -> + Enable = case Operation of disable -> false; enable -> true end, + case emqx_trace:update(Name, Enable) of + ok -> {ok, #{enable => Enable, name => Name}}; + {error, not_found} -> ?NOT_FOUND(Name) + end. + +%% if HTTP request headers include accept-encoding: gzip and file size > 300 bytes. +%% cowboy_compress_h will auto encode gzip format. +download_zip_log(#{name := Name}, _Param) -> + case emqx_trace:get_trace_filename(Name) of + {ok, TraceLog} -> + TraceFiles = collect_trace_file(TraceLog), + ZipDir = emqx_trace:zip_dir(), + Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), + ZipFileName = ZipDir ++ TraceLog, + {ok, ZipFile} = zip:zip(ZipFileName, Zips), + emqx_trace:delete_files_after_send(ZipFileName, Zips), + {ok, #{}, {sendfile, 0, filelib:file_size(ZipFile), ZipFile}}; + {error, Reason} -> + {error, Reason} + end. + +group_trace_file(ZipDir, TraceLog, TraceFiles) -> + lists:foldl(fun(Res, Acc) -> + case Res of + {ok, Node, Bin} -> + ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, + ok = file:write_file(ZipName, Bin), + [ZipName | Acc]; + {error, Node, Reason} -> + ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), + Acc + end + end, [], TraceFiles). + +collect_trace_file(TraceLog) -> + Nodes = ekka_mnesia:running_nodes(), + {Files, BadNodes} = rpc:multicall(Nodes, emqx_trace, trace_file, [TraceLog], 60000), + BadNodes =/= [] andalso ?LOG(error, "download log rpc failed on ~p", [BadNodes]), + Files. + +%% _page as position and _limit as bytes for front-end reusing components +stream_log_file(#{name := Name}, Params) -> + Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), + Position0 = proplists:get_value(<<"_page">>, Params, <<"0">>), + Bytes0 = proplists:get_value(<<"_limit">>, Params, <<"500">>), + Node = binary_to_existing_atom(Node0), + Position = binary_to_integer(Position0), + Bytes = binary_to_integer(Bytes0), + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + {ok, Bin} -> + Meta = #{<<"page">> => Position + byte_size(Bin), <<"limit">> => Bytes}, + {ok, #{meta => Meta, items => Bin}}; + eof -> + Meta = #{<<"page">> => Position, <<"limit">> => Bytes}, + {ok, #{meta => Meta, items => <<"">>}}; + {error, Reason} -> + logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), + {error, Reason} + end. + +%% this is an rpc call for stream_log_file/2 +read_trace_file(Name, Position, Limit) -> + TraceDir = emqx_trace:trace_dir(), + {ok, AllFiles} = file:list_dir(TraceDir), + TracePrefix = "trace_" ++ binary_to_list(Name) ++ "_", + Filter = fun(FileName) -> nomatch =/= string:prefix(FileName, TracePrefix) end, + case lists:filter(Filter, AllFiles) of + [TraceFile] -> + TracePath = filename:join([TraceDir, TraceFile]), + read_file(TracePath, Position, Limit); + [] -> {error, not_found} + end. + +read_file(Path, Offset, Bytes) -> + {ok, IoDevice} = file:open(Path, [read, raw, binary]), + try + _ = case Offset of + 0 -> ok; + _ -> file:position(IoDevice, {bof, Offset}) + end, + file:read(IoDevice, Bytes) + after + file:close(IoDevice) + end. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl new file mode 100644 index 000000000..1bd28d888 --- /dev/null +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -0,0 +1,353 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_trace_SUITE). + +%% API +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-record(emqx_trace, { + name, + type, + topic, + clientid, + packets = [], + enable = true, + start_at, + end_at, + log_size = #{} + }). + +-define(PACKETS, ['CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL' + , 'PUBCOMP', 'SUBSCRIBE', 'SUBACK', 'UNSUBSCRIBE', 'UNSUBACK' + , 'PINGREQ', 'PINGRESP', 'DISCONNECT', 'AUTH']). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_plugin_libs), + emqx_ct_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([]). + +t_base_create_delete(_Config) -> + ok = emqx_trace:clear(), + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + End = to_rfc3339(Now + 30 * 60), + Name = <<"name1">>, + ClientId = <<"test-device">>, + Packets = [atom_to_binary(E) || E <- ?PACKETS], + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, + {<<"clientid">>, ClientId}, + {<<"packets">>, Packets}, + {<<"start_at">>, Start}, + {<<"end_at">>, End} + ], + AnotherTrace = lists:keyreplace(<<"name">>, 1, Trace, {<<"name">>, <<"AnotherName">>}), + ok = emqx_trace:create(Trace), + ?assertEqual({error, {already_existed, Name}}, emqx_trace:create(Trace)), + ?assertEqual({error, {duplicate_condition, Name}}, emqx_trace:create(AnotherTrace)), + [TraceRec] = emqx_trace:list(), + Expect = #emqx_trace{ + name = Name, + type = clientid, + topic = undefined, + clientid = ClientId, + packets = ?PACKETS, + start_at = Now, + end_at = Now + 30 * 60 + }, + ?assertEqual(Expect, TraceRec), + ExpectFormat = [ + #{ + clientid => <<"test-device">>, + enable => true, + type => clientid, + packets => ?PACKETS, + name => <<"name1">>, + start_at => Start, + end_at => End, + log_size => #{}, + topic => undefined + } + ], + ?assertEqual(ExpectFormat, emqx_trace:format([TraceRec])), + ?assertEqual(ok, emqx_trace:delete(Name)), + ?assertEqual({error, not_found}, emqx_trace:delete(Name)), + ?assertEqual([], emqx_trace:list()), + ok. + +t_create_size_max(_Config) -> + emqx_trace:clear(), + lists:map(fun(Seq) -> + Name = list_to_binary("name" ++ integer_to_list(Seq)), + Trace = [{<<"name">>, Name}, {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], + ok = emqx_trace:create(Trace) + end, lists:seq(1, 30)), + Trace31 = [{<<"name">>, <<"name31">>}, {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/31">>}], + {error, _} = emqx_trace:create(Trace31), + ok = emqx_trace:delete(<<"name30">>), + ok = emqx_trace:create(Trace31), + ?assertEqual(30, erlang:length(emqx_trace:list())), + ok. + +t_create_failed(_Config) -> + ok = emqx_trace:clear(), + UnknownField = [{<<"unknown">>, 12}], + {error, Reason1} = emqx_trace:create(UnknownField), + ?assertEqual(<<"unknown field: {<<\"unknown\">>,12}">>, iolist_to_binary(Reason1)), + + InvalidTopic = [{<<"topic">>, "#/#//"}], + {error, Reason2} = emqx_trace:create(InvalidTopic), + ?assertEqual(<<"#/#// invalid by function_clause">>, iolist_to_binary(Reason2)), + + InvalidStart = [{<<"start_at">>, <<"2021-12-3:12">>}], + {error, Reason3} = emqx_trace:create(InvalidStart), + ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, + iolist_to_binary(Reason3)), + + InvalidEnd = [{<<"end_at">>, <<"2021-12-3:12">>}], + {error, Reason4} = emqx_trace:create(InvalidEnd), + ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, + iolist_to_binary(Reason4)), + + InvalidPackets = [{<<"packets">>, [<<"publish">>]}], + {error, Reason5} = emqx_trace:create(InvalidPackets), + ?assertEqual(<<"unsupport packets: [publish]">>, iolist_to_binary(Reason5)), + + InvalidPackets2 = [{<<"packets">>, <<"publish">>}], + {error, Reason6} = emqx_trace:create(InvalidPackets2), + ?assertEqual(<<"unsupport packets: <<\"publish\">>">>, iolist_to_binary(Reason6)), + + {error, Reason7} = emqx_trace:create([{<<"name">>, <<"test">>}, {<<"type">>, <<"clientid">>}]), + ?assertEqual(<<"topic/clientid cannot be both empty">>, iolist_to_binary(Reason7)), + + InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, + {<<"type">>, <<"clientid">>}], + {error, Reason9} = emqx_trace:create(InvalidPackets4), + ?assertEqual(<<"name cannot contain /">>, iolist_to_binary(Reason9)), + + ?assertEqual({error, "type required"}, emqx_trace:create([{<<"name">>, <<"test-name">>}, + {<<"packets">>, []}, {<<"clientid">>, <<"good">>}])), + + ?assertEqual({error, "incorrect type: only support clientid/topic"}, + emqx_trace:create([{<<"name">>, <<"test-name">>}, + {<<"packets">>, []}, {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), + ok. + +t_create_default(_Config) -> + ok = emqx_trace:clear(), + {error, "name required"} = emqx_trace:create([]), + ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, + {<<"type">>, <<"clientid">>}, {<<"packets">>, []}, {<<"clientid">>, <<"good">>}]), + [#emqx_trace{packets = Packets}] = emqx_trace:list(), + ?assertEqual([], Packets), + ok = emqx_trace:clear(), + Trace = [ + {<<"name">>, <<"test-name">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"start_at">>, <<"2021-10-28T10:54:47+08:00">>}, + {<<"end_at">>, <<"2021-10-27T10:54:47+08:00">>} + ], + {error, "end_at time has already passed"} = emqx_trace:create(Trace), + Now = erlang:system_time(second), + Trace2 = [ + {<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"start_at">>, to_rfc3339(Now + 10)}, + {<<"end_at">>, to_rfc3339(Now + 3)} + ], + {error, "failed by start_at >= end_at"} = emqx_trace:create(Trace2), + ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>}]), + [#emqx_trace{start_at = Start, end_at = End}] = emqx_trace:list(), + ?assertEqual(10 * 60, End - Start), + ?assertEqual(true, Start - erlang:system_time(second) < 5), + ok. + +t_update_enable(_Config) -> + ok = emqx_trace:clear(), + Name = <<"test-name">>, + Now = erlang:system_time(second), + End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), + ok = emqx_trace:create([{<<"name">>, Name}, {<<"packets">>, [<<"PUBLISH">>]}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), + [#emqx_trace{enable = Enable}] = emqx_trace:list(), + ?assertEqual(Enable, true), + ok = emqx_trace:update(Name, false), + [#emqx_trace{enable = false}] = emqx_trace:list(), + ok = emqx_trace:update(Name, false), + [#emqx_trace{enable = false}] = emqx_trace:list(), + ok = emqx_trace:update(Name, true), + [#emqx_trace{enable = true}] = emqx_trace:list(), + ok = emqx_trace:update(Name, false), + [#emqx_trace{enable = false}] = emqx_trace:list(), + ?assertEqual({error, not_found}, emqx_trace:update(<<"Name not found">>, true)), + ct:sleep(2100), + ?assertEqual({error, finished}, emqx_trace:update(Name, true)), + ok. + +t_load_state(_Config) -> + emqx_trace:clear(), + load(), + Now = erlang:system_time(second), + Running = [{<<"name">>, <<"Running">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/1">>}, {<<"start_at">>, to_rfc3339(Now - 1)}, + {<<"end_at">>, to_rfc3339(Now + 2)}], + Waiting = [{<<"name">>, <<"Waiting">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/2">>}, {<<"start_at">>, to_rfc3339(Now + 3)}, + {<<"end_at">>, to_rfc3339(Now + 8)}], + Finished = [{<<"name">>, <<"Finished">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/3">>}, {<<"start_at">>, to_rfc3339(Now - 5)}, + {<<"end_at">>, to_rfc3339(Now)}], + ok = emqx_trace:create(Running), + ok = emqx_trace:create(Waiting), + {error, "end_at time has already passed"} = emqx_trace:create(Finished), + Traces = emqx_trace:format(emqx_trace:list()), + ?assertEqual(2, erlang:length(Traces)), + Enables = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces), + ExpectEnables = [{<<"Running">>, true}, {<<"Waiting">>, true}], + ?assertEqual(ExpectEnables, lists:sort(Enables)), + ct:sleep(3500), + Traces2 = emqx_trace:format(emqx_trace:list()), + ?assertEqual(2, erlang:length(Traces2)), + Enables2 = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces2), + ExpectEnables2 = [{<<"Running">>, false}, {<<"Waiting">>, true}], + ?assertEqual(ExpectEnables2, lists:sort(Enables2)), + unload(), + ok. + +t_client_event(_Config) -> + application:set_env(emqx, allow_anonymous, true), + emqx_trace:clear(), + ClientId = <<"client-test">>, + load(), + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + Name = <<"test_client_id_event">>, + ok = emqx_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + emqtt:ping(Client), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]), + ct:sleep(200), + ok = emqx_trace:create([{<<"name">>, <<"test_topic">>}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Bin} = file:read_file(emqx_trace:log_file(Name, Now)), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ct:sleep(200), + {ok, Bin2} = file:read_file(emqx_trace:log_file(Name, Now)), + {ok, Bin3} = file:read_file(emqx_trace:log_file(<<"test_topic">>, Now)), + ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), + ?assert(erlang:byte_size(Bin) > 0), + ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)), + ?assert(erlang:byte_size(Bin3) > 0), + unload(), + ok. + +t_get_log_filename(_Config) -> + ok = emqx_trace:clear(), + load(), + Now = erlang:system_time(second), + Start = calendar:system_time_to_rfc3339(Now), + End = calendar:system_time_to_rfc3339(Now + 2), + Name = <<"name1">>, + ClientId = <<"test-device">>, + Packets = [atom_to_binary(E) || E <- ?PACKETS], + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, + {<<"clientid">>, ClientId}, + {<<"packets">>, Packets}, + {<<"start_at">>, list_to_binary(Start)}, + {<<"end_at">>, list_to_binary(End)} + ], + ok = emqx_trace:create(Trace), + ?assertEqual({error, not_found}, emqx_trace:get_trace_filename(<<"test">>)), + ?assertEqual(ok, element(1, emqx_trace:get_trace_filename(Name))), + ct:sleep(3000), + ?assertEqual(ok, element(1, emqx_trace:get_trace_filename(Name))), + unload(), + ok. + +t_trace_file(_Config) -> + FileName = "test.log", + Content = <<"test \n test">>, + TraceDir = emqx_trace:trace_dir(), + File = filename:join(TraceDir, FileName), + ok = file:write_file(File, Content), + {ok, Node, Bin} = emqx_trace:trace_file(FileName), + ?assertEqual(Node, atom_to_list(node())), + ?assertEqual(Content, Bin), + ok = file:delete(File), + ok. + +t_download_log(_Config) -> + emqx_trace:clear(), + load(), + ClientId = <<"client-test">>, + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + Name = <<"test_client_id">>, + ok = emqx_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], + ct:sleep(100), + {ok, #{}, {sendfile, 0, ZipFileSize, _ZipFile}} = + emqx_trace_api:download_zip_log(#{name => Name}, []), + ?assert(ZipFileSize > 0), + ok = emqtt:disconnect(Client), + unload(), + ok. + +to_rfc3339(Second) -> + list_to_binary(calendar:system_time_to_rfc3339(Second)). + +load() -> + emqx_trace:start_link(). + +unload() -> + gen_server:stop(emqx_trace). diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl index 40a978a37..59318a5a1 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl @@ -41,7 +41,8 @@ start_listeners() -> lists:foreach(fun(Listener) -> start_listener(Listener) end, listeners()). -%% Start HTTP Listener + +%% Start HTTP(S) Listener start_listener({Proto, Port, Options}) -> Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, @@ -88,7 +89,7 @@ listener_name(Proto) -> http_handlers() -> Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), [{"/api/v4/", - minirest:handler(#{apps => Plugins ++ [emqx_modules, emqx_plugin_libs], + minirest:handler(#{apps => Plugins ++ [emqx_modules, emqx_plugin_libs], filter => fun ?MODULE:filter/1}), [{authorization, fun ?MODULE:is_authorized/1}]}]. diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 550f1e9de..71a7692be 100644 --- a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -166,4 +166,3 @@ api_path(Path) -> json(Data) -> {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. - diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace.erl b/lib-ce/emqx_modules/src/emqx_mod_trace.erl index 737ee7e97..03d468a82 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace.erl @@ -16,493 +16,24 @@ -module(emqx_mod_trace). --behaviour(gen_server). -behaviour(emqx_gen_mod). - -include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("kernel/include/file.hrl"). - --logger_header("[Trace]"). - -%% Mnesia bootstrap --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). -export([ load/1 , unload/1 , description/0 ]). --export([ start_link/0 - , list/0 - , list/1 - , get_trace_filename/1 - , create/1 - , delete/1 - , clear/0 - , update/2 - ]). - --export([ format/1 - , zip_dir/0 - , trace_dir/0 - , trace_file/1 - , delete_files_after_send/2 - ]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). - --define(TRACE, ?MODULE). --define(PACKETS, tuple_to_list(?TYPE_NAMES)). --define(MAX_SIZE, 30). - --ifdef(TEST). --export([log_file/2]). --endif. - --record(?TRACE, - { name :: binary() | undefined | '_' - , type :: clientid | topic | undefined | '_' - , topic :: emqx_types:topic() | undefined | '_' - , clientid :: emqx_types:clientid() | undefined | '_' - , packets = [] :: list() | '_' - , enable = true :: boolean() | '_' - , start_at :: integer() | undefined | binary() | '_' - , end_at :: integer() | undefined | binary() | '_' - , log_size = #{} :: map() | '_' - }). - -mnesia(boot) -> - ok = ekka_mnesia:create_table(?TRACE, [ - {type, set}, - {disc_copies, [node()]}, - {record_name, ?TRACE}, - {attributes, record_info(fields, ?TRACE)}]); -mnesia(copy) -> - ok = ekka_mnesia:copy_table(?TRACE, disc_copies). - +-spec description() -> string(). description() -> "EMQ X Trace Module". -spec load(any()) -> ok. load(_Env) -> - emqx_mod_sup:start_child(?MODULE, worker). + emqx_mod_sup:start_child(emqx_trace, worker). -spec unload(any()) -> ok. unload(_Env) -> - _ = emqx_mod_sup:stop_child(?MODULE), - stop_all_trace_handler(). - --spec(start_link() -> emqx_types:startlink_ret()). -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - --spec list() -> [tuple()]. -list() -> - ets:match_object(?TRACE, #?TRACE{_ = '_'}). - --spec list(boolean()) -> [tuple()]. -list(Enable) -> - ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). - --spec create([{Key :: binary(), Value :: binary()}]) -> - ok | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. -create(Trace) -> - case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of - true -> - case to_trace(Trace) of - {ok, TraceRec} -> create_new_trace(TraceRec); - {error, Reason} -> {error, Reason} - end; - false -> - {error, """The number of traces created has reached the maximum, -please delete the useless ones first"""} - end. - --spec delete(Name :: binary()) -> ok | {error, not_found}. -delete(Name) -> - Tran = fun() -> - case mnesia:read(?TRACE, Name) of - [_] -> mnesia:delete(?TRACE, Name, write); - [] -> mnesia:abort(not_found) - end - end, - transaction(Tran). - --spec clear() -> ok | {error, Reason :: term()}. -clear() -> - case mnesia:clear_table(?TRACE) of - {atomic, ok} -> ok; - {aborted, Reason} -> {error, Reason} - end. - --spec update(Name :: binary(), Enable :: boolean()) -> - ok | {error, not_found | finished}. -update(Name, Enable) -> - Tran = fun() -> - case mnesia:read(?TRACE, Name) of - [] -> mnesia:abort(not_found); - [#?TRACE{enable = Enable}] -> ok; - [Rec] -> - case erlang:system_time(second) >= Rec#?TRACE.end_at of - false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write); - true -> mnesia:abort(finished) - end - end - end, - transaction(Tran). - --spec get_trace_filename(Name :: binary()) -> - {ok, FileName :: string()} | {error, not_found}. -get_trace_filename(Name) -> - Tran = fun() -> - case mnesia:read(?TRACE, Name, read) of - [] -> mnesia:abort(not_found); - [#?TRACE{start_at = Start}] -> {ok, filename(Name, Start)} - end end, - transaction(Tran). - --spec trace_file(File :: list()) -> - {ok, Node :: list(), Binary :: binary()} | - {error, Node :: list(), Reason :: term()}. -trace_file(File) -> - FileName = filename:join(trace_dir(), File), - Node = atom_to_list(node()), - case file:read_file(FileName) of - {ok, Bin} -> {ok, Node, Bin}; - {error, Reason} -> {error, Node, Reason} - end. - -delete_files_after_send(TraceLog, Zips) -> - gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}). - --spec format(list(#?TRACE{})) -> list(map()). -format(Traces) -> - Fields = record_info(fields, ?TRACE), - lists:map(fun(Trace0 = #?TRACE{start_at = StartAt, end_at = EndAt}) -> - Trace = Trace0#?TRACE{ - start_at = list_to_binary(calendar:system_time_to_rfc3339(StartAt)), - end_at = list_to_binary(calendar:system_time_to_rfc3339(EndAt)) - }, - [_ | Values] = tuple_to_list(Trace), - maps:from_list(lists:zip(Fields, Values)) - end, Traces). - -init([]) -> - erlang:process_flag(trap_exit, true), - OriginLogLevel = emqx_logger:get_primary_log_level(), - ok = filelib:ensure_dir(trace_dir()), - ok = filelib:ensure_dir(zip_dir()), - {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), - Traces = get_enable_trace(), - ok = update_log_primary_level(Traces, OriginLogLevel), - TRef = update_trace(Traces), - {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ok, State}. - -handle_cast({delete_tag, Pid, Files}, State = #{monitors := Monitors}) -> - erlang:monitor(process, Pid), - {noreply, State#{monitors => Monitors#{Pid => Files}}}; -handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), - {noreply, State}. - -handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitors}) -> - case maps:take(Pid, Monitors) of - error -> {noreply, State}; - {Files, NewMonitors} -> - lists:foreach(fun(F) -> file:delete(F) end, Files), - {noreply, State#{monitors => NewMonitors}} - end; -handle_info({timeout, TRef, update_trace}, - #{timer := TRef, primary_log_level := OriginLogLevel} = State) -> - Traces = get_enable_trace(), - ok = update_log_primary_level(Traces, OriginLogLevel), - NextTRef = update_trace(Traces), - {noreply, State#{timer => NextTRef}}; - -handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef), - handle_info({timeout, TRef, update_trace}, State); - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> - ok = set_log_primary_level(OriginLogLevel), - _ = mnesia:unsubscribe({table, ?TRACE, simple}), - emqx_misc:cancel_timer(TRef), - stop_all_trace_handler(), - _ = file:del_dir_r(zip_dir()), - ok. - -code_change(_, State, _Extra) -> - {ok, State}. - -create_new_trace(Trace) -> - Tran = fun() -> - case mnesia:read(?TRACE, Trace#?TRACE.name) of - [] -> - #?TRACE{start_at = StartAt, topic = Topic, - clientid = ClientId, packets = Packets} = Trace, - Match = #?TRACE{_ = '_', start_at = StartAt, topic = Topic, - clientid = ClientId, packets = Packets}, - case mnesia:match_object(?TRACE, Match, read) of - [] -> mnesia:write(?TRACE, Trace, write); - [#?TRACE{name = Name}] -> mnesia:abort({duplicate_condition, Name}) - end; - [#?TRACE{name = Name}] -> mnesia:abort({already_existed, Name}) - end - end, - transaction(Tran). - -update_trace(Traces) -> - Now = erlang:system_time(second), - {_Waiting, Running, Finished} = classify_by_time(Traces, Now), - disable_finished(Finished), - Started = already_running(), - {NeedRunning, AllStarted} = start_trace(Running, Started), - NeedStop = AllStarted -- NeedRunning, - ok = stop_trace(NeedStop, Started), - clean_stale_trace_files(), - NextTime = find_closest_time(Traces, Now), - emqx_misc:start_timer(NextTime, update_trace). - -stop_all_trace_handler() -> - lists:foreach(fun(#{type := Type, name := Name} = Trace) -> - _ = emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name) - end - , already_running()). - -already_running() -> - emqx_tracer:lookup_traces(). - -get_enable_trace() -> - {atomic, Traces} = - mnesia:transaction(fun() -> - mnesia:match_object(?TRACE, #?TRACE{enable = true, _ = '_'}, read) - end), - Traces. - -find_closest_time(Traces, Now) -> - Sec = - lists:foldl( - fun(#?TRACE{start_at = Start, end_at = End}, Closest) - when Start >= Now andalso Now < End -> %% running - min(End - Now, Closest); - (#?TRACE{start_at = Start}, Closest) when Start < Now -> %% waiting - min(Now - Start, Closest); - (_, Closest) -> Closest %% finished - end, 60 * 15, Traces), - timer:seconds(Sec). - -disable_finished([]) -> ok; -disable_finished(Traces) -> - NameWithLogSize = - lists:map(fun(#?TRACE{name = Name, start_at = StartAt}) -> - FileSize = filelib:file_size(log_file(Name, StartAt)), - {Name, FileSize} end, Traces), - transaction(fun() -> - lists:map(fun({Name, LogSize}) -> - case mnesia:read(?TRACE, Name, write) of - [] -> ok; - [Trace = #?TRACE{log_size = Logs}] -> - mnesia:write(?TRACE, Trace#?TRACE{enable = false, - log_size = Logs#{node() => LogSize}}, write) - end end, NameWithLogSize) - end). - -start_trace(Traces, Started0) -> - Started = lists:map(fun(#{name := Name}) -> Name end, Started0), - lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> - case lists:member(Name, StartedAcc) of - true -> {[Name | Running], StartedAcc}; - false -> - case start_trace(Trace) of - ok -> {[Name | Running], [Name | StartedAcc]}; - Error -> - ?LOG(error, "(~p)start trace failed by:~p", [Name, Error]), - {[Name | Running], StartedAcc} - end - end - end, {[], Started}, Traces). - -start_trace(Trace) -> - #?TRACE{name = Name - , type = Type - , clientid = ClientId - , topic = Topic - , packets = Packets - , start_at = Start - } = Trace, - Who0 = #{name => Name, labels => Packets}, - Who = - case Type of - topic -> Who0#{type => topic, topic => Topic}; - clientid -> Who0#{type => clientid, clientid => ClientId} - end, - case emqx_tracer:start_trace(Who, debug, log_file(Name, Start)) of - ok -> ok; - {error, {already_exist, _}} -> ok; - {error, Reason} -> {error, Reason} - end. - -stop_trace(Finished, Started) -> - lists:foreach(fun(#{name := Name, type := Type} = Trace) -> - case lists:member(Name, Finished) of - true -> emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name); - false -> ok - end - end, Started). - -clean_stale_trace_files() -> - TraceDir = trace_dir(), - case file:list_dir(TraceDir) of - {ok, AllFiles} when AllFiles =/= ["zip"] -> - FileFun = fun(#?TRACE{name = Name, start_at = StartAt}) -> filename(Name, StartAt) end, - KeepFiles = lists:map(FileFun, list()), - case AllFiles -- ["zip" | KeepFiles] of - [] -> ok; - DeleteFiles -> - DelFun = fun(F) -> file:delete(filename:join(TraceDir, F)) end, - lists:foreach(DelFun, DeleteFiles) - end; - _ -> ok - end. - -classify_by_time(Traces, Now) -> - classify_by_time(Traces, Now, [], [], []). - -classify_by_time([], _Now, Wait, Run, Finish) -> {Wait, Run, Finish}; -classify_by_time([Trace = #?TRACE{start_at = Start} | Traces], - Now, Wait, Run, Finish) when Start > Now -> - classify_by_time(Traces, Now, [Trace | Wait], Run, Finish); -classify_by_time([Trace = #?TRACE{end_at = End} | Traces], - Now, Wait, Run, Finish) when End =< Now -> - classify_by_time(Traces, Now, Wait, Run, [Trace | Finish]); -classify_by_time([Trace | Traces], Now, Wait, Run, Finish) -> - classify_by_time(Traces, Now, Wait, [Trace | Run], Finish). - -to_trace(TraceList) -> - case to_trace(TraceList, #?TRACE{}) of - {error, Reason} -> {error, Reason}; - {ok, #?TRACE{name = undefined}} -> - {error, "name required"}; - {ok, #?TRACE{type = undefined}} -> - {error, "type required"}; - {ok, #?TRACE{topic = undefined, clientid = undefined}} -> - {error, "topic/clientid cannot be both empty"}; - {ok, Trace} -> - case fill_default(Trace) of - #?TRACE{start_at = Start, end_at = End} when End =< Start -> - {error, "failed by start_at >= end_at"}; - Trace1 -> {ok, Trace1} - end - end. - -fill_default(Trace = #?TRACE{start_at = undefined}) -> - fill_default(Trace#?TRACE{start_at = erlang:system_time(second)}); -fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> - fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60}); -fill_default(Trace) -> Trace. - -to_trace([], Rec) -> {ok, Rec}; -to_trace([{<<"name">>, Name} | Trace], Rec) -> - case binary:match(Name, [<<"/">>], []) of - nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); - _ -> {error, "name cannot contain /"} - end; -to_trace([{<<"type">>, Type} | Trace], Rec) -> - case lists:member(Type, [<<"clientid">>, <<"topic">>]) of - true -> to_trace(Trace, Rec#?TRACE{type = binary_to_existing_atom(Type)}); - false -> {error, "incorrect type: only support clientid/topic"} - end; -to_trace([{<<"topic">>, Topic} | Trace], Rec) -> - case validate_topic(Topic) of - ok -> to_trace(Trace, Rec#?TRACE{topic = Topic}); - {error, Reason} -> {error, Reason} - end; -to_trace([{<<"start_at">>, StartAt} | Trace], Rec) -> - case to_system_second(StartAt) of - {ok, Sec} -> to_trace(Trace, Rec#?TRACE{start_at = Sec}); - {error, Reason} -> {error, Reason} - end; -to_trace([{<<"end_at">>, EndAt} | Trace], Rec) -> - Now = erlang:system_time(second), - case to_system_second(EndAt) of - {ok, Sec} when Sec > Now -> - to_trace(Trace, Rec#?TRACE{end_at = Sec}); - {ok, _Sec} -> - {error, "end_at time has already passed"}; - {error, Reason} -> - {error, Reason} - end; -to_trace([{<<"clientid">>, ClientId} | Trace], Rec) -> - to_trace(Trace, Rec#?TRACE{clientid = ClientId}); -to_trace([{<<"packets">>, PacketList} | Trace], Rec) -> - case to_packets(PacketList) of - {ok, Packets} -> to_trace(Trace, Rec#?TRACE{packets = Packets}); - {error, Reason} -> {error, io_lib:format("unsupport packets: ~p", [Reason])} - end; -to_trace([Unknown | _Trace], _Rec) -> {error, io_lib:format("unknown field: ~p", [Unknown])}. - -validate_topic(TopicName) -> - try emqx_topic:validate(name, TopicName) of - true -> ok - catch - error:Error -> - {error, io_lib:format("~s invalid by ~p", [TopicName, Error])} - end. - -to_system_second(At) -> - try - Sec = calendar:rfc3339_to_system_time(binary_to_list(At), [{unit, second}]), - {ok, Sec} - catch error: {badmatch, _} -> - {error, ["The rfc3339 specification not satisfied: ", At]} - end. - -to_packets(Packets) when is_list(Packets) -> - AtomTypes = lists:map(fun(Type) -> binary_to_existing_atom(Type) end, Packets), - case lists:filter(fun(T) -> not lists:member(T, ?PACKETS) end, AtomTypes) of - [] -> {ok, AtomTypes}; - InvalidE -> {error, InvalidE} - end; -to_packets(Packets) -> {error, Packets}. - -zip_dir() -> - trace_dir() ++ "zip/". - -trace_dir() -> - filename:join(emqx:get_env(data_dir), "trace") ++ "/". - -log_file(Name, Start) -> - filename:join(trace_dir(), filename(Name, Start)). - -filename(Name, Start) -> - [Time, _] = string:split(calendar:system_time_to_rfc3339(Start), "T", leading), - lists:flatten(["trace_", binary_to_list(Name), "_", Time, ".log"]). - -transaction(Tran) -> - case mnesia:transaction(Tran) of - {atomic, Res} -> Res; - {aborted, Reason} -> {error, Reason} - end. - -update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel); -update_log_primary_level(_, _) -> set_log_primary_level(debug). - -set_log_primary_level(NewLevel) -> - case NewLevel =/= emqx_logger:get_primary_log_level() of - true -> emqx_logger:set_primary_log_level(NewLevel); - false -> ok - end. + emqx_mod_sup:stop_child(emqx_trace). diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl index 6d2813b96..a3abbedf7 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -15,7 +15,6 @@ %%-------------------------------------------------------------------- -module(emqx_mod_trace_api). --include_lib("emqx/include/logger.hrl"). %% API -export([ list_trace/2 @@ -26,10 +25,6 @@ , download_zip_log/2 , stream_log_file/2 ]). --export([read_trace_file/3]). - --define(TO_BIN(_B_), iolist_to_binary(_B_)). --define(RETURN_NOT_FOUND(N), return({error, 'NOT_FOUND', ?TO_BIN([N, "NOT FOUND"])})). -import(minirest, [return/1]). @@ -75,117 +70,26 @@ func => stream_log_file, descr => "download trace's log"}). -list_trace(_, Params) -> - List = - case Params of - [{<<"enable">>, Enable}] -> emqx_mod_trace:list(Enable); - _ -> emqx_mod_trace:list() - end, - return({ok, emqx_mod_trace:format(List)}). +list_trace(Path, Params) -> + return(emqx_trace_api:list_trace(Path, Params)). -create_trace(_, Param) -> - case emqx_mod_trace:create(Param) of - ok -> return(ok); - {error, {already_existed, Name}} -> - return({error, 'ALREADY_EXISTED', ?TO_BIN([Name, "Already Exists"])}); - {error, {duplicate_condition, Name}} -> - return({error, 'DUPLICATE_CONDITION', ?TO_BIN([Name, "Duplication Condition"])}); - {error, Reason} -> - return({error, 'INCORRECT_PARAMS', ?TO_BIN(Reason)}) +create_trace(Path, Params) -> + return(emqx_trace_api:create_trace(Path, Params)). + +delete_trace(Path, Params) -> + return(emqx_trace_api:delete_trace(Path, Params)). + +clear_traces(Path, Params) -> + return(emqx_trace_api:clear_traces(Path, Params)). + +update_trace(Path, Params) -> + return(emqx_trace_api:update_trace(Path, Params)). + +download_zip_log(Path, Params) -> + case emqx_trace_api:download_zip_log(Path, Params) of + {ok, _Header, _File}= Return -> Return; + {error, _Reason} = Err -> return(Err) end. -delete_trace(#{name := Name}, _Param) -> - case emqx_mod_trace:delete(Name) of - ok -> return(ok); - {error, not_found} -> ?RETURN_NOT_FOUND(Name) - end. - -clear_traces(_, _) -> - return(emqx_mod_trace:clear()). - -update_trace(#{name := Name, operation := Operation}, _Param) -> - Enable = case Operation of disable -> false; enable -> true end, - case emqx_mod_trace:update(Name, Enable) of - ok -> return({ok, #{enable => Enable, name => Name}}); - {error, not_found} -> ?RETURN_NOT_FOUND(Name) - end. - -%% if HTTP request headers include accept-encoding: gzip and file size > 300 bytes. -%% cowboy_compress_h will auto encode gzip format. -download_zip_log(#{name := Name}, _Param) -> - case emqx_mod_trace:get_trace_filename(Name) of - {ok, TraceLog} -> - TraceFiles = collect_trace_file(TraceLog), - ZipDir = emqx_mod_trace:zip_dir(), - Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), - ZipFileName = ZipDir ++ TraceLog, - {ok, ZipFile} = zip:zip(ZipFileName, Zips), - emqx_mod_trace:delete_files_after_send(ZipFileName, Zips), - {ok, #{}, {sendfile, 0, filelib:file_size(ZipFile), ZipFile}}; - {error, Reason} -> - return({error, Reason}) - end. - -group_trace_file(ZipDir, TraceLog, TraceFiles) -> - lists:foldl(fun(Res, Acc) -> - case Res of - {ok, Node, Bin} -> - ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, - ok = file:write_file(ZipName, Bin), - [ZipName | Acc]; - {error, Node, Reason} -> - ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), - Acc - end - end, [], TraceFiles). - -collect_trace_file(TraceLog) -> - Nodes = ekka_mnesia:running_nodes(), - {Files, BadNodes} = rpc:multicall(Nodes, emqx_mod_trace, trace_file, [TraceLog], 60000), - BadNodes =/= [] andalso ?LOG(error, "download log rpc failed on ~p", [BadNodes]), - Files. - -%% _page as position and _limit as bytes for front-end reusing components -stream_log_file(#{name := Name}, Params) -> - Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), - Position0 = proplists:get_value(<<"_page">>, Params, <<"0">>), - Bytes0 = proplists:get_value(<<"_limit">>, Params, <<"500">>), - Node = binary_to_existing_atom(Node0), - Position = binary_to_integer(Position0), - Bytes = binary_to_integer(Bytes0), - case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of - {ok, Bin} -> - Meta = #{<<"page">> => Position + byte_size(Bin), <<"limit">> => Bytes}, - return({ok, #{meta => Meta, items => Bin}}); - eof -> - Meta = #{<<"page">> => Position, <<"limit">> => Bytes}, - return({ok, #{meta => Meta, items => <<"">>}}); - {error, Reason} -> - logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), - return({error, Reason}) - end. - -%% this is an rpc call for stream_log_file/2 -read_trace_file(Name, Position, Limit) -> - TraceDir = emqx_mod_trace:trace_dir(), - {ok, AllFiles} = file:list_dir(TraceDir), - TracePrefix = "trace_" ++ binary_to_list(Name) ++ "_", - Filter = fun(FileName) -> nomatch =/= string:prefix(FileName, TracePrefix) end, - case lists:filter(Filter, AllFiles) of - [TraceFile] -> - TracePath = filename:join([TraceDir, TraceFile]), - read_file(TracePath, Position, Limit); - [] -> {error, not_found} - end. - -read_file(Path, Offset, Bytes) -> - {ok, IoDevice} = file:open(Path, [read, raw, binary]), - try - _ = case Offset of - 0 -> ok; - _ -> file:position(IoDevice, {bof, Offset}) - end, - file:read(IoDevice, Bytes) - after - file:close(IoDevice) - end. +stream_log_file(Path, Params) -> + return(emqx_trace_api:stream_log_file(Path, Params)). diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl deleted file mode 100644 index c643c7908..000000000 --- a/lib-ce/emqx_modules/test/emqx_mod_trace_SUITE.erl +++ /dev/null @@ -1,478 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_mod_trace_SUITE). - -%% API --compile(export_all). --compile(nowarn_export_all). - --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). - --define(HOST, "http://127.0.0.1:18083/"). --define(API_VERSION, "v4"). --define(BASE_PATH, "api"). - --record(emqx_mod_trace, { - name, - type, - topic, - clientid, - packets = [], - enable = true, - start_at, - end_at, - log_size = #{} - }). - --define(PACKETS, ['CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL' - , 'PUBCOMP', 'SUBSCRIBE', 'SUBACK', 'UNSUBSCRIBE', 'UNSUBACK' - , 'PINGREQ', 'PINGRESP', 'DISCONNECT', 'AUTH']). - -%%-------------------------------------------------------------------- -%% Setups -%%-------------------------------------------------------------------- - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_modules, emqx_dashboard]), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules, emqx_dashboard]). - -t_base_create_delete(_Config) -> - ok = emqx_mod_trace:clear(), - Now = erlang:system_time(second), - Start = to_rfc3339(Now), - End = to_rfc3339(Now + 30 * 60), - Name = <<"name1">>, - ClientId = <<"test-device">>, - Packets = [atom_to_binary(E) || E <- ?PACKETS], - Trace = [ - {<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, - {<<"clientid">>, ClientId}, - {<<"packets">>, Packets}, - {<<"start_at">>, Start}, - {<<"end_at">>, End} - ], - AnotherTrace = lists:keyreplace(<<"name">>, 1, Trace, {<<"name">>, <<"AnotherName">>}), - ok = emqx_mod_trace:create(Trace), - ?assertEqual({error, {already_existed, Name}}, emqx_mod_trace:create(Trace)), - ?assertEqual({error, {duplicate_condition, Name}}, emqx_mod_trace:create(AnotherTrace)), - [TraceRec] = emqx_mod_trace:list(), - Expect = #emqx_mod_trace{ - name = Name, - type = clientid, - topic = undefined, - clientid = ClientId, - packets = ?PACKETS, - start_at = Now, - end_at = Now + 30 * 60 - }, - ?assertEqual(Expect, TraceRec), - ExpectFormat = [ - #{ - clientid => <<"test-device">>, - enable => true, - type => clientid, - packets => ?PACKETS, - name => <<"name1">>, - start_at => Start, - end_at => End, - log_size => #{}, - topic => undefined - } - ], - ?assertEqual(ExpectFormat, emqx_mod_trace:format([TraceRec])), - ?assertEqual(ok, emqx_mod_trace:delete(Name)), - ?assertEqual({error, not_found}, emqx_mod_trace:delete(Name)), - ?assertEqual([], emqx_mod_trace:list()), - ok. - -t_create_size_max(_Config) -> - emqx_mod_trace:clear(), - lists:map(fun(Seq) -> - Name = list_to_binary("name" ++ integer_to_list(Seq)), - Trace = [{<<"name">>, Name}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, - {<<"topic">>, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], - ok = emqx_mod_trace:create(Trace) - end, lists:seq(1, 30)), - Trace31 = [{<<"name">>, <<"name31">>}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/31">>}], - {error, _} = emqx_mod_trace:create(Trace31), - ok = emqx_mod_trace:delete(<<"name30">>), - ok = emqx_mod_trace:create(Trace31), - ?assertEqual(30, erlang:length(emqx_mod_trace:list())), - ok. - -t_create_failed(_Config) -> - ok = emqx_mod_trace:clear(), - UnknownField = [{<<"unknown">>, 12}], - {error, Reason1} = emqx_mod_trace:create(UnknownField), - ?assertEqual(<<"unknown field: {<<\"unknown\">>,12}">>, iolist_to_binary(Reason1)), - - InvalidTopic = [{<<"topic">>, "#/#//"}], - {error, Reason2} = emqx_mod_trace:create(InvalidTopic), - ?assertEqual(<<"#/#// invalid by function_clause">>, iolist_to_binary(Reason2)), - - InvalidStart = [{<<"start_at">>, <<"2021-12-3:12">>}], - {error, Reason3} = emqx_mod_trace:create(InvalidStart), - ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, - iolist_to_binary(Reason3)), - - InvalidEnd = [{<<"end_at">>, <<"2021-12-3:12">>}], - {error, Reason4} = emqx_mod_trace:create(InvalidEnd), - ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, - iolist_to_binary(Reason4)), - - InvalidPackets = [{<<"packets">>, [<<"publish">>]}], - {error, Reason5} = emqx_mod_trace:create(InvalidPackets), - ?assertEqual(<<"unsupport packets: [publish]">>, iolist_to_binary(Reason5)), - - InvalidPackets2 = [{<<"packets">>, <<"publish">>}], - {error, Reason6} = emqx_mod_trace:create(InvalidPackets2), - ?assertEqual(<<"unsupport packets: <<\"publish\">>">>, iolist_to_binary(Reason6)), - - {error, Reason7} = emqx_mod_trace:create([{<<"name">>, <<"test">>}, {<<"type">>, <<"clientid">>}]), - ?assertEqual(<<"topic/clientid cannot be both empty">>, iolist_to_binary(Reason7)), - - InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, {<<"type">>, <<"clientid">>}], - {error, Reason9} = emqx_mod_trace:create(InvalidPackets4), - ?assertEqual(<<"name cannot contain /">>, iolist_to_binary(Reason9)), - - ?assertEqual({error, "type required"}, emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, - {<<"packets">>, []}, {<<"clientid">>, <<"good">>}])), - - ?assertEqual({error, "incorrect type: only support clientid/topic"}, - emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, - {<<"packets">>, []}, {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), - ok. - -t_create_default(_Config) -> - ok = emqx_mod_trace:clear(), - {error, "name required"} = emqx_mod_trace:create([]), - ok = emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, - {<<"type">>, <<"clientid">>}, {<<"packets">>, []}, {<<"clientid">>, <<"good">>}]), - [#emqx_mod_trace{packets = Packets}] = emqx_mod_trace:list(), - ?assertEqual([], Packets), - ok = emqx_mod_trace:clear(), - Trace = [ - {<<"name">>, <<"test-name">>}, - {<<"packets">>, [<<"PUBLISH">>]}, - {<<"type">>, <<"topic">>}, - {<<"topic">>, <<"/x/y/z">>}, - {<<"start_at">>, <<"2021-10-28T10:54:47+08:00">>}, - {<<"end_at">>, <<"2021-10-27T10:54:47+08:00">>} - ], - {error, "end_at time has already passed"} = emqx_mod_trace:create(Trace), - Now = erlang:system_time(second), - Trace2 = [ - {<<"name">>, <<"test-name">>}, - {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, - {<<"topic">>, <<"/x/y/z">>}, - {<<"start_at">>, to_rfc3339(Now + 10)}, - {<<"end_at">>, to_rfc3339(Now + 3)} - ], - {error, "failed by start_at >= end_at"} = emqx_mod_trace:create(Trace2), - ok = emqx_mod_trace:create([{<<"name">>, <<"test-name">>}, - {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>}]), - [#emqx_mod_trace{start_at = Start, end_at = End}] = emqx_mod_trace:list(), - ?assertEqual(10 * 60, End - Start), - ?assertEqual(true, Start - erlang:system_time(second) < 5), - ok. - -t_update_enable(_Config) -> - ok = emqx_mod_trace:clear(), - Name = <<"test-name">>, - Now = erlang:system_time(second), - End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), - ok = emqx_mod_trace:create([{<<"name">>, Name}, {<<"packets">>, [<<"PUBLISH">>]}, - {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), - [#emqx_mod_trace{enable = Enable}] = emqx_mod_trace:list(), - ?assertEqual(Enable, true), - ok = emqx_mod_trace:update(Name, false), - [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), - ok = emqx_mod_trace:update(Name, false), - [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), - ok = emqx_mod_trace:update(Name, true), - [#emqx_mod_trace{enable = true}] = emqx_mod_trace:list(), - ok = emqx_mod_trace:update(Name, false), - [#emqx_mod_trace{enable = false}] = emqx_mod_trace:list(), - ?assertEqual({error, not_found}, emqx_mod_trace:update(<<"Name not found">>, true)), - ct:sleep(2100), - ?assertEqual({error, finished}, emqx_mod_trace:update(Name, true)), - ok. - -t_load_state(_Config) -> - emqx_mod_trace:clear(), - ok = emqx_mod_trace:load(test), - Now = erlang:system_time(second), - Running = [{<<"name">>, <<"Running">>}, {<<"type">>, <<"topic">>}, - {<<"topic">>, <<"/x/y/1">>}, {<<"start_at">>, to_rfc3339(Now - 1)}, {<<"end_at">>, to_rfc3339(Now + 2)}], - Waiting = [{<<"name">>, <<"Waiting">>}, {<<"type">>, <<"topic">>}, - {<<"topic">>, <<"/x/y/2">>}, {<<"start_at">>, to_rfc3339(Now + 3)}, {<<"end_at">>, to_rfc3339(Now + 8)}], - Finished = [{<<"name">>, <<"Finished">>}, {<<"type">>, <<"topic">>}, - {<<"topic">>, <<"/x/y/3">>}, {<<"start_at">>, to_rfc3339(Now - 5)}, {<<"end_at">>, to_rfc3339(Now)}], - ok = emqx_mod_trace:create(Running), - ok = emqx_mod_trace:create(Waiting), - {error, "end_at time has already passed"} = emqx_mod_trace:create(Finished), - Traces = emqx_mod_trace:format(emqx_mod_trace:list()), - ?assertEqual(2, erlang:length(Traces)), - Enables = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces), - ExpectEnables = [{<<"Running">>, true}, {<<"Waiting">>, true}], - ?assertEqual(ExpectEnables, lists:sort(Enables)), - ct:sleep(3500), - Traces2 = emqx_mod_trace:format(emqx_mod_trace:list()), - ?assertEqual(2, erlang:length(Traces2)), - Enables2 = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces2), - ExpectEnables2 = [{<<"Running">>, false}, {<<"Waiting">>, true}], - ?assertEqual(ExpectEnables2, lists:sort(Enables2)), - ok = emqx_mod_trace:unload(test), - ok. - -t_client_event(_Config) -> - application:set_env(emqx, allow_anonymous, true), - emqx_mod_trace:clear(), - ClientId = <<"client-test">>, - ok = emqx_mod_trace:load(test), - Now = erlang:system_time(second), - Start = to_rfc3339(Now), - Name = <<"test_client_id_event">>, - ok = emqx_mod_trace:create([{<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), - ct:sleep(200), - {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), - {ok, _} = emqtt:connect(Client), - emqtt:ping(Client), - ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]), - ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]), - ct:sleep(200), - ok = emqx_mod_trace:create([{<<"name">>, <<"test_topic">>}, - {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]), - ct:sleep(200), - {ok, Bin} = file:read_file(emqx_mod_trace:log_file(Name, Now)), - ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]), - ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]), - ok = emqtt:disconnect(Client), - ct:sleep(200), - {ok, Bin2} = file:read_file(emqx_mod_trace:log_file(Name, Now)), - {ok, Bin3} = file:read_file(emqx_mod_trace:log_file(<<"test_topic">>, Now)), - ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), - ?assert(erlang:byte_size(Bin) > 0), - ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)), - ?assert(erlang:byte_size(Bin3) > 0), - ok = emqx_mod_trace:unload(test), - ok. - -t_get_log_filename(_Config) -> - ok = emqx_mod_trace:clear(), - ok = emqx_mod_trace:load(test), - Now = erlang:system_time(second), - Start = calendar:system_time_to_rfc3339(Now), - End = calendar:system_time_to_rfc3339(Now + 2), - Name = <<"name1">>, - ClientId = <<"test-device">>, - Packets = [atom_to_binary(E) || E <- ?PACKETS], - Trace = [ - {<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, - {<<"clientid">>, ClientId}, - {<<"packets">>, Packets}, - {<<"start_at">>, list_to_binary(Start)}, - {<<"end_at">>, list_to_binary(End)} - ], - ok = emqx_mod_trace:create(Trace), - ?assertEqual({error, not_found}, emqx_mod_trace:get_trace_filename(<<"test">>)), - ?assertEqual(ok, element(1, emqx_mod_trace:get_trace_filename(Name))), - ct:sleep(3000), - ?assertEqual(ok, element(1, emqx_mod_trace:get_trace_filename(Name))), - ok = emqx_mod_trace:unload(test), - ok. - -t_trace_file(_Config) -> - FileName = "test.log", - Content = <<"test \n test">>, - TraceDir = emqx_mod_trace:trace_dir(), - File = filename:join(TraceDir, FileName), - ok = file:write_file(File, Content), - {ok, Node, Bin} = emqx_mod_trace:trace_file(FileName), - ?assertEqual(Node, atom_to_list(node())), - ?assertEqual(Content, Bin), - ok = file:delete(File), - ok. - -t_http_test(_Config) -> - emqx_mod_trace:clear(), - emqx_mod_trace:load(test), - Header = auth_header_(), - %% list - {ok, Empty} = request_api(get, api_path("trace"), Header), - ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(Empty)), - %% create - ErrorTrace = #{}, - {ok, Error} = request_api(post, api_path("trace"), Header, ErrorTrace), - ?assertEqual(#{<<"message">> => <<"unknown field: {}">>, <<"code">> => <<"INCORRECT_PARAMS">>}, json(Error)), - - Name = <<"test-name">>, - Trace = [ - {<<"name">>, Name}, - {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, - {<<"topic">>, <<"/x/y/z">>} - ], - - {ok, Create} = request_api(post, api_path("trace"), Header, Trace), - ?assertEqual(#{<<"code">> => 0}, json(Create)), - - {ok, List} = request_api(get, api_path("trace"), Header), - #{<<"code">> := 0, <<"data">> := [Data]} = json(List), - ?assertEqual(Name, maps:get(<<"name">>, Data)), - - %% update - {ok, Update} = request_api(put, api_path("trace/test-name/disable"), Header, #{}), - ?assertEqual(#{<<"code">> => 0, - <<"data">> => #{<<"enable">> => false, - <<"name">> => <<"test-name">>}}, json(Update)), - - {ok, List1} = request_api(get, api_path("trace"), Header), - #{<<"code">> := 0, <<"data">> := [Data1]} = json(List1), - ?assertEqual(false, maps:get(<<"enable">>, Data1)), - - %% delete - {ok, Delete} = request_api(delete, api_path("trace/test-name"), Header), - ?assertEqual(#{<<"code">> => 0}, json(Delete)), - - {ok, DeleteNotFound} = request_api(delete, api_path("trace/test-name"), Header), - ?assertEqual(#{<<"code">> => <<"NOT_FOUND">>, - <<"message">> => <<"test-nameNOT FOUND">>}, json(DeleteNotFound)), - - {ok, List2} = request_api(get, api_path("trace"), Header), - ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(List2)), - - %% clear - {ok, Create1} = request_api(post, api_path("trace"), Header, Trace), - ?assertEqual(#{<<"code">> => 0}, json(Create1)), - - {ok, Clear} = request_api(delete, api_path("trace"), Header), - ?assertEqual(#{<<"code">> => 0}, json(Clear)), - - emqx_mod_trace:unload(test), - ok. - -t_download_log(_Config) -> - emqx_mod_trace:clear(), - emqx_mod_trace:load(test), - ClientId = <<"client-test">>, - Now = erlang:system_time(second), - Start = to_rfc3339(Now), - Name = <<"test_client_id">>, - ok = emqx_mod_trace:create([{<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), - {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), - {ok, _} = emqtt:connect(Client), - [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], - ct:sleep(100), - {ok, #{}, {sendfile, 0, ZipFileSize, _ZipFile}} = - emqx_mod_trace_api:download_zip_log(#{name => Name}, []), - ?assert(ZipFileSize > 0), - %% download zip file failed by server_closed occasionally? - %Header = auth_header_(), - %{ok, ZipBin} = request_api(get, api_path("trace/test_client_id/download"), Header), - %{ok, ZipHandler} = zip:zip_open(ZipBin), - %{ok, [ZipName]} = zip:zip_get(ZipHandler), - %?assertNotEqual(nomatch, string:find(ZipName, "test@127.0.0.1")), - %{ok, _} = file:read_file(emqx_mod_trace:log_file(<<"test_client_id">>, Now)), - ok = emqtt:disconnect(Client), - emqx_mod_trace:unload(test), - ok. - -t_stream_log(_Config) -> - application:set_env(emqx, allow_anonymous, true), - emqx_mod_trace:clear(), - emqx_mod_trace:load(test), - ClientId = <<"client-stream">>, - Now = erlang:system_time(second), - Name = <<"test_stream_log">>, - Start = to_rfc3339(Now - 10), - ok = emqx_mod_trace:create([{<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), - ct:sleep(200), - {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), - {ok, _} = emqtt:connect(Client), - [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], - emqtt:publish(Client, <<"/good">>, #{}, <<"ghood1">>, [{qos, 0}]), - emqtt:publish(Client, <<"/good">>, #{}, <<"ghood2">>, [{qos, 0}]), - ok = emqtt:disconnect(Client), - ct:sleep(200), - File = emqx_mod_trace:log_file(Name, Now), - ct:pal("FileName: ~p", [File]), - {ok, FileBin} = file:read_file(File), - ct:pal("FileBin: ~p ~s", [byte_size(FileBin), FileBin]), - Header = auth_header_(), - {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?_limit=10"), Header), - #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta, <<"items">> := Bin}} = json(Binary), - ?assertEqual(10, byte_size(Bin)), - ?assertEqual(#{<<"page">> => 10, <<"limit">> => 10}, Meta), - {ok, Binary1} = request_api(get, api_path("trace/test_stream_log/log?_page=20&_limit=10"), Header), - #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta1, <<"items">> := Bin1}} = json(Binary1), - ?assertEqual(#{<<"page">> => 30, <<"limit">> => 10}, Meta1), - ?assertEqual(10, byte_size(Bin1)), - emqx_mod_trace:unload(test), - ok. - -to_rfc3339(Second) -> - list_to_binary(calendar:system_time_to_rfc3339(Second)). - -auth_header_() -> - auth_header_("admin", "public"). - -auth_header_(User, Pass) -> - Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), - {"Authorization", "Basic " ++ Encoded}. - -request_api(Method, Url, Auth) -> do_request_api(Method, {Url, [Auth]}). - -request_api(Method, Url, Auth, Body) -> - Request = {Url, [Auth], "application/json", emqx_json:encode(Body)}, - do_request_api(Method, Request). - -do_request_api(Method, Request) -> - ct:pal("Method: ~p, Request: ~p", [Method, Request]), - case httpc:request(Method, Request, [], [{body_format, binary}]) of - {error, socket_closed_remotely} -> - {error, socket_closed_remotely}; - {error,{shutdown, server_closed}} -> - {error, server_closed}; - {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } - when Code =:= 200 orelse Code =:= 201 -> - {ok, Return}; - {ok, {Reason, _, _}} -> - {error, Reason} - end. - -api_path(Path) -> - ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION, Path]). - -json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl new file mode 100644 index 000000000..609a2d93c --- /dev/null +++ b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl @@ -0,0 +1,179 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mod_trace_api_SUITE). + +%% API +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-define(HOST, "http://127.0.0.1:18083/"). +-define(API_VERSION, "v4"). +-define(BASE_PATH, "api"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_plugin_libs), + emqx_ct_helpers:start_apps([emqx_modules, emqx_dashboard]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules, emqx_dashboard]). + +t_http_test(_Config) -> + emqx_trace:clear(), + load(), + Header = auth_header_(), + %% list + {ok, Empty} = request_api(get, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(Empty)), + %% create + ErrorTrace = #{}, + {ok, Error} = request_api(post, api_path("trace"), Header, ErrorTrace), + ?assertEqual(#{<<"message">> => <<"unknown field: {}">>, + <<"code">> => <<"INCORRECT_PARAMS">>}, json(Error)), + + Name = <<"test-name">>, + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"topic">>}, + {<<"packets">>, [<<"PUBLISH">>]}, + {<<"topic">>, <<"/x/y/z">>} + ], + + {ok, Create} = request_api(post, api_path("trace"), Header, Trace), + ?assertEqual(#{<<"code">> => 0}, json(Create)), + + {ok, List} = request_api(get, api_path("trace"), Header), + #{<<"code">> := 0, <<"data">> := [Data]} = json(List), + ?assertEqual(Name, maps:get(<<"name">>, Data)), + + %% update + {ok, Update} = request_api(put, api_path("trace/test-name/disable"), Header, #{}), + ?assertEqual(#{<<"code">> => 0, + <<"data">> => #{<<"enable">> => false, + <<"name">> => <<"test-name">>}}, json(Update)), + + {ok, List1} = request_api(get, api_path("trace"), Header), + #{<<"code">> := 0, <<"data">> := [Data1]} = json(List1), + ?assertEqual(false, maps:get(<<"enable">>, Data1)), + + %% delete + {ok, Delete} = request_api(delete, api_path("trace/test-name"), Header), + ?assertEqual(#{<<"code">> => 0}, json(Delete)), + + {ok, DeleteNotFound} = request_api(delete, api_path("trace/test-name"), Header), + ?assertEqual(#{<<"code">> => <<"NOT_FOUND">>, + <<"message">> => <<"test-nameNOT FOUND">>}, json(DeleteNotFound)), + + {ok, List2} = request_api(get, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(List2)), + + %% clear + {ok, Create1} = request_api(post, api_path("trace"), Header, Trace), + ?assertEqual(#{<<"code">> => 0}, json(Create1)), + + {ok, Clear} = request_api(delete, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0}, json(Clear)), + + unload(), + ok. + +t_stream_log(_Config) -> + application:set_env(emqx, allow_anonymous, true), + emqx_trace:clear(), + load(), + ClientId = <<"client-stream">>, + Now = erlang:system_time(second), + Name = <<"test_stream_log">>, + Start = to_rfc3339(Now - 10), + ok = emqx_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], + emqtt:publish(Client, <<"/good">>, #{}, <<"ghood1">>, [{qos, 0}]), + emqtt:publish(Client, <<"/good">>, #{}, <<"ghood2">>, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ct:sleep(200), + File = emqx_trace:log_file(Name, Now), + ct:pal("FileName: ~p", [File]), + {ok, FileBin} = file:read_file(File), + ct:pal("FileBin: ~p ~s", [byte_size(FileBin), FileBin]), + Header = auth_header_(), + {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?_limit=10"), Header), + #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta, <<"items">> := Bin}} = json(Binary), + ?assertEqual(10, byte_size(Bin)), + ?assertEqual(#{<<"page">> => 10, <<"limit">> => 10}, Meta), + Path = api_path("trace/test_stream_log/log?_page=20&_limit=10"), + {ok, Binary1} = request_api(get, Path, Header), + #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta1, <<"items">> := Bin1}} = json(Binary1), + ?assertEqual(#{<<"page">> => 30, <<"limit">> => 10}, Meta1), + ?assertEqual(10, byte_size(Bin1)), + unload(), + ok. + +to_rfc3339(Second) -> + list_to_binary(calendar:system_time_to_rfc3339(Second)). + +auth_header_() -> + auth_header_("admin", "public"). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), + {"Authorization", "Basic " ++ Encoded}. + +request_api(Method, Url, Auth) -> do_request_api(Method, {Url, [Auth]}). + +request_api(Method, Url, Auth, Body) -> + Request = {Url, [Auth], "application/json", emqx_json:encode(Body)}, + do_request_api(Method, Request). + +do_request_api(Method, Request) -> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], [{body_format, binary}]) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {error,{shutdown, server_closed}} -> + {error, server_closed}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +api_path(Path) -> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION, Path]). + +json(Data) -> + {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. + +load() -> + emqx_trace:start_link(). + +unload() -> + gen_server:stop(emqx_trace). From a4a7cac647909116ea0b2094b97ab0917c7755e5 Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 12 Nov 2021 14:00:48 +0800 Subject: [PATCH 018/363] fix(eqmx_st_statistics): add ignore_before_create in config (#6140) 1. allows not to process the message before the session is created to solve the problem caused by clean session = false 2. fix some elvis errors --- .../emqx_st_statistics/emqx_st_statistics.erl | 27 ++++++++---- etc/emqx.conf | 5 +++ priv/emqx.schema | 5 +++ src/emqx_session.erl | 43 ++++++++++--------- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl index ac1604c9b..21cf2692e 100644 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl @@ -20,11 +20,11 @@ -include_lib("include/emqx.hrl"). -include_lib("include/logger.hrl"). --include("include/emqx_st_statistics.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_st_statistics.hrl"). -logger_header("[SLOW TOPICS]"). --export([ start_link/1, on_publish_done/3, enable/0 +-export([ start_link/1, on_publish_done/5, enable/0 , disable/0, clear_history/0 ]). @@ -90,8 +90,13 @@ start_link(Env) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). --spec on_publish_done(message(), pos_integer(), counters:counters_ref()) -> ok. -on_publish_done(#message{timestamp = Timestamp} = Msg, Threshold, Counter) -> +-spec on_publish_done(message(), + pos_integer(), boolean(), pos_integer(), counters:counters_ref()) -> ok. +on_publish_done(#message{timestamp = Timestamp}, Created, IgnoreBeforeCreate, _, _) + when IgnoreBeforeCreate, Timestamp < Created -> + ok; + +on_publish_done(#message{timestamp = Timestamp} = Msg, _, _, Threshold, Counter) -> case ?NOW - Timestamp of Elapsed when Elapsed > Threshold -> case get_log_quota(Counter) of @@ -125,7 +130,8 @@ init([Env]) -> Counter = counters:new(1, [write_concurrency]), set_log_quota(Env, Counter), Threshold = get_value(threshold_time, Env), - load(Threshold, Counter), + IgnoreBeforeCreate = get_value(ignore_before_create, Env), + load(IgnoreBeforeCreate, Threshold, Counter), {ok, #{config => Env, period => 1, last_tick_at => ?NOW, @@ -139,7 +145,8 @@ handle_call({enable, Enable}, _From, State; true -> Threshold = get_value(threshold_time, Cfg), - load(Threshold, Counter), + IgnoreBeforeCreate = get_value(ignore_before_create, Cfg), + load(IgnoreBeforeCreate, Threshold, Counter), State#{enable := true}; _ -> unload(), @@ -319,12 +326,14 @@ publish(TickTime, Cfg, Notices) -> }), ok. -load(Threshold, Counter) -> - _ = emqx:hook('message.publish_done', fun ?MODULE:on_publish_done/3, [Threshold, Counter]), +load(IgnoreBeforeCreate, Threshold, Counter) -> + _ = emqx:hook('message.publish_done', + fun ?MODULE:on_publish_done/5, + [IgnoreBeforeCreate, Threshold, Counter]), ok. unload() -> - emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/3). + emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/5). -spec get_topic(proplists:proplist()) -> binary(). get_topic(Cfg) -> diff --git a/etc/emqx.conf b/etc/emqx.conf index a94afd961..7b81e40cc 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2219,6 +2219,11 @@ module.presence.qos = 1 ## Default: 10 seconds #module.st_statistics.threshold_time = 10S +## ignore the messages that before than session created +## +## Default: true +#module.st_statistics.ignore_before_create = true + ## Time window of slow topics statistics ## ## Value: 5 minutes diff --git a/priv/emqx.schema b/priv/emqx.schema index 69f3acc41..602de56b9 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2193,6 +2193,11 @@ end}. {datatype, {duration, ms}} ]}. +{mapping, "module.st_statistics.ignore_before_create", "emqx.modules", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + {mapping, "module.st_statistics.time_window", "emqx.modules", [ {default, "5M"}, {datatype, {duration, ms}} diff --git a/src/emqx_session.erl b/src/emqx_session.erl index db09bf8c9..d4b671e71 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -269,7 +269,8 @@ unsubscribe(ClientInfo, TopicFilter, UnSubOpts, Session = #session{subscriptions case maps:find(TopicFilter, Subs) of {ok, SubOpts} -> ok = emqx_broker:unsubscribe(TopicFilter), - ok = emqx_hooks:run('session.unsubscribed', [ClientInfo, TopicFilter, maps:merge(SubOpts, UnSubOpts)]), + ok = emqx_hooks:run('session.unsubscribed', + [ClientInfo, TopicFilter, maps:merge(SubOpts, UnSubOpts)]), {ok, Session#session{subscriptions = maps:remove(TopicFilter, Subs)}}; error -> {error, ?RC_NO_SUBSCRIPTION_EXISTED} @@ -316,10 +317,10 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, -> {ok, emqx_types:message(), session()} | {ok, emqx_types:message(), replies(), session()} | {error, emqx_types:reason_code()}). -puback(PacketId, Session = #session{inflight = Inflight}) -> +puback(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - emqx:run_hook('message.publish_done', [Msg]), + emqx:run_hook('message.publish_done', [Msg, CreatedAt]), Inflight1 = emqx_inflight:delete(PacketId, Inflight), return_with(Msg, dequeue(Session#session{inflight = Inflight1})); {value, {_Pubrel, _Ts}} -> @@ -341,11 +342,11 @@ return_with(Msg, {ok, Publishes, Session}) -> -spec(pubrec(emqx_types:packet_id(), session()) -> {ok, emqx_types:message(), session()} | {error, emqx_types:reason_code()}). -pubrec(PacketId, Session = #session{inflight = Inflight}) -> +pubrec(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> %% execute hook here, because message record will be replaced by pubrel - emqx:run_hook('message.publish_done', [Msg]), + emqx:run_hook('message.publish_done', [Msg, CreatedAt]), Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; {value, {pubrel, _Ts}} -> @@ -408,7 +409,7 @@ dequeue(Cnt, Msgs, Q) -> case emqx_message:is_expired(Msg) of true -> ok = inc_expired_cnt(delivery), dequeue(Cnt, Msgs, Q1); - false -> dequeue(acc_cnt(Msg, Cnt), [Msg|Msgs], Q1) + false -> dequeue(acc_cnt(Msg, Cnt), [Msg | Msgs], Q1) end end. @@ -438,11 +439,11 @@ deliver([Msg | More], Acc, Session) -> {ok, Session1} -> deliver(More, Acc, Session1); {ok, [Publish], Session1} -> - deliver(More, [Publish|Acc], Session1) + deliver(More, [Publish | Acc], Session1) end. deliver_msg(Msg = #message{qos = ?QOS_0}, Session) -> - emqx:run_hook('message.publish_done', [Msg]), + emqx:run_hook('message.publish_done', [Msg, Session#session.created_at]), {ok, [{undefined, maybe_ack(Msg)}], Session}; deliver_msg(Msg = #message{qos = QoS}, Session = @@ -461,7 +462,7 @@ deliver_msg(Msg = #message{qos = QoS}, Session = {ok, [Publish], next_pkt_id(Session1)} end. --spec(enqueue(list(emqx_types:deliver())|emqx_types:message(), +-spec(enqueue(list(emqx_types:deliver()) | emqx_types:message(), session()) -> session()). enqueue([Deliver], Session) -> %% Optimize Enrich = enrich_fun(Session), @@ -512,23 +513,23 @@ get_subopts(Topic, SubMap) -> end. enrich_subopts([], Msg, _Session) -> Msg; -enrich_subopts([{nl, 1}|Opts], Msg, Session) -> +enrich_subopts([{nl, 1} | Opts], Msg, Session) -> enrich_subopts(Opts, emqx_message:set_flag(nl, Msg), Session); -enrich_subopts([{nl, 0}|Opts], Msg, Session) -> +enrich_subopts([{nl, 0} | Opts], Msg, Session) -> enrich_subopts(Opts, Msg, Session); -enrich_subopts([{qos, SubQoS}|Opts], Msg = #message{qos = PubQoS}, +enrich_subopts([{qos, SubQoS} | Opts], Msg = #message{qos = PubQoS}, Session = #session{upgrade_qos = true}) -> enrich_subopts(Opts, Msg#message{qos = max(SubQoS, PubQoS)}, Session); -enrich_subopts([{qos, SubQoS}|Opts], Msg = #message{qos = PubQoS}, +enrich_subopts([{qos, SubQoS} | Opts], Msg = #message{qos = PubQoS}, Session = #session{upgrade_qos = false}) -> enrich_subopts(Opts, Msg#message{qos = min(SubQoS, PubQoS)}, Session); -enrich_subopts([{rap, 1}|Opts], Msg, Session) -> +enrich_subopts([{rap, 1} | Opts], Msg, Session) -> enrich_subopts(Opts, Msg, Session); -enrich_subopts([{rap, 0}|Opts], Msg = #message{headers = #{retained := true}}, Session) -> +enrich_subopts([{rap, 0} | Opts], Msg = #message{headers = #{retained := true}}, Session) -> enrich_subopts(Opts, Msg, Session); -enrich_subopts([{rap, 0}|Opts], Msg, Session) -> +enrich_subopts([{rap, 0} | Opts], Msg, Session) -> enrich_subopts(Opts, emqx_message:set_flag(retain, false, Msg), Session); -enrich_subopts([{subid, SubId}|Opts], Msg, Session) -> +enrich_subopts([{subid, SubId} | Opts], Msg, Session) -> Props = emqx_message:get_header(properties, Msg, #{}), Msg1 = emqx_message:set_header(properties, Props#{'Subscription-Identifier' => SubId}, Msg), enrich_subopts(Opts, Msg1, Session). @@ -556,8 +557,8 @@ retry(Session = #session{inflight = Inflight}) -> retry_delivery([], Acc, _Now, Session = #session{retry_interval = Interval}) -> {ok, lists:reverse(Acc), Interval, Session}; -retry_delivery([{PacketId, {Msg, Ts}}|More], Acc, Now, Session = - #session{retry_interval = Interval, inflight = Inflight}) -> +retry_delivery([{PacketId, {Msg, Ts}} | More], Acc, Now, Session = + #session{retry_interval = Interval, inflight = Inflight}) -> case (Age = age(Now, Ts)) >= Interval of true -> {Acc1, Inflight1} = retry_delivery(PacketId, Msg, Now, Acc, Inflight), @@ -574,12 +575,12 @@ retry_delivery(PacketId, Msg, Now, Acc, Inflight) when is_record(Msg, message) - false -> Msg1 = emqx_message:set_flag(dup, true, Msg), Inflight1 = emqx_inflight:update(PacketId, {Msg1, Now}, Inflight), - {[{PacketId, Msg1}|Acc], Inflight1} + {[{PacketId, Msg1} | Acc], Inflight1} end; retry_delivery(PacketId, pubrel, Now, Acc, Inflight) -> Inflight1 = emqx_inflight:update(PacketId, {pubrel, Now}, Inflight), - {[{pubrel, PacketId}|Acc], Inflight1}. + {[{pubrel, PacketId} | Acc], Inflight1}. %%-------------------------------------------------------------------- %% Expire Awaiting Rel From e7bbe98a7a6a095a7d8ee1fea84d424012717d3c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 12 Nov 2021 13:53:34 +0800 Subject: [PATCH 019/363] fix(ekka): update ekka to 0.8.2 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index a38d74e88..cc147dec3 100644 --- a/rebar.config +++ b/rebar.config @@ -43,7 +43,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.4"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.6.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} From 66d0c44e3612a3446970b121f30efbac57b78abe Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 12 Nov 2021 14:51:02 +0800 Subject: [PATCH 020/363] fix(emqx_retainer): refresh the timestamp when dispatch retained message (#6148) --- apps/emqx_retainer/src/emqx_retainer.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 340e6929d..8884731f3 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -78,7 +78,8 @@ dispatch(Pid, Topic) -> false -> read_messages(Topic); true -> match_messages(Topic) end, - [Pid ! {deliver, Topic, Msg} || Msg <- sort_retained(Msgs)]. + Now = erlang:system_time(millisecond), + [Pid ! {deliver, Topic, Msg#message{timestamp = Now}} || Msg <- sort_retained(Msgs)]. %% RETAIN flag set to 1 and payload containing zero bytes on_message_publish(Msg = #message{flags = #{retain := true}, From f3de1bdb7715e828cdb185c272f74790f3b8b24b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 12 Nov 2021 15:48:30 +0800 Subject: [PATCH 021/363] fix: make sure enable boolean() (#6147) --- apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 1bc71654e..a5e5b2906 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -34,7 +34,7 @@ list_trace(_, Params) -> List = case Params of - [{<<"enable">>, Enable}] -> emqx_trace:list(Enable); + [{<<"enable">>, Enable}] -> emqx_trace:list(binary_to_existing_atom(Enable)); _ -> emqx_trace:list() end, {ok, emqx_trace:format(List)}. From e9ce8e7586143175590e5805c6c30ec15e04270b Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 12 Nov 2021 18:31:55 +0800 Subject: [PATCH 022/363] fix(emqx_retainer): revert the "refresh the timestamp when dispatch retained message" This reverts commit 66d0c44e3612a3446970b121f30efbac57b78abe. --- apps/emqx_retainer/src/emqx_retainer.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 8884731f3..340e6929d 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -78,8 +78,7 @@ dispatch(Pid, Topic) -> false -> read_messages(Topic); true -> match_messages(Topic) end, - Now = erlang:system_time(millisecond), - [Pid ! {deliver, Topic, Msg#message{timestamp = Now}} || Msg <- sort_retained(Msgs)]. + [Pid ! {deliver, Topic, Msg} || Msg <- sort_retained(Msgs)]. %% RETAIN flag set to 1 and payload containing zero bytes on_message_publish(Msg = #message{flags = #{retain := true}, From 8bf6668e4c0475fadea06ef79e0b4e068bd40971 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 12 Nov 2021 20:11:59 +0800 Subject: [PATCH 023/363] Set keepalive via http api (#6143) * feat: set keepalive over http api * chore: elvis warning * fix: bump retainer to 4.4.0 --- apps/emqx_management/src/emqx_mgmt.erl | 47 +++--- .../src/emqx_mgmt_api_clients.erl | 58 +++++-- .../test/emqx_mgmt_api_SUITE.erl | 143 +++++++++++++----- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- src/emqx_channel.erl | 23 ++- src/emqx_keepalive.erl | 8 +- 6 files changed, 200 insertions(+), 81 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index bfba34603..93f24cd37 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -22,6 +22,9 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_mgmt]}}]). +-elvis([{elvis_style, god_modules, #{ignore => [emqx_mgmt]}}]). + %% Nodes and Brokers API -export([ list_nodes/0 , lookup_node/1 @@ -49,6 +52,7 @@ , clean_acl_cache_all/1 , set_ratelimit_policy/2 , set_quota_policy/2 + , set_keepalive/2 ]). %% Internal funcs @@ -142,7 +146,8 @@ node_info(Node) when Node =:= node() -> memory_used => proplists:get_value(total, Memory), process_available => erlang:system_info(process_limit), process_used => erlang:system_info(process_count), - max_fds => proplists:get_value(max_fds, lists:usort(lists:flatten(erlang:system_info(check_io)))), + max_fds => proplists:get_value(max_fds, + lists:usort(lists:flatten(erlang:system_info(check_io)))), connections => ets:info(emqx_channel, size), node_status => 'Running', uptime => iolist_to_binary(proplists:get_value(uptime, BrokerInfo)), @@ -196,10 +201,12 @@ get_stats(Node) -> %%-------------------------------------------------------------------- lookup_client({clientid, ClientId}, FormatFun) -> - lists:append([lookup_client(Node, {clientid, ClientId}, FormatFun) || Node <- ekka_mnesia:running_nodes()]); + lists:append([lookup_client(Node, {clientid, ClientId}, FormatFun) + || Node <- ekka_mnesia:running_nodes()]); lookup_client({username, Username}, FormatFun) -> - lists:append([lookup_client(Node, {username, Username}, FormatFun) || Node <- ekka_mnesia:running_nodes()]). + lists:append([lookup_client(Node, {username, Username}, FormatFun) + || Node <- ekka_mnesia:running_nodes()]). lookup_client(Node, {clientid, ClientId}, {M,F}) when Node =:= node() -> lists:append(lists:map( @@ -222,10 +229,7 @@ lookup_client(Node, {username, Username}, FormatFun) -> kickout_client(ClientId) -> Results = [kickout_client(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. + has_any_ok(Results). kickout_client(Node, ClientId) when Node =:= node() -> emqx_cm:kick_session(ClientId); @@ -238,10 +242,7 @@ list_acl_cache(ClientId) -> clean_acl_cache(ClientId) -> Results = [clean_acl_cache(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. + has_any_ok(Results). clean_acl_cache(Node, ClientId) when Node =:= node() -> case emqx_cm:lookup_channels(ClientId) of @@ -273,6 +274,9 @@ set_ratelimit_policy(ClientId, Policy) -> set_quota_policy(ClientId, Policy) -> call_client(ClientId, {quota, Policy}). +set_keepalive(ClientId, Interval) -> + call_client(ClientId, {keepalive, Interval}). + %% @private call_client(ClientId, Req) -> Results = [call_client(Node, ClientId, Req) || Node <- ekka_mnesia:running_nodes()], @@ -281,7 +285,7 @@ call_client(ClientId, Req) -> end, Results), case Expected of [] -> {error, not_found}; - [Result|_] -> Result + [Result | _] -> Result end. %% @private @@ -313,7 +317,8 @@ list_subscriptions(Node) -> rpc_call(Node, list_subscriptions, [Node]). list_subscriptions_via_topic(Topic, FormatFun) -> - lists:append([list_subscriptions_via_topic(Node, Topic, FormatFun) || Node <- ekka_mnesia:running_nodes()]). + lists:append([list_subscriptions_via_topic(Node, Topic, FormatFun) + || Node <- ekka_mnesia:running_nodes()]). list_subscriptions_via_topic(Node, Topic, {M,F}) when Node =:= node() -> MatchSpec = [{{{'_', '$1'}, '_'}, [{'=:=','$1', Topic}], ['$_']}], @@ -436,7 +441,8 @@ list_listeners(Node) when Node =:= node() -> Http = lists:map(fun({Protocol, Opts}) -> #{protocol => Protocol, listen_on => proplists:get_value(port, Opts), - acceptors => maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0), + acceptors => maps:get(num_acceptors, + proplists:get_value(transport_options, Opts, #{}), 0), max_conns => proplists:get_value(max_connections, Opts), current_conns => proplists:get_value(all_connections, Opts), shutdown_count => []} @@ -483,9 +489,10 @@ add_duration_field(Alarms) -> add_duration_field([], _Now, Acc) -> Acc; -add_duration_field([Alarm = #{activated := true, activate_at := ActivateAt}| Rest], Now, Acc) -> +add_duration_field([Alarm = #{activated := true, activate_at := ActivateAt} | Rest], Now, Acc) -> add_duration_field(Rest, Now, [Alarm#{duration => Now - ActivateAt} | Acc]); -add_duration_field([Alarm = #{activated := false, activate_at := ActivateAt, deactivate_at := DeactivateAt}| Rest], Now, Acc) -> +add_duration_field([Alarm = #{activated := false, + activate_at := ActivateAt, deactivate_at := DeactivateAt} | Rest], Now, Acc) -> add_duration_field(Rest, Now, [Alarm#{duration => DeactivateAt - ActivateAt} | Acc]). %%-------------------------------------------------------------------- @@ -560,7 +567,7 @@ check_row_limit(Tables) -> check_row_limit([], _Limit) -> ok; -check_row_limit([Tab|Tables], Limit) -> +check_row_limit([Tab | Tables], Limit) -> case table_size(Tab) > Limit of true -> false; false -> check_row_limit(Tables, Limit) @@ -571,4 +578,8 @@ max_row_limit() -> table_size(Tab) -> ets:info(Tab, size). - +has_any_ok(Results) -> + case lists:any(fun(Item) -> Item =:= ok end, Results) of + true -> ok; + false -> lists:last(Results) + end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 1ddd87a3d..5aacc7895 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -117,6 +117,12 @@ func => clean_quota, descr => "Clear the quota policy"}). +-rest_api(#{name => set_keepalive, + method => 'PUT', + path => "/clients/:bin:clientid/keepalive", + func => set_keepalive, + descr => "Set the client keepalive"}). + -import(emqx_mgmt_util, [ ntoa/1 , strftime/1 ]). @@ -130,23 +136,24 @@ , set_quota_policy/2 , clean_ratelimit/2 , clean_quota/2 + , set_keepalive/2 ]). -export([ query/3 , format_channel_info/1 ]). --define(query_fun, {?MODULE, query}). --define(format_fun, {?MODULE, format_channel_info}). +-define(QUERY_FUN, {?MODULE, query}). +-define(FORMAT_FUN, {?MODULE, format_channel_info}). list(Bindings, Params) when map_size(Bindings) == 0 -> fence(fun() -> - emqx_mgmt_api:cluster_query(Params, ?CLIENT_QS_SCHEMA, ?query_fun) + emqx_mgmt_api:cluster_query(Params, ?CLIENT_QS_SCHEMA, ?QUERY_FUN) end); list(#{node := Node}, Params) when Node =:= node() -> fence(fun() -> - emqx_mgmt_api:node_query(Node, Params, ?CLIENT_QS_SCHEMA, ?query_fun) + emqx_mgmt_api:node_query(Node, Params, ?CLIENT_QS_SCHEMA, ?QUERY_FUN) end); list(Bindings = #{node := Node}, Params) -> @@ -169,16 +176,20 @@ fence(Func) -> end. lookup(#{node := Node, clientid := ClientId}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client(Node, {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client(Node, + {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?FORMAT_FUN)}); lookup(#{clientid := ClientId}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client({clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client( + {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?FORMAT_FUN)}); lookup(#{node := Node, username := Username}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client(Node, {username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client(Node, + {username, emqx_mgmt_util:urldecode(Username)}, ?FORMAT_FUN)}); lookup(#{username := Username}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client({username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}). + minirest:return({ok, emqx_mgmt:lookup_client({username, + emqx_mgmt_util:urldecode(Username)}, ?FORMAT_FUN)}). kickout(#{clientid := ClientId}, _Params) -> case emqx_mgmt:kickout_client(emqx_mgmt_util:urldecode(ClientId)) of @@ -204,7 +215,7 @@ list_acl_cache(#{clientid := ClientId}, _Params) -> set_ratelimit_policy(#{clientid := ClientId}, Params) -> P = [{conn_bytes_in, proplists:get_value(<<"conn_bytes_in">>, Params)}, {conn_messages_in, proplists:get_value(<<"conn_messages_in">>, Params)}], - case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of + case filter_ratelimit_params(P) of [] -> minirest:return(); Policy -> case emqx_mgmt:set_ratelimit_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of @@ -223,7 +234,7 @@ clean_ratelimit(#{clientid := ClientId}, _Params) -> set_quota_policy(#{clientid := ClientId}, Params) -> P = [{conn_messages_routing, proplists:get_value(<<"conn_messages_routing">>, Params)}], - case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of + case filter_ratelimit_params(P) of [] -> minirest:return(); Policy -> case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of @@ -233,6 +244,7 @@ set_quota_policy(#{clientid := ClientId}, Params) -> end end. + clean_quota(#{clientid := ClientId}, _Params) -> case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), []) of ok -> minirest:return(); @@ -240,6 +252,19 @@ clean_quota(#{clientid := ClientId}, _Params) -> {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end. +set_keepalive(#{clientid := ClientId}, Params) -> + case proplists:get_value(<<"interval">>, Params) of + undefined -> + minirest:return({error, ?ERROR7, params_not_found}); + Interval0 -> + Interval = binary_to_integer(Interval0), + case emqx_mgmt:set_keepalive(emqx_mgmt_util:urldecode(ClientId), Interval) of + ok -> minirest:return(); + {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) + end + end. + %% @private %% S = 100,1s %% | 100KB, 1m @@ -266,7 +291,7 @@ format_channel_info({_Key, Info, Stats0}) -> ConnInfo = maps:get(conninfo, Info, #{}), Session = case maps:get(session, Info, #{}) of undefined -> #{}; - _Sess -> _Sess + Sess -> Sess end, SessCreated = maps:get(created_at, Session, maps:get(connected_at, ConnInfo)), Connected = case maps:get(conn_state, Info, connected) of @@ -306,7 +331,8 @@ format(Data) when is_map(Data)-> created_at => iolist_to_binary(strftime(CreatedAt div 1000))}, case maps:get(disconnected_at, Data, undefined) of undefined -> #{}; - DisconnectedAt -> #{disconnected_at => iolist_to_binary(strftime(DisconnectedAt div 1000))} + DisconnectedAt -> #{disconnected_at => + iolist_to_binary(strftime(DisconnectedAt div 1000))} end). format_acl_cache({{PubSub, Topic}, {AclResult, Timestamp}}) -> @@ -326,7 +352,8 @@ query({Qs, []}, Start, Limit) -> query({Qs, Fuzzy}, Start, Limit) -> Ms = qs2ms(Qs), MatchFun = match_fun(Ms, Fuzzy), - emqx_mgmt_api:traverse_table(emqx_channel_info, MatchFun, Start, Limit, fun format_channel_info/1). + emqx_mgmt_api:traverse_table(emqx_channel_info, MatchFun, + Start, Limit, fun format_channel_info/1). %%-------------------------------------------------------------------- %% Match funcs @@ -352,7 +379,7 @@ escape(B) when is_binary(B) -> run_fuzzy_match(_, []) -> true; -run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> +run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE} | Fuzzy]) -> Val = case maps:get(Key, ClientInfo, "") of undefined -> ""; V -> V @@ -406,6 +433,9 @@ ms(connected_at, X) -> ms(created_at, X) -> #{session => #{created_at => X}}. +filter_ratelimit_params(P) -> + [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined]. + %%-------------------------------------------------------------------- %% EUnits %%-------------------------------------------------------------------- diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index e45acfd42..f3bc42f3a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -29,6 +29,8 @@ -define(HOST, "http://127.0.0.1:8081/"). +-elvis([{elvis_style, line_length, disable}]). + -define(API_VERSION, "v4"). -define(BASE_PATH, "api"). @@ -76,30 +78,40 @@ t_alarms(_) -> ?assert(is_existing(alarm2, emqx_alarm:get_alarms(activated))), {ok, Return1} = request_api(get, api_path(["alarms/activated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return1))))), + ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return1))))), emqx_alarm:deactivate(alarm1), {ok, Return2} = request_api(get, api_path(["alarms"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return2))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return2))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return2))))), + ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return2))))), {ok, Return3} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return3))))), - ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return3))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return3))))), + ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return3))))), emqx_alarm:deactivate(alarm2), {ok, Return4} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return4))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return4))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return4))))), + ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return4))))), {ok, _} = request_api(delete, api_path(["alarms/deactivated"]), auth_header_()), {ok, Return5} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assertNot(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))), - ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))). + ?assertNot(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return5))))), + ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return5))))). t_apps(_) -> AppId = <<"123456">>, @@ -153,7 +165,8 @@ t_banned(_) -> [Banned] = get(<<"data">>, Result), ?assertEqual(Who, maps:get(<<"who">>, Banned)), - {ok, _} = request_api(delete, api_path(["banned", "clientid", binary_to_list(Who)]), auth_header_()), + {ok, _} = request_api(delete, api_path(["banned", "clientid", binary_to_list(Who)]), + auth_header_()), {ok, Result2} = request_api(get, api_path(["banned"]), auth_header_()), ?assertEqual([], get(<<"data">>, Result2)). @@ -205,40 +218,50 @@ t_clients(_) -> meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, kickout_client, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), + {ok, MeckRet1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), + auth_header_()), ?assertEqual(?ERROR1, get(<<"code">>, MeckRet1)), meck:expect(emqx_mgmt, clean_acl_cache, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet2} = request_api(delete, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), + {ok, MeckRet2} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR1, get(<<"code">>, MeckRet2)), meck:expect(emqx_mgmt, list_acl_cache, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet3} = request_api(get, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), + {ok, MeckRet3} = request_api(get, + api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR1, get(<<"code">>, MeckRet3)), meck:unload(emqx_mgmt), - {ok, Ok} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), + {ok, Ok} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), ?assertEqual(?SUCCESS, get(<<"code">>, Ok)), timer:sleep(300), - {ok, Ok1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), + {ok, Ok1} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), ?assertEqual(?SUCCESS, get(<<"code">>, Ok1)), - {ok, Clients6} = request_api(get, api_path(["clients"]), "_limit=100&_page=1", auth_header_()), + {ok, Clients6} = request_api(get, + api_path(["clients"]), "_limit=100&_page=1", auth_header_()), ?assertEqual(1, maps:get(<<"count">>, get(<<"meta">>, Clients6))), - {ok, NotFound1} = request_api(get, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), + {ok, NotFound1} = request_api(get, + api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR12, get(<<"code">>, NotFound1)), - {ok, NotFound2} = request_api(delete, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), + {ok, NotFound2} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR12, get(<<"code">>, NotFound2)), - {ok, EmptyAclCache} = request_api(get, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), + {ok, EmptyAclCache} = request_api(get, + api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), ?assertEqual(0, length(get(<<"data">>, EmptyAclCache))), - {ok, Ok1} = request_api(delete, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), + {ok, Ok1} = request_api(delete, + api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), ?assertEqual(?SUCCESS, get(<<"code">>, Ok1)). receive_exit(0) -> @@ -257,7 +280,8 @@ receive_exit(Count) -> t_listeners(_) -> {ok, _} = request_api(get, api_path(["listeners"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "listeners"]), auth_header_()), + {ok, _} = request_api(get, + api_path(["nodes", atom_to_list(node()), "listeners"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, list_listeners, 0, fun() -> [{node(), {error, undefined}}] end), {ok, Return} = request_api(get, api_path(["listeners"]), auth_header_()), @@ -268,10 +292,12 @@ t_listeners(_) -> t_metrics(_) -> {ok, _} = request_api(get, api_path(["metrics"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), + {ok, _} = request_api(get, + api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, get_metrics, 1, fun(_) -> {error, undefined} end), - {ok, "{\"message\":\"undefined\"}"} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), + {ok, "{\"message\":\"undefined\"}"} = + request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), meck:unload(emqx_mgmt). t_nodes(_) -> @@ -348,7 +374,8 @@ t_acl_cache(_) -> {ok, _} = emqtt:connect(C1), {ok, _, _} = emqtt:subscribe(C1, Topic, 2), %% get acl cache, should not be empty - {ok, Result} = request_api(get, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), + {ok, Result} = request_api(get, + api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), #{<<"code">> := 0, <<"data">> := Caches} = jiffy:decode(list_to_binary(Result), [return_maps]), ?assert(length(Caches) > 0), ?assertMatch(#{<<"access">> := <<"subscribe">>, @@ -356,11 +383,14 @@ t_acl_cache(_) -> <<"result">> := <<"allow">>, <<"updated_time">> := _}, hd(Caches)), %% clear acl cache - {ok, Result2} = request_api(delete, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), + {ok, Result2} = request_api(delete, + api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), ?assertMatch(#{<<"code">> := 0}, jiffy:decode(list_to_binary(Result2), [return_maps])), %% get acl cache again, after the acl cache is cleared - {ok, Result3} = request_api(get, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), - #{<<"code">> := 0, <<"data">> := Caches3} = jiffy:decode(list_to_binary(Result3), [return_maps]), + {ok, Result3} = request_api(get, + api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), + #{<<"code">> := 0, <<"data">> := Caches3} + = jiffy:decode(list_to_binary(Result3), [return_maps]), ?assertEqual(0, length(Caches3)), ok = emqtt:disconnect(C1). @@ -482,12 +512,15 @@ t_pubsub(_) -> Topic_list = [<<"mytopic1">>, <<"mytopic2">>], [ {ok, _, [2]} = emqtt:subscribe(C1, Topics, 2) || Topics <- Topic_list], - Body1 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2} || Topics <- Topic_list], + Body1 = [ #{<<"clientid">> => ClientId, + <<"topic">> => Topics, <<"qos">> => 2} || Topics <- Topic_list], {ok, Data1} = request_api(post, api_path(["mqtt/subscribe_batch"]), [], auth_header_(), Body1), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data1), [return_maps]))), %% tests publish_batch - Body2 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2, <<"retain">> => <<"false">>, <<"payload">> => #{body => "hello world"}} || Topics <- Topic_list ], + Body2 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2, + <<"retain">> => <<"false">>, <<"payload">> => #{body => "hello world"}} + || Topics <- Topic_list ], {ok, Data2} = request_api(post, api_path(["mqtt/publish_batch"]), [], auth_header_(), Body2), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data2), [return_maps]))), [ ?assert(receive @@ -499,7 +532,8 @@ t_pubsub(_) -> %% tests unsubscribe_batch Body3 = [#{<<"clientid">> => ClientId, <<"topic">> => Topics} || Topics <- Topic_list], - {ok, Data3} = request_api(post, api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), + {ok, Data3} = request_api(post, + api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data3), [return_maps]))), ok = emqtt:disconnect(C1), @@ -523,7 +557,8 @@ t_routes_and_subscriptions(_) -> ?assertEqual([], get(<<"data">>, NonRoute)), {ok, NonSubscription} = request_api(get, api_path(["subscriptions"]), auth_header_()), ?assertEqual([], get(<<"data">>, NonSubscription)), - {ok, NonSubscription1} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), + {ok, NonSubscription1} = request_api(get, + api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), ?assertEqual([], get(<<"data">>, NonSubscription1)), {ok, NonSubscription2} = request_api(get, api_path(["subscriptions", binary_to_list(ClientId)]), @@ -552,11 +587,14 @@ t_routes_and_subscriptions(_) -> ?assertEqual(Topic, maps:get(<<"topic">>, Subscription)), ?assertEqual(ClientId, maps:get(<<"clientid">>, Subscription)), - {ok, Result3} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), + {ok, Result3} = request_api(get, + api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), - {ok, Result4} = request_api(get, api_path(["subscriptions", binary_to_list(ClientId)]), auth_header_()), + {ok, Result4} = request_api(get, + api_path(["subscriptions", binary_to_list(ClientId)]), auth_header_()), [Subscription] = get(<<"data">>, Result4), - {ok, Result4} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions", binary_to_list(ClientId)]) + {ok, Result4} = request_api(get, + api_path(["nodes", atom_to_list(node()), "subscriptions", binary_to_list(ClientId)]) , auth_header_()), ok = emqtt:disconnect(C1). @@ -566,7 +604,8 @@ t_stats(_) -> {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, get_stats, 1, fun(_) -> {error, undefined} end), - {ok, Return} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), + {ok, Return} = request_api(get, + api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), ?assertEqual(<<"undefined">>, get(<<"message">>, Return)), meck:unload(emqx_mgmt). @@ -578,10 +617,15 @@ t_data(_) -> {ok, Data} = request_api(post, api_path(["data","export"]), [], auth_header_(), [#{}]), #{<<"filename">> := Filename, <<"node">> := Node} = emqx_ct_http:get_http_data(Data), {ok, DataList} = request_api(get, api_path(["data","export"]), auth_header_()), - ?assertEqual(true, lists:member(emqx_ct_http:get_http_data(Data), emqx_ct_http:get_http_data(DataList))), + ?assertEqual(true, + lists:member(emqx_ct_http:get_http_data(Data), emqx_ct_http:get_http_data(DataList))), - ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename, <<"node">> => Node})), - ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename})), + ?assertMatch({ok, _}, request_api(post, + api_path(["data","import"]), [], auth_header_(), + #{<<"filename">> => Filename, <<"node">> => Node})), + ?assertMatch({ok, _}, + request_api(post, api_path(["data","import"]), [], auth_header_(), + #{<<"filename">> => Filename})), application:stop(emqx_rule_engine), application:stop(emqx_dahboard), ok. @@ -596,9 +640,26 @@ t_data_import_content(_) -> Dir = emqx:get_env(data_dir), {ok, Bin} = file:read_file(filename:join(Dir, Filename)), Content = emqx_json:decode(Bin), - ?assertMatch({ok, "{\"code\":0}"}, request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), + ?assertMatch({ok, "{\"code\":0}"}, + request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), application:stop(emqx_rule_engine), - application:stop(emqx_dahboard). + application:stop(emqx_dashboard). + +t_keepalive(_Config) -> + application:ensure_all_started(emqx_dashboard), + Username = "user_keepalive", + ClientId = "client_keepalive", + AuthHeader = auth_header_(), + Path = api_path(["clients", ClientId, "keepalive"]), + {ok, NotFound} = request_api(put, Path, "interval=5", AuthHeader, [#{}]), + ?assertEqual("{\"message\":\"not_found\",\"code\":112}", NotFound), + {ok, C1} = emqtt:start_link(#{username => Username, clientid => ClientId}), + {ok, _} = emqtt:connect(C1), + {ok, Ok} = request_api(put, Path, "interval=5", AuthHeader, [#{}]), + ?assertEqual("{\"code\":0}", Ok), + emqtt:disconnect(C1), + application:stop(emqx_dashboard), + ok. request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index c3ffb9f90..a423fb9b7 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 5f164d99e..4c49a1cb3 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -102,7 +102,7 @@ -type(reply() :: {outgoing, emqx_types:packet()} | {outgoing, [emqx_types:packet()]} - | {event, conn_state()|updated} + | {event, conn_state() | updated} | {close, Reason :: atom()}). -type(replies() :: emqx_types:packet() | reply() | [reply()]). @@ -131,7 +131,7 @@ info(Channel) -> maps:from_list(info(?INFO_KEYS, Channel)). --spec(info(list(atom())|atom(), channel()) -> term()). +-spec(info(list(atom()) | atom(), channel()) -> term()). info(Keys, Channel) when is_list(Keys) -> [{Key, info(Key, Channel)} || Key <- Keys]; info(conninfo, #channel{conninfo = ConnInfo}) -> @@ -619,7 +619,7 @@ ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> -compile({inline, [puback_reason_code/1]}). puback_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS; -puback_reason_code([_|_]) -> ?RC_SUCCESS. +puback_reason_code([_ | _]) -> ?RC_SUCCESS. -compile({inline, [after_message_acked/3]}). after_message_acked(ClientInfo, Msg, PubAckProps) -> @@ -638,7 +638,7 @@ process_subscribe(TopicFilters, SubProps, Channel) -> process_subscribe([], _SubProps, Channel, Acc) -> {lists:reverse(Acc), Channel}; -process_subscribe([Topic = {TopicFilter, SubOpts}|More], SubProps, Channel, Acc) -> +process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Acc) -> case check_sub_caps(TopicFilter, SubOpts, Channel) of ok -> {ReasonCode, NChannel} = do_subscribe(TopicFilter, @@ -676,9 +676,9 @@ process_unsubscribe(TopicFilters, UnSubProps, Channel) -> process_unsubscribe([], _UnSubProps, Channel, Acc) -> {lists:reverse(Acc), Channel}; -process_unsubscribe([{TopicFilter, SubOpts}|More], UnSubProps, Channel, Acc) -> +process_unsubscribe([{TopicFilter, SubOpts} | More], UnSubProps, Channel, Acc) -> {RC, NChannel} = do_unsubscribe(TopicFilter, SubOpts#{unsub_props => UnSubProps}, Channel), - process_unsubscribe(More, UnSubProps, NChannel, [RC|Acc]). + process_unsubscribe(More, UnSubProps, NChannel, [RC | Acc]). do_unsubscribe(TopicFilter, SubOpts, Channel = #channel{clientinfo = ClientInfo = #{mountpoint := MountPoint}, @@ -938,6 +938,17 @@ handle_call({quota, Policy}, Channel) -> Quota = emqx_limiter:init(Zone, Policy), reply(ok, Channel#channel{quota = Quota}); +handle_call({keepalive, Interval}, Channel = #channel{keepalive = KeepAlive, + conninfo = ConnInfo}) -> + ClientId = info(clientid, Channel), + NKeepalive = emqx_keepalive:set(interval, Interval * 1000, KeepAlive), + NConnInfo = maps:put(keepalive, Interval, ConnInfo), + NChannel = Channel#channel{keepalive = NKeepalive, conninfo = NConnInfo}, + SockInfo = maps:get(sockinfo, emqx_cm:get_chan_info(ClientId), #{}), + ChanInfo1 = info(NChannel), + emqx_cm:set_chan_info(ClientId, ChanInfo1#{sockinfo => SockInfo}), + reply(ok, reset_timer(alive_timer, NChannel)); + handle_call(Req, Channel) -> ?LOG(error, "Unexpected call: ~p", [Req]), reply(ignored, Channel). diff --git a/src/emqx_keepalive.erl b/src/emqx_keepalive.erl index 8fba00f50..216999acc 100644 --- a/src/emqx_keepalive.erl +++ b/src/emqx_keepalive.erl @@ -20,9 +20,11 @@ , info/1 , info/2 , check/2 + , set/3 ]). -export_type([keepalive/0]). +-elvis([{elvis_style, no_if_expression, disable}]). -record(keepalive, { interval :: pos_integer(), @@ -49,7 +51,7 @@ info(#keepalive{interval = Interval, repeat => Repeat }. --spec(info(interval|statval|repeat, keepalive()) +-spec(info(interval | statval | repeat, keepalive()) -> non_neg_integer()). info(interval, #keepalive{interval = Interval}) -> Interval; @@ -71,3 +73,7 @@ check(NewVal, KeepAlive = #keepalive{statval = OldVal, true -> {error, timeout} end. +%% @doc Update keepalive's interval +-spec(set(interval, non_neg_integer(), keepalive()) -> keepalive()). +set(interval, Interval, KeepAlive) -> + KeepAlive#keepalive{interval = Interval}. From ad4d3fc652330ac1bc041c6872715041f070732b Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 12 Nov 2021 21:26:44 +0800 Subject: [PATCH 024/363] chore(emqx_retainer): refresh the timestamp when dispatch retained message (#6155) 1. refresh the timestamp when dispatch retained message 2. fix some elvis style error --- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- apps/emqx_retainer/src/emqx_retainer.erl | 37 +++++++++++++++----- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index a423fb9b7..4ef423b78 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "4.4.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 340e6929d..fcc5652cc 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -78,7 +78,8 @@ dispatch(Pid, Topic) -> false -> read_messages(Topic); true -> match_messages(Topic) end, - [Pid ! {deliver, Topic, Msg} || Msg <- sort_retained(Msgs)]. + Now = erlang:system_time(millisecond), + [Pid ! {deliver, Topic, refresh_timestamp_expiry(Msg, Now)} || Msg <- sort_retained(Msgs)]. %% RETAIN flag set to 1 and payload containing zero bytes on_message_publish(Msg = #message{flags = #{retain := true}, @@ -146,7 +147,7 @@ init([Env]) -> ok end, StatsFun = emqx_stats:statsfun('retained.count', 'retained.max'), - {ok, StatsTimer} = timer:send_interval(timer:seconds(1), stats), + StatsTimer = erlang:send_after(timer:seconds(1), self(), stats), State = #state{stats_fun = StatsFun, stats_timer = StatsTimer}, {ok, start_expire_timer(proplists:get_value(expiry_interval, Env, 0), State)}. @@ -155,7 +156,7 @@ start_expire_timer(0, State) -> start_expire_timer(undefined, State) -> State; start_expire_timer(Ms, State) -> - {ok, Timer} = timer:send_interval(Ms, expire), + Timer = erlang:send_after(Ms, self(), stats), State#state{expiry_timer = Timer}. handle_call(Req, _From, State) -> @@ -194,7 +195,7 @@ sort_retained([]) -> []; sort_retained([Msg]) -> [Msg]; sort_retained(Msgs) -> lists:sort(fun(#message{timestamp = Ts1}, #message{timestamp = Ts2}) -> - Ts1 =< Ts2 + Ts1 =< Ts2 end, Msgs). store_retained(Msg = #message{topic = Topic, payload = Payload}, Env) -> @@ -209,11 +210,13 @@ store_retained(Msg = #message{topic = Topic, payload = Payload}, Env) -> fun() -> case mnesia:read(?TAB, Topic) of [_] -> - mnesia:write(?TAB, #retained{topic = topic2tokens(Topic), - msg = Msg, - expiry_time = get_expiry_time(Msg, Env)}, write); + mnesia:write(?TAB, + #retained{topic = topic2tokens(Topic), + msg = Msg, + expiry_time = get_expiry_time(Msg, Env)}, write); [] -> - ?LOG(error, "Cannot retain message(topic=~s) for table is full!", [Topic]) + ?LOG(error, + "Cannot retain message(topic=~s) for table is full!", [Topic]) end end), ok; @@ -234,7 +237,8 @@ is_too_big(Size, Env) -> get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := 0}}}, _Env) -> 0; -get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, timestamp = Ts}, _Env) -> +get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, + timestamp = Ts}, _Env) -> Ts + Interval * 1000; get_expiry_time(#message{timestamp = Ts}, Env) -> case proplists:get_value(expiry_interval, Env, 0) of @@ -303,3 +307,18 @@ condition(Ws) -> false -> Ws1; _ -> (Ws1 -- ['#']) ++ '_' end. + +-spec(refresh_timestamp_expiry(emqx_types:message(), pos_integer()) -> emqx_types:message()). +refresh_timestamp_expiry(Msg = #message{headers = + #{properties := + #{'Message-Expiry-Interval' := Interval} = Props}, + timestamp = CreatedAt}, + Now) -> + Elapsed = max(0, Now - CreatedAt), + Interval1 = max(1, Interval - (Elapsed div 1000)), + emqx_message:set_header(properties, + Props#{'Message-Expiry-Interval' => Interval1}, + Msg#message{timestamp = Now}); + +refresh_timestamp_expiry(Msg, Now) -> + Msg#message{timestamp = Now}. From 6fb3ff1f9f183c444c9126dc06e025d4d8fe9fd2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 5 Nov 2021 18:32:43 +0800 Subject: [PATCH 025/363] feat(exhook): expose headers for on_messages_publish hook --- apps/emqx_exhook/priv/protos/exhook.proto | 25 ++++++++++ apps/emqx_exhook/src/emqx_exhook_handler.erl | 48 +++++++++++++++++-- .../emqx_exhook/test/emqx_exhook_demo_svr.erl | 11 +++-- src/emqx_types.erl | 1 + 4 files changed, 78 insertions(+), 7 deletions(-) diff --git a/apps/emqx_exhook/priv/protos/exhook.proto b/apps/emqx_exhook/priv/protos/exhook.proto index 72ba26581..639066c6a 100644 --- a/apps/emqx_exhook/priv/protos/exhook.proto +++ b/apps/emqx_exhook/priv/protos/exhook.proto @@ -358,6 +358,31 @@ message Message { bytes payload = 6; uint64 timestamp = 7; + + // The key of header can be: + // - username: + // * Readonly + // * The username of sender client + // * Value type: utf8 string + // - protocol: + // * Readonly + // * The protocol name of sender client + // * Value type: string enum with "mqtt", "mqtt-sn", ... + // - peerhost: + // * Readonly + // * The peerhost of sender client + // * Value type: ip address string + // - allow_publish: + // * Writable + // * Whether to allow the message to be published by emqx + // * Value type: string enum with "true", "false", default is "true" + // + // Notes: All header may be missing, which means that the message does not + // carry these headers. We can guarantee that clients coming from MQTT, + // MQTT-SN, CoAP, LwM2M and other natively supported protocol clients will + // carry these headers, but there is no guarantee that messages published + // by other means will do, e.g. messages published by HTTP-API + map headers = 8; } message Property { diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index f3964dc42..efdf7f8b9 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -258,17 +258,57 @@ clientinfo(ClientInfo = cn => maybe(maps:get(cn, ClientInfo, undefined)), dn => maybe(maps:get(dn, ClientInfo, undefined))}. -message(#message{id = Id, qos = Qos, from = From, topic = Topic, payload = Payload, timestamp = Ts}) -> +message(#message{id = Id, qos = Qos, from = From, topic = Topic, + payload = Payload, timestamp = Ts, headers = Headers}) -> #{node => stringfy(node()), id => emqx_guid:to_hexstr(Id), qos => Qos, from => stringfy(From), topic => Topic, payload => Payload, - timestamp => Ts}. + timestamp => Ts, + headers => headers(Headers) + }. -assign_to_message(#{qos := Qos, topic := Topic, payload := Payload}, Message) -> - Message#message{qos = Qos, topic = Topic, payload = Payload}. +headers(undefined) -> + #{}; +headers(Headers) -> + Ls = [username, protocol, peerhost, allow_publish], + maps:fold( + fun + (_, undefined, Acc) -> + Acc; %% Ignore undefined value + (K, V, Acc) -> + case lists:member(K, Ls) of + true -> + Acc#{atom_to_binary(K) => bin(K, V)}; + _ -> + Acc + end + end, #{}, Headers). + +bin(K, V) when K == username; + K == protocol; + K == allow_publish -> + bin(V); +bin(peerhost, V) -> + bin(inet:ntoa(V)). + +bin(V) when is_binary(V) -> V; +bin(V) when is_atom(V) -> atom_to_binary(V); +bin(V) when is_list(V) -> iolist_to_binary(V). + +assign_to_message(InMessage = #{qos := Qos, topic := Topic, + payload := Payload}, Message) -> + NMsg = Message#message{qos = Qos, topic = Topic, payload = Payload}, + enrich_header(maps:get(headers, InMessage, #{}), NMsg). + +enrich_header(Headers, Message) -> + AllowPub = case maps:get(<<"allow_publish">>, Headers, <<"true">>) of + <<"false">> -> false; + _ -> true + end, + emqx_message:set_header(allow_publish, AllowPub, Message). topicfilters(Tfs) when is_list(Tfs) -> [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs]. diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index c2db04dd4..fd8a5f9a3 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -295,14 +295,14 @@ on_session_terminated(Req, Md) -> | {error, grpc_cowboy_h:error_response()}. on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), + io:format(standard_error, "fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), %% some cases for testing case From of <<"baduser">> -> - NMsg = Msg#{qos => 0, + NMsg = deny(Msg#{qos => 0, topic => <<"">>, payload => <<"">> - }, + }), {ok, #{type => 'STOP_AND_RETURN', value => {message, NMsg}}, Md}; <<"gooduser">> -> @@ -314,6 +314,11 @@ on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> {ok, #{type => 'IGNORE'}, Md} end. +deny(Msg) -> + NHeader = maps:put(<<"allow_publish">>, <<"false">>, + maps:get(headers, Msg, #{})), + maps:put(headers, NHeader, Msg). + -spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata()) -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} | {error, grpc_cowboy_h:error_response()}. diff --git a/src/emqx_types.erl b/src/emqx_types.erl index 3e53eafd1..410ed5a27 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -193,6 +193,7 @@ username => username(), peerhost => peerhost(), properties => properties(), + allow_publish => boolean(), atom() => term()}). -type(banned() :: #banned{}). From ef2a5c1dc71722d35aafc307d516a99503c7e5cd Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 11:11:16 +0800 Subject: [PATCH 026/363] chore(exhook): fix diaylzer warnings --- apps/emqx_exhook/src/emqx_exhook_handler.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index efdf7f8b9..45e6bf082 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -270,8 +270,6 @@ message(#message{id = Id, qos = Qos, from = From, topic = Topic, headers => headers(Headers) }. -headers(undefined) -> - #{}; headers(Headers) -> Ls = [username, protocol, peerhost, allow_publish], maps:fold( From 012c7415926d2cf11cc1e52917c4774dbb89021e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 11:11:37 +0800 Subject: [PATCH 027/363] chore(exhook): update appup.src --- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.appup.src | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 46223d212..b386bcaca 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,6 +1,6 @@ {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.3.4"}, + {vsn, "4.3.5"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index d6a699c33..e78afef1e 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,15 +1,17 @@ %% -*-: erlang -*- {VSN, - [ - {<<"4.3.[0-3]">>, [ - {restart_application, emqx_exhook} - ]}, - {<<".*">>, []} + [{"4.3.4", + [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}]}, + {<<"4\\.3\\.[0-3]+">>, + [{restart_application,emqx_exhook}]}, + {<<".*">>, []} ], - [ - {<<"4.3.[0-3]">>, [ - {restart_application, emqx_exhook} - ]}, - {<<".*">>, []} + [{"4.3.4", + [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}]}, + {<<"4\\.3\\.[0-3]+">>, + [{restart_application,emqx_exhook}]}, + {<<".*">>, []} ] }. From b756e7d17a4e73213329d518cd5e0217355cf268 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 14:15:32 +0800 Subject: [PATCH 028/363] chore: upgrade grpc to 0.6.4 --- apps/emqx_exhook/rebar.config | 2 +- apps/emqx_exproto/rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index 3529b6314..d1cc4d778 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -5,7 +5,7 @@ ]}. {deps, - [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.3"}}} + [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} ]}. {grpc, diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index 4ad1aa192..da868de82 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -13,7 +13,7 @@ ]}. {deps, - [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.3"}}} + [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} ]}. {grpc, From c170d076e32ad72db408a85a41644db2c8df94e2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 14:16:26 +0800 Subject: [PATCH 029/363] feat(exhook): expose process pool_size for grpc client --- apps/emqx_exhook/etc/emqx_exhook.conf | 5 +++++ apps/emqx_exhook/priv/emqx_exhook.schema | 4 ++++ apps/emqx_exhook/src/emqx_exhook_mngr.erl | 11 +++++++++++ apps/emqx_exhook/src/emqx_exhook_server.erl | 3 ++- apps/emqx_exhook/src/emqx_exhook_sup.erl | 5 +++-- 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/apps/emqx_exhook/etc/emqx_exhook.conf b/apps/emqx_exhook/etc/emqx_exhook.conf index ffb71e43b..23895f902 100644 --- a/apps/emqx_exhook/etc/emqx_exhook.conf +++ b/apps/emqx_exhook/etc/emqx_exhook.conf @@ -24,6 +24,11 @@ ## Value: false | Duration #exhook.auto_reconnect = 60s +## The process pool size for gRPC client +## +## Default: Equals cpu cores +## Value: Integer +#exhook.pool_size = 16 ##-------------------------------------------------------------------- ## The Hook callback servers diff --git a/apps/emqx_exhook/priv/emqx_exhook.schema b/apps/emqx_exhook/priv/emqx_exhook.schema index d11001c0d..f55913d72 100644 --- a/apps/emqx_exhook/priv/emqx_exhook.schema +++ b/apps/emqx_exhook/priv/emqx_exhook.schema @@ -26,6 +26,10 @@ end end}. +{mapping, "exhook.pool_size", "emqx_exhook.pool_size", [ + {datatype, integer} +]}. + {mapping, "exhook.server.$name.url", "emqx_exhook.servers", [ {datatype, string} ]}. diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl index cadd5eb37..54e106f13 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mngr.erl @@ -36,6 +36,8 @@ , server/1 , put_request_failed_action/1 , get_request_failed_action/0 + , put_pool_size/1 + , get_pool_size/0 ]). %% gen_server callbacks @@ -117,6 +119,9 @@ init([Servers, AutoReconnect, ReqOpts0]) -> put_request_failed_action( maps:get(request_failed_action, ReqOpts0, deny) ), + put_pool_size( + maps:get(pool_size, ReqOpts0, erlang:system_info(schedulers)) + ), %% Load the hook servers ReqOpts = maps:without([request_failed_action], ReqOpts0), @@ -286,6 +291,12 @@ put_request_failed_action(Val) -> get_request_failed_action() -> persistent_term:get({?APP, request_failed_action}). +put_pool_size(Val) -> + persistent_term:put({?APP, pool_size}, Val). + +get_pool_size() -> + persistent_term:get({?APP, pool_size}). + save(Name, ServerState) -> Saved = persistent_term:get(?APP, []), persistent_term:put(?APP, lists:reverse([Name | Saved])), diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 7df5b643c..123cbb558 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -131,7 +131,8 @@ channel_opts(Opts) -> transport_opts => SslOpts}}; _ -> #{} end, - {SvrAddr, ClientOpts}. + NClientOpts = ClientOpts#{pool_size => emqx_exhook_mngr:get_pool_size()}, + {SvrAddr, NClientOpts}. format_http_uri(Scheme, Host0, Port) -> Host = case is_tuple(Host0) of diff --git a/apps/emqx_exhook/src/emqx_exhook_sup.erl b/apps/emqx_exhook/src/emqx_exhook_sup.erl index e9c405de0..c92fd6ca4 100644 --- a/apps/emqx_exhook/src/emqx_exhook_sup.erl +++ b/apps/emqx_exhook/src/emqx_exhook_sup.erl @@ -54,7 +54,8 @@ auto_reconnect() -> request_options() -> #{timeout => env(request_timeout, 5000), - request_failed_action => env(request_failed_action, deny) + request_failed_action => env(request_failed_action, deny), + pool_size => env(pool_size, erlang:system_info(schedulers)) }. env(Key, Def) -> @@ -67,7 +68,7 @@ env(Key, Def) -> -spec start_grpc_client_channel( string(), uri_string:uri_string(), - grpc_client:options()) -> {ok, pid()} | {error, term()}. + grpc_client_sup:options()) -> {ok, pid()} | {error, term()}. start_grpc_client_channel(Name, SvrAddr, Options) -> grpc_client_sup:create_channel_pool(Name, SvrAddr, Options). From 44008b9a6da0aa39354be5c7a96197297942023e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 15:05:29 +0800 Subject: [PATCH 030/363] chore: fix compiling warnings --- apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index 0947bdaca..794eedd6b 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -171,7 +171,7 @@ t_subscribe_case02(_) -> {ok, Socket} = gen_udp:open(0, [binary]), ClientId = ?CLIENTID, - send_connect_msg(Socket, ?CLIENTID), + send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), Topic1 = ?PREDEF_TOPIC_NAME1, From 5922521e499b8bb2b551e5ca2b8564daefd49310 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Nov 2021 15:05:54 +0800 Subject: [PATCH 031/363] test(props): cover messages headers --- apps/emqx_exhook/src/emqx_exhook_handler.erl | 14 +++++++++----- .../emqx_exhook/test/emqx_exhook_demo_svr.erl | 9 +++++++-- .../test/props/prop_exhook_hooks.erl | 19 +++++++++++++------ 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index 45e6bf082..ea6cdf5bc 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -50,6 +50,7 @@ %% Utils -export([ message/1 + , headers/1 , stringfy/1 , merge_responsed_bool/2 , merge_responsed_message/2 @@ -302,11 +303,14 @@ assign_to_message(InMessage = #{qos := Qos, topic := Topic, enrich_header(maps:get(headers, InMessage, #{}), NMsg). enrich_header(Headers, Message) -> - AllowPub = case maps:get(<<"allow_publish">>, Headers, <<"true">>) of - <<"false">> -> false; - _ -> true - end, - emqx_message:set_header(allow_publish, AllowPub, Message). + case maps:get(<<"allow_publish">>, Headers, undefined) of + <<"false">> -> + emqx_message:set_header(allow_publish, false, Message); + <<"true">> -> + emqx_message:set_header(allow_publish, true, Message); + _ -> + Message + end. topicfilters(Tfs) when is_list(Tfs) -> [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs]. diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index fd8a5f9a3..6e9691c3e 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -306,8 +306,8 @@ on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> {ok, #{type => 'STOP_AND_RETURN', value => {message, NMsg}}, Md}; <<"gooduser">> -> - NMsg = Msg#{topic => From, - payload => From}, + NMsg = allow(Msg#{topic => From, + payload => From}), {ok, #{type => 'STOP_AND_RETURN', value => {message, NMsg}}, Md}; _ -> @@ -319,6 +319,11 @@ deny(Msg) -> maps:get(headers, Msg, #{})), maps:put(headers, NHeader, Msg). +allow(Msg) -> + NHeader = maps:put(<<"allow_publish">>, <<"true">>, + maps:get(headers, Msg, #{})), + maps:put(headers, NHeader, Msg). + -spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata()) -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} | {error, grpc_cowboy_h:error_response()}. diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index 24f45c8b0..88eba8f11 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -299,19 +299,24 @@ prop_message_publish() -> _ -> ExpectedOutMsg = case emqx_message:from(Msg) of <<"baduser">> -> - MsgMap = emqx_message:to_map(Msg), + MsgMap = #{headers := Headers} + = emqx_message:to_map(Msg), emqx_message:from_map( MsgMap#{qos => 0, topic => <<"">>, - payload => <<"">> + payload => <<"">>, + headers => maps:put(allow_publish, false, Headers) }); <<"gooduser">> = From -> - MsgMap = emqx_message:to_map(Msg), + MsgMap = #{headers := Headers} + = emqx_message:to_map(Msg), emqx_message:from_map( MsgMap#{topic => From, - payload => From + payload => From, + headers => maps:put(allow_publish, true, Headers) }); - _ -> Msg + _ -> + Msg end, ?assertEqual(ExpectedOutMsg, OutMsg), @@ -464,7 +469,9 @@ from_message(Msg) -> from => stringfy(emqx_message:from(Msg)), topic => emqx_message:topic(Msg), payload => emqx_message:payload(Msg), - timestamp => emqx_message:timestamp(Msg) + timestamp => emqx_message:timestamp(Msg), + headers => emqx_exhook_handler:headers( + emqx_message:get_headers(Msg)) }. %%-------------------------------------------------------------------- From 641f36514ff6292705e31e89de2c503f6a24db22 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 9 Nov 2021 11:52:47 +0800 Subject: [PATCH 032/363] chore: fix elvis warnings --- apps/emqx_exhook/src/emqx_exhook_handler.erl | 17 +- apps/emqx_exhook/src/emqx_exhook_mngr.erl | 6 +- apps/emqx_exhook/src/emqx_exhook_server.erl | 36 +- apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 375 ++++++++++++++----- src/emqx_types.erl | 2 +- 5 files changed, 307 insertions(+), 129 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index ea6cdf5bc..63d41d8eb 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -63,6 +63,8 @@ , call_fold/3 ]). +-elvis([{elvis_style, god_modules, disable}]). + %%-------------------------------------------------------------------- %% Clients %%-------------------------------------------------------------------- @@ -341,11 +343,7 @@ merge_responsed_bool(_Req, #{type := 'IGNORE'}) -> ignore; merge_responsed_bool(Req, #{type := Type, value := {bool_result, NewBool}}) when is_boolean(NewBool) -> - NReq = Req#{result => NewBool}, - case Type of - 'CONTINUE' -> {ok, NReq}; - 'STOP_AND_RETURN' -> {stop, NReq} - end; + {ret(Type), Req#{result => NewBool}}; merge_responsed_bool(_Req, Resp) -> ?LOG(warning, "Unknown responsed value ~0p to merge to callback chain", [Resp]), ignore. @@ -353,11 +351,10 @@ merge_responsed_bool(_Req, Resp) -> merge_responsed_message(_Req, #{type := 'IGNORE'}) -> ignore; merge_responsed_message(Req, #{type := Type, value := {message, NMessage}}) -> - NReq = Req#{message => NMessage}, - case Type of - 'CONTINUE' -> {ok, NReq}; - 'STOP_AND_RETURN' -> {stop, NReq} - end; + {ret(Type), Req#{message => NMessage}}; merge_responsed_message(_Req, Resp) -> ?LOG(warning, "Unknown responsed value ~0p to merge to callback chain", [Resp]), ignore. + +ret('CONTINUE') -> ok; +ret('STOP_AND_RETURN') -> stop. diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl index 54e106f13..0f3c8b8ab 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mngr.erl @@ -86,11 +86,11 @@ start_link(Servers, AutoReconnect, ReqOpts) -> gen_server:start_link(?MODULE, [Servers, AutoReconnect, ReqOpts], []). --spec enable(pid(), atom()|string()) -> ok | {error, term()}. +-spec enable(pid(), atom() | string()) -> ok | {error, term()}. enable(Pid, Name) -> call(Pid, {load, Name}). --spec disable(pid(), atom()|string()) -> ok | {error, term()}. +-spec disable(pid(), atom() | string()) -> ok | {error, term()}. disable(Pid, Name) -> call(Pid, {unload, Name}). @@ -141,7 +141,7 @@ load_all_servers(Servers, ReqOpts) -> load_all_servers(Servers, ReqOpts, #{}, #{}). load_all_servers([], _Request, Waiting, Running) -> {Waiting, Running}; -load_all_servers([{Name, Options}|More], ReqOpts, Waiting, Running) -> +load_all_servers([{Name, Options} | More], ReqOpts, Waiting, Running) -> {NWaiting, NRunning} = case emqx_exhook_server:load(Name, Options, ReqOpts) of {ok, ServerState} -> diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 123cbb558..9e88d774d 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -77,6 +77,8 @@ -dialyzer({nowarn_function, [inc_metrics/2]}). +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + %%-------------------------------------------------------------------- %% Load/Unload APIs %%-------------------------------------------------------------------- @@ -125,7 +127,11 @@ channel_opts(Opts) -> SvrAddr = format_http_uri(Scheme, Host, Port), ClientOpts = case Scheme of https -> - SslOpts = lists:keydelete(ssl, 1, proplists:get_value(ssl_options, Opts, [])), + SslOpts = lists:keydelete( + ssl, + 1, + proplists:get_value(ssl_options, Opts, []) + ), #{gun_opts => #{transport => ssl, transport_opts => SslOpts}}; @@ -175,16 +181,18 @@ resovle_hookspec(HookSpecs) when is_list(HookSpecs) -> case maps:get(name, HookSpec, undefined) of undefined -> Acc; Name0 -> - Name = try binary_to_existing_atom(Name0, utf8) catch T:R:_ -> {T,R} end, - case lists:member(Name, AvailableHooks) of - true -> - case lists:member(Name, MessageHooks) of - true -> - Acc#{Name => #{topics => maps:get(topics, HookSpec, [])}}; - _ -> - Acc#{Name => #{}} - end; - _ -> error({unknown_hookpoint, Name}) + Name = try + binary_to_existing_atom(Name0, utf8) + catch T:R:_ -> {T,R} + end, + case {lists:member(Name, AvailableHooks), + lists:member(Name, MessageHooks)} of + {false, _} -> + error({unknown_hookpoint, Name}); + {true, false} -> + Acc#{Name => #{}}; + {true, true} -> + Acc#{Name => #{topics => maps:get(topics, HookSpec, [])}} end end end, #{}, HookSpecs). @@ -256,7 +264,7 @@ call(Hookpoint, Req, #server{name = ChannName, options = ReqOpts, %% @private inc_metrics(IncFun, Name) when is_function(IncFun) -> %% BACKW: e4.2.0-e4.2.2 - {env, [Prefix|_]} = erlang:fun_info(IncFun, env), + {env, [Prefix | _]} = erlang:fun_info(IncFun, env), inc_metrics(Prefix, Name); inc_metrics(Prefix, Name) when is_list(Prefix) -> emqx_metrics:inc(list_to_atom(Prefix ++ atom_to_list(Name))). @@ -272,8 +280,8 @@ do_call(ChannName, Fun, Req, ReqOpts) -> Options = ReqOpts#{channel => ChannName}, ?LOG(debug, "Call ~0p:~0p(~0p, ~0p)", [?PB_CLIENT_MOD, Fun, Req, Options]), case catch apply(?PB_CLIENT_MOD, Fun, [Req, Options]) of - {ok, Resp, _Metadata} -> - ?LOG(debug, "Response {ok, ~0p, ~0p}", [Resp, _Metadata]), + {ok, Resp, Metadata} -> + ?LOG(debug, "Response {ok, ~0p, ~0p}", [Resp, Metadata]), {ok, Resp}; {error, {Code, Msg}, _Metadata} -> ?LOG(error, "CALL ~0p:~0p(~0p, ~0p) response errcode: ~0p, errmsg: ~0p", diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index 794eedd6b..27215cd4f 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -50,6 +50,9 @@ %% erlang:system_time should be unique and random enough -define(CLIENTID, iolist_to_binary([atom_to_list(?FUNCTION_NAME), "-", integer_to_list(erlang:system_time())])). + +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -66,8 +69,10 @@ end_per_suite(_) -> emqx_ct_helpers:stop_apps([emqx_sn]). set_special_confs(emqx) -> - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); + application:set_env( + emqx, + plugins_loaded_file, + emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); set_special_confs(emqx_sn) -> application:set_env(emqx_sn, enable_qos3, ?ENABLE_QOS3), application:set_env(emqx_sn, enable_stats, true), @@ -113,7 +118,8 @@ t_subscribe(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), TopicName1 = <<"abcD">>, send_register_msg(Socket, TopicName1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, @@ -145,7 +151,8 @@ t_subscribe_case01(_) -> TopicName1 = <<"abcD">>, send_register_msg(Socket, TopicName1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, @@ -166,7 +173,7 @@ t_subscribe_case02(_) -> Will = 0, CleanSession = 0, MsgId = 1, - TopicId = ?PREDEF_TOPIC_ID1, %this TopicId is the predefined topic id corresponding to ?PREDEF_TOPIC_NAME1 + TopicId = ?PREDEF_TOPIC_ID1, ReturnCode = 0, {ok, Socket} = gen_udp:open(0, [binary]), @@ -176,7 +183,8 @@ t_subscribe_case02(_) -> Topic1 = ?PREDEF_TOPIC_NAME1, send_register_msg(Socket, Topic1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, @@ -206,9 +214,11 @@ t_subscribe_case03(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"te">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, + CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_short_topic(Socket, <<"te">>, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -217,8 +227,12 @@ t_subscribe_case03(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). -%%In this case We use predefined topic name to register and subcribe, and expect to receive the corresponding predefined topic id but not a new generated topic id from broker. We design this case to illustrate -%% emqx_sn_gateway's compatibility of dealing with predefined and normal topics. Once we give more restrictions to different topic id type, this case would be deleted or modified. +%% In this case We use predefined topic name to register and subcribe, and +%% expect to receive the corresponding predefined topic id but not a new +%% generated topic id from broker. We design this case to illustrate +%% emqx_sn_gateway's compatibility of dealing with predefined and normal topics. +%% Once we give more restrictions to different topic id type, this case would +%% be deleted or modified. t_subscribe_case04(_) -> Dup = 0, QoS = 0, @@ -226,7 +240,7 @@ t_subscribe_case04(_) -> Will = 0, CleanSession = 0, MsgId = 1, - TopicId = ?PREDEF_TOPIC_ID1, %this TopicId is the predefined topic id corresponding to ?PREDEF_TOPIC_NAME1 + TopicId = ?PREDEF_TOPIC_ID1, ReturnCode = 0, {ok, Socket} = gen_udp:open(0, [binary]), ClientId = ?CLIENTID, @@ -234,10 +248,14 @@ t_subscribe_case04(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), Topic1 = ?PREDEF_TOPIC_NAME1, send_register_msg(Socket, Topic1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, Topic1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_normal_topic(Socket, Topic1, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -264,19 +282,30 @@ t_subscribe_case05(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_register_msg(Socket, <<"abcD">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"abcD">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/sport/#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/a/+/water">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/Tom/Home">>, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, @@ -306,19 +335,32 @@ t_subscribe_case06(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_register_msg(Socket, <<"abc">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/blue/#">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/blue/+/white">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/$sys/rain">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId2:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId2:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_subscribe_msg_short_topic(Socket, QoS, <<"Q2">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_normal_topic(Socket, <<"Q2">>, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -342,8 +384,11 @@ t_subscribe_case07(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket) + ), send_unsubscribe_msg_predefined_topic(Socket, TopicId2, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -365,8 +410,11 @@ t_subscribe_case08(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_reserved_topic(Socket, QoS, TopicId2, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -390,15 +438,20 @@ t_publish_negqos_case09(_) -> send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, NegQoS, MsgId1, TopicId1, Payload1), timer:sleep(100), case ?ENABLE_QOS3 of true -> - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What) end, @@ -431,7 +484,9 @@ t_publish_qos0_case01(_) -> send_publish_msg_normal_topic(Socket, QoS, MsgId1, TopicId1, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -453,15 +508,20 @@ t_publish_qos0_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId1, PredefTopicId, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, PredefTopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, + PredefTopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -484,15 +544,20 @@ t_publish_qos0_case3(_) -> Topic = <<"/a/b/c">>, send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId1, TopicId, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -514,8 +579,11 @@ t_publish_qos0_case04(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, <<"#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 2, Payload1 = <<20, 21, 22, 23>>, @@ -523,7 +591,9 @@ t_publish_qos0_case04(_) -> send_publish_msg_short_topic(Socket, QoS, MsgId1, Topic, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, Topic/binary, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, + Topic/binary, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -544,8 +614,11 @@ t_publish_qos0_case05(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"/#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -567,15 +640,20 @@ t_publish_qos0_case06(_) -> Topic = <<"abc">>, send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, QoS, MsgId1, TopicId1, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -597,16 +675,25 @@ t_publish_qos1_case01(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), - ?assertEqual(<<7, ?SN_PUBACK, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicId1:16, + MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), timer:sleep(100), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), gen_udp:close(Socket). @@ -625,12 +712,18 @@ t_publish_qos1_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId, PredefTopicId, Payload1), - ?assertEqual(<<7, ?SN_PUBACK, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, PredefTopicId:16, + MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), timer:sleep(100), send_disconnect_msg(Socket, undefined), @@ -645,7 +738,10 @@ t_publish_qos1_case03(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_publish_msg_predefined_topic(Socket, QoS, MsgId, tid(5), <<20, 21, 22, 23>>), - ?assertEqual(<<7, ?SN_PUBACK, TopicId5:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicId5:16, + MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -664,15 +760,20 @@ t_publish_qos1_case04(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"ab">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Topic = <<"ab">>, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_short_topic(Socket, QoS, MsgId, Topic, Payload1), <> = Topic, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), timer:sleep(100), send_disconnect_msg(Socket, undefined), @@ -692,13 +793,18 @@ t_publish_qos1_case05(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, <<"ab">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/#">>, <<20, 21, 22, 23>>), <> = <<"/#">>, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -724,7 +830,10 @@ t_publish_qos1_case06(_) -> send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/+">>, <<20, 21, 22, 23>>), <> = <<"/+">>, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -751,7 +860,11 @@ t_publish_qos2_case01(_) -> send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -773,15 +886,21 @@ t_publish_qos2_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId, PredefTopicId, Payload1), ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC :2, PredefTopicId:16, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, + PredefTopicId:16, 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -812,7 +931,11 @@ t_publish_qos2_case03(_) -> ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC :2, <<"/a">>/binary, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, + "/a", 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -1083,7 +1206,11 @@ t_asleep_test03_to_awake_qos1_dl_msg(_) -> send_register_msg(Socket, TopicName1, MsgId1), ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId1:16, 0:8>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state send_disconnect_msg(Socket, 1), @@ -1109,7 +1236,10 @@ t_asleep_test03_to_awake_qos1_dl_msg(_) -> %% the broker should sent dl msgs to the awake client before sending the pingresp UdpData = receive_response(Socket), - MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId1, Payload1}, UdpData), + MsgId_udp = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicId1, Payload1}, UdpData), send_puback_msg(Socket, TopicId1, MsgId_udp), %% check the pingresp is received at last @@ -1141,8 +1271,11 @@ t_asleep_test04_to_awake_qos1_dl_msg(_) -> CleanSession = 0, ReturnCode = 0, send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1,CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state send_disconnect_msg(Socket, 1), @@ -1176,11 +1309,17 @@ t_asleep_test04_to_awake_qos1_dl_msg(_) -> send_regack_msg(Socket, TopicIdNew, MsgId3), UdpData2 = receive_response(Socket), - MsgId_udp2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2), + MsgId_udp2 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload1}, UdpData2), send_puback_msg(Socket, TopicIdNew, MsgId_udp2), UdpData3 = receive_response(Socket), - MsgId_udp3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData3), + MsgId_udp3 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload2}, UdpData3), send_puback_msg(Socket, TopicIdNew, MsgId_udp3), ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), @@ -1216,8 +1355,11 @@ t_asleep_test05_to_awake_qos1_dl_msg(_) -> CleanSession = 0, ReturnCode = 0, send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state SleepDuration = 30, @@ -1250,21 +1392,28 @@ t_asleep_test05_to_awake_qos1_dl_msg(_) -> send_regack_msg(Socket, TopicIdNew, MsgId_reg), UdpData2 = receive_response(Socket), - MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), + MsgId2 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload2}, UdpData2), send_puback_msg(Socket, TopicIdNew, MsgId2), timer:sleep(50), UdpData3 = wrap_receive_response(Socket), - MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), + MsgId3 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload3}, UdpData3), send_puback_msg(Socket, TopicIdNew, MsgId3), timer:sleep(50), case receive_response(Socket) of <<2,23>> -> ok; UdpData4 -> - MsgId4 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, - CleanSession, ?SN_NORMAL_TOPIC, - TopicIdNew, Payload4}, UdpData4), + MsgId4 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload4}, UdpData4), send_puback_msg(Socket, TopicIdNew, MsgId4) end, ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), @@ -1322,7 +1471,10 @@ t_asleep_test06_to_awake_qos2_dl_msg(_) -> send_pingreq_msg(Socket, ClientId), UdpData = wrap_receive_response(Socket), - MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId_tom, Payload1}, UdpData), + MsgId_udp = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicId_tom, Payload1}, UdpData), send_pubrec_msg(Socket, MsgId_udp), ?assertMatch(<<_:8, ?SN_PUBREL:8, _/binary>>, receive_response(Socket)), send_pubcomp_msg(Socket, MsgId_udp), @@ -1357,8 +1509,11 @@ t_asleep_test07_to_connected(_) -> send_register_msg(Socket, TopicName_tom, MsgId1), TopicId_tom = check_regack_msg_on_udp(MsgId1, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId_tom, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId_tom:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1,CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId_tom:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state send_disconnect_msg(Socket, SleepDuration), @@ -1436,8 +1591,11 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> CleanSession = 0, ReturnCode = 0, send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1,CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state SleepDuration = 30, send_disconnect_msg(Socket, SleepDuration), @@ -1471,7 +1629,10 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> udp_receive_timeout -> ok; UdpData2 -> - MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), + MsgId2 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload2}, UdpData2), send_puback_msg(Socket, TopicIdNew, MsgId2) end, timer:sleep(100), @@ -1480,7 +1641,10 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> udp_receive_timeout -> ok; UdpData3 -> - MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), + MsgId3 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload3}, UdpData3), send_puback_msg(Socket, TopicIdNew, MsgId3) end, timer:sleep(100), @@ -1489,16 +1653,18 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> udp_receive_timeout -> ok; UdpData4 -> - MsgId4 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, - CleanSession, ?SN_NORMAL_TOPIC, - TopicIdNew, Payload4}, UdpData4), + MsgId4 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload4}, UdpData4), send_puback_msg(Socket, TopicIdNew, MsgId4) end, ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), %% send PINGREQ again to enter awake state send_pingreq_msg(Socket, ClientId), - %% will not receive any buffered PUBLISH messages buffered before last awake, only receive PINGRESP here + %% will not receive any buffered PUBLISH messages buffered before last + %% awake, only receive PINGRESP here ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), gen_udp:close(Socket). @@ -1901,8 +2067,12 @@ check_dispatched_message(Dup, QoS, Retain, TopicIdType, TopicId, Payload, Socket PubMsg = receive_response(Socket), Length = 7 + byte_size(Payload), ?LOG("check_dispatched_message ~p~n", [PubMsg]), - ?LOG("expected ~p xx ~p~n", [<>, Payload]), - <> = PubMsg, + ?LOG("expected ~p xx ~p~n", + [<>, Payload]), + <> = PubMsg, case QoS of 0 -> ok; 1 -> send_puback_msg(Socket, TopicId, MsgId); @@ -1914,11 +2084,14 @@ check_dispatched_message(Dup, QoS, Retain, TopicIdType, TopicId, Payload, Socket get_udp_broadcast_address() -> "255.255.255.255". -check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, TopicType, TopicId, Payload}, UdpData) -> +check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, + CleanSession, TopicType, TopicId, Payload}, UdpData) -> <> = UdpData, ct:pal("UdpData: ~p, Payload: ~p, PayloadIn: ~p", [UdpData, Payload, PayloadIn]), Size9 = byte_size(Payload) + 7, - Eexp = <>, + Eexp = <>, ?assertEqual(Eexp, HeaderUdp), % mqtt-sn header should be same ?assertEqual(Payload, PayloadIn), % payload should be same MsgId. diff --git a/src/emqx_types.erl b/src/emqx_types.erl index 410ed5a27..3d5fb66a5 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -147,7 +147,7 @@ dn => binary(), atom() => term() }). --type(clientid() :: binary()|atom()). +-type(clientid() :: binary() | atom()). -type(username() :: maybe(binary())). -type(password() :: maybe(binary())). -type(peerhost() :: inet:ip_address()). From f2d99017a04bfb54d07fb30f646ab3cfcf064e59 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 9 Nov 2021 15:25:58 +0800 Subject: [PATCH 033/363] chore(exhook): update appup.src --- apps/emqx_exhook/src/emqx_exhook.appup.src | 10 ++++++++-- apps/emqx_exhook/test/emqx_exhook_demo_svr.erl | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index e78afef1e..ea8aacb7a 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -2,14 +2,20 @@ {VSN, [{"4.3.4", [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}, + {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-3]+">>, [{restart_application,emqx_exhook}]}, {<<".*">>, []} ], [{"4.3.4", [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}, + {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-3]+">>, [{restart_application,emqx_exhook}]}, {<<".*">>, []} diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index 6e9691c3e..1d3896e14 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -295,7 +295,7 @@ on_session_terminated(Req, Md) -> | {error, grpc_cowboy_h:error_response()}. on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> ?MODULE:in({?FUNCTION_NAME, Req}), - io:format(standard_error, "fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), + %io:format(standard_error, "fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), %% some cases for testing case From of <<"baduser">> -> From 7d06e48b4bfdf28964a2f03f469819316bd78535 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 9 Nov 2021 17:46:59 +0800 Subject: [PATCH 034/363] chore: remove needless catch Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_exhook/src/emqx_exhook_server.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 9e88d774d..276f5a638 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -183,7 +183,7 @@ resovle_hookspec(HookSpecs) when is_list(HookSpecs) -> Name0 -> Name = try binary_to_existing_atom(Name0, utf8) - catch T:R:_ -> {T,R} + catch T:R -> {T,R} end, case {lists:member(Name, AvailableHooks), lists:member(Name, MessageHooks)} of From 8dfc8ed96b133776235cccdef13b534f502f6bd1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 10 Nov 2021 16:24:42 +0800 Subject: [PATCH 035/363] chore: fill message headers --- apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl | 18 +++++++--- .../emqx_exproto/src/emqx_exproto_channel.erl | 29 ++++++++++++---- apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl | 18 ++++++++-- apps/emqx_stomp/src/emqx_stomp_protocol.erl | 34 +++++++++++++------ 4 files changed, 76 insertions(+), 23 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl index d465f9ca3..89fb411b2 100644 --- a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl @@ -244,15 +244,26 @@ chann_publish(Topic, Payload, State = #state{clientid = ClientId}) -> case emqx_access_control:check_acl(clientinfo(State), publish, Topic) of allow -> _ = emqx_broker:publish( - emqx_message:set_flag(retain, false, - emqx_message:make(ClientId, ?QOS_0, Topic, Payload))), - ok; + packet_to_message(Topic, Payload, State)), ok; deny -> ?LOG(warning, "publish to ~p by clientid ~p failed due to acl check.", [Topic, ClientId]), {error, forbidden} end. +packet_to_message(Topic, Payload, + #state{clientid = ClientId, + username = Username, + peername = {PeerHost, _}}) -> + Message = emqx_message:set_flag( + retain, false, + emqx_message:make(ClientId, ?QOS_0, Topic, Payload) + ), + emqx_message:set_headers( + #{ proto_ver => 1 + , protocol => coap + , username => Username + , peerhost => PeerHost}, Message). %%-------------------------------------------------------------------- %% Deliver @@ -384,4 +395,3 @@ clientinfo(#state{peername = {PeerHost, _}, mountpoint => undefined, ws_cookie => undefined }. - diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 67f85f932..d8ceae4bd 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -340,17 +340,14 @@ handle_call({unsubscribe, TopicFilter}, handle_call({publish, Topic, Qos, Payload}, Channel = #channel{ conn_state = connected, - clientinfo = ClientInfo - = #{clientid := From, - mountpoint := Mountpoint}}) -> + clientinfo = ClientInfo}) -> case is_acl_enabled(ClientInfo) andalso emqx_access_control:check_acl(ClientInfo, publish, Topic) of deny -> {reply, {error, ?RESP_PERMISSION_DENY, <<"ACL deny">>}, Channel}; _ -> - Msg = emqx_message:make(From, Qos, Topic, Payload), - NMsg = emqx_mountpoint:mount(Mountpoint, Msg), - _ = emqx:publish(NMsg), + Msg = packet_to_message(Topic, Qos, Payload, Channel), + _ = emqx:publish(Msg), {reply, ok, Channel} end; @@ -419,6 +416,24 @@ is_anonymous(_AuthResult) -> false. clean_anonymous_clients() -> ets:delete(?CHAN_CONN_TAB, ?CHANMOCK(self())). +packet_to_message(Topic, Qos, Payload, + #channel{ + conninfo = #{proto_ver := ProtoVer}, + clientinfo = #{ + protocol := Protocol, + clientid := ClientId, + username := Username, + peerhost := PeerHost, + mountpoint := Mountpoint}}) -> + Msg = emqx_message:make( + ClientId, Qos, + Topic, Payload, #{}, + #{proto_ver => ProtoVer, + protocol => Protocol, + username => Username, + peerhost => PeerHost}), + emqx_mountpoint:mount(Mountpoint, Msg). + %%-------------------------------------------------------------------- %% Sub/UnSub %%-------------------------------------------------------------------- @@ -591,6 +606,8 @@ default_conninfo(ConnInfo) -> ConnInfo#{clean_start => true, clientid => undefined, username => undefined, + proto_name => undefined, + proto_ver => undefined, conn_props => #{}, connected => true, connected_at => erlang:system_time(millisecond), diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index 34c72dcca..4a5fafdb0 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -235,8 +235,20 @@ unsubscribe(Topic, Lwm2mState = #lwm2m_state{endpoint_name = _EndpointName}) -> emqx_broker:unsubscribe(Topic), emqx_hooks:run('session.unsubscribed', [clientinfo(Lwm2mState), Topic, Opts]). -publish(Topic, Payload, Qos, EndpointName) -> - emqx_broker:publish(emqx_message:set_flag(retain, false, emqx_message:make(EndpointName, Qos, Topic, Payload))). +publish(Topic, Payload, Qos, + #lwm2m_state{ + version = ProtoVer, + peername = {PeerHost, _}, + endpoint_name = EndpointName}) -> + Message = emqx_message:set_flag( + retain, false, + emqx_message:make(EndpointName, Qos, Topic, Payload) + ), + NMessage = emqx_message:set_headers( + #{proto_ver => ProtoVer, + protocol => lwm2m, + peerhost => PeerHost}, Message), + emqx_broker:publish(NMessage). time_now() -> erlang:system_time(millisecond). @@ -281,7 +293,7 @@ do_send_to_broker(EventType, #{<<"data">> := Data} = Payload, #lwm2m_state{endpo emqx_lwm2m_cm:register_cmd(EndpointName, ReqPath, EventType, {Code, CodeMsg, Content}), NewPayload = maps:put(<<"msgType">>, EventType, Payload), Topic = uplink_topic(EventType, Lwm2mState), - publish(Topic, emqx_json:encode(NewPayload), _Qos = 0, Lwm2mState#lwm2m_state.endpoint_name). + publish(Topic, emqx_json:encode(NewPayload), _Qos = 0, Lwm2mState). %%-------------------------------------------------------------------- %% Auto Observe diff --git a/apps/emqx_stomp/src/emqx_stomp_protocol.erl b/apps/emqx_stomp/src/emqx_stomp_protocol.erl index fc211be10..3dcc6052f 100644 --- a/apps/emqx_stomp/src/emqx_stomp_protocol.erl +++ b/apps/emqx_stomp/src/emqx_stomp_protocol.erl @@ -588,15 +588,29 @@ next_ackid() -> put(ackid, AckId + 1), AckId. -make_mqtt_message(Topic, Headers, Body) -> - Msg = emqx_message:make(stomp, Topic, Body), - Headers1 = lists:foldl(fun(Key, Headers0) -> - proplists:delete(Key, Headers0) - end, Headers, [<<"destination">>, - <<"content-length">>, - <<"content-type">>, - <<"transaction">>, - <<"receipt">>]), +make_mqtt_message(Topic, Headers, Body, + #pstate{ + conninfo = #{proto_ver := ProtoVer}, + clientinfo = #{ + protocol := Protocol, + clientid := ClientId, + username := Username, + peerhost := PeerHost}}) -> + Msg = emqx_message:make( + ClientId, ?QOS_0, + Topic, Body, #{}, + #{proto_ver => ProtoVer, + protocol => Protocol, + username => Username, + peerhost => PeerHost}), + Headers1 = lists:foldl( + fun(Key, Headers0) -> + proplists:delete(Key, Headers0) + end, Headers, [<<"destination">>, + <<"content-length">>, + <<"content-type">>, + <<"transaction">>, + <<"receipt">>]), emqx_message:set_headers(#{stomp_headers => Headers1}, Msg). receipt_id(Headers) -> @@ -611,7 +625,7 @@ handle_recv_send_frame(#stomp_frame{command = <<"SEND">>, headers = Headers, bod allow -> _ = maybe_send_receipt(receipt_id(Headers), State), _ = emqx_broker:publish( - make_mqtt_message(Topic, Headers, iolist_to_binary(Body)) + make_mqtt_message(Topic, Headers, iolist_to_binary(Body), State) ), State; deny -> From f194ae65d299347a098f637a78c0141e61e7f011 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 10 Nov 2021 16:53:03 +0800 Subject: [PATCH 036/363] chore: update appup.src --- apps/emqx_coap/src/emqx_coap.app.src | 2 +- apps/emqx_coap/src/emqx_coap.appup.src | 9 +++++++++ apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.appup.src | 8 ++++++-- apps/emqx_lwm2m/src/emqx_lwm2m.app.src | 2 +- apps/emqx_lwm2m/src/emqx_lwm2m.appup.src | 6 ++++-- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.appup.src | 10 ++++++++-- 8 files changed, 31 insertions(+), 10 deletions(-) create mode 100644 apps/emqx_coap/src/emqx_coap.appup.src diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src index 2b5fcbb6a..dce5ef4f4 100644 --- a/apps/emqx_coap/src/emqx_coap.app.src +++ b/apps/emqx_coap/src/emqx_coap.app.src @@ -1,6 +1,6 @@ {application, emqx_coap, [{description, "EMQ X CoAP Gateway"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gen_coap]}, diff --git a/apps/emqx_coap/src/emqx_coap.appup.src b/apps/emqx_coap/src/emqx_coap.appup.src new file mode 100644 index 000000000..b73e26be6 --- /dev/null +++ b/apps/emqx_coap/src/emqx_coap.appup.src @@ -0,0 +1,9 @@ +%% -*-: erlang -*- +{VSN, + [{"4.3.0",[ + {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []}]}, + {<<".*">>, []}], + [{"4.3.0",[ + {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []}]}, + {<<".*">>, []}] +}. diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index f7cab4c2e..0ed54e6fd 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.4"}, %% 4.3.3 is used by ee + {vsn, "4.3.5"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index e0a021af5..7da28e773 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,6 +1,8 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.3", + [{"4.3.4", + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {"4.3.3", [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {"4.3.2", @@ -12,7 +14,9 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.3", + [{"4.3.4", + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {"4.3.3", [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {"4.3.2", diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src index 551cf8d07..b929ad854 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -1,6 +1,6 @@ {application,emqx_lwm2m, [{description,"EMQ X LwM2M Gateway"}, - {vsn, "4.3.4"}, % strict semver, bump manually! + {vsn, "4.3.5"}, % strict semver, bump manually! {modules,[]}, {registered,[emqx_lwm2m_sup]}, {applications,[kernel,stdlib,lwm2m_coap]}, diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src index 600cf236b..0cd98db0d 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src @@ -1,5 +1,5 @@ %% -*-: erlang -*- -{"4.3.4", +{VSN, [ {<<"4\\.3\\.[0-1]">>, [ {restart_application, emqx_lwm2m} @@ -7,7 +7,9 @@ {"4.3.2", [ {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []} ]}, - {"4.3.3", []} %% only config change + {<<"4\\.3\\.[3-4]">>, [ + {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} + ]} ], [ {<<"4\\.3\\.[0-1]">>, [ diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index d2ecae53b..c2f4b57d3 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_stomp, [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_stomp_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.appup.src b/apps/emqx_stomp/src/emqx_stomp.appup.src index bf4603e52..dce441b3c 100644 --- a/apps/emqx_stomp/src/emqx_stomp.appup.src +++ b/apps/emqx_stomp/src/emqx_stomp.appup.src @@ -1,10 +1,16 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.1",[{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + [{"4.3.2",[{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}]}, + {"4.3.1",[ + {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{restart_application,emqx_stomp}]}, {<<".*">>,[]}], - [{"4.3.1",[{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + [{"4.3.2",[{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}]}, + {"4.3.1",[ + {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{restart_application,emqx_stomp}]}, {<<".*">>,[]}]}. From 2be33b33e3d9ef36bab0ab2e87985b62d2c0fd48 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 10 Nov 2021 17:05:27 +0800 Subject: [PATCH 037/363] chore: fix elvis warnings --- apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl | 4 +- .../emqx_exproto/src/emqx_exproto_channel.erl | 2 - apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl | 47 ++++++++++++------- apps/emqx_stomp/src/emqx_stomp_protocol.erl | 34 ++++++++++---- 4 files changed, 57 insertions(+), 30 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl index 89fb411b2..f7f0b4eda 100644 --- a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl @@ -139,7 +139,7 @@ handle_call({subscribe, Topic, CoapPid}, _From, State=#state{sub_topics = TopicL NewTopics = proplists:delete(Topic, TopicList), IsWild = emqx_topic:wildcard(Topic), {reply, chann_subscribe(Topic, State), State#state{sub_topics = - [{Topic, {IsWild, CoapPid}}|NewTopics]}, hibernate}; + [{Topic, {IsWild, CoapPid}} | NewTopics]}, hibernate}; handle_call({unsubscribe, Topic, _CoapPid}, _From, State=#state{sub_topics = TopicList}) -> NewTopics = proplists:delete(Topic, TopicList), @@ -281,7 +281,7 @@ do_deliver({Topic, Payload}, Subscribers) -> deliver_to_coap(_TopicName, _Payload, []) -> ok; -deliver_to_coap(TopicName, Payload, [{TopicFilter, {IsWild, CoapPid}}|T]) -> +deliver_to_coap(TopicName, Payload, [{TopicFilter, {IsWild, CoapPid}} | T]) -> Matched = case IsWild of true -> emqx_topic:match(TopicName, TopicFilter); false -> TopicName =:= TopicFilter diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index d8ceae4bd..44b8b64d3 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -606,8 +606,6 @@ default_conninfo(ConnInfo) -> ConnInfo#{clean_start => true, clientid => undefined, username => undefined, - proto_name => undefined, - proto_ver => undefined, conn_props => #{}, connected => true, connected_at => erlang:system_time(millisecond), diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index 4a5fafdb0..7f40041a5 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -74,7 +74,8 @@ call(Pid, Msg, Timeout) -> Error -> {error, Error} end. -init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> := LifeTime, <<"lwm2m">> := Ver}) -> +init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, + RegInfo = #{<<"lt">> := LifeTime, <<"lwm2m">> := Ver}) -> Mountpoint = proplists:get_value(mountpoint, lwm2m_coap_responder:options(), ""), Lwm2mState = #lwm2m_state{peername = Peername, endpoint_name = EndpointName, @@ -103,9 +104,10 @@ init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> emqx_cm:register_channel(EndpointName, CoapPid, conninfo(Lwm2mState1)) end), emqx_cm:insert_channel_info(EndpointName, info(Lwm2mState1), stats(Lwm2mState1)), - emqx_lwm2m_cm:register_channel(EndpointName, RegInfo, LifeTime, Ver, Peername), + emqx_lwm2m_cm:register_channel(EndpointName, RegInfo, LifeTime, Ver, Peername), - {ok, Lwm2mState1#lwm2m_state{life_timer = emqx_lwm2m_timer:start_timer(LifeTime, {life_timer, expired})}}; + NTimer = emqx_lwm2m_timer:start_timer(LifeTime, {life_timer, expired}), + {ok, Lwm2mState1#lwm2m_state{life_timer = NTimer}}; {error, Error} -> _ = run_hooks('client.connack', [conninfo(Lwm2mState), not_authorized], undefined), {error, Error} @@ -133,7 +135,7 @@ update_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, regi %% - report the registration info update, but only when objectList is updated. case NewRegInfo of #{<<"objectList">> := _} -> - emqx_lwm2m_cm:update_reg_info(Epn, NewRegInfo), + emqx_lwm2m_cm:update_reg_info(Epn, NewRegInfo), send_to_broker(<<"update">>, #{<<"data">> => UpdatedRegInfo}, Lwm2mState); _ -> ok end @@ -186,7 +188,8 @@ deliver(#message{topic = Topic, payload = Payload}, started_at = StartedAt, endpoint_name = EndpointName}) -> IsCacheMode = is_cache_mode(RegInfo, StartedAt), - ?LOG(debug, "Get MQTT message from broker, IsCacheModeNow?: ~p, Topic: ~p, Payload: ~p", [IsCacheMode, Topic, Payload]), + ?LOG(debug, "Get MQTT message from broker, IsCacheModeNow?: ~p, " + "Topic: ~p, Payload: ~p", [IsCacheMode, Topic, Payload]), AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), deliver_to_coap(AlternatePath, Payload, CoapPid, IsCacheMode, EndpointName), Lwm2mState. @@ -256,7 +259,8 @@ time_now() -> erlang:system_time(millisecond). %% Deliver downlink message to coap %%-------------------------------------------------------------------- -deliver_to_coap(AlternatePath, JsonData, CoapPid, CacheMode, EndpointName) when is_binary(JsonData)-> +deliver_to_coap(AlternatePath, JsonData, + CoapPid, CacheMode, EndpointName) when is_binary(JsonData)-> try TermData = emqx_json:decode(JsonData, [return_maps]), deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) @@ -285,7 +289,8 @@ deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) when send_to_broker(EventType, Payload = #{}, Lwm2mState) -> do_send_to_broker(EventType, Payload, Lwm2mState). -do_send_to_broker(EventType, #{<<"data">> := Data} = Payload, #lwm2m_state{endpoint_name = EndpointName} = Lwm2mState) -> +do_send_to_broker(EventType, #{<<"data">> := Data} = Payload, + #lwm2m_state{endpoint_name = EndpointName} = Lwm2mState) -> ReqPath = maps:get(<<"reqPath">>, Data, undefined), Code = maps:get(<<"code">>, Data, undefined), CodeMsg = maps:get(<<"codeMsg">>, Data, undefined), @@ -327,18 +332,27 @@ auto_observe(AlternatePath, ObjectList, CoapPid, EndpointName) -> observe_object_list(AlternatePath, ObjectList, CoapPid, EndpointName) -> lists:foreach(fun(ObjectPath) -> - [ObjId| LastPath] = emqx_lwm2m_cmd_handler:path_list(ObjectPath), + [ObjId | LastPath] = emqx_lwm2m_cmd_handler:path_list(ObjectPath), case ObjId of <<"19">> -> [ObjInsId | _LastPath1] = LastPath, case ObjInsId of <<"0">> -> - observe_object_slowly(AlternatePath, <<"/19/0/0">>, CoapPid, 100, EndpointName); + observe_object_slowly( + AlternatePath, <<"/19/0/0">>, + CoapPid, 100, EndpointName + ); _ -> - observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) + observe_object_slowly( + AlternatePath, ObjectPath, + CoapPid, 100, EndpointName + ) end; _ -> - observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) + observe_object_slowly( + AlternatePath, ObjectPath, + CoapPid, 100, EndpointName + ) end end, ObjectList). @@ -392,11 +406,12 @@ get_cached_downlink_messages() -> is_cache_mode(RegInfo, StartedAt) -> case is_psm(RegInfo) orelse is_qmode(RegInfo) of true -> - QModeTimeWind = proplists:get_value(qmode_time_window, lwm2m_coap_responder:options(), 22), - Now = time_now(), - if (Now - StartedAt) >= QModeTimeWind -> true; - true -> false - end; + QModeTimeWind = proplists:get_value( + qmode_time_window, + lwm2m_coap_responder:options(), + 22 + ), + (time_now() - StartedAt) >= QModeTimeWind; false -> false end. diff --git a/apps/emqx_stomp/src/emqx_stomp_protocol.erl b/apps/emqx_stomp/src/emqx_stomp_protocol.erl index 3dcc6052f..4e371d9b0 100644 --- a/apps/emqx_stomp/src/emqx_stomp_protocol.erl +++ b/apps/emqx_stomp/src/emqx_stomp_protocol.erl @@ -108,6 +108,8 @@ , init/2 ]}). +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + -type(pstate() :: #pstate{}). %% @doc Init protocol @@ -132,8 +134,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, AllowAnonymous = get_value(allow_anonymous, Opts, false), DefaultUser = get_value(default_user, Opts), - - #pstate{ + #pstate{ conninfo = NConnInfo, clientinfo = ClientInfo, heartfun = HeartFun, @@ -165,7 +166,7 @@ default_conninfo(ConnInfo) -> info(State) -> maps:from_list(info(?INFO_KEYS, State)). --spec info(list(atom())|atom(), pstate()) -> term(). +-spec info(list(atom()) | atom(), pstate()) -> term(). info(Keys, State) when is_list(Keys) -> [{Key, info(Key, State)} || Key <- Keys]; info(conninfo, #pstate{conninfo = ConnInfo}) -> @@ -288,7 +289,12 @@ received(#stomp_frame{command = <<"CONNECT">>}, State = #pstate{connected = true received(Frame = #stomp_frame{command = <<"SEND">>, headers = Headers}, State) -> case header(<<"transaction">>, Headers) of undefined -> {ok, handle_recv_send_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_send_frame/2, [Frame]}, receipt_id(Headers), State) + TransactionId -> + add_action(TransactionId, + {fun ?MODULE:handle_recv_send_frame/2, [Frame]}, + receipt_id(Headers), + State + ) end; received(#stomp_frame{command = <<"SUBSCRIBE">>, headers = Headers}, @@ -346,7 +352,11 @@ received(#stomp_frame{command = <<"UNSUBSCRIBE">>, headers = Headers}, received(Frame = #stomp_frame{command = <<"ACK">>, headers = Headers}, State) -> case header(<<"transaction">>, Headers) of undefined -> {ok, handle_recv_ack_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_ack_frame/2, [Frame]}, receipt_id(Headers), State) + TransactionId -> + add_action(TransactionId, + {fun ?MODULE:handle_recv_ack_frame/2, [Frame]}, + receipt_id(Headers), + State) end; %% NACK @@ -357,7 +367,11 @@ received(Frame = #stomp_frame{command = <<"ACK">>, headers = Headers}, State) -> received(Frame = #stomp_frame{command = <<"NACK">>, headers = Headers}, State) -> case header(<<"transaction">>, Headers) of undefined -> {ok, handle_recv_nack_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_nack_frame/2, [Frame]}, receipt_id(Headers), State) + TransactionId -> + add_action(TransactionId, + {fun ?MODULE:handle_recv_nack_frame/2, [Frame]}, + receipt_id(Headers), + State) end; %% BEGIN @@ -516,9 +530,9 @@ negotiate_version(Accepts) -> negotiate_version(Ver, []) -> {error, <<"Supported protocol versions < ", Ver/binary>>}; -negotiate_version(Ver, [AcceptVer|_]) when Ver >= AcceptVer -> +negotiate_version(Ver, [AcceptVer | _]) when Ver >= AcceptVer -> {ok, AcceptVer}; -negotiate_version(Ver, [_|T]) -> +negotiate_version(Ver, [_ | T]) -> negotiate_version(Ver, T). check_login(Login, _, AllowAnonymous, _) @@ -537,7 +551,7 @@ check_login(Login, Passcode, _, DefaultUser) -> add_action(Id, Action, ReceiptId, State = #pstate{transaction = Trans}) -> case maps:get(Id, Trans, undefined) of {Ts, Actions} -> - NTrans = Trans#{Id => {Ts, [Action|Actions]}}, + NTrans = Trans#{Id => {Ts, [Action | Actions]}}, {ok, State#pstate{transaction = NTrans}}; _ -> send(error_frame(ReceiptId, ["Transaction ", Id, " not found"]), State) @@ -713,7 +727,7 @@ find_sub_by_id(Id, Subs) -> end, Subs), case maps:to_list(Found) of [] -> undefined; - [Sub|_] -> Sub + [Sub | _] -> Sub end. is_acl_enabled(_) -> From 6b40048d297f8f6d4728eef4dcc89e7bb8344565 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 10 Nov 2021 18:40:40 +0800 Subject: [PATCH 038/363] chore: put the pool_size default value to avoid hot upgrade failure --- apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl | 6 ++++-- apps/emqx_exhook/src/emqx_exhook_mngr.erl | 4 +++- apps/emqx_exhook/test/emqx_exhook_demo_svr.erl | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl index f7f0b4eda..a6ea09881 100644 --- a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl @@ -58,6 +58,8 @@ -define(SUBOPTS, #{rh => 0, rap => 0, nl => 0, qos => ?QOS_0, is_new => false}). +-define(PROTO_VER, 1). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -260,7 +262,7 @@ packet_to_message(Topic, Payload, emqx_message:make(ClientId, ?QOS_0, Topic, Payload) ), emqx_message:set_headers( - #{ proto_ver => 1 + #{ proto_ver => ?PROTO_VER , protocol => coap , username => Username , peerhost => PeerHost}, Message). @@ -335,7 +337,7 @@ conninfo(#state{peername = Peername, peercert => nossl, %% TODO: dtls conn_mod => ?MODULE, proto_name => <<"CoAP">>, - proto_ver => 1, + proto_ver => ?PROTO_VER, clean_start => true, clientid => ClientId, username => undefined, diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl index 0f3c8b8ab..d4c493cb8 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mngr.erl @@ -295,7 +295,9 @@ put_pool_size(Val) -> persistent_term:put({?APP, pool_size}, Val). get_pool_size() -> - persistent_term:get({?APP, pool_size}). + %% Avoid the scenario that the parameter is not set after + %% the hot upgrade completed. + persistent_term:get({?APP, pool_size}, erlang:system_info(schedulers)). save(Name, ServerState) -> Saved = persistent_term:get(?APP, []), diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index 1d3896e14..b05748856 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -295,7 +295,7 @@ on_session_terminated(Req, Md) -> | {error, grpc_cowboy_h:error_response()}. on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> ?MODULE:in({?FUNCTION_NAME, Req}), - %io:format(standard_error, "fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), + %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), %% some cases for testing case From of <<"baduser">> -> From 08cf0326b318a589fb082fab0f0bc975548bcfe8 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 10 Nov 2021 19:18:42 +0800 Subject: [PATCH 039/363] chore(exhook): bump version to 4.4.0 --- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.appup.src | 20 ++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index b386bcaca..715060df4 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,6 +1,6 @@ {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.3.5"}, + {vsn, "4.4.0"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index ea8aacb7a..0fe0ea78f 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,23 +1,7 @@ %% -*-: erlang -*- {VSN, - [{"4.3.4", - [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}, - {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-3]+">>, - [{restart_application,emqx_exhook}]}, - {<<".*">>, []} + [{<<".*">>, []} ], - [{"4.3.4", - [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[emqx_exhook_pb]}, - {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-3]+">>, - [{restart_application,emqx_exhook}]}, - {<<".*">>, []} + [{<<".*">>, []} ] }. From f7bdd6defe43d3df4d553c9fc321e467f1bc423e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 12 Nov 2021 15:35:08 +0800 Subject: [PATCH 040/363] chore(lwm2m): fix bad appup.src --- apps/emqx_lwm2m/src/emqx_lwm2m.appup.src | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src index 0cd98db0d..6656eb149 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src @@ -7,7 +7,8 @@ {"4.3.2", [ {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []} ]}, - {<<"4\\.3\\.[3-4]">>, [ + {"4.3.3", []}, %% only config change + {"4.3.4", [ {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} ]} ], @@ -18,6 +19,9 @@ {"4.3.2", [ {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []} ]}, - {"4.3.3", []} %% only config change + {"4.3.3", []}, %% only config change + {"4.3.4", [ + {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} + ]} ] }. From 25f504c90ac27d7685e598aab179c2885b033efe Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 12 Nov 2021 15:59:35 +0800 Subject: [PATCH 041/363] feat(mongo srv): support srv for mongodb authentication --- apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 9 +- .../priv/emqx_auth_mongo.schema | 37 +++++--- .../src/emqx_auth_mongo.app.src | 2 +- .../src/emqx_auth_mongo_sup.erl | 93 ++++++++++++++++++- 4 files changed, 121 insertions(+), 20 deletions(-) diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index 2a3d038f0..3d3d0ee5b 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -5,7 +5,13 @@ ## MongoDB Topology Type. ## ## Value: single | unknown | sharded | rs -auth.mongo.type = single +auth.mongo.type = + +## Whether to use SRV and TXT records. +## +## Value: true | false +## Default: false +auth.mongo.srv_record = false ## The set name if type is rs. ## @@ -37,7 +43,6 @@ auth.mongo.pool = 8 ## MongoDB AuthSource ## ## Value: String -## Default: mqtt ## auth.mongo.auth_source = admin ## MongoDB database diff --git a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema index 8a2ff98b3..17a83c37c 100644 --- a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema +++ b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema @@ -6,8 +6,12 @@ {datatype, {enum, [single, unknown, sharded, rs]}} ]}. +{mapping, "auth.mongo.srv_record", "emqx_auth_mongo.server", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + {mapping, "auth.mongo.rs_set_name", "emqx_auth_mongo.server", [ - {default, "mqtt"}, {datatype, string} ]}. @@ -41,7 +45,6 @@ ]}. {mapping, "auth.mongo.auth_source", "emqx_auth_mongo.server", [ - {default, "mqtt"}, {datatype, string} ]}. @@ -101,9 +104,9 @@ ]}. {translation, "emqx_auth_mongo.server", fun(Conf) -> - H = cuttlefish:conf_get("auth.mongo.server", Conf), - Hosts = string:tokens(H, ","), - Type0 = cuttlefish:conf_get("auth.mongo.type", Conf), + SrvRecord = cuttlefish:conf_get("auth.mongo.srv_record", Conf, false), + Server = cuttlefish:conf_get("auth.mongo.server", Conf), + Type = cuttlefish:conf_get("auth.mongo.type", Conf), Pool = cuttlefish:conf_get("auth.mongo.pool", Conf), %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 Login = cuttlefish:conf_get("auth.mongo.username", Conf, @@ -111,7 +114,10 @@ ), Passwd = cuttlefish:conf_get("auth.mongo.password", Conf), DB = cuttlefish:conf_get("auth.mongo.database", Conf), - AuthSrc = cuttlefish:conf_get("auth.mongo.auth_source", Conf), + AuthSource = case cuttlefish:conf_get("auth.mongo.auth_source", Conf, undefined) of + undefined -> []; + AuthSource0 -> [{auth_source, list_to_binary(AuthSource0)}] + end, R = cuttlefish:conf_get("auth.mongo.w_mode", Conf), W = cuttlefish:conf_get("auth.mongo.r_mode", Conf), Login0 = case Login =:= [] of @@ -156,8 +162,8 @@ false -> [] end, - WorkerOptions = [{database, list_to_binary(DB)}, {auth_source, list_to_binary(AuthSrc)}] - ++ Login0 ++ Passwd0 ++ W0 ++ R0 ++ Ssl, + WorkerOptions = [{database, list_to_binary(DB)}] + ++ Login0 ++ Passwd0 ++ W0 ++ R0 ++ Ssl ++ AuthSource, Vars = cuttlefish_variable:fuzzy_matches(["auth", "mongo", "topology", "$name"], Conf), Options = lists:map(fun({_, Name}) -> @@ -174,16 +180,17 @@ {list_to_atom(Name2), cuttlefish:conf_get("auth.mongo.topology."++Name, Conf)} end, Vars), - Type = case Type0 =:= rs of - true -> {Type0, list_to_binary(cuttlefish:conf_get("auth.mongo.rs_set_name", Conf))}; - false -> Type0 - end, - [{type, Type}, - {hosts, Hosts}, + ReplicaSet = case cuttlefish:conf_get("auth.mongo.rs_set_name", Conf, undefined) of + undefined -> []; + ReplicaSet0 -> [{rs_set_name, list_to_binary(ReplicaSet0)}] + end, + [{srv_record, SrvRecord}, + {type, Type}, + {server, Server}, {options, Options}, {worker_options, WorkerOptions}, {auto_reconnect, 1}, - {pool_size, Pool}] + {pool_size, Pool}] ++ ReplicaSet end}. %% The mongodb operation timeout is specified by the value of `cursor_timeout` from application config, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src index cc4e72ef3..ab0b4ff56 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mongo, [{description, "EMQ X Authentication/ACL with MongoDB"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mongo_sup]}, {applications, [kernel,stdlib,mongodb,ecpool]}, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl index 3f27cb1dd..55263494a 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl @@ -28,7 +28,96 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - {ok, PoolEnv} = application:get_env(?APP, server), - PoolSpec = ecpool:pool_spec(?APP, ?APP, ?APP, PoolEnv), + {ok, Opts} = application:get_env(?APP, server), + NOpts = may_parse_srv_and_txt_records(Opts), + PoolSpec = ecpool:pool_spec(?APP, ?APP, ?APP, NOpts), {ok, {{one_for_all, 10, 100}, [PoolSpec]}}. +may_parse_srv_and_txt_records(Opts) when is_list(Opts) -> + maps:to_list(may_parse_srv_and_txt_records(maps:from_list(Opts))); + +may_parse_srv_and_txt_records(#{type := Type, + srv_record := false, + server := Server} = Opts) -> + Hosts = to_hosts(Server), + case Type =:= rs of + true -> + case maps:get(rs_set_name, Opts, undefined) of + undefined -> + error({missing_parameter, rs_set_name}); + ReplicaSet -> + Opts#{type => {rs, ReplicaSet}, + hosts => Hosts} + end; + false -> + Opts#{hosts => Hosts} + end; + +may_parse_srv_and_txt_records(#{type := Type, + srv_record := true, + server := Server, + worker_options := WorkerOptions} = Opts) -> + Hosts = parse_srv_records(Server), + Opts0 = parse_txt_records(Type, Server), + NWorkerOptions = maps:to_list(maps:merge(maps:from_list(WorkerOptions), maps:with([auth_source], Opts0))), + NOpts = Opts#{hosts => Hosts, worker_options => NWorkerOptions}, + case Type =:= rs of + true -> + case maps:get(rs_set_name, Opts0, maps:get(rs_set_name, NOpts, undefined)) of + undefined -> + error({missing_parameter, rs_set_name}); + ReplicaSet -> + NOpts#{type => {Type, ReplicaSet}} + end; + false -> + NOpts + end. + +to_hosts(Server) -> + [string:trim(H) || H <- string:tokens(Server, ",")]. + +parse_srv_records(Server) -> + case inet_res:lookup("_mongodb._tcp." ++ Server, in, srv) of + [] -> + error(service_not_found); + Services -> + [Host ++ ":" ++ integer_to_list(Port) || {_, _, Port, Host} <- Services] + end. + +parse_txt_records(Type, Server) -> + case inet_res:lookup(Server, in, txt) of + [] -> + #{}; + [[QueryString]] -> + case uri_string:dissect_query(QueryString) of + {error, _, _} -> + error({invalid_txt_record, invalid_query_string}); + Options -> + Fields = case Type of + rs -> ["authSource", "replicaSet"]; + _ -> ["authSource"] + end, + take_and_convert(Fields, Options) + end; + _ -> + error({invalid_txt_record, multiple_records}) + end. + +take_and_convert(Fields, Options) -> + take_and_convert(Fields, Options, #{}). + +take_and_convert([], [_ | _], _Acc) -> + error({invalid_txt_record, invalid_option}); +take_and_convert([], [], Acc) -> + Acc; +take_and_convert([Field | More], Options, Acc) -> + case lists:keytake(Field, 1, Options) of + {value, {"authSource", V}, NOptions} -> + take_and_convert(More, NOptions, Acc#{auth_source => list_to_binary(V)}); + {value, {"replicaSet", V}, NOptions} -> + take_and_convert(More, NOptions, Acc#{rs_set_name => list_to_binary(V)}); + {value, _, _} -> + error({invalid_txt_record, invalid_option}); + false -> + take_and_convert(More, Options, Acc) + end. From cb185389577263c17af0f3348b82c4f3fc50464a Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 15 Nov 2021 09:41:03 +0800 Subject: [PATCH 042/363] fix(mong srv): fix wrong configuration --- apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index 3d3d0ee5b..8baddae19 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -5,7 +5,7 @@ ## MongoDB Topology Type. ## ## Value: single | unknown | sharded | rs -auth.mongo.type = +auth.mongo.type = single ## Whether to use SRV and TXT records. ## From 454f609aa6cb057772f56fdb7533ab9036ebd2ed Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 12 Nov 2021 14:18:32 +0800 Subject: [PATCH 043/363] build: emqx package name scheme include otp version --- Makefile | 1 + build | 5 ++++- deploy/packages/deb/Makefile | 2 +- deploy/packages/rpm/Makefile | 2 +- scripts/get-otp-vsn.sh | 7 +++++++ 5 files changed, 14 insertions(+), 3 deletions(-) create mode 100755 scripts/get-otp-vsn.sh diff --git a/Makefile b/Makefile index 17ceb9481..11edd8929 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ REBAR_VERSION = 3.14.3-emqx-8 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts +export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3 diff --git a/build b/build index be4f88672..dbbfcbdfb 100755 --- a/build +++ b/build @@ -12,6 +12,9 @@ ARTIFACT="$2" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" +export OTP_VSN + PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN @@ -116,7 +119,7 @@ make_zip() { log "ERROR: $tarball is not found" fi local zipball - zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" + zipball="${pkgpath}/${PROFILE}-${OTP_VSN}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" tar zxf "${tarball}" -C "${tard}/emqx" ## try to be portable for zip packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 48124d780..6f3956409 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -8,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 5a6e6bee4..e8c01e935 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -16,7 +16,7 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) ifeq ($(RPM_REL),) # no tail RPM_REL := 1 diff --git a/scripts/get-otp-vsn.sh b/scripts/get-otp-vsn.sh new file mode 100755 index 000000000..fd5b4fd0b --- /dev/null +++ b/scripts/get-otp-vsn.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail + +vsn=$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell) + +echo "otp${vsn}" From 7e91a47be8b1ae68ded8d7ddb22350be8a8eebc9 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 12 Nov 2021 14:50:56 +0800 Subject: [PATCH 044/363] ci: update workflows --- .github/workflows/build_packages.yaml | 53 +++++++++++---------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index befa37912..efaf14bfc 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -212,6 +212,8 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + otp: + - 23.3.4.8-1 arch: - amd64 - arm64 @@ -248,15 +250,11 @@ jobs: shell: bash steps: - - name: prepare docker - run: | - mkdir -p $HOME/.docker - echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json - echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker info - docker buildx create --use --name mybuild - docker run --rm --privileged tonistiigi/binfmt --install all + - uses: docker/setup-buildx-action@v1 + - uses: docker/setup-qemu-action@v1 + with: + image: tonistiigi/binfmt:latest + platforms: all - uses: actions/download-artifact@v2 with: name: source @@ -265,6 +263,7 @@ jobs: run: unzip -q source.zip - name: downloads old emqx zip packages env: + OTP_VSN: otp${{ matrix.otp }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} @@ -283,36 +282,28 @@ jobs: cd source/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do - if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 - echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1 + if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip) | grep -oE "^[23]+")" ];then + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256 + echo "$(cat ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256) ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip" | sha256sum -c || exit 1 fi done - name: build emqx packages env: - ERL_OTP: erl23.2.7.2-emqx-2 + OTP: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} + working-directory: source run: | - set -e -u - cd source - docker buildx build --no-cache \ - --platform=linux/$ARCH \ - -t cross_build_emqx_for_$SYSTEM \ - -f .ci/build_packages/Dockerfile \ - --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \ - --build-arg EMQX_NAME=$PROFILE \ - --output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar . - - mkdir -p /tmp/packages/$PROFILE - tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/* - mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/ - rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar - - docker rm -f $(docker ps -a -q) - docker volume prune -f + docker run -i --rm \ + -v $(pwd):/emqx \ + --workdir /emqx \ + --platform linux/$ARCH \ + ghcr.io/emqx/emqx-builder/4.4:$OTP-$SYSTEM \ + bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ + make $PROFILE-pkg || cat rebar3.crashdump; \ + EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" - name: create sha256 env: PROFILE: ${{ matrix.profile}} From f46084438b1994084137dd07a4322ebf20a1084e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 28 Oct 2021 09:19:30 +0800 Subject: [PATCH 045/363] chore(cluster): add new type for dns auto cluster Signed-off-by: zhanghongtong --- etc/emqx.conf | 5 +++++ priv/emqx.schema | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 7b81e40cc..78d383930 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -101,6 +101,11 @@ cluster.autoclean = 5m ## Value: String ## cluster.dns.app = emqx +## Type of dns record. +## +## Value: Value: a | srv +## cluster.dns.type = a + ##-------------------------------------------------------------------- ## Cluster using etcd diff --git a/priv/emqx.schema b/priv/emqx.schema index 602de56b9..f8df59da9 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -96,6 +96,11 @@ {datatype, string} ]}. +{mapping, "cluster.dns.type", "ekka.cluster_discovery", [ + {datatype, {enum, [a, srv]}}, + {default, a} +]}. + %%-------------------------------------------------------------------- %% Cluster using etcd @@ -171,7 +176,8 @@ {loop, cuttlefish:conf_get("cluster.mcast.loop", Conf, true)}]; (dns) -> [{name, cuttlefish:conf_get("cluster.dns.name", Conf)}, - {app, cuttlefish:conf_get("cluster.dns.app", Conf)}]; + {app, cuttlefish:conf_get("cluster.dns.app", Conf)}, + {type, cuttlefish:conf_get("cluster.dns.type", Conf)}]; (etcd) -> SslOpts = fun(Conf) -> Options = cuttlefish_variable:filter_by_prefix("cluster.etcd.ssl", Conf), From 87a2667e35ec6ee62b8918f4d66c04f4f2b2e938 Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 15 Nov 2021 10:57:59 +0800 Subject: [PATCH 046/363] fix(emqx_retainer): fix timer message error (#6156) * fix(emqx_retainer): fix timer message error --- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- apps/emqx_retainer/src/emqx_retainer.erl | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 4ef423b78..a423fb9b7 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.4.1"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index fcc5652cc..b6de9d9f6 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -156,7 +156,7 @@ start_expire_timer(0, State) -> start_expire_timer(undefined, State) -> State; start_expire_timer(Ms, State) -> - Timer = erlang:send_after(Ms, self(), stats), + Timer = erlang:send_after(Ms, self(), {expire, Ms}), State#state{expiry_timer = Timer}. handle_call(Req, _From, State) -> @@ -168,12 +168,14 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info(stats, State = #state{stats_fun = StatsFun}) -> + StatsTimer = erlang:send_after(timer:seconds(1), self(), stats), StatsFun(retained_count()), - {noreply, State, hibernate}; + {noreply, State#state{stats_timer = StatsTimer}, hibernate}; -handle_info(expire, State) -> +handle_info({expire, Ms} = Expire, State) -> + Timer = erlang:send_after(Ms, self(), Expire), ok = expire_messages(), - {noreply, State, hibernate}; + {noreply, State#state{expiry_timer = Timer}, hibernate}; handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), From 0357f7ad853db6b83b1c92978885dacd3cbbb2e4 Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 15 Nov 2021 11:00:04 +0800 Subject: [PATCH 047/363] improve(emqx_st_statistics): optimize the parameters of on_publish_done (#6151) * fix(emqx_st_statistics): optimize the parameters of on_publish_done --- .../emqx_st_statistics/emqx_st_statistics.erl | 35 +++++++++++++------ src/emqx_session.erl | 9 +++-- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl index 21cf2692e..f22aec41c 100644 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl @@ -24,7 +24,7 @@ -logger_header("[SLOW TOPICS]"). --export([ start_link/1, on_publish_done/5, enable/0 +-export([ start_link/1, on_publish_done/3, enable/0 , disable/0, clear_history/0 ]). @@ -42,7 +42,7 @@ -type state() :: #{ config := proplist:proplist() , period := pos_integer() , last_tick_at := pos_integer() - , counter := counters:counter_ref() + , counter := counters:counters_ref() , enable := boolean() }. @@ -70,6 +70,13 @@ -type slow_log() :: #slow_log{}. -type top_k_map() :: #{emqx_types:topic() => top_k()}. +-type publish_done_env() :: #{ ignore_before_create := boolean() + , threshold := pos_integer() + , counter := counters:counters_ref() + }. + +-type publish_done_args() :: #{session_rebirth_time => pos_integer()}. + -ifdef(TEST). -define(TOPK_ACCESS, public). -else. @@ -90,13 +97,16 @@ start_link(Env) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). --spec on_publish_done(message(), - pos_integer(), boolean(), pos_integer(), counters:counters_ref()) -> ok. -on_publish_done(#message{timestamp = Timestamp}, Created, IgnoreBeforeCreate, _, _) +-spec on_publish_done(message(), publish_done_args(), publish_done_env()) -> ok. +on_publish_done(#message{timestamp = Timestamp}, + #{session_rebirth_time := Created}, + #{ignore_before_create := IgnoreBeforeCreate}) when IgnoreBeforeCreate, Timestamp < Created -> ok; -on_publish_done(#message{timestamp = Timestamp} = Msg, _, _, Threshold, Counter) -> +on_publish_done(#message{timestamp = Timestamp} = Msg, + _, + #{threshold := Threshold, counter := Counter}) -> case ?NOW - Timestamp of Elapsed when Elapsed > Threshold -> case get_log_quota(Counter) of @@ -202,7 +212,7 @@ init_topk_tab(_) -> , {read_concurrency, true} ]). --spec get_log_quota(counter:counter_ref()) -> boolean(). +-spec get_log_quota(counters:counters_ref()) -> boolean(). get_log_quota(Counter) -> case counters:get(Counter, ?QUOTA_IDX) of Quota when Quota > 0 -> @@ -212,7 +222,7 @@ get_log_quota(Counter) -> false end. --spec set_log_quota(proplists:proplist(), counter:counter_ref()) -> ok. +-spec set_log_quota(proplists:proplist(), counters:counters_ref()) -> ok. set_log_quota(Cfg, Counter) -> MaxLogNum = get_value(max_log_num, Cfg), counters:put(Counter, ?QUOTA_IDX, MaxLogNum). @@ -328,12 +338,15 @@ publish(TickTime, Cfg, Notices) -> load(IgnoreBeforeCreate, Threshold, Counter) -> _ = emqx:hook('message.publish_done', - fun ?MODULE:on_publish_done/5, - [IgnoreBeforeCreate, Threshold, Counter]), + fun ?MODULE:on_publish_done/3, + [#{ignore_before_create => IgnoreBeforeCreate, + threshold => Threshold, + counter => Counter} + ]), ok. unload() -> - emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/5). + emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/3). -spec get_topic(proplists:proplist()) -> binary(). get_topic(Cfg) -> diff --git a/src/emqx_session.erl b/src/emqx_session.erl index d4b671e71..6122982ae 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -320,7 +320,8 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, puback(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - emqx:run_hook('message.publish_done', [Msg, CreatedAt]), + emqx:run_hook('message.publish_done', + [Msg, #{session_rebirth_time => CreatedAt}]), Inflight1 = emqx_inflight:delete(PacketId, Inflight), return_with(Msg, dequeue(Session#session{inflight = Inflight1})); {value, {_Pubrel, _Ts}} -> @@ -346,7 +347,8 @@ pubrec(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt} case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> %% execute hook here, because message record will be replaced by pubrel - emqx:run_hook('message.publish_done', [Msg, CreatedAt]), + emqx:run_hook('message.publish_done', + [Msg, #{session_rebirth_time => CreatedAt}]), Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; {value, {pubrel, _Ts}} -> @@ -443,7 +445,8 @@ deliver([Msg | More], Acc, Session) -> end. deliver_msg(Msg = #message{qos = ?QOS_0}, Session) -> - emqx:run_hook('message.publish_done', [Msg, Session#session.created_at]), + emqx:run_hook('message.publish_done', + [Msg, #{session_rebirth_time => Session#session.created_at}]), {ok, [{undefined, maybe_ack(Msg)}], Session}; deliver_msg(Msg = #message{qos = QoS}, Session = From 6bd5fd218ec421b144f7c75211f46e12ca4b1da2 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 15 Nov 2021 11:02:45 +0800 Subject: [PATCH 048/363] chore: limit/page to position/bytes (#6161) --- .../src/emqx_trace/emqx_trace_api.erl | 25 ++++++++++++------- .../test/emqx_mod_trace_api_SUITE.erl | 8 +++--- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index a5e5b2906..3af272776 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -16,6 +16,7 @@ -module(emqx_trace_api). -include_lib("emqx/include/logger.hrl"). +-include_lib("kernel/include/file.hrl"). %% API -export([ list_trace/2 @@ -75,7 +76,7 @@ download_zip_log(#{name := Name}, _Param) -> ZipDir = emqx_trace:zip_dir(), Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), ZipFileName = ZipDir ++ TraceLog, - {ok, ZipFile} = zip:zip(ZipFileName, Zips), + {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]), emqx_trace:delete_files_after_send(ZipFileName, Zips), {ok, #{}, {sendfile, 0, filelib:file_size(ZipFile), ZipFile}}; {error, Reason} -> @@ -88,7 +89,7 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> {ok, Node, Bin} -> ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, ok = file:write_file(ZipName, Bin), - [ZipName | Acc]; + [Node ++ "-" ++ TraceLog | Acc]; {error, Node, Reason} -> ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), Acc @@ -101,20 +102,19 @@ collect_trace_file(TraceLog) -> BadNodes =/= [] andalso ?LOG(error, "download log rpc failed on ~p", [BadNodes]), Files. -%% _page as position and _limit as bytes for front-end reusing components stream_log_file(#{name := Name}, Params) -> Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), - Position0 = proplists:get_value(<<"_page">>, Params, <<"0">>), - Bytes0 = proplists:get_value(<<"_limit">>, Params, <<"500">>), + Position0 = proplists:get_value(<<"position">>, Params, <<"0">>), + Bytes0 = proplists:get_value(<<"bytes">>, Params, <<"1000">>), Node = binary_to_existing_atom(Node0), Position = binary_to_integer(Position0), Bytes = binary_to_integer(Bytes0), case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of {ok, Bin} -> - Meta = #{<<"page">> => Position + byte_size(Bin), <<"limit">> => Bytes}, + Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => Bin}}; - eof -> - Meta = #{<<"page">> => Position, <<"limit">> => Bytes}, + {eof, Size} -> + Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => <<"">>}}; {error, Reason} -> logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), @@ -134,6 +134,7 @@ read_trace_file(Name, Position, Limit) -> [] -> {error, not_found} end. +-dialyzer({nowarn_function, read_file/3}). read_file(Path, Offset, Bytes) -> {ok, IoDevice} = file:open(Path, [read, raw, binary]), try @@ -141,7 +142,13 @@ read_file(Path, Offset, Bytes) -> 0 -> ok; _ -> file:position(IoDevice, {bof, Offset}) end, - file:read(IoDevice, Bytes) + case file:read(IoDevice, Bytes) of + {ok, Bin} -> {ok, Bin}; + {error, Reason} -> {error, Reason}; + eof -> + #file_info{size = Size} = file:read_file_info(IoDevice), + {eof, Size} + end after file:close(IoDevice) end. diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl index 609a2d93c..fc786dbd0 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl @@ -124,14 +124,14 @@ t_stream_log(_Config) -> {ok, FileBin} = file:read_file(File), ct:pal("FileBin: ~p ~s", [byte_size(FileBin), FileBin]), Header = auth_header_(), - {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?_limit=10"), Header), + {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?bytes=10"), Header), #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta, <<"items">> := Bin}} = json(Binary), ?assertEqual(10, byte_size(Bin)), - ?assertEqual(#{<<"page">> => 10, <<"limit">> => 10}, Meta), - Path = api_path("trace/test_stream_log/log?_page=20&_limit=10"), + ?assertEqual(#{<<"position">> => 10, <<"bytes">> => 10}, Meta), + Path = api_path("trace/test_stream_log/log?position=20&bytes=10"), {ok, Binary1} = request_api(get, Path, Header), #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta1, <<"items">> := Bin1}} = json(Binary1), - ?assertEqual(#{<<"page">> => 30, <<"limit">> => 10}, Meta1), + ?assertEqual(#{<<"position">> => 30, <<"bytes">> => 10}, Meta1), ?assertEqual(10, byte_size(Bin1)), unload(), ok. From cace53a02b8b9e535be158955029db6c8c19abac Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 12 Nov 2021 17:24:25 +0800 Subject: [PATCH 049/363] build: update docker build --- .github/workflows/build_packages.yaml | 67 ++++---- .github/workflows/run_automate_tests.yaml | 59 +++---- .github/workflows/run_fvt_tests.yaml | 15 +- Makefile | 11 +- build | 17 ++ deploy/docker/Dockerfile | 21 +-- docker.mk | 188 ---------------------- 7 files changed, 104 insertions(+), 274 deletions(-) delete mode 100644 docker.mk diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index efaf14bfc..4e696dab2 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -308,8 +308,8 @@ jobs: env: PROFILE: ${{ matrix.profile}} run: | - if [ -d /tmp/packages/$PROFILE ]; then - cd /tmp/packages/$PROFILE + if [ -d _packages/$PROFILE ]; then + cd _packages/$PROFILE for var in $(ls emqx-* ); do bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" done @@ -319,7 +319,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') with: name: ${{ matrix.profile }} - path: /tmp/packages/${{ matrix.profile }}/. + path: _packages/${{ matrix.profile }}/. docker: runs-on: ubuntu-20.04 @@ -329,6 +329,8 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + otp: + - 23.3.4.8-1 arch: - [amd64, x86_64] - [arm64v8, aarch64] @@ -348,22 +350,42 @@ jobs: path: . - name: unzip source code run: unzip -q source.zip - - name: build emqx docker image - env: - PROFILE: ${{ matrix.profile }} - ARCH: ${{ matrix.arch[0] }} - QEMU_ARCH: ${{ matrix.arch[1] }} - run: | - sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - cd source - sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker - cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd - - - uses: actions/upload-artifact@v1 - if: startsWith(github.ref, 'refs/tags/') + - uses: docker/setup-buildx-action@v1 + - uses: docker/setup-qemu-action@v1 with: - name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/. + image: tonistiigi/binfmt:latest + platforms: all + - uses: docker/metadata-action@v3 + id: meta + with: + images: ${{ github.repository_owner }}/${{ matrix.profile }} + flavor: | + latest=${{ !github.event.release.prerelease }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=semver,pattern={{version}}-otp${{ matrix.otp }} + type=semver,pattern={{major}}.{{minor}}-otp${{ matrix.otp }} + - uses: docker/login-action@v1 + if: github.event_name == 'release' + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - uses: docker/build-push-action@v2 + with: + push: ${{ github.event_name == 'release' }} + pull: true + no-cache: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:${{ matrix.otp }}-alpine3.14 + RUN_FROM=alpine:3.14 + EMQX_NAME=${{ matrix.profile }} + file: source/deploy/docker/Dockerfile + context: source delete-artifact: runs-on: ubuntu-20.04 @@ -443,15 +465,6 @@ jobs: -X POST \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - - name: push docker image to docker hub - if: github.event_name == 'release' - run: | - set -e -x -u - sudo make docker-prepare - cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd - - echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin - sudo TARGET=emqx/${{ matrix.profile }} make docker-push - sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list - name: update repo.emqx.io if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' run: | diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index fb213c713..4bea3f128 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -10,14 +10,19 @@ on: jobs: build: + strategy: + matrix: + otp: + - 23.3.4.8-1 + runs-on: ubuntu-latest outputs: - imgname: ${{ steps.build_docker.outputs.imgname}} - version: ${{ steps.build_docker.outputs.version}} + imgname: ${{ steps.prepare.outputs.imgname}} + version: ${{ steps.prepare.outputs.version}} steps: - uses: actions/checkout@v2 - - name: build docker - id: build_docker + - name: prepare + id: prepare run: | if [ -f EMQX_ENTERPRISE ]; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials @@ -25,18 +30,20 @@ jobs: echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee make clean - fi - make docker - echo "::set-output name=version::$(./pkg-vsn.sh)" - if [ -f EMQX_ENTERPRISE ]; then echo "::set-output name=imgname::emqx-ee" + echo "::set-output name=version::$(./pkg-vsn.sh)" else echo "::set-output name=imgname::emqx" + echo "::set-output name=version::$(./pkg-vsn.sh)" fi + - name: build docker image + run: | + make ${{ steps.prepare.outputs.imgname }}-docker + docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz - uses: actions/upload-artifact@v2 with: - name: emqx-docker-image-zip - path: _packages/${{ steps.build_docker.outputs.imgname }}/${{ steps.build_docker.outputs.imgname }}-docker-${{ steps.build_docker.outputs.version }}.zip + name: image + path: image.tar.gz webhook: runs-on: ubuntu-latest @@ -52,15 +59,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname}} - version: ${{ needs.build.outputs.version }} run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: @@ -152,15 +155,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname }} - version: ${{ needs.build.outputs.version }} run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: @@ -259,15 +258,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname }} - version: ${{ needs.build.outputs.version }} run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: @@ -355,15 +350,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image-zip + name: image path: /tmp - name: load docker image - env: - imgname: ${{ needs.build.outputs.imgname }} - version: ${{ needs.build.outputs.version }} run: | - unzip -q /tmp/${imgname}-docker-${version}.zip -d /tmp - docker load < /tmp/${imgname}-docker-${version} + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 80fced0e4..8eecc9528 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -25,13 +25,15 @@ jobs: echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV + echo "PROFILE=emqx-ee" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV else echo "TARGET=emqx/emqx" >> $GITHUB_ENV + echo "PROFILE=emqx" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi - name: make emqx image - run: make docker + run: make ${PROFILE}-docker - name: run emqx timeout-minutes: 5 run: | @@ -79,11 +81,15 @@ jobs: echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV + echo "PROFILE=emqx-ee" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV else echo "TARGET=emqx/emqx" >> $GITHUB_ENV + echo "PROFILE=emqx" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi - name: make emqx image - run: make docker + run: make ${PROFILE}-docker - name: install k3s env: KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" @@ -105,11 +111,10 @@ jobs: KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" timeout-minutes: 5 run: | - version=$(./pkg-vsn.sh) - sudo docker save ${TARGET}:$version -o emqx.tar.gz + sudo docker save ${TARGET}:${EMQX_TAG} -o emqx.tar.gz sudo k3s ctr image import emqx.tar.gz - sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml + sed -i -r "s/^appVersion: .*$/appVersion: \"${EMQX_TAG}\"/g" deploy/charts/emqx/Chart.yaml sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml helm install emqx \ diff --git a/Makefile b/Makefile index 11edd8929..e758892e9 100644 --- a/Makefile +++ b/Makefile @@ -145,11 +145,18 @@ $1: $1-rel endef $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt)))) +## docker target is to create docker instructions +.PHONY: $(REL_PROFILES:%=%-docker) +define gen-docker-target +$1-docker: $(COMMON_DEPS) + @$(BUILD) $1 docker +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) + .PHONY: run run: $(PROFILE) quickrun .PHONY: quickrun quickrun: ./_build/$(PROFILE)/rel/emqx/bin/emqx console - -include docker.mk diff --git a/build b/build index dbbfcbdfb..7caf2579d 100755 --- a/build +++ b/build @@ -127,6 +127,20 @@ make_zip() { (cd "${tard}" && zip -qr - emqx) > "${zipball}" } +make_docker() { + ## Build Docker image + echo "DOCKER BUILD: Build Docker image." + echo "DOCKER BUILD: build version -> $PKG_VSN." + echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " + + docker build --no-cache \ + --build-arg BUILD_FROM="ghcr.io/emqx/emqx-builder/4.4:${OTP:-23.3.4.8-1}-alpine3.14" \ + --build-arg RUN_FROM="alpine:3.14" \ + --build-arg EMQX_NAME="$PROFILE" \ + --tag "emqx/$PROFILE:$PKG_VSN" \ + -f deploy/docker/Dockerfile . +} + log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in @@ -147,6 +161,9 @@ case "$ARTIFACT" in make -C "deploy/packages/${PKGERDIR}" clean EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}" ;; + docker) + make_docker + ;; *) log "Unknown artifact $ARTIFACT" exit 1 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index e362c6b73..ae11ad424 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,10 +1,7 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-amd64 -ARG RUN_FROM=alpine:3.12 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:23.3.4.8-1-alpine3.14 +ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder -ARG QEMU_ARCH=x86_64 -COPY tmp/qemu-$QEMU_ARCH-stati* /usr/bin/ - RUN apk add --no-cache \ git \ curl \ @@ -32,21 +29,9 @@ RUN cd /emqx \ FROM $RUN_FROM -# Basic build-time metadata as defined at http://label-schema.org -LABEL org.label-schema.docker.dockerfile="Dockerfile" \ - org.label-schema.license="GNU" \ - org.label-schema.name="emqx" \ - org.label-schema.version=${PKG_VSN} \ - org.label-schema.description="EMQ (Erlang MQTT Broker) is a distributed, massively scalable, highly extensible MQTT messaging broker written in Erlang/OTP." \ - org.label-schema.url="https://emqx.io" \ - org.label-schema.vcs-type="Git" \ - org.label-schema.vcs-url="https://github.com/emqx/emqx" \ - maintainer="EMQ X Team " - -ARG QEMU_ARCH=x86_64 ARG EMQX_NAME=emqx -COPY deploy/docker/docker-entrypoint.sh tmp/qemu-$QEMU_ARCH-stati* /usr/bin/ +COPY deploy/docker/docker-entrypoint.sh /usr/bin/ COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx RUN ln -s /opt/emqx/bin/* /usr/local/bin/ diff --git a/docker.mk b/docker.mk deleted file mode 100644 index 22f2b6e4f..000000000 --- a/docker.mk +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/make -f -# -*- makefile -*- - -## default globals. -## when built with `make docker` command the default profile is either emqx or emqx-ee (for enterprise) -## or the TARGET varialbe can be set beforehand to force a different name -TARGET ?= emqx/$(PROFILE) -QEMU_ARCH ?= x86_64 -ARCH ?= amd64 -QEMU_VERSION ?= v5.0.0-2 -OS ?= alpine -export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) - -ifeq ($(findstring emqx-ee, $(TARGET)), emqx-ee) - ARCH_LIST := amd64 arm64v8 arm32v7 - EMQX_NAME := emqx-ee -else ifeq ($(findstring emqx-edge, $(TARGET)), emqx-edge) - ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x - EMQX_NAME := emqx-edge -else - ARCH_LIST := amd64 arm64v8 arm32v7 i386 s390x - EMQX_NAME := emqx -endif - -.PHONY: docker -docker: docker-build docker-tag docker-save - -.PHONY: docker-prepare -docker-prepare: - ## Prepare the machine before any code installation scripts - # @echo "PREPARE: Setting up dependencies." - # @apt update -y - # @apt install --only-upgrade docker-ce -y - - ## Update docker configuration to enable docker manifest command - @echo "PREPARE: Updating docker configuration" - @mkdir -p $$HOME/.docker - - # enable experimental to use docker manifest command - @echo '{ "experimental": "enabled" }' | tee $$HOME/.docker/config.json - # enable experimental - @echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | tee /etc/docker/daemon.json - @service docker restart - -.PHONY: docker-build -docker-build: - ## Build Docker image - @echo "DOCKER BUILD: Build Docker image." - @echo "DOCKER BUILD: build version -> $(PKG_VSN)." - @echo "DOCKER BUILD: arch - $(ARCH)." - @echo "DOCKER BUILD: qemu arch - $(QEMU_ARCH)." - @echo "DOCKER BUILD: docker repo - $(TARGET) " - @echo "DOCKER BUILD: emqx name - $(EMQX_NAME)." - - ## Prepare qemu to build images other then x86_64 on travis - @echo "PREPARE: Qemu" \ - && docker run --rm --privileged multiarch/qemu-user-static:register --reset - - @mkdir -p tmp \ - && cd tmp \ - && curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \ - && tar xzf qemu-$(QEMU_ARCH)-static.tar.gz \ - && cd - - - @docker build --no-cache \ - --build-arg PKG_VSN=$(PKG_VSN) \ - --build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine-$(ARCH) \ - --build-arg RUN_FROM=$(ARCH)/alpine:3.12 \ - --build-arg EMQX_NAME=$(EMQX_NAME) \ - --build-arg QEMU_ARCH=$(QEMU_ARCH) \ - --tag $(TARGET):build-$(OS)-$(ARCH) \ - -f deploy/docker/Dockerfile . - -.PHONY: docker-tag -docker-tag: - @echo "DOCKER TAG: Tag Docker image." - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):build-$(OS)-$${arch})" ]; then \ - docker tag $(TARGET):build-$(OS)-$${arch} $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \ - echo "DOCKER TAG: $(TARGET):$(PKG_VSN)-$(OS)-$${arch}"; \ - if [ $${arch} = amd64 ]; then \ - docker tag $(TARGET):$(PKG_VSN)-$(OS)-amd64 $(TARGET):$(PKG_VSN); \ - echo "DOCKER TAG: $(TARGET):$(PKG_VSN)"; \ - fi; \ - fi; \ - done - -.PHONY: docker-save -docker-save: - @echo "DOCKER SAVE: Save Docker image." - - @mkdir -p _packages/$(EMQX_NAME) - - @if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \ - docker save $(TARGET):$(PKG_VSN) > $(EMQX_NAME)-docker-$(PKG_VSN); \ - zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN).zip $(EMQX_NAME)-docker-$(PKG_VSN); \ - mv ./$(EMQX_NAME)-docker-$(PKG_VSN).zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN).zip; \ - fi - - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker save $(TARGET):$(PKG_VSN)-$(OS)-$${arch} > $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \ - zip -r -m $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip $(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}; \ - mv ./$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip _packages/$(EMQX_NAME)/$(EMQX_NAME)-docker-$(PKG_VSN)-$(OS)-$${arch}.zip; \ - fi; \ - done - -.PHONY: docker-push -docker-push: - @echo "DOCKER PUSH: Push Docker image."; - @echo "DOCKER PUSH: pushing - $(TARGET):$(PKG_VSN)."; - - @if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then \ - docker push $(TARGET):$(PKG_VSN); \ - docker tag $(TARGET):$(PKG_VSN) $(TARGET):latest; \ - docker push $(TARGET):latest; \ - fi; - - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker push $(TARGET):$(PKG_VSN)-$(OS)-$${arch}; \ - fi; \ - done - -.PHONY: docker-manifest-list -docker-manifest-list: - version="docker manifest create --amend $(TARGET):$(PKG_VSN)"; \ - latest="docker manifest create --amend $(TARGET):latest"; \ - for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ];then \ - version="$${version} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \ - latest="$${latest} $(TARGET):$(PKG_VSN)-$(OS)-$${arch} "; \ - fi; \ - done; \ - eval $$version; \ - eval $$latest; - - for arch in $(ARCH_LIST); do \ - case $${arch} in \ - "amd64") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-amd64 --os=linux --arch=amd64; \ - fi; \ - ;; \ - "arm64v8") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm64v8 --os=linux --arch=arm64 --variant=v8; \ - fi; \ - ;; \ - "arm32v7") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-arm32v7 --os=linux --arch=arm --variant=v7; \ - fi; \ - ;; \ - "i386") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-i386 --os=linux --arch=386; \ - fi; \ - ;; \ - "s390x") \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker manifest annotate $(TARGET):$(PKG_VSN) $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \ - docker manifest annotate $(TARGET):latest $(TARGET):$(PKG_VSN)-$(OS)-s390x --os=linux --arch=s390x; \ - fi; \ - ;; \ - esac; \ - done; - - docker manifest inspect $(TARGET):$(PKG_VSN) - docker manifest push $(TARGET):$(PKG_VSN); - docker manifest inspect $(TARGET):latest - docker manifest push $(TARGET):latest; - -.PHONY: docker-clean -docker-clean: - @echo "DOCKER CLEAN: Clean Docker image." - - @if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN))" ]; then docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)); fi - - @for arch in $(ARCH_LIST); do \ - if [ -n "$$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch})" ]; then \ - docker rmi -f $$(docker images -q $(TARGET):$(PKG_VSN)-$(OS)-$${arch}); \ - fi \ - done From 23e2bd62c59a65d4f38ae661cb3e166c49839692 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 15 Nov 2021 15:53:49 +0800 Subject: [PATCH 050/363] fix: there should not be multiple layers of directories when download trace zip file (#6165) --- apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl | 4 ++-- apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl | 5 ++--- lib-ce/emqx_modules/src/emqx_mod_trace_api.erl | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 3af272776..e6c87d69f 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -75,10 +75,10 @@ download_zip_log(#{name := Name}, _Param) -> TraceFiles = collect_trace_file(TraceLog), ZipDir = emqx_trace:zip_dir(), Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), - ZipFileName = ZipDir ++ TraceLog, + ZipFileName = ZipDir ++ binary_to_list(Name) ++ ".zip", {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]), emqx_trace:delete_files_after_send(ZipFileName, Zips), - {ok, #{}, {sendfile, 0, filelib:file_size(ZipFile), ZipFile}}; + {ok, ZipFile}; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index 1bd28d888..169fd50bc 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -336,9 +336,8 @@ t_download_log(_Config) -> {ok, _} = emqtt:connect(Client), [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], ct:sleep(100), - {ok, #{}, {sendfile, 0, ZipFileSize, _ZipFile}} = - emqx_trace_api:download_zip_log(#{name => Name}, []), - ?assert(ZipFileSize > 0), + {ok, ZipFile} = emqx_trace_api:download_zip_log(#{name => Name}, []), + ?assert(filelib:file_size(ZipFile) > 0), ok = emqtt:disconnect(Client), unload(), ok. diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl index a3abbedf7..99fb97796 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -87,7 +87,7 @@ update_trace(Path, Params) -> download_zip_log(Path, Params) -> case emqx_trace_api:download_zip_log(Path, Params) of - {ok, _Header, _File}= Return -> Return; + {ok, File} -> minirest:return_file(File); {error, _Reason} = Err -> return(Err) end. From 61c68ddb35a269821ae30d2daf259583691d7766 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Mon, 15 Nov 2021 10:27:08 +0100 Subject: [PATCH 051/363] feat(rpc): Bump gen_rpc version --- priv/emqx.schema | 24 ++++++++++++++++++++++++ rebar.config | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/priv/emqx.schema b/priv/emqx.schema index 602de56b9..b4f008a6c 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -362,11 +362,35 @@ end}. ]}. %% RPC server port. +{mapping, "rpc.driver", "gen_rpc.driver", +[ {default, tcp} +, {datatype, {enum, [tcp, ssl]}} +]}. + {mapping, "rpc.tcp_server_port", "gen_rpc.tcp_server_port", [ {default, 5369}, {datatype, integer} ]}. +%% RPC SSL server port. +{mapping, "rpc.enable_ssl", "gen_rpc.ssl_server_port", [ + {default, 5369}, + {datatype, integer} +]}. + +%% RPC SSL certificates +{mapping, "rpc.certfile", "gen_rpc.certfile", [ + {datatype, string} +]}. + +{mapping, "rpc.keyfile", "gen_rpc.keyfile", [ + {datatype, string} +]}. + +{mapping, "rpc.cacertfile", "gen_rpc.cacertfile", [ + {datatype, string} +]}. + %% Number of tcp connections when connecting to RPC server {mapping, "rpc.tcp_client_num", "gen_rpc.tcp_client_num", [ {default, 0}, diff --git a/rebar.config b/rebar.config index cc147dec3..52f07a12a 100644 --- a/rebar.config +++ b/rebar.config @@ -44,7 +44,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.2"}}} - , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.6.0"}}} + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} From 96d2615cc84137368902b0ea09db21e71c0cb2b5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 15 Nov 2021 17:41:22 +0800 Subject: [PATCH 052/363] fix: return error code when trace log not foundd (#6168) --- lib-ce/emqx_modules/src/emqx_mod_trace_api.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl index 99fb97796..1daab1520 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -88,8 +88,11 @@ update_trace(Path, Params) -> download_zip_log(Path, Params) -> case emqx_trace_api:download_zip_log(Path, Params) of {ok, File} -> minirest:return_file(File); - {error, _Reason} = Err -> return(Err) + {error, Reason} -> return({error, 'NOT_FOUND', Reason}) end. stream_log_file(Path, Params) -> - return(emqx_trace_api:stream_log_file(Path, Params)). + case emqx_trace_api:stream_log_file(Path, Params) of + {ok, File} -> return({ok, File}); + {error, Reason} -> return({error, 'NOT_FOUND', Reason}) + end. From fce93c5a17ec0773a85fd99f3eed5293fb396ce7 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 15 Nov 2021 17:44:23 +0800 Subject: [PATCH 053/363] feat(migration): improve modules migration and add test cases --- .../src/emqx_mgmt_data_backup.erl | 54 ++++++++++++--- ...emqx_mongo_auth_module_migration_SUITE.erl | 65 +++++++++++++++++++ .../e4.2.8.json | 1 + .../e4.3.5.json | 1 + 4 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl create mode 100644 apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json create mode 100644 apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 6e467a8ba..4aa66b0fc 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -237,10 +237,12 @@ import_resource(#{<<"id">> := Id, config => Config, created_at => NCreatedAt, description => Desc}). + import_resources_and_rules(Resources, Rules, FromVersion) when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse - FromVersion =:= "4.2" -> + FromVersion =:= "4.2" orelse + FromVersion =:= "4.3" -> Configs = lists:foldl(fun compatible_version/2 , [], Resources), lists:foreach(fun(#{<<"actions">> := Actions} = Rule) -> NActions = apply_new_config(Actions, Configs), @@ -305,6 +307,17 @@ compatible_version(#{<<"id">> := ID, {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)), [{ID, #{headers => NHeaders, method => Method}} | Acc]; + +compatible_version(#{<<"id">> := ID, + <<"type">> := Type, + <<"config">> := Config} = Resource, Acc) + when Type =:= <<"backend_mongo_single">> + orelse Type =:= <<"backend_mongo_sharded">> + orelse Type =:= <<"backend_mongo_rs">> -> + NewConfig = maps:merge(#{<<"srv_record">> => false}, Config), + {ok, _Resource} = import_resource(Resource#{<<"config">> := NewConfig}), + [{ID, NewConfig} | Acc]; + % normal version compatible_version(Resource, Acc) -> {ok, _Resource} = import_resource(Resource), @@ -511,16 +524,39 @@ import_modules(Modules) -> undefined -> ok; _ -> - lists:foreach(fun(#{<<"id">> := Id, - <<"type">> := Type, - <<"config">> := Config, - <<"enabled">> := Enabled, - <<"created_at">> := CreatedAt, - <<"description">> := Description}) -> - _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) - end, Modules) + NModules = migrate_modules(Modules), + lists:foreach(fun(#{<<"id">> := Id, + <<"type">> := Type, + <<"config">> := Config, + <<"enabled">> := Enabled, + <<"created_at">> := CreatedAt, + <<"description">> := Description}) -> + _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) + end, NModules) end. +migrate_modules(Modules) -> + migrate_modules(Modules, []). + +migrate_modules([], Acc) -> + lists:reverse(Acc); +migrate_modules([#{<<"type">> := <<"mongo_authentication">>, + <<"config">> := Config} = Module | More], Acc) -> + WMode = case maps:get(<<"w_mode">>, Config, <<"unsafe">>) of + <<"undef">> -> <<"unsafe">>; + Other -> Other + end, + RMode = case maps:get(<<"r_mode">>, Config, <<"master">>) of + <<"undef">> -> <<"master">>; + <<"slave-ok">> -> <<"slave_ok">>; + Other0 -> Other0 + end, + NConfig = Config#{<<"srv_record">> => false, + <<"w_mode">> => WMode, + <<"r_mode">> => RMode}, + migrate_modules(More, [Module#{<<"config">> => NConfig} | Acc]); +migrate_modules([Module | More], Acc) -> + migrate_modules(More, [Module | Acc]). import_schemas(Schemas) -> case ets:info(emqx_schema) of diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl new file mode 100644 index 000000000..3333c3087 --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mongo_auth_module_migration_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx_modules/include/emqx_modules.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_modules_spec), + emqx_ct_helpers:start_apps([emqx_management, emqx_modules]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]), + application:unload(emqx_modules_spec), + ok. + +-ifdef(EMQX_ENTERPRISE). + +t_import_4_2(Config) -> + ?assertMatch(ok, import("e4.2.8.json", Config)), + timer:sleep(100), + + MongoAuthNModule = emqx_modules_registry:find_module_by_type(mongo_authentication), + ?assertNotEqual(not_found, MongoAuthNModule), + ?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule), + delete_modules(). + +t_import_4_3(Config) -> + ?assertMatch(ok, import("e4.3.5.json", Config)), + timer:sleep(100), + + MongoAuthNModule = emqx_modules_registry:find_module_by_type(mongo_authentication), + ?assertNotEqual(not_found, MongoAuthNModule), + ?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule), + delete_modules(). + +-endif. + +import(File, Config) -> + Filename = filename:join(proplists:get_value(data_dir, Config), File), + emqx_mgmt_data_backup:import(Filename, "{}"). + +delete_modules() -> + [emqx_modules_registry:remove_module(Mod) || Mod <- emqx_modules_registry:get_modules()]. diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json new file mode 100644 index 000000000..0ea956a93 --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json @@ -0,0 +1 @@ +{"version":"4.2","date":"2021-11-15 01:52:40","modules":[{"id":"module:79002e0f","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:34834081","type":"presence","config":{"qos":0},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:f6eb69d1","type":"recon","config":{},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:7ae737b2","type":"mongo_authentication","config":{"w_mode":"undef","verify":false,"type":"single","super_query_selector":"","super_query_field":"","super_query_collection":"","ssl":false,"server":"127.0.0.1:27017","r_mode":"undef","pool_size":8,"password":"public","login":"admin","keyfile":{"filename":"","file":""},"database":"mqtt","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auth_source":"admin","auth_query_selector":"username=%u","auth_query_password_hash":"sha256","auth_query_password_field":"password","auth_query_collection":"mqtt_user","acl_query_selectors":[],"acl_query_collection":"mqtt_acl"},"enabled":false,"created_at":1636941148794,"description":""},{"id":"module:e8c63201","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1636941076704,"description":""}],"rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"qP5m2iS9qnn51gHoGLbaiMo/GwE=","tags":"administrator"}],"auth_mnesia":[],"acl_mnesia":[],"schemas":[],"configs":[],"listeners_state":[]} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json new file mode 100644 index 000000000..48828a9c0 --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json @@ -0,0 +1 @@ +{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"/mWV4UgV0xmVUZX4qdIXQvxXZB0=","tags":"administrator"}],"auth_mnesia":[],"acl_mnesia":[],"modules":[{"id":"module:5881add2","type":"mongo_authentication","config":{"w_mode":"undef","verify":false,"type":"single","super_query_selector":"","super_query_field":"","super_query_collection":"","ssl":false,"server":"127.0.0.1:27017","r_mode":"undef","pool_size":8,"password":"public","login":"admin","keyfile":{"filename":"","file":""},"database":"mqtt","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auth_source":"admin","auth_query_selector":"username=%u","auth_query_password_hash":"sha256","auth_query_password_field":"password","auth_query_collection":"mqtt_user","acl_query_selectors":[],"acl_query_collection":"mqtt_acl"},"enabled":false,"created_at":1636942609573,"description":""},{"id":"module:2adb6480","type":"presence","config":{"qos":0},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:24fabe8a","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:22c70ab8","type":"recon","config":{},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:a59f9a4a","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1636942586725,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2021-11-15 10:16:56"} \ No newline at end of file From 2f279b36075739b3ee0c07f6c8dfa9a523e7dc1b Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 15 Nov 2021 18:14:51 +0800 Subject: [PATCH 054/363] test(migration): fix code scope --- .../test/emqx_mongo_auth_module_migration_SUITE.erl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl index 3333c3087..3a697a17d 100644 --- a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl @@ -20,11 +20,16 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). + +-ifdef(EMQX_ENTERPRISE). -include_lib("emqx_modules/include/emqx_modules.hrl"). +-endif. all() -> emqx_ct:all(?MODULE). +-ifdef(EMQX_ENTERPRISE). + init_per_suite(Config) -> application:load(emqx_modules_spec), emqx_ct_helpers:start_apps([emqx_management, emqx_modules]), @@ -35,8 +40,6 @@ end_per_suite(_Config) -> application:unload(emqx_modules_spec), ok. --ifdef(EMQX_ENTERPRISE). - t_import_4_2(Config) -> ?assertMatch(ok, import("e4.2.8.json", Config)), timer:sleep(100), @@ -55,11 +58,11 @@ t_import_4_3(Config) -> ?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule), delete_modules(). --endif. - import(File, Config) -> Filename = filename:join(proplists:get_value(data_dir, Config), File), emqx_mgmt_data_backup:import(Filename, "{}"). delete_modules() -> [emqx_modules_registry:remove_module(Mod) || Mod <- emqx_modules_registry:get_modules()]. + +-endif. From ca1458d4d7e1040d6fcc7597d5d68b36a316e652 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Nov 2021 15:34:48 +0100 Subject: [PATCH 055/363] chore(emqx_rule_engine): bump app vsn to 4.4.0 --- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 66af0da21..98e5487e2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.6"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, From a6032d54352d975d1e169bd55c9384faf98c2d35 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Nov 2021 23:02:28 +0100 Subject: [PATCH 056/363] build: update default otp versions --- .tool-versions | 2 +- build | 6 ++++-- deploy/docker/Dockerfile | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.tool-versions b/.tool-versions index 200caa93b..a42915120 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 23.2.7.2-emqx-2 +erlang 23.3.4.9-2 diff --git a/build b/build index 7caf2579d..5cd022dcc 100755 --- a/build +++ b/build @@ -14,6 +14,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" export OTP_VSN +EMQX_RUNNER_IMAGE='alpine:3.14' +EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE}}" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN @@ -134,8 +136,8 @@ make_docker() { echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " docker build --no-cache \ - --build-arg BUILD_FROM="ghcr.io/emqx/emqx-builder/4.4:${OTP:-23.3.4.8-1}-alpine3.14" \ - --build-arg RUN_FROM="alpine:3.14" \ + --build-arg BUILD_FROM="${EMQX_BUILDER}" \ + --build-arg RUN_FROM="{EMQX_RUNNER_IMAGE}" \ --build-arg EMQX_NAME="$PROFILE" \ --tag "emqx/$PROFILE:$PKG_VSN" \ -f deploy/docker/Dockerfile . diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index ae11ad424..b48011bfd 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:23.3.4.8-1-alpine3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder From f711f78c5c591dec92adee63a8cdd672b5c52903 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Nov 2021 23:12:31 +0100 Subject: [PATCH 057/363] build: pin otp version 23.3.4.9-2 --- .github/workflows/build_packages.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 4e696dab2..85d14bbfb 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -11,7 +11,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -134,7 +134,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: - - 23.2.7.2-emqx-2 + - 23.3.4.9-2 exclude: - profile: emqx-edge @@ -213,7 +213,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.8-1 + - 23.3.4.9-2 arch: - amd64 - arm64 @@ -300,7 +300,7 @@ jobs: -v $(pwd):/emqx \ --workdir /emqx \ --platform linux/$ARCH \ - ghcr.io/emqx/emqx-builder/4.4:$OTP-$SYSTEM \ + ghcr.io/emqx/emqx-builder/4.4-1:$OTP-$SYSTEM \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ make $PROFILE-pkg || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" @@ -330,7 +330,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.8-1 + - 23.3.4.9-2 arch: - [amd64, x86_64] - [arm64v8, aarch64] @@ -381,7 +381,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile From d350281270cae5fe9423edeaa9c3f7a6b2fbdd42 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Nov 2021 23:33:23 +0100 Subject: [PATCH 058/363] build: pin ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 --- .ci/build_packages/Dockerfile | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/build_slim_packages.yaml | 6 +++--- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index 6f9a12159..0a7c7496d 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index a502aeb6e..0415ac142 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index bf85578c5..5ee2a63dd 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -15,12 +15,12 @@ jobs: strategy: matrix: erl_otp: - - erl23.2.7.2-emqx-2 + - 23.3.4.9-2 os: - ubuntu20.04 - centos7 - container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 @@ -58,7 +58,7 @@ jobs: strategy: matrix: erl_otp: - - 23.2.7.2-emqx-2 + - 23.3.4.9-2 steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 0aa8f7903..5ebd2f69d 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 855d9463c..4cd2a914b 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 8eecc9528..4bc0d13b2 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -185,7 +185,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -234,7 +234,7 @@ jobs: relup_test_build: needs: relup_test_plan runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 3fbe88c40..9d002138e 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_static_analysis: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 steps: - uses: actions/checkout@v2 @@ -27,7 +27,7 @@ jobs: run_proper_test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 steps: - uses: actions/checkout@v2 From b68f01e7e718b4f4db66f6cad2d795f75a9b90d7 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 15 Nov 2021 23:42:19 +0100 Subject: [PATCH 059/363] build: do not print 'otp' prefix for otp version --- scripts/get-otp-vsn.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/get-otp-vsn.sh b/scripts/get-otp-vsn.sh index fd5b4fd0b..a791318dc 100755 --- a/scripts/get-otp-vsn.sh +++ b/scripts/get-otp-vsn.sh @@ -2,6 +2,4 @@ set -euo pipefail -vsn=$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell) - -echo "otp${vsn}" +erl -noshell -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' From c97c6aefc9dfb718a707a2322c82ac579dcd3e86 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 00:46:58 +0100 Subject: [PATCH 060/363] build: rename zip package and rpm deb packages --- .ci/fvt_tests/local_relup_test_run.sh | 4 ++++ .ci/fvt_tests/relup.lux | 8 +++++--- .github/workflows/build_packages.yaml | 11 ++++++----- .github/workflows/run_fvt_tests.yaml | 2 ++ build | 9 ++++----- deploy/packages/deb/Makefile | 2 +- deploy/packages/deb/debian/control | 2 +- deploy/packages/rpm/Makefile | 9 +++++---- deploy/packages/rpm/emqx.spec | 2 +- 9 files changed, 29 insertions(+), 20 deletions(-) diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh index 7ef40b0f2..576ed6d02 100755 --- a/.ci/fvt_tests/local_relup_test_run.sh +++ b/.ci/fvt_tests/local_relup_test_run.sh @@ -15,6 +15,8 @@ PROFILE="$1" VSN="$2" OLD_VSN="$3" PACKAGE_PATH="$4" +FROM_OTP_VSN="${5:-23.3.4.9-2}" +TO_OTP_VSN="${6:-23.3.4.9-2}" TEMPDIR=$(mktemp -d) trap '{ rm -rf -- "$TEMPDIR"; }' EXIT @@ -37,4 +39,6 @@ exec docker run \ --var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ + --var FROM_OTP_VSN="$FROM_OTP_VSN" \ + --var TO_OTP_VSN="$TO_OTP_VSN" \ relup.lux diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 2940f5ce0..6da46a706 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -3,6 +3,8 @@ [config var=ONE_MORE_EMQX_PATH] [config var=VSN] [config var=OLD_VSN] +[config var=FROM_OTP_VSN] +[config var=TO_OTP_VSN] [config shell_cmd=/bin/bash] [config timeout=600000] @@ -19,7 +21,7 @@ [shell emqx] !cd $PACKAGE_PATH - !unzip -q -o $PROFILE-ubuntu20.04-$(echo $OLD_VSN | sed -r 's/[v|e]//g')-amd64.zip + !unzip -q -o $PROFILE-$(echo $OLD_VSN | sed -r 's/[v|e]//g')-otp${FROM_OTP_VSN}-ubuntu20.04-amd64.zip ?SH-PROMPT !cd emqx @@ -80,7 +82,7 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/ !./bin/emqx install $VSN ?Made release permanent: "$VSN" @@ -105,7 +107,7 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/ !./bin/emqx install $VSN ?Made release permanent: "$VSN" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 85d14bbfb..f468c8c04 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -263,7 +263,7 @@ jobs: run: unzip -q source.zip - name: downloads old emqx zip packages env: - OTP_VSN: otp${{ matrix.otp }} + OTP_VSN: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} @@ -282,10 +282,11 @@ jobs: cd source/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do - if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip) | grep -oE "^[23]+")" ];then - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256 - echo "$(cat ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip.sha256) ${PROFILE}-${OTP_VSN}-${SYSTEM}-${tag#[e|v]}-${ARCH}.zip" | sha256sum -c || exit 1 + package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" + if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256 + echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1 fi done - name: build emqx packages diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 4bc0d13b2..82fc2b64c 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -321,6 +321,8 @@ jobs: --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ + --var FROM_OTP_VSN="23.3.4.9-2" \ + --var TO_OTP_VSN="23.3.4.9-2" \ emqx_built/.ci/fvt_tests/relup.lux - uses: actions/upload-artifact@v2 name: Save debug data diff --git a/build b/build index 5cd022dcc..50a2df1de 100755 --- a/build +++ b/build @@ -12,10 +12,9 @@ ARTIFACT="$2" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" -OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" -export OTP_VSN EMQX_RUNNER_IMAGE='alpine:3.14' -EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE}}" +EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')" +EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN @@ -81,7 +80,7 @@ make_relup() { rm -rf "$tmp_dir" fi releases+=( "$base_vsn" ) - done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f) + done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f) fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -121,7 +120,7 @@ make_zip() { log "ERROR: $tarball is not found" fi local zipball - zipball="${pkgpath}/${PROFILE}-${OTP_VSN}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" + zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" tar zxf "${tarball}" -C "${tard}/emqx" ## try to be portable for zip packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 6f3956409..1f709d860 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -8,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/deb/debian/control b/deploy/packages/deb/debian/control index e35535c12..b3794036c 100644 --- a/deploy/packages/deb/debian/control +++ b/deploy/packages/deb/debian/control @@ -4,7 +4,7 @@ Priority: optional Maintainer: emqx Build-Depends: debhelper (>=9) Standards-Version: 3.9.6 -Homepage: https://www.emqx.io +Homepage: https://www.emqx.com Package: emqx Architecture: any diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index e8c01e935..0e6cf9c5d 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -6,7 +6,7 @@ dash := - none := space := $(none) $(none) RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?") -RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]") +RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]+") ARCH ?= amd64 ifeq ($(ARCH),mips64) @@ -16,12 +16,12 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -TARGET_PKG := $(EMQX_NAME)-$(OTP_VSN)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) ifeq ($(RPM_REL),) # no tail RPM_REL := 1 endif -SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM)-$(shell uname -m) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -47,7 +47,8 @@ all: | $(BUILT) --define "_service_dst $(SERVICE_DST)" \ --define "_post_addition $(POST_ADDITION)" \ --define "_preun_addition $(PREUN_ADDITION)" \ - --define "_ostype -$(SYSTEM)" \ + --define "_ostype $(SYSTEM)" \ + --define "_otp_vsn $(OTP_VSN)" \ --define "_sharedstatedir /var/lib" \ emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index 882b7753e..0c309a8d0 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -5,7 +5,7 @@ %define _log_dir %{_var}/log/%{_name} %define _lib_home /usr/lib/%{_name} %define _var_home %{_sharedstatedir}/%{_name} -%define _build_name_fmt %{_arch}/%{_name}%{?_ostype}-%{_version}-%{_release}.%{_arch}.rpm +%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}-otp%{_otp_vsn}-%{?_ostype}.%{_arch}.rpm %define _build_id_links none Name: %{_package_name} From 3f1fc64a98acfdccf05c17c1f16b67ff40480c22 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 01:10:52 +0100 Subject: [PATCH 061/363] build: fix source rpm package name --- deploy/packages/rpm/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 0e6cf9c5d..5ad80cd37 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -21,7 +21,7 @@ ifeq ($(RPM_REL),) # no tail RPM_REL := 1 endif -SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM)-$(shell uname -m) +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM).$(shell uname -m) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C From 755dd11b070cf8f0d3909cc3d0cafc942cdb1df5 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 01:28:42 +0100 Subject: [PATCH 062/363] build: always pull image before build --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 50a2df1de..08580de70 100755 --- a/build +++ b/build @@ -134,7 +134,7 @@ make_docker() { echo "DOCKER BUILD: build version -> $PKG_VSN." echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " - docker build --no-cache \ + docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ --build-arg RUN_FROM="{EMQX_RUNNER_IMAGE}" \ --build-arg EMQX_NAME="$PROFILE" \ From a178b6cc083617d24cdf5f50836a1af90b0a6e89 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Nov 2021 11:27:05 +0800 Subject: [PATCH 063/363] ci: fix make docker error for github action --- .github/workflows/run_automate_tests.yaml | 7 ++----- .github/workflows/run_fvt_tests.yaml | 14 ++++++-------- build | 14 ++++++++------ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index 4bea3f128..e4680be78 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -10,11 +10,6 @@ on: jobs: build: - strategy: - matrix: - otp: - - 23.3.4.8-1 - runs-on: ubuntu-latest outputs: imgname: ${{ steps.prepare.outputs.imgname}} @@ -37,6 +32,8 @@ jobs: echo "::set-output name=version::$(./pkg-vsn.sh)" fi - name: build docker image + env: + OTP_VSN: 23.3.4.9-2 run: | make ${{ steps.prepare.outputs.imgname }}-docker docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 82fc2b64c..2aece7312 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -13,10 +13,6 @@ jobs: steps: - uses: actions/checkout@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang - with: - otp-version: 23.2 - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -24,6 +20,7 @@ jobs: git config --global credential.helper store echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee + make clean echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV echo "PROFILE=emqx-ee" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV @@ -33,6 +30,8 @@ jobs: echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi - name: make emqx image + env: + OTP_VSN: 23.3.4.9-2 run: make ${PROFILE}-docker - name: run emqx timeout-minutes: 5 @@ -69,10 +68,6 @@ jobs: steps: - uses: actions/checkout@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang - with: - otp-version: 23.2 - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -80,6 +75,7 @@ jobs: git config --global credential.helper store echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee + make clean echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV echo "PROFILE=emqx-ee" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV @@ -89,6 +85,8 @@ jobs: echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi - name: make emqx image + env: + OTP_VSN: 23.3.4.9-2 run: make ${PROFILE}-docker - name: install k3s env: diff --git a/build b/build index 08580de70..f29dbc985 100755 --- a/build +++ b/build @@ -12,10 +12,6 @@ ARTIFACT="$2" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" -EMQX_RUNNER_IMAGE='alpine:3.14' -EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')" -EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" - PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN @@ -129,14 +125,20 @@ make_zip() { } make_docker() { + EMQX_RUNNER_IMAGE='alpine:3.14' + EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')" + EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" + ## Build Docker image echo "DOCKER BUILD: Build Docker image." - echo "DOCKER BUILD: build version -> $PKG_VSN." echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " + echo "DOCKER BUILD: build version -> $PKG_VSN." + echo "DOCKER BUILD: build from -> $EMQX_BUILDER." + echo "DOCKER BUILD: runner from -> $EMQX_RUNNER_IMAGE." docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ - --build-arg RUN_FROM="{EMQX_RUNNER_IMAGE}" \ + --build-arg RUN_FROM="${EMQX_RUNNER_IMAGE}" \ --build-arg EMQX_NAME="$PROFILE" \ --tag "emqx/$PROFILE:$PKG_VSN" \ -f deploy/docker/Dockerfile . From 2dc63cffea4f039c7b1132a437777efdd80ff168 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 08:37:53 +0100 Subject: [PATCH 064/363] fix(emqx_misc): call gen_tcp to decide if ipv6_probe is supported --- src/emqx_misc.erl | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index 2b77d0455..01495460f 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -66,16 +66,9 @@ maybe_parse_ip(Host) -> %% @doc Add `ipv6_probe' socket option if it's supported. ipv6_probe(Opts) -> - case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of - unknown -> - %% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64 - OtpVsn = emqx_vm:get_otp_version(), - Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])), - _ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool), - ipv6_probe(Bool, Opts); - Bool -> - ipv6_probe(Bool, Opts) - end. + Bool = try gen_tcp:ipv6_probe() + catch _ : _ -> false end, + ipv6_probe(Bool, Opts). ipv6_probe(false, Opts) -> Opts; ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts]. From 2251159c4fe613c10c416a5bd22702da719c0fc4 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Nov 2021 17:51:38 +0800 Subject: [PATCH 065/363] build: pin otp version to 23.3.4.9-3 and builder version to 4.4-2 --- .github/workflows/build_packages.yaml | 12 ++++++------ .github/workflows/build_slim_packages.yaml | 6 +++--- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_automate_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 12 ++++++------ .github/workflows/run_test_cases.yaml | 4 ++-- build | 2 +- deploy/docker/Dockerfile | 2 +- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f468c8c04..83c10a4c9 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -11,7 +11,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -134,7 +134,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: - - 23.3.4.9-2 + - 23.3.4.9-3 exclude: - profile: emqx-edge @@ -213,7 +213,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.9-2 + - 23.3.4.9-3 arch: - amd64 - arm64 @@ -301,7 +301,7 @@ jobs: -v $(pwd):/emqx \ --workdir /emqx \ --platform linux/$ARCH \ - ghcr.io/emqx/emqx-builder/4.4-1:$OTP-$SYSTEM \ + ghcr.io/emqx/emqx-builder/4.4-2:$OTP-$SYSTEM \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ make $PROFILE-pkg || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" @@ -331,7 +331,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.9-2 + - 23.3.4.9-3 arch: - [amd64, x86_64] - [arm64v8, aarch64] @@ -382,7 +382,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 5ee2a63dd..5888dff8b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -15,12 +15,12 @@ jobs: strategy: matrix: erl_otp: - - 23.3.4.9-2 + - 23.3.4.9-3 os: - ubuntu20.04 - centos7 - container: ghcr.io/emqx/emqx-builder/4.4-1:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-2:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 @@ -58,7 +58,7 @@ jobs: strategy: matrix: erl_otp: - - 23.3.4.9-2 + - 23.3.4.9-3 steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 5ebd2f69d..b8c4a5c18 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 4cd2a914b..5f452194c 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index e4680be78..ca7a84612 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -33,7 +33,7 @@ jobs: fi - name: build docker image env: - OTP_VSN: 23.3.4.9-2 + OTP_VSN: 23.3.4.9-3 run: | make ${{ steps.prepare.outputs.imgname }}-docker docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 2aece7312..a16586bd9 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -31,7 +31,7 @@ jobs: fi - name: make emqx image env: - OTP_VSN: 23.3.4.9-2 + OTP_VSN: 23.3.4.9-3 run: make ${PROFILE}-docker - name: run emqx timeout-minutes: 5 @@ -86,7 +86,7 @@ jobs: fi - name: make emqx image env: - OTP_VSN: 23.3.4.9-2 + OTP_VSN: 23.3.4.9-3 run: make ${PROFILE}-docker - name: install k3s env: @@ -183,7 +183,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -232,7 +232,7 @@ jobs: relup_test_build: needs: relup_test_plan runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 defaults: run: shell: bash @@ -319,8 +319,8 @@ jobs: --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ - --var FROM_OTP_VSN="23.3.4.9-2" \ - --var TO_OTP_VSN="23.3.4.9-2" \ + --var FROM_OTP_VSN="23.3.4.9-3" \ + --var TO_OTP_VSN="23.3.4.9-3" \ emqx_built/.ci/fvt_tests/relup.lux - uses: actions/upload-artifact@v2 name: Save debug data diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 9d002138e..2f5570e28 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_static_analysis: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 steps: - uses: actions/checkout@v2 @@ -27,7 +27,7 @@ jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/build b/build index f29dbc985..24e79dda3 100755 --- a/build +++ b/build @@ -127,7 +127,7 @@ make_zip() { make_docker() { EMQX_RUNNER_IMAGE='alpine:3.14' EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')" - EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-1:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" + EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-2:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" ## Build Docker image echo "DOCKER BUILD: Build Docker image." diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index b48011bfd..88f80bdf1 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-alpine3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder From 39c564c072d7934c29d543dc7a72c510a80ded71 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Nov 2021 17:55:27 +0800 Subject: [PATCH 066/363] ci: rename windows packae --- .github/workflows/build_packages.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 83c10a4c9..46133e95b 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -65,6 +65,8 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + otp: + - 23.2 exclude: - profile: emqx-edge @@ -79,7 +81,7 @@ jobs: - uses: gleam-lang/setup-erlang@v1.1.0 id: install_erlang with: - otp-version: 23.2 + otp-version: ${{ matrix.otp }} - name: build env: PYTHON: python @@ -90,10 +92,10 @@ jobs: $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" - $pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip" + $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip" } else { - $pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip" + $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" } cd source ## We do not build/release bcrypt for windows package From 284d1223728e43f796bdcfb6c04cd076f4993029 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 16 Nov 2021 18:33:21 +0800 Subject: [PATCH 067/363] ci: pin otp version to 23.3.4.9-3 and builder version to 4.4-2 --- .ci/build_packages/Dockerfile | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .ci/fvt_tests/local_relup_test_run.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index 0a7c7496d..1063a2c13 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 0415ac142..8bd6455e9 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-1:23.3.4.9-2-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh index 576ed6d02..8a563a217 100755 --- a/.ci/fvt_tests/local_relup_test_run.sh +++ b/.ci/fvt_tests/local_relup_test_run.sh @@ -15,8 +15,8 @@ PROFILE="$1" VSN="$2" OLD_VSN="$3" PACKAGE_PATH="$4" -FROM_OTP_VSN="${5:-23.3.4.9-2}" -TO_OTP_VSN="${6:-23.3.4.9-2}" +FROM_OTP_VSN="${5:-23.3.4.9-3}" +TO_OTP_VSN="${6:-23.3.4.9-3}" TEMPDIR=$(mktemp -d) trap '{ rm -rf -- "$TEMPDIR"; }' EXIT From db802ad04ff347d5fafc3fe3691fd6094449b549 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 11:35:13 +0100 Subject: [PATCH 068/363] chore: update toos-versions to pin 23.3.4.9-3 --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index a42915120..757309f18 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 23.3.4.9-2 +erlang 23.3.4.9-3 From 4d9854012e959800ed69384d772f0b35fdab7ef1 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 14:08:58 +0100 Subject: [PATCH 069/363] build: parameterise path to Dockerfile --- Makefile | 1 + build | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e758892e9..4b15adfc7 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3 +export DOCKERFILE=deploy/docker/Dockerfile ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif diff --git a/build b/build index 24e79dda3..9375dfc8c 100755 --- a/build +++ b/build @@ -141,7 +141,7 @@ make_docker() { --build-arg RUN_FROM="${EMQX_RUNNER_IMAGE}" \ --build-arg EMQX_NAME="$PROFILE" \ --tag "emqx/$PROFILE:$PKG_VSN" \ - -f deploy/docker/Dockerfile . + -f "${DOCKERFILE}" . } log "building artifact=$ARTIFACT for profile=$PROFILE" From ced24290110c0f7fe7e145582afe8187aa5fc54d Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 14:36:36 +0100 Subject: [PATCH 070/363] fix: bump new feature lib-ce apps to 4.4 --- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- scripts/apps-version-check.sh | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index e78f1c3f6..1604198dc 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.7"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index 47a3d8888..87c1c083f 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.4"}, + {vsn, "4.4.0"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 7c2ea5eb2..241d0fc15 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -2,6 +2,7 @@ set -euo pipefail latest_release=$(git describe --abbrev=0 --tags) +echo "Compare base: $latest_release" bad_app_count=0 From a070708e8d8b2ebed55d8785653e06fcc70f6200 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 18:59:23 +0100 Subject: [PATCH 071/363] build: Move otp version number to RPM's release part --- deploy/packages/rpm/Makefile | 13 ++++--------- deploy/packages/rpm/emqx.spec | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 5ad80cd37..acee8b51c 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -5,8 +5,9 @@ BUILT := $(SRCDIR)/BUILT dash := - none := space := $(none) $(none) -RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?") -RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]+") +## RPM does not allow '-' in version nubmer and release string, replace with '_' +RPM_VSN := $(subst -,_,$(PKG_VSN)) +RPM_REL := otp$(subst -,_,$(OTP_VSN)) ARCH ?= amd64 ifeq ($(ARCH),mips64) @@ -17,11 +18,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) -ifeq ($(RPM_REL),) - # no tail - RPM_REL := 1 -endif -SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL)-otp$(OTP_VSN)-$(SYSTEM).$(shell uname -m) +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -47,8 +44,6 @@ all: | $(BUILT) --define "_service_dst $(SERVICE_DST)" \ --define "_post_addition $(POST_ADDITION)" \ --define "_preun_addition $(PREUN_ADDITION)" \ - --define "_ostype $(SYSTEM)" \ - --define "_otp_vsn $(OTP_VSN)" \ --define "_sharedstatedir /var/lib" \ emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index 0c309a8d0..ef63b936c 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -5,7 +5,7 @@ %define _log_dir %{_var}/log/%{_name} %define _lib_home /usr/lib/%{_name} %define _var_home %{_sharedstatedir}/%{_name} -%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}-otp%{_otp_vsn}-%{?_ostype}.%{_arch}.rpm +%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}.%{_arch}.rpm %define _build_id_links none Name: %{_package_name} From ca1ece3db0d4ba33fef5fb9fd5bbb87cbd70d2ae Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 17 Nov 2021 11:17:10 +0800 Subject: [PATCH 072/363] feat(publish-api): Publish api supports user-properties parameters --- .../src/emqx_mgmt_api_pubsub.erl | 54 ++++++++++++------- .../test/emqx_mgmt_api_SUITE.erl | 24 +++++++-- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index 53ca022bb..1d89f237a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -71,8 +71,8 @@ subscribe(_Bindings, Params) -> publish(_Bindings, Params) -> logger:debug("API publish Params:~p", [Params]), - {ClientId, Topic, Qos, Retain, Payload} = parse_publish_params(Params), - case do_publish(ClientId, Topic, Qos, Retain, Payload) of + {ClientId, Topic, Qos, Retain, Payload, UserProps} = parse_publish_params(Params), + case do_publish(ClientId, Topic, Qos, Retain, Payload, UserProps) of {ok, MsgIds} -> case proplists:get_value(<<"return">>, Params, undefined) of undefined -> minirest:return(ok); @@ -114,7 +114,8 @@ loop_subscribe([Params | ParamsN], Acc) -> {_, Code0, _Reason} -> Code0 end, Result = #{clientid => ClientId, - topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + topic => resp_topic(proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_subscribe(ParamsN, [Result | Acc]). @@ -123,12 +124,13 @@ loop_publish(Params) -> loop_publish([], Result) -> lists:reverse(Result); loop_publish([Params | ParamsN], Acc) -> - {ClientId, Topic, Qos, Retain, Payload} = parse_publish_params(Params), - Code = case do_publish(ClientId, Topic, Qos, Retain, Payload) of + {ClientId, Topic, Qos, Retain, Payload, UserProps} = parse_publish_params(Params), + Code = case do_publish(ClientId, Topic, Qos, Retain, Payload, UserProps) of {ok, _} -> 0; {_, Code0, _} -> Code0 end, - Result = #{topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + Result = #{topic => resp_topic(proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_publish(ParamsN, [Result | Acc]). @@ -143,7 +145,8 @@ loop_unsubscribe([Params | ParamsN], Acc) -> {_, Code0, _} -> Code0 end, Result = #{clientid => ClientId, - topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + topic => resp_topic(proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_unsubscribe(ParamsN, [Result | Acc]). @@ -158,14 +161,17 @@ do_subscribe(ClientId, Topics, QoS) -> _ -> ok end. -do_publish(ClientId, _Topics, _Qos, _Retain, _Payload) when not (is_binary(ClientId) or (ClientId =:= undefined)) -> +do_publish(ClientId, _Topics, _Qos, _Retain, _Payload, _UserProps) + when not (is_binary(ClientId) or (ClientId =:= undefined)) -> {ok, ?ERROR8, <<"bad clientid: must be string">>}; -do_publish(_ClientId, [], _Qos, _Retain, _Payload) -> +do_publish(_ClientId, [], _Qos, _Retain, _Payload, _UserProps) -> {ok, ?ERROR15, bad_topic}; -do_publish(ClientId, Topics, Qos, Retain, Payload) -> +do_publish(ClientId, Topics, Qos, Retain, Payload, UserProps) -> MsgIds = lists:map(fun(Topic) -> Msg = emqx_message:make(ClientId, Qos, Topic, Payload), - _ = emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}}), + UserProps1 = #{'User-Property' => UserProps}, + _ = emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}, + headers = #{properties => UserProps1}}), emqx_guid:to_hexstr(Msg#message.id) end, Topics), {ok, MsgIds}. @@ -185,19 +191,22 @@ do_unsubscribe(ClientId, Topic) -> parse_subscribe_params(Params) -> ClientId = proplists:get_value(<<"clientid">>, Params), - Topics = topics(filter, proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + Topics = topics(filter, proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), QoS = proplists:get_value(<<"qos">>, Params, 0), {ClientId, Topics, QoS}. parse_publish_params(Params) -> - Topics = topics(name, proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), - ClientId = proplists:get_value(<<"clientid">>, Params), - Payload = decode_payload(proplists:get_value(<<"payload">>, Params, <<>>), - proplists:get_value(<<"encoding">>, Params, <<"plain">>)), - Qos = proplists:get_value(<<"qos">>, Params, 0), - Retain = proplists:get_value(<<"retain">>, Params, false), - Payload1 = maybe_maps_to_binary(Payload), - {ClientId, Topics, Qos, Retain, Payload1}. + Topics = topics(name, proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), + ClientId = proplists:get_value(<<"clientid">>, Params), + Payload = decode_payload(proplists:get_value(<<"payload">>, Params, <<>>), + proplists:get_value(<<"encoding">>, Params, <<"plain">>)), + Qos = proplists:get_value(<<"qos">>, Params, 0), + Retain = proplists:get_value(<<"retain">>, Params, false), + Payload1 = maybe_maps_to_binary(Payload), + UserProps = check_user_props(proplists:get_value(<<"user_properties">>, Params, [])), + {ClientId, Topics, Qos, Retain, Payload1, UserProps}. parse_unsubscribe_params(Params) -> ClientId = proplists:get_value(<<"clientid">>, Params), @@ -251,3 +260,8 @@ maybe_maps_to_binary(Payload) -> _C : _E : S -> error({encode_payload_fail, S}) end. + +check_user_props(UserProps) when is_list(UserProps) -> + UserProps; +check_user_props(UserProps) -> + error({user_properties_type_error, UserProps}). diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index f3bc42f3a..a2d3aef0b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -401,7 +401,7 @@ t_pubsub(_) -> ClientId = <<"client1">>, Options = #{clientid => ClientId, - proto_ver => 5}, + proto_ver => v5}, Topic = <<"mytopic">>, {ok, C1} = emqtt:start_link(Options), {ok, _} = emqtt:connect(C1), @@ -536,11 +536,29 @@ t_pubsub(_) -> api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data3), [return_maps]))), + {ok, _, [1]} = emqtt:subscribe(C1, <<"mytopic">>, qos1), + timer:sleep(50), + + %% user properties + {ok, Code} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), + #{<<"clientid">> => ClientId, + <<"topic">> => <<"mytopic">>, + <<"qos">> => 1, + <<"payload">> => <<"hello world">>, + <<"user_properties">> => #{<<"porp_1">> => <<"porp_1">>}}), + ?assert(receive + {publish, #{payload := <<"hello world">>, + properties := #{'User-Property' := [{<<"porp_1">>,<<"porp_1">>}]}}} -> + true + after 100 -> + false + end), + ok = emqtt:disconnect(C1), - ?assertEqual(3, emqx_metrics:val('messages.qos1.received') - Qos1Received), + ?assertEqual(4, emqx_metrics:val('messages.qos1.received') - Qos1Received), ?assertEqual(2, emqx_metrics:val('messages.qos2.received') - Qos2Received), - ?assertEqual(5, emqx_metrics:val('messages.received') - Received). + ?assertEqual(6, emqx_metrics:val('messages.received') - Received). loop([]) -> []; From 45965a3e7108a94048a4b410fdd7c26deef04a00 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 17 Nov 2021 10:08:38 +0800 Subject: [PATCH 073/363] ci: update emqx-ci-helper version --- .github/workflows/build_packages.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 46133e95b..ef152604f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -475,7 +475,7 @@ jobs: -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ + -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - name: update repo.emqx.io if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' @@ -484,7 +484,7 @@ jobs: -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ + -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - name: update homebrew packages if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' @@ -494,7 +494,7 @@ jobs: -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ + -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" fi - uses: geekyeggo/delete-artifact@v1 From af5f93d81a1b5bf8f9d9aad3e20f1319177fafdf Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 17 Nov 2021 10:09:28 +0800 Subject: [PATCH 074/363] build: show macos version --- scripts/get-distro.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index ae52abba3..00e95e1d8 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -6,7 +6,9 @@ set -euo pipefail if [ "$(uname -s)" = 'Darwin' ]; then - echo 'macos' + DIST='macos' + VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') + SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" elif [ "$(uname -s)" = 'Linux' ]; then if grep -q -i 'centos' /etc/*-release; then DIST='centos' @@ -15,5 +17,6 @@ elif [ "$(uname -s)" = 'Linux' ]; then DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" fi - echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g' + SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" fi +echo "$SYSTEM" From 834240a76078205025a75047081b63f05bd2873b Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 17 Nov 2021 15:20:03 +0800 Subject: [PATCH 075/363] ci: cancel otp vsn for docker image tag add otp vsn for docker labels --- .github/workflows/build_packages.yaml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index ef152604f..73600e91f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -334,17 +334,6 @@ jobs: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - 23.3.4.9-3 - arch: - - [amd64, x86_64] - - [arm64v8, aarch64] - - [arm32v7, arm] - - [i386, i386] - - [s390x, s390x] - exclude: - - profile: emqx-ee - arch: [i386, i386] - - profile: emqx-ee - arch: [s390x, s390x] steps: - uses: actions/download-artifact@v2 @@ -368,8 +357,10 @@ jobs: type=ref,event=branch type=ref,event=pr type=ref,event=tag - type=semver,pattern={{version}}-otp${{ matrix.otp }} - type=semver,pattern={{major}}.{{minor}}-otp${{ matrix.otp }} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + labels: + org.opencontainers.image.otp.version=${{ matrix.otp }} - uses: docker/login-action@v1 if: github.event_name == 'release' with: From 35164951e21ebbb053df798fbdb4438017c71d23 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 16 Nov 2021 23:18:30 +0100 Subject: [PATCH 076/363] feat(docker): add flexible docker build The defulat docker build (e.g. make emqx-docker) is based on alpine image and it builds EMQ X from source code. This is not flexible enough when we want to quickly run some tests in a docker container. The new docker build (e.g. make emqx-docker-testing) by default takes the built zip package, and extract it in a very primitive base image such as ubuntu:20.04 and centos:8 --- Makefile | 15 +++++++- build | 62 ++++++++++++++++++++++++++------ deploy/docker/Dockerfile.testing | 43 ++++++++++++++++++++++ 3 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 deploy/docker/Dockerfile.testing diff --git a/Makefile b/Makefile index 4b15adfc7..1f8f17bbd 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3 -export DOCKERFILE=deploy/docker/Dockerfile +export DOCKERFILE := deploy/docker/Dockerfile +export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif @@ -155,6 +156,18 @@ endef ALL_ZIPS = $(REL_PROFILES) $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) +## emqx-docker-testing +## emqx-ee-docker-testing +## is to directly copy a unzipped zip-package to a +## base image such as ubuntu20.04. Mostly for testing +.PHONY: $(REL_PROFILES:%=%-docker-testing) +define gen-docker-target-testing +$1-docker-testing: $(COMMON_DEPS) + @$(BUILD) $1 docker-testing +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt)))) + .PHONY: run run: $(PROFILE) quickrun diff --git a/build b/build index 9375dfc8c..50997f76c 100755 --- a/build +++ b/build @@ -124,26 +124,63 @@ make_zip() { (cd "${tard}" && zip -qr - emqx) > "${zipball}" } +## This function builds the default docker image based on alpine:3.14 (by default) make_docker() { - EMQX_RUNNER_IMAGE='alpine:3.14' - EMQX_RUNNER_IMAGE_COMPACT="$(echo $EMQX_RUNNER_IMAGE | tr -d ':')" + EMQX_RUNNER_IMAGE="${EMQX_RUNNER_IMAGE:-alpine:3.14}" + EMQX_RUNNER_IMAGE_COMPACT="$(echo "$EMQX_RUNNER_IMAGE" | tr -d ':')" EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-2:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" - - ## Build Docker image - echo "DOCKER BUILD: Build Docker image." - echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " - echo "DOCKER BUILD: build version -> $PKG_VSN." - echo "DOCKER BUILD: build from -> $EMQX_BUILDER." - echo "DOCKER BUILD: runner from -> $EMQX_RUNNER_IMAGE." - + set -x docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ --build-arg RUN_FROM="${EMQX_RUNNER_IMAGE}" \ --build-arg EMQX_NAME="$PROFILE" \ - --tag "emqx/$PROFILE:$PKG_VSN" \ + --tag "emqx/$PROFILE:${PKG_VSN}" \ -f "${DOCKERFILE}" . } +## This function accepts any base docker image, +## a emqx zip-image, and a image tag (for the image to be built), +## to build a docker image which runs EMQ X +## +## Export below variables to quickly build an image +## +## Name Default Example +## --------------------------------------------------------------------- +## EMQX_BASE_IMAGE current os centos:7 +## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.zip +## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag +## +make_docker_testing() { + if [ -z "${EMQX_BASE_IMAGE:-}" ]; then + case "$SYSTEM" in + ubuntu20*) + EMQX_BASE_IMAGE="ubuntu:20.04" + ;; + centos8) + EMQX_BASE_IMAGE="centos:8" + ;; + *) + echo "Unsupported testing base image for $SYSTEM" + exit 1 + ;; + esac + fi + EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" + local defaultzip + defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" + local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}" + if [ ! -f "$zip" ]; then + log "ERROR: $zip not built?" + exit 1 + fi + set -x + docker build \ + --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \ + --build-arg EMQX_ZIP_PACKAGE="${zip}" \ + --tag "$EMQX_IMAGE_TAG" \ + -f "${DOCKERFILE_TESTING}" . +} + log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in @@ -167,6 +204,9 @@ case "$ARTIFACT" in docker) make_docker ;; + docker-testing) + make_docker_testing + ;; *) log "Unknown artifact $ARTIFACT" exit 1 diff --git a/deploy/docker/Dockerfile.testing b/deploy/docker/Dockerfile.testing new file mode 100644 index 000000000..02490272d --- /dev/null +++ b/deploy/docker/Dockerfile.testing @@ -0,0 +1,43 @@ +ARG BUILD_FROM +FROM ${BUILD_FROM} + +## all we need is the unzip command +RUN if command -v yum; then yum update -y && yum install -y unzip; fi +RUN if command -v apt-get; then apt-get update -y && apt-get install unzip; fi + +ARG EMQX_ZIP_PACKAGE +COPY ${EMQX_ZIP_PACKAGE} /opt/emqx.zip +RUN unzip -q /opt/emqx.zip -d /opt/ && rm /opt/emqx.zip + +COPY deploy/docker/docker-entrypoint.sh /usr/bin/ +RUN ln -s /opt/emqx/bin/* /usr/local/bin/ + +WORKDIR /opt/emqx + +RUN adduser -u 1000 emqx +RUN echo "emqx ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers + +RUN chgrp -Rf emqx /opt/emqx && chmod -Rf g+w /opt/emqx \ + && chown -Rf emqx /opt/emqx + +USER emqx + +VOLUME ["/opt/emqx/log", "/opt/emqx/data", "/opt/emqx/etc"] + +# emqx will occupy these port: +# - 1883 port for MQTT +# - 8081 for mgmt API +# - 8083 for WebSocket/HTTP +# - 8084 for WSS/HTTPS +# - 8883 port for MQTT(SSL) +# - 11883 port for internal MQTT/TCP +# - 18083 for dashboard +# - 4369 epmd (Erlang-distrbution port mapper daemon) listener (deprecated) +# - 4370 default Erlang distrbution port +# - 5369 for gen_rpc port mapping +# - 6369 6370 for distributed node +EXPOSE 1883 8081 8083 8084 8883 11883 18083 4369 4370 5369 6369 6370 + +ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] + +CMD ["/opt/emqx/bin/emqx", "foreground"] From 2a55a712d167b518b52fec9b047c627fc2baccf8 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 17 Nov 2021 13:58:09 +0100 Subject: [PATCH 077/363] build: prepare for 4.4-alpha.1 release --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 4b7da70d2..58cc11811 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.0-alpha.1"}). +-define(EMQX_RELEASE, {opensource, "4.4-alpha.1"}). -else. From eb0f4a543dfb970a114a752106228d7dbca7e463 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 18 Nov 2021 10:35:03 +0800 Subject: [PATCH 078/363] ci: fix upload artifact error --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 73600e91f..a06d7de08 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -322,7 +322,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') with: name: ${{ matrix.profile }} - path: _packages/${{ matrix.profile }}/. + path: source/_packages/${{ matrix.profile }}/. docker: runs-on: ubuntu-20.04 From 093a93a7ecd5b472cd54f60f8ea521606b5a4e2c Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 18 Nov 2021 14:27:37 +0800 Subject: [PATCH 079/363] ci: fix not found package when check sha256 --- .github/workflows/build_packages.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index a06d7de08..6f4aab000 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -308,6 +308,7 @@ jobs: make $PROFILE-pkg || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" - name: create sha256 + working-directory: source env: PROFILE: ${{ matrix.profile}} run: | @@ -358,7 +359,6 @@ jobs: type=ref,event=pr type=ref,event=tag type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} labels: org.opencontainers.image.otp.version=${{ matrix.otp }} - uses: docker/login-action@v1 @@ -368,7 +368,7 @@ jobs: password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/build-push-action@v2 with: - push: ${{ github.event_name == 'release' }} + push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} pull: true no-cache: true platforms: linux/amd64,linux/arm64 From 42333882c8b378a45a8d7048f65775124a621a42 Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 18 Nov 2021 14:41:59 +0800 Subject: [PATCH 080/363] fix(emqx_st_statistics): fix unsafe rank range (#6207) * fix(emqx_st_statistics): fix unsafe rank range --- .../emqx_st_statistics_api.erl | 46 +++++---- .../test/emqx_st_statistics_api_SUITE.erl | 98 ++++++++++++------- 2 files changed, 92 insertions(+), 52 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl index ba13d70e2..dc74dcf36 100644 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl @@ -51,25 +51,33 @@ get_history(_Bindings, Params) -> Limit = erlang:binary_to_integer(LimitT), Start = (Page - 1) * Limit + 1, Size = ets:info(?TOPK_TAB, size), - EndT = Start + Limit - 1, - End = erlang:min(EndT, Size), - Infos = lists:foldl(fun(Rank, Acc) -> - [#top_k{topic = Topic - , average_count = Count - , average_elapsed = Elapsed}] = ets:lookup(?TOPK_TAB, Rank), - - Info =[ {rank, Rank} - , {topic, Topic} - , {count, Count} - , {elapsed, Elapsed}], - - [Info | Acc] - end, - [], - lists:seq(Start, End)), - + End = Start + Limit - 1, + {HasNext, Count, Infos} = get_history(Start, End, Size), return({ok, #{meta => #{page => Page, limit => Limit, - hasnext => End < Size, - count => End - Start + 1}, + hasnext => HasNext, + count => Count}, data => Infos}}). + + +get_history(Start, _End, Size) when Start > Size -> + {false, 0, []}; + +get_history(Start, End, Size) when End > Size -> + get_history(Start, Size, Size); + +get_history(Start, End, Size) -> + Fold = fun(Rank, Acc) -> + [#top_k{topic = Topic + , average_count = Count + , average_elapsed = Elapsed}] = ets:lookup(?TOPK_TAB, Rank), + + Info = [ {rank, Rank} + , {topic, Topic} + , {count, Count} + , {elapsed, Elapsed}], + + [Info | Acc] + end, + Infos = lists:foldl(Fold, [], lists:seq(Start, End)), + {End < Size, End - Start + 1, Infos}. diff --git a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl index 3cf4dafd8..0fb9e904b 100644 --- a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl @@ -55,23 +55,6 @@ end_per_testcase(_, Config) -> emqx_mod_st_statistics:unload(undefined), Config. -get(Key, ResponseBody) -> - maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). - -lookup_alarm(Name, [#{<<"name">> := Name} | _More]) -> - true; -lookup_alarm(Name, [_Alarm | More]) -> - lookup_alarm(Name, More); -lookup_alarm(_Name, []) -> - false. - -is_existing(Name, [#{name := Name} | _More]) -> - true; -is_existing(Name, [_Alarm | More]) -> - is_existing(Name, More); -is_existing(_Name, []) -> - false. - t_get_history(_) -> ets:insert(?TOPK_TAB, #top_k{rank = 1, topic = <<"test">>, @@ -81,19 +64,54 @@ t_get_history(_) -> {ok, Data} = request_api(get, api_path(["slow_topic"]), "_page=1&_limit=10", auth_header_()), - Return = #{meta => #{page => 1, - limit => 10, - hasnext => false, - count => 1}, - data => [#{topic => <<"test">>, - rank => 1, - elapsed => 1500, - count => 12}], - code => 0}, + ShouldRet = #{meta => #{page => 1, + limit => 10, + hasnext => false, + count => 1}, + data => [#{topic => <<"test">>, + rank => 1, + elapsed => 1500, + count => 12}], + code => 0}, - ShouldBe = emqx_json:encode(Return), + Ret = decode(Data), - ?assertEqual(ShouldBe, erlang:list_to_binary(Data)). + ?assertEqual(ShouldRet, Ret). + +t_rank_range(_) -> + Insert = fun(Rank) -> + ets:insert(?TOPK_TAB, + #top_k{rank = Rank, + topic = <<"test">>, + average_count = 12, + average_elapsed = 1500}) + end, + lists:foreach(Insert, lists:seq(1, 15)), + + timer:sleep(100), + + {ok, Data} = request_api(get, api_path(["slow_topic"]), "_page=1&_limit=10", + auth_header_()), + + Meta1 = #{page => 1, limit => 10, hasnext => true, count => 10}, + Ret1 = decode(Data), + ?assertEqual(Meta1, maps:get(meta, Ret1)), + + %% End > Size + {ok, Data2} = request_api(get, api_path(["slow_topic"]), "_page=2&_limit=10", + auth_header_()), + + Meta2 = #{page => 2, limit => 10, hasnext => false, count => 5}, + Ret2 = decode(Data2), + ?assertEqual(Meta2, maps:get(meta, Ret2)), + + %% Start > Size + {ok, Data3} = request_api(get, api_path(["slow_topic"]), "_page=3&_limit=10", + auth_header_()), + + Meta3 = #{page => 3, limit => 10, hasnext => false, count => 0}, + Ret3 = decode(Data3), + ?assertEqual(Meta3, maps:get(meta, Ret3)). t_clear(_) -> ets:insert(?TOPK_TAB, #top_k{rank = 1, @@ -106,6 +124,25 @@ t_clear(_) -> ?assertEqual(0, ets:info(?TOPK_TAB, size)). +decode(Data) -> + Pairs = emqx_json:decode(Data), + to_maps(Pairs). + +to_maps([H | _] = List) when is_tuple(H) -> + to_maps(List, #{}); + +to_maps([_ | _] = List) -> + [to_maps(X) || X <- List]; + +to_maps(V) -> V. + +to_maps([{K, V} | T], Map) -> + AtomKey = erlang:binary_to_atom(K), + to_maps(T, Map#{AtomKey => to_maps(V)}); + +to_maps([], Map) -> + Map. + request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). @@ -148,8 +185,3 @@ auth_header_(User, Pass) -> api_path(Parts)-> ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). - -filter(List, Key, Value) -> - lists:filter(fun(Item) -> - maps:get(Key, Item) == Value - end, List). From d1cf526f34d7c160fbca2b6498501a55ae55c7f1 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 18 Nov 2021 14:43:55 +0800 Subject: [PATCH 081/363] ci: fix permission error when check sha256 --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 6f4aab000..e1f3e8988 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -315,7 +315,7 @@ jobs: if [ -d _packages/$PROFILE ]; then cd _packages/$PROFILE for var in $(ls emqx-* ); do - bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" + sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" done cd - fi From daeac6edf465aa0d561d0c35af9b92926e551f37 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 18 Nov 2021 15:26:36 +0800 Subject: [PATCH 082/363] chore(release): update emqx release version --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 58cc11811..3e96f1aa7 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4-alpha.1"}). +-define(EMQX_RELEASE, {opensource, "4.4-alpha.2"}). -else. From 099e2a8752aa9f317cda9a4b022f86dc1b131a3f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 19 Nov 2021 21:54:04 +0100 Subject: [PATCH 083/363] ci: port changes made in master branch --- .github/workflows/build_packages.yaml | 63 +++++++++++++--------- .github/workflows/build_slim_packages.yaml | 21 ++++++-- 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e1f3e8988..b2104e618 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -1,5 +1,9 @@ name: Cross build packages +concurrency: + group: build-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + on: schedule: - cron: '0 */6 * * *' @@ -11,11 +15,12 @@ on: jobs: prepare: runs-on: ubuntu-20.04 + # prepare source with any OTP version, no need for a matrix container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 outputs: - profiles: ${{ steps.set_profile.outputs.profiles}} - old_vsns: ${{ steps.set_profile.outputs.old_vsns}} + profiles: ${{ steps.set_profile.outputs.profiles }} + old_vsns: ${{ steps.set_profile.outputs.old_vsns }} steps: - uses: actions/checkout@v2 @@ -25,8 +30,8 @@ jobs: - name: set profile id: set_profile shell: bash + working-directory: source run: | - cd source vsn="$(./pkg-vsn.sh)" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -43,7 +48,7 @@ jobs: run: | make -C source deps-all zip -ryq source.zip source/* source/.[^.]* - - name: get_all_deps + - name: get_all_deps_ee if: endsWith(github.repository, 'enterprise') run: | echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials @@ -63,6 +68,7 @@ jobs: if: endsWith(github.repository, 'emqx') strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: @@ -78,14 +84,16 @@ jobs: - name: unzip source code run: Expand-Archive -Path source.zip -DestinationPath ./ - uses: ilammy/msvc-dev-cmd@v1 - - uses: gleam-lang/setup-erlang@v1.1.0 + - uses: gleam-lang/setup-erlang@v1.1.2 id: install_erlang + ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows with: otp-version: ${{ matrix.otp }} - name: build env: PYTHON: python DIAGNOSTIC: 1 + working-directory: source run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" @@ -97,7 +105,6 @@ jobs: else { $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" } - cd source ## We do not build/release bcrypt for windows package Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ if (Test-Path rebar.lock) { @@ -114,8 +121,8 @@ jobs: Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256 - name: run emqx timeout-minutes: 1 + working-directory: source run: | - cd source ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start Start-Sleep -s 5 ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop @@ -128,18 +135,19 @@ jobs: path: source/_packages/${{ matrix.profile }}/. mac: - runs-on: macos-10.15 - needs: prepare - strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: - 23.3.4.9-3 + macos: + - macos-11 + - macos-10.15 exclude: - profile: emqx-edge - + runs-on: ${{ matrix.macos }} steps: - uses: actions/download-artifact@v2 with: @@ -157,7 +165,7 @@ jobs: id: cache with: path: ~/.kerl - key: erl${{ matrix.erl_otp }}-macos10.15 + key: erl${{ matrix.erl_otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 @@ -169,18 +177,19 @@ jobs: kerl build ${{ matrix.erl_otp }} kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} - name: build + working-directory: source run: | . $HOME/.kerl/${{ matrix.erl_otp }}/activate - cd source make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 rm -rf _build/${{ matrix.profile }}/lib make ${{ matrix.profile }}-zip - name: test + working-directory: source run: | - cd source - pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip) - unzip -q _packages/${{ matrix.profile }}/$pkg_name + set -x + pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) + unzip -q $pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' @@ -199,11 +208,11 @@ jobs: ./emqx/bin/emqx_ctl status ./emqx/bin/emqx stop rm -rf emqx - openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}' > ./_packages/${{ matrix.profile }}/$pkg_name.sha256 + openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256 - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-${{ matrix.otp }} path: source/_packages/${{ matrix.profile }}/. linux: @@ -212,6 +221,7 @@ jobs: needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: @@ -270,6 +280,7 @@ jobs: ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }} + working-directory: source run: | set -e -x -u broker=$PROFILE @@ -280,8 +291,8 @@ jobs: export ARCH="arm" fi - mkdir -p source/_upgrade_base - cd source/_upgrade_base + mkdir -p _upgrade_base + cd _upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" @@ -322,15 +333,15 @@ jobs: - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-${{ matrix.otp }} path: source/_packages/${{ matrix.profile }}/. docker: runs-on: ubuntu-20.04 - needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: @@ -399,6 +410,8 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + otp: + - 23.3.4.9-3 steps: - uses: actions/checkout@v2 @@ -409,7 +422,7 @@ jobs: echo 'EOF' >> $GITHUB_ENV - uses: actions/download-artifact@v2 with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-${{ matrix.otp }} path: ./_packages/${{ matrix.profile }} - name: install dos2unix run: sudo apt-get update && sudo apt install -y dos2unix @@ -460,7 +473,7 @@ jobs: -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - name: update repo.emqx.io - if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' + if: github.event_name == 'release' && matrix.profile == 'emqx-ee' run: | curl --silent --show-error \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ @@ -469,7 +482,7 @@ jobs: -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - name: update repo.emqx.io - if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' + if: github.event_name == 'release' && matrix.profile == 'emqx' run: | curl --silent --show-error \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 5888dff8b..23d0e3e53 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -1,5 +1,10 @@ name: Build slim packages +concurrency: + group: slim-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + + on: push: tags: @@ -13,6 +18,7 @@ jobs: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: erl_otp: - 23.3.4.9-3 @@ -43,7 +49,7 @@ jobs: with: name: rebar3.crashdump path: ./rebar3.crashdump - - name: pakcages test + - name: packages test run: | export CODE_PATH=$GITHUB_WORKSPACE .ci/build_packages/tests.sh @@ -53,12 +59,17 @@ jobs: path: _packages/**/*.zip mac: - runs-on: macos-10.15 strategy: + fail-fast: false matrix: erl_otp: - 23.3.4.9-3 + macos: + - macos-11 + - macos-10.15 + + runs-on: ${{ matrix.macos }} steps: - uses: actions/checkout@v1 @@ -82,7 +93,7 @@ jobs: id: cache with: path: ~/.kerl - key: erl${{ matrix.erl_otp }}-macos10.15 + key: otp-${{ matrix.erl_otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 @@ -106,8 +117,8 @@ jobs: path: ./rebar3.crashdump - name: test run: | - pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) - unzip -q _packages/${EMQX_NAME}/$pkg_name + pkg_name=$(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip) + unzip -q $pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' From 4767b41eb7f0b4efc9af7f81c1cbb3a95c0195df Mon Sep 17 00:00:00 2001 From: lafirest Date: Sun, 21 Nov 2021 18:55:43 +0800 Subject: [PATCH 084/363] fix(emqx_st_statistics): fix initial value error (#6224) * fix(emqx_st_statistics): fix initial value error --- .../src/emqx_st_statistics/emqx_st_statistics.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl index f22aec41c..668cb3926 100644 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl +++ b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl @@ -56,8 +56,8 @@ }. -record(slow_log, { topic :: emqx_types:topic() - , count :: pos_integer() - , elapsed :: pos_integer() + , count :: integer() %% 0 will be used in initial value + , elapsed :: integer() }). -type message() :: #message{}. @@ -233,8 +233,8 @@ update_log(#message{topic = Topic}, Elapsed) -> Topic, [{#slow_log.count, 1}, {#slow_log.elapsed, Elapsed}], #slow_log{topic = Topic, - count = 1, - elapsed = Elapsed}), + count = 0, + elapsed = 0}), ok. -spec do_notification(state()) -> true. From 62dc72c85949b0ec472719b09b9d414288362037 Mon Sep 17 00:00:00 2001 From: Turtle Date: Mon, 22 Nov 2021 11:04:21 +0800 Subject: [PATCH 085/363] feat(sql_rule): test rule add User-Property information --- apps/emqx_rule_engine/src/emqx_rule_events.erl | 3 ++- .../src/emqx_rule_sqltester.erl | 17 ++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 97e40439d..72f8345bf 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -454,7 +454,8 @@ columns_with_exam('message.publish') -> , {<<"topic">>, <<"t/a">>} , {<<"qos">>, 1} , {<<"flags">>, #{}} - , {<<"headers">>, undefined} + , {<<"headers">>, #{<<"properties">> => #{<<"User-Property">> => + #{'prop_key' => <<"prop_val">>}}}} , {<<"publish_received_at">>, erlang:system_time(millisecond)} , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 760205d62..747de87d7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -98,21 +98,8 @@ sql_test_action() -> fill_default_values(Event, Context) -> maps:merge(envs_examp(Event), Context). -envs_examp(<<"$events/", _/binary>> = EVENT_TOPIC) -> +envs_examp(EVENT_TOPIC) -> EventName = emqx_rule_events:event_name(EVENT_TOPIC), emqx_rule_maps:atom_key_map( maps:from_list( - emqx_rule_events:columns_with_exam(EventName))); -envs_examp(_) -> - #{id => emqx_guid:to_hexstr(emqx_guid:gen()), - clientid => <<"c_emqx">>, - username => <<"u_emqx">>, - payload => <<"{\"id\": 1, \"name\": \"ha\"}">>, - peerhost => <<"127.0.0.1">>, - topic => <<"t/a">>, - qos => 1, - flags => #{sys => true, event => true}, - publish_received_at => emqx_rule_utils:now_ms(), - timestamp => emqx_rule_utils:now_ms(), - node => node() - }. + emqx_rule_events:columns_with_exam(EventName))). From d76275d17d75cf278d412343e698fbfd8701aa87 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 22 Nov 2021 14:59:11 +0800 Subject: [PATCH 086/363] feat: add support ip_address trace options --- apps/emqx_management/src/emqx_mgmt_cli.erl | 142 ++++++++-- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 113 ++++---- .../src/emqx_trace/emqx_trace.erl | 196 ++++++------- .../src/emqx_trace/emqx_trace_api.erl | 110 ++++++-- .../test/emqx_trace_SUITE.erl | 102 +++---- .../emqx_modules/src/emqx_mod_trace_api.erl | 14 +- .../test/emqx_mod_trace_api_SUITE.erl | 18 +- src/emqx_broker.erl | 9 +- src/emqx_channel.erl | 3 +- src/emqx_logger.erl | 49 ++-- src/emqx_trace_handler.erl | 206 ++++++++++++++ src/emqx_tracer.erl | 261 ------------------ test/emqx_trace_handler_SUITE.erl | 191 +++++++++++++ test/emqx_tracer_SUITE.erl | 171 ------------ 14 files changed, 840 insertions(+), 745 deletions(-) create mode 100644 src/emqx_trace_handler.erl delete mode 100644 src/emqx_tracer.erl create mode 100644 test/emqx_trace_handler_SUITE.erl delete mode 100644 test/emqx_tracer_SUITE.erl diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 97cb182e6..b6468fa7c 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -21,7 +21,7 @@ -include("emqx_mgmt.hrl"). --elvis([{elvis_style, invalid_dynamic_call, #{ ignore => [emqx_mgmt_cli]}}]). +-elvis([{elvis_style, invalid_dynamic_call, disable}]). -define(PRINT_CMD(Cmd, Desc), io:format("~-48s# ~s~n", [Cmd, Desc])). @@ -38,6 +38,7 @@ , vm/1 , mnesia/1 , trace/1 + , traces/1 , log/1 , mgmt/1 , data/1 @@ -421,39 +422,43 @@ log(_) -> trace(["list"]) -> lists:foreach(fun(Trace) -> - #{type := Type, level := Level, dst := Dst} = Trace, - Who = maps:get(Type, Trace), - emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Who, Level, Dst]) - end, emqx_tracer:lookup_traces()); + #{type := Type, filter := Filter, level := Level, dst := Dst} = Trace, + emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Filter, Level, Dst]) + end, emqx_trace_handler:running()); -trace(["stop", "client", ClientId]) -> - trace_off(clientid, ClientId); +trace(["stop", Operation, ClientId]) -> + case trace_type(Operation) of + {ok, Type} -> trace_off(Type, ClientId); + error -> trace([]) + end; -trace(["start", "client", ClientId, LogFile]) -> - trace_on(clientid, ClientId, all, LogFile); +trace(["start", Operation, ClientId, LogFile]) -> + trace(["start", Operation, ClientId, LogFile, "all"]); -trace(["start", "client", ClientId, LogFile, Level]) -> - trace_on(clientid, ClientId, list_to_atom(Level), LogFile); - -trace(["stop", "topic", Topic]) -> - trace_off(topic, Topic); - -trace(["start", "topic", Topic, LogFile]) -> - trace_on(topic, Topic, all, LogFile); - -trace(["start", "topic", Topic, LogFile, Level]) -> - trace_on(topic, Topic, list_to_atom(Level), LogFile); +trace(["start", Operation, ClientId, LogFile, Level]) -> + case trace_type(Operation) of + {ok, Type} -> trace_on(Type, ClientId, list_to_existing_atom(Level), LogFile); + error -> trace([]) + end; trace(_) -> - emqx_ctl:usage([{"trace list", "List all traces started"}, - {"trace start client []", "Traces for a client"}, - {"trace stop client ", "Stop tracing for a client"}, - {"trace start topic [] ", "Traces for a topic"}, - {"trace stop topic ", "Stop tracing for a topic"}]). + emqx_ctl:usage([{"trace list", "List all traces started on local node"}, + {"trace start client []", + "Traces for a client on local node"}, + {"trace stop client ", + "Stop tracing for a client on local node"}, + {"trace start topic [] ", + "Traces for a topic on local node"}, + {"trace stop topic ", + "Stop tracing for a topic on local node"}, + {"trace start ip_address [] ", + "Traces for a client ip on local node"}, + {"trace stop ip_addresss ", + "Stop tracing for a client ip on local node"} + ]). --dialyzer({nowarn_function, [trace_on/4, trace_off/2]}). trace_on(Who, Name, Level, LogFile) -> - case emqx_tracer:start_trace(Who, Name, Level, LogFile) of + case emqx_trace_handler:install(Who, Name, Level, LogFile) of ok -> emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]); {error, Error} -> @@ -461,13 +466,94 @@ trace_on(Who, Name, Level, LogFile) -> end. trace_off(Who, Name) -> - case emqx_tracer:stop_trace(Who, Name) of + case emqx_trace_handler:uninstall(Who, Name) of ok -> emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]); {error, Error} -> emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Name, Error]) end. +%%-------------------------------------------------------------------- +%% @doc Trace Cluster Command +traces(["list"]) -> + {ok, List} = emqx_trace_api:list_trace(get, []), + case List of + [] -> + emqx_ctl:print("Cluster Trace is empty~n", []); + _ -> + lists:foreach(fun(Trace) -> + #{type := Type, name := Name, status := Status, + log_size := LogSize} = Trace, + emqx_ctl:print("Trace(~s: ~s=~s, ~s, LogSize:~p)~n", + [Name, Type, maps:get(Type, Trace), Status, LogSize]) + end, List) + end, + length(List); + +traces(["stop", Name]) -> + trace_cluster_off(Name); + +traces(["delete", Name]) -> + trace_cluster_del(Name); + +traces(["start", Name, Operation, Filter]) -> + traces(["start", Name, Operation, Filter, "900"]); + +traces(["start", Name, Operation, Filter, DurationS]) -> + case trace_type(Operation) of + {ok, Type} -> trace_cluster_on(Name, Type, Filter, DurationS); + error -> traces([]) + end; + +traces(_) -> + emqx_ctl:usage([{"traces list", "List all cluster traces started"}, + {"traces start client ", "Traces for a client in cluster"}, + {"traces start topic ", "Traces for a topic in cluster"}, + {"traces start ip_address ", "Traces for a IP in cluster"}, + {"traces stop ", "Stop trace in cluster"}, + {"traces delete ", "Delete trace in cluster"} + ]). + +trace_cluster_on(Name, Type, Filter, DurationS0) -> + case erlang:whereis(emqx_trace) of + undefined -> + emqx_ctl:print("[error] Tracer module not started~n" + "Please run `emqx_ctl modules start tracer` " + "or `emqx_ctl modules start emqx_mod_trace` first~n", []); + _ -> + DurationS = list_to_integer(DurationS0), + Now = erlang:system_time(second), + Trace = #{ name => list_to_binary(Name) + , type => atom_to_binary(Type) + , Type => list_to_binary(Filter) + , start_at => list_to_binary(calendar:system_time_to_rfc3339(Now)) + , end_at => list_to_binary(calendar:system_time_to_rfc3339(Now + DurationS)) + }, + case emqx_trace:create(Trace) of + ok -> + emqx_ctl:print("Cluster_trace ~p ~s ~s successfully~n", [Type, Filter, Name]); + {error, Error} -> + emqx_ctl:print("[error] Cluster_trace ~s ~s=~s ~p~n", + [Name, Type, Filter, Error]) + end + end. + +trace_cluster_del(Name) -> + case emqx_trace:delete(list_to_binary(Name)) of + ok -> emqx_ctl:print("Del cluster_trace ~s successfully~n", [Name]); + {error, Error} -> emqx_ctl:print("[error] Del cluster_trace ~s: ~p~n", [Name, Error]) + end. + +trace_cluster_off(Name) -> + case emqx_trace:update(list_to_binary(Name), false) of + ok -> emqx_ctl:print("Stop cluster_trace ~s successfully~n", [Name]); + {error, Error} -> emqx_ctl:print("[error] Stop cluster_trace ~s: ~p~n", [Name, Error]) + end. + +trace_type("client") -> {ok, clientid}; +trace_type("topic") -> {ok, topic}; +trace_type("ip_address") -> {ok, ip_address}; +trace_type(_) -> error. %%-------------------------------------------------------------------- %% @doc Listeners Command diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 77d46b744..434e96e21 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -45,6 +45,7 @@ groups() -> t_vm_cmd, t_plugins_cmd, t_trace_cmd, + t_traces_cmd, t_broker_cmd, t_router_cmd, t_subscriptions_cmd, @@ -64,6 +65,23 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_helpers:stop_apps(apps()). +init_per_testcase(t_plugins_cmd, Config) -> + meck:new(emqx_plugins, [non_strict, passthrough]), + meck:expect(emqx_plugins, load, fun(_) -> ok end), + meck:expect(emqx_plugins, unload, fun(_) -> ok end), + meck:expect(emqx_plugins, reload, fun(_) -> ok end), + mock_print(), + Config; +init_per_testcase(_Case, Config) -> + mock_print(), + Config. + +end_per_testcase(t_plugins_cmd, _Config) -> + meck:unload(emqx_plugins), + unmock_print(); +end_per_testcase(_Case, _Config) -> + unmock_print(). + t_app(_Config) -> {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>), ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret)), @@ -96,7 +114,6 @@ t_app(_Config) -> ok. t_log_cmd(_) -> - mock_print(), lists:foreach(fun(Level) -> emqx_mgmt_cli:log(["primary-level", Level]), ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["primary-level"])) @@ -109,12 +126,9 @@ t_log_cmd(_) -> ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["handlers", "set-level", atom_to_list(Id), Level])) end, ?LOG_LEVELS) - || #{id := Id} <- emqx_logger:get_log_handlers()], - meck:unload(). + || #{id := Id} <- emqx_logger:get_log_handlers()]. t_mgmt_cmd(_) -> - % ct:pal("start testing the mgmt command"), - mock_print(), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( ["lookup", "emqx_appid"]), "Not Found.")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( @@ -127,28 +141,19 @@ t_mgmt_cmd(_) -> ["update", "emqx_appid", "ts"]), "update successfully")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( ["delete", "emqx_appid"]), "ok")), - ok = emqx_mgmt_cli:mgmt(["list"]), - meck:unload(). + ok = emqx_mgmt_cli:mgmt(["list"]). t_status_cmd(_) -> - % ct:pal("start testing status command"), - mock_print(), %% init internal status seem to be always 'starting' when running ct tests - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([]), "Node\s.*@.*\sis\sstart(ed|ing)")), - meck:unload(). + ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([]), "Node\s.*@.*\sis\sstart(ed|ing)")). t_broker_cmd(_) -> - % ct:pal("start testing the broker command"), - mock_print(), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([]), "sysdescr")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker(["stats"]), "subscriptions.shared")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker(["metrics"]), "bytes.sent")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([undefined]), "broker")), - meck:unload(). + ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([undefined]), "broker")). t_clients_cmd(_) -> - % ct:pal("start testing the client command"), - mock_print(), process_flag(trap_exit, true), {ok, T} = emqtt:start_link([{clientid, <<"client12">>}, {username, <<"testuser1">>}, @@ -164,7 +169,6 @@ t_clients_cmd(_) -> receive {'EXIT', T, _} -> ok - % ct:pal("Connection closed: ~p~n", [Reason]) after 500 -> erlang:error("Client is not kick") @@ -179,10 +183,11 @@ t_clients_cmd(_) -> {ok, Connack, <<>>, _} = raw_recv_pase(Bin), timer:sleep(300), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client13"]), "client13")), - meck:unload(). + % emqx_mgmt_cli:clients(["kick", "client13"]), % timer:sleep(500), % ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client13"]), "Not Found")). + ok. raw_recv_pase(Packet) -> emqx_frame:parse(Packet). @@ -191,8 +196,6 @@ raw_send_serialize(Packet) -> emqx_frame:serialize(Packet). t_vm_cmd(_) -> - % ct:pal("start testing the vm command"), - mock_print(), [[?assertMatch({match, _}, re:run(Result, Name)) || Result <- emqx_mgmt_cli:vm([Name])] || Name <- ["load", "memory", "process", "io", "ports"]], @@ -205,12 +208,9 @@ t_vm_cmd(_) -> [?assertMatch({match, _}, re:run(Result, "io")) || Result <- emqx_mgmt_cli:vm(["io"])], [?assertMatch({match, _}, re:run(Result, "ports")) - || Result <- emqx_mgmt_cli:vm(["ports"])], - unmock_print(). + || Result <- emqx_mgmt_cli:vm(["ports"])]. t_trace_cmd(_) -> - % ct:pal("start testing the trace command"), - mock_print(), logger:set_primary_config(level, debug), {ok, T} = emqtt:start_link([{clientid, <<"client">>}, {username, <<"testuser">>}, @@ -237,12 +237,34 @@ t_trace_cmd(_) -> Trace7 = emqx_mgmt_cli:trace(["start", "topic", "a/b/c", "log/clientid_trace.log", "error"]), ?assertMatch({match, _}, re:run(Trace7, "successfully")), - logger:set_primary_config(level, error), - unmock_print(). + logger:set_primary_config(level, error). + +t_traces_cmd(_) -> + emqx_trace:mnesia(boot), + Count1 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(0, Count1), + Error1 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), + ?assertMatch({match, _}, re:run(Error1, "Tracer module not started")), + emqx_trace:start_link(), + Trace1 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), + ?assertMatch({match, _}, re:run(Trace1, "successfully")), + Count2 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(1, Count2), + Error2 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), + ?assertMatch({match, _}, re:run(Error2, "already_existed")), + Trace2 = emqx_mgmt_cli:traces(["stop", "test-name"]), + ?assertMatch({match, _}, re:run(Trace2, "successfully")), + Count3 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(1, Count3), + Trace3 = emqx_mgmt_cli:traces(["delete", "test-name"]), + ?assertMatch({match, _}, re:run(Trace3, "successfully")), + Count4 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(0, Count4), + Error3 = emqx_mgmt_cli:traces(["delete", "test-name"]), + ?assertMatch({match, _}, re:run(Error3, "not_found")), + ok. t_router_cmd(_) -> - % ct:pal("start testing the router command"), - mock_print(), {ok, T} = emqtt:start_link([{clientid, <<"client1">>}, {username, <<"testuser1">>}, {password, <<"pass1">>} @@ -257,12 +279,9 @@ t_router_cmd(_) -> emqtt:connect(T1), emqtt:subscribe(T1, <<"a/b/c/d">>), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["list"]), "a/b/c | a/b/c")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["show", "a/b/c"]), "a/b/c")), - unmock_print(). + ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["show", "a/b/c"]), "a/b/c")). t_subscriptions_cmd(_) -> - % ct:pal("Start testing the subscriptions command"), - mock_print(), {ok, T3} = emqtt:start_link([{clientid, <<"client">>}, {username, <<"testuser">>}, {password, <<"pass">>} @@ -273,22 +292,18 @@ t_subscriptions_cmd(_) -> [?assertMatch({match, _} , re:run(Result, "b/b/c")) || Result <- emqx_mgmt_cli:subscriptions(["show", <<"client">>])], ?assertEqual(emqx_mgmt_cli:subscriptions(["add", "client", "b/b/c", "0"]), "ok~n"), - ?assertEqual(emqx_mgmt_cli:subscriptions(["del", "client", "b/b/c"]), "ok~n"), - unmock_print(). + ?assertEqual(emqx_mgmt_cli:subscriptions(["del", "client", "b/b/c"]), "ok~n"). t_listeners_cmd_old(_) -> ok = emqx_listeners:ensure_all_started(), - mock_print(), ?assertEqual(emqx_mgmt_cli:listeners([]), ok), ?assertEqual( "Stop mqtt:wss:external listener on 0.0.0.0:8084 successfully.\n", emqx_mgmt_cli:listeners(["stop", "wss", "8084"]) - ), - unmock_print(). + ). t_listeners_cmd_new(_) -> ok = emqx_listeners:ensure_all_started(), - mock_print(), ?assertEqual(emqx_mgmt_cli:listeners([]), ok), ?assertEqual( "Stop mqtt:wss:external listener on 0.0.0.0:8084 successfully.\n", @@ -304,16 +319,11 @@ t_listeners_cmd_new(_) -> ), ?assertEqual( emqx_mgmt_cli:listeners(["restart", "bad:listener:identifier"]), - "Failed to restart bad:listener:identifier listener: {no_such_listener,\"bad:listener:identifier\"}\n" - ), - unmock_print(). + "Failed to restart bad:listener:identifier listener: " + "{no_such_listener,\"bad:listener:identifier\"}\n" + ). t_plugins_cmd(_) -> - mock_print(), - meck:new(emqx_plugins, [non_strict, passthrough]), - meck:expect(emqx_plugins, load, fun(_) -> ok end), - meck:expect(emqx_plugins, unload, fun(_) -> ok end), - meck:expect(emqx_plugins, reload, fun(_) -> ok end), ?assertEqual(emqx_mgmt_cli:plugins(["list"]), ok), ?assertEqual( emqx_mgmt_cli:plugins(["unload", "emqx_auth_mnesia"]), @@ -326,11 +336,9 @@ t_plugins_cmd(_) -> ?assertEqual( emqx_mgmt_cli:plugins(["unload", "emqx_management"]), "Plugin emqx_management can not be unloaded.~n" - ), - unmock_print(). + ). t_cli(_) -> - mock_print(), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([""]), "status")), [?assertMatch({match, _}, re:run(Value, "broker")) || Value <- emqx_mgmt_cli:broker([""])], @@ -352,9 +360,10 @@ t_cli(_) -> || Value <- emqx_mgmt_cli:mnesia([""])], [?assertMatch({match, _}, re:run(Value, "trace")) || Value <- emqx_mgmt_cli:trace([""])], + [?assertMatch({match, _}, re:run(Value, "traces")) + || Value <- emqx_mgmt_cli:traces([""])], [?assertMatch({match, _}, re:run(Value, "mgmt")) - || Value <- emqx_mgmt_cli:mgmt([""])], - unmock_print(). + || Value <- emqx_mgmt_cli:mgmt([""])]. mock_print() -> catch meck:unload(emqx_ctl), diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index eb2ed7276..eb41703fa 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -18,10 +18,9 @@ -behaviour(gen_server). -include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --logger_header("[Trace]"). +-logger_header("[Tracer]"). %% Mnesia bootstrap -export([mnesia/1]). @@ -29,6 +28,11 @@ -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). +-export([ publish/1 + , subscribe/3 + , unsubscribe/2 + ]). + -export([ start_link/0 , list/0 , list/1 @@ -41,6 +45,7 @@ -export([ format/1 , zip_dir/0 + , filename/2 , trace_dir/0 , trace_file/1 , delete_files_after_send/2 @@ -49,23 +54,22 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -define(TRACE, ?MODULE). --define(PACKETS, tuple_to_list(?TYPE_NAMES)). -define(MAX_SIZE, 30). -ifdef(TEST). -export([log_file/2]). -endif. +-export_type([ip_address/0]). +-type ip_address() :: string(). + -record(?TRACE, { name :: binary() | undefined | '_' - , type :: clientid | topic | undefined | '_' - , topic :: emqx_types:topic() | undefined | '_' - , clientid :: emqx_types:clientid() | undefined | '_' - , packets = [] :: list() | '_' + , type :: clientid | topic | ip_address | undefined | '_' + , filter :: emqx_types:topic() | emqx_types:clientid() | ip_address() | undefined | '_' , enable = true :: boolean() | '_' - , start_at :: integer() | undefined | binary() | '_' - , end_at :: integer() | undefined | binary() | '_' - , log_size = #{} :: map() | '_' + , start_at :: integer() | undefined | '_' + , end_at :: integer() | undefined | '_' }). mnesia(boot) -> @@ -77,6 +81,31 @@ mnesia(boot) -> mnesia(copy) -> ok = ekka_mnesia:copy_table(?TRACE, disc_copies). +publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore; +publish(#message{from = From, topic = Topic, payload = Payload}) when + is_binary(From); is_atom(From) -> + emqx_logger:info( + #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "PUBLISH to ~s: ~0p", + [Topic, Payload] + ). + +subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore; +subscribe(Topic, SubId, SubOpts) -> + emqx_logger:info( + #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "~ts SUBSCRIBE ~ts: Options: ~0p", + [SubId, Topic, SubOpts] + ). + +unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; +unsubscribe(Topic, SubOpts) -> + emqx_logger:info( + #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "~ts UNSUBSCRIBE ~ts: Options: ~0p", + [maps:get(subid, SubOpts, ""), Topic, SubOpts] + ). + -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). @@ -89,18 +118,18 @@ list() -> list(Enable) -> ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). --spec create([{Key :: binary(), Value :: binary()}]) -> +-spec create([{Key :: binary(), Value :: binary()}] | #{atom() => binary()}) -> ok | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. create(Trace) -> case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of true -> case to_trace(Trace) of - {ok, TraceRec} -> create_new_trace(TraceRec); + {ok, TraceRec} -> insert_new_trace(TraceRec); {error, Reason} -> {error, Reason} end; false -> - {error, """The number of traces created has reached the maximum, -please delete the useless ones first"""} + {error, "The number of traces created has reached the maximum" + " please delete the useless ones first"} end. -spec delete(Name :: binary()) -> ok | {error, not_found}. @@ -163,12 +192,8 @@ delete_files_after_send(TraceLog, Zips) -> -spec format(list(#?TRACE{})) -> list(map()). format(Traces) -> Fields = record_info(fields, ?TRACE), - lists:map(fun(Trace0 = #?TRACE{start_at = StartAt, end_at = EndAt}) -> - Trace = Trace0#?TRACE{ - start_at = list_to_binary(calendar:system_time_to_rfc3339(StartAt)), - end_at = list_to_binary(calendar:system_time_to_rfc3339(EndAt)) - }, - [_ | Values] = tuple_to_list(Trace), + lists:map(fun(Trace0 = #?TRACE{}) -> + [_ | Values] = tuple_to_list(Trace0), maps:from_list(lists:zip(Fields, Values)) end, Traces). @@ -198,7 +223,7 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitor case maps:take(Pid, Monitors) of error -> {noreply, State}; {Files, NewMonitors} -> - lists:foreach(fun(F) -> file:delete(F) end, Files), + lists:foreach(fun file:delete/1, Files), {noreply, State#{monitors => NewMonitors}} end; handle_info({timeout, TRef, update_trace}, @@ -227,14 +252,12 @@ terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> code_change(_, State, _Extra) -> {ok, State}. -create_new_trace(Trace) -> +insert_new_trace(Trace) -> Tran = fun() -> case mnesia:read(?TRACE, Trace#?TRACE.name) of [] -> - #?TRACE{start_at = StartAt, topic = Topic, - clientid = ClientId, packets = Packets} = Trace, - Match = #?TRACE{_ = '_', start_at = StartAt, topic = Topic, - clientid = ClientId, packets = Packets}, + #?TRACE{start_at = StartAt, type = Type, filter = Filter} = Trace, + Match = #?TRACE{_ = '_', start_at = StartAt, type = Type, filter = Filter}, case mnesia:match_object(?TRACE, Match, read) of [] -> mnesia:write(?TRACE, Trace, write); [#?TRACE{name = Name}] -> mnesia:abort({duplicate_condition, Name}) @@ -248,7 +271,7 @@ update_trace(Traces) -> Now = erlang:system_time(second), {_Waiting, Running, Finished} = classify_by_time(Traces, Now), disable_finished(Finished), - Started = already_running(), + Started = emqx_trace_handler:running(), {NeedRunning, AllStarted} = start_trace(Running, Started), NeedStop = AllStarted -- NeedRunning, ok = stop_trace(NeedStop, Started), @@ -257,14 +280,8 @@ update_trace(Traces) -> emqx_misc:start_timer(NextTime, update_trace). stop_all_trace_handler() -> - lists:foreach(fun(#{type := Type, name := Name} = Trace) -> - _ = emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name) - end - , already_running()). - -already_running() -> - emqx_tracer:lookup_traces(). - + lists:foreach(fun(#{id := Id}) -> emqx_trace_handler:uninstall(Id) end, + emqx_trace_handler:running()). get_enable_trace() -> {atomic, Traces} = mnesia:transaction(fun() -> @@ -286,31 +303,24 @@ find_closest_time(Traces, Now) -> disable_finished([]) -> ok; disable_finished(Traces) -> - NameWithLogSize = - lists:map(fun(#?TRACE{name = Name, start_at = StartAt}) -> - FileSize = filelib:file_size(log_file(Name, StartAt)), - {Name, FileSize} end, Traces), transaction(fun() -> - lists:map(fun({Name, LogSize}) -> + lists:map(fun(#?TRACE{name = Name}) -> case mnesia:read(?TRACE, Name, write) of [] -> ok; - [Trace = #?TRACE{log_size = Logs}] -> - mnesia:write(?TRACE, Trace#?TRACE{enable = false, - log_size = Logs#{node() => LogSize}}, write) - end end, NameWithLogSize) + [Trace] -> mnesia:write(?TRACE, Trace#?TRACE{enable = false}, write) + end end, Traces) end). start_trace(Traces, Started0) -> Started = lists:map(fun(#{name := Name}) -> Name end, Started0), lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> case lists:member(Name, StartedAcc) of - true -> {[Name | Running], StartedAcc}; + true -> + {[Name | Running], StartedAcc}; false -> case start_trace(Trace) of ok -> {[Name | Running], [Name | StartedAcc]}; - Error -> - ?LOG(error, "(~p)start trace failed by:~p", [Name, Error]), - {[Name | Running], StartedAcc} + {error, _Reason} -> {[Name | Running], StartedAcc} end end end, {[], Started}, Traces). @@ -318,27 +328,16 @@ start_trace(Traces, Started0) -> start_trace(Trace) -> #?TRACE{name = Name , type = Type - , clientid = ClientId - , topic = Topic - , packets = Packets + , filter = Filter , start_at = Start } = Trace, - Who0 = #{name => Name, labels => Packets}, - Who = - case Type of - topic -> Who0#{type => topic, topic => Topic}; - clientid -> Who0#{type => clientid, clientid => ClientId} - end, - case emqx_tracer:start_trace(Who, debug, log_file(Name, Start)) of - ok -> ok; - {error, {already_exist, _}} -> ok; - {error, Reason} -> {error, Reason} - end. + Who = #{name => Name, type => Type, filter => Filter}, + emqx_trace_handler:install(Who, debug, log_file(Name, Start)). stop_trace(Finished, Started) -> - lists:foreach(fun(#{name := Name, type := Type} = Trace) -> + lists:foreach(fun(#{name := Name, type := Type}) -> case lists:member(Name, Finished) of - true -> emqx_tracer:stop_trace(Type, maps:get(Type, Trace), Name); + true -> emqx_trace_handler:uninstall(Type, Name); false -> ok end end, Started). @@ -371,23 +370,31 @@ classify_by_time([Trace = #?TRACE{end_at = End} | Traces], classify_by_time([Trace | Traces], Now, Wait, Run, Finish) -> classify_by_time(Traces, Now, Wait, [Trace | Run], Finish). -to_trace(TraceList) -> - case to_trace(TraceList, #?TRACE{}) of +to_trace(TraceParam) -> + case to_trace(ensure_proplists(TraceParam), #?TRACE{}) of {error, Reason} -> {error, Reason}; {ok, #?TRACE{name = undefined}} -> {error, "name required"}; {ok, #?TRACE{type = undefined}} -> - {error, "type required"}; - {ok, #?TRACE{topic = undefined, clientid = undefined}} -> - {error, "topic/clientid cannot be both empty"}; - {ok, Trace} -> - case fill_default(Trace) of + {error, "type=[topic,clientid,ip_address] required"}; + {ok, #?TRACE{filter = undefined}} -> + {error, "topic/clientid/ip_address filter required"}; + {ok, TraceRec0} -> + case fill_default(TraceRec0) of #?TRACE{start_at = Start, end_at = End} when End =< Start -> {error, "failed by start_at >= end_at"}; - Trace1 -> {ok, Trace1} + TraceRec -> {ok, TraceRec} end end. +ensure_proplists(#{} = Trace) -> maps:to_list(Trace); +ensure_proplists(Trace) when is_list(Trace) -> + lists:foldl( + fun({K, V}, Acc) when is_binary(K) -> [{binary_to_existing_atom(K), V} | Acc]; + ({K, V}, Acc) when is_atom(K) -> [{K, V} | Acc]; + (_, Acc) -> Acc + end, [], Trace). + fill_default(Trace = #?TRACE{start_at = undefined}) -> fill_default(Trace#?TRACE{start_at = erlang:system_time(second)}); fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> @@ -395,27 +402,35 @@ fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> fill_default(Trace) -> Trace. to_trace([], Rec) -> {ok, Rec}; -to_trace([{<<"name">>, Name} | Trace], Rec) -> +to_trace([{name, Name} | Trace], Rec) -> case binary:match(Name, [<<"/">>], []) of - nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); + nomatch when byte_size(Name) < 200 -> to_trace(Trace, Rec#?TRACE{name = Name}); + nomatch -> {error, "name(latin1) length must < 200"}; _ -> {error, "name cannot contain /"} end; -to_trace([{<<"type">>, Type} | Trace], Rec) -> - case lists:member(Type, [<<"clientid">>, <<"topic">>]) of +to_trace([{type, Type} | Trace], Rec) -> + case lists:member(Type, [<<"clientid">>, <<"topic">>, <<"ip_address">>]) of true -> to_trace(Trace, Rec#?TRACE{type = binary_to_existing_atom(Type)}); - false -> {error, "incorrect type: only support clientid/topic"} + false -> {error, "incorrect type: only support clientid/topic/ip_address"} end; -to_trace([{<<"topic">>, Topic} | Trace], Rec) -> +to_trace([{topic, Topic} | Trace], Rec) -> case validate_topic(Topic) of - ok -> to_trace(Trace, Rec#?TRACE{topic = Topic}); + ok -> to_trace(Trace, Rec#?TRACE{filter = Topic}); {error, Reason} -> {error, Reason} end; -to_trace([{<<"start_at">>, StartAt} | Trace], Rec) -> +to_trace([{clientid, ClientId} | Trace], Rec) -> + to_trace(Trace, Rec#?TRACE{filter = ClientId}); +to_trace([{ip_address, IP} | Trace], Rec) -> + case inet:parse_address(binary_to_list(IP)) of + {ok, _} -> to_trace(Trace, Rec#?TRACE{filter = binary_to_list(IP)}); + {error, Reason} -> {error, lists:flatten(io_lib:format("ip address: ~p", [Reason]))} + end; +to_trace([{start_at, StartAt} | Trace], Rec) -> case to_system_second(StartAt) of {ok, Sec} -> to_trace(Trace, Rec#?TRACE{start_at = Sec}); {error, Reason} -> {error, Reason} end; -to_trace([{<<"end_at">>, EndAt} | Trace], Rec) -> +to_trace([{end_at, EndAt} | Trace], Rec) -> Now = erlang:system_time(second), case to_system_second(EndAt) of {ok, Sec} when Sec > Now -> @@ -425,21 +440,14 @@ to_trace([{<<"end_at">>, EndAt} | Trace], Rec) -> {error, Reason} -> {error, Reason} end; -to_trace([{<<"clientid">>, ClientId} | Trace], Rec) -> - to_trace(Trace, Rec#?TRACE{clientid = ClientId}); -to_trace([{<<"packets">>, PacketList} | Trace], Rec) -> - case to_packets(PacketList) of - {ok, Packets} -> to_trace(Trace, Rec#?TRACE{packets = Packets}); - {error, Reason} -> {error, io_lib:format("unsupport packets: ~p", [Reason])} - end; to_trace([Unknown | _Trace], _Rec) -> {error, io_lib:format("unknown field: ~p", [Unknown])}. validate_topic(TopicName) -> - try emqx_topic:validate(name, TopicName) of + try emqx_topic:validate(filter, TopicName) of true -> ok catch error:Error -> - {error, io_lib:format("~s invalid by ~p", [TopicName, Error])} + {error, io_lib:format("topic: ~s invalid by ~p", [TopicName, Error])} end. to_system_second(At) -> @@ -450,14 +458,6 @@ to_system_second(At) -> {error, ["The rfc3339 specification not satisfied: ", At]} end. -to_packets(Packets) when is_list(Packets) -> - AtomTypes = lists:map(fun(Type) -> binary_to_existing_atom(Type) end, Packets), - case lists:filter(fun(T) -> not lists:member(T, ?PACKETS) end, AtomTypes) of - [] -> {ok, AtomTypes}; - InvalidE -> {error, InvalidE} - end; -to_packets(Packets) -> {error, Packets}. - zip_dir() -> trace_dir() ++ "zip/". diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index e6c87d69f..7d982c00d 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -27,18 +27,37 @@ , download_zip_log/2 , stream_log_file/2 ]). --export([read_trace_file/3]). +-export([ read_trace_file/3 + , get_trace_size/0 + ]). -define(TO_BIN(_B_), iolist_to_binary(_B_)). --define(NOT_FOUND(N), {error, 'NOT_FOUND', ?TO_BIN([N, "NOT FOUND"])}). +-define(NOT_FOUND(N), {error, 'NOT_FOUND', ?TO_BIN([N, " NOT FOUND"])}). -list_trace(_, Params) -> - List = - case Params of - [{<<"enable">>, Enable}] -> emqx_trace:list(binary_to_existing_atom(Enable)); - _ -> emqx_trace:list() - end, - {ok, emqx_trace:format(List)}. +list_trace(_, _Params) -> + case emqx_trace:list() of + [] -> {ok, []}; + List0 -> + List = lists:sort(fun(#{start_at := A}, #{start_at := B}) -> A > B end, List0), + Nodes = ekka_mnesia:running_nodes(), + TraceSize = cluster_call(?MODULE, get_trace_size, [], 30000), + AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize), + Now = erlang:system_time(second), + Traces = + lists:map(fun(Trace = #{name := Name, start_at := Start, + end_at := End, enable := Enable, type := Type, filter := Filter}) -> + FileName = emqx_trace:filename(Name, Start), + LogSize = collect_file_size(Nodes, FileName, AllFileSize), + Trace0 = maps:without([enable, filter], Trace), + Trace0#{ log_size => LogSize + , Type => Filter + , start_at => list_to_binary(calendar:system_time_to_rfc3339(Start)) + , end_at => list_to_binary(calendar:system_time_to_rfc3339(End)) + , status => status(Enable, Start, End, Now) + } + end, emqx_trace:format(List)), + {ok, Traces} + end. create_trace(_, Param) -> case emqx_trace:create(Param) of @@ -97,28 +116,48 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> end, [], TraceFiles). collect_trace_file(TraceLog) -> + cluster_call(emqx_trace, trace_file, [TraceLog], 60000). + +cluster_call(Mod, Fun, Args, Timeout) -> Nodes = ekka_mnesia:running_nodes(), - {Files, BadNodes} = rpc:multicall(Nodes, emqx_trace, trace_file, [TraceLog], 60000), - BadNodes =/= [] andalso ?LOG(error, "download log rpc failed on ~p", [BadNodes]), - Files. + {GoodRes, BadNodes} = rpc:multicall(Nodes, Mod, Fun, Args, Timeout), + BadNodes =/= [] andalso ?LOG(error, "rpc call failed on ~p ~p", [BadNodes, {Mod, Fun, Args}]), + GoodRes. stream_log_file(#{name := Name}, Params) -> Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), Position0 = proplists:get_value(<<"position">>, Params, <<"0">>), Bytes0 = proplists:get_value(<<"bytes">>, Params, <<"1000">>), - Node = binary_to_existing_atom(Node0), - Position = binary_to_integer(Position0), - Bytes = binary_to_integer(Bytes0), - case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of - {ok, Bin} -> - Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, - {ok, #{meta => Meta, items => Bin}}; - {eof, Size} -> - Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, - {ok, #{meta => Meta, items => <<"">>}}; - {error, Reason} -> - logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), - {error, Reason} + case to_node(Node0) of + {ok, Node} -> + Position = binary_to_integer(Position0), + Bytes = binary_to_integer(Bytes0), + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + {ok, Bin} -> + Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, + {ok, #{meta => Meta, items => Bin}}; + {eof, Size} -> + Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, + {ok, #{meta => Meta, items => <<"">>}}; + {error, Reason} -> + logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), + {error, Reason}; + {badrpc, nodedown} -> + {error, "BadRpc node down"} + end; + {error, Reason} -> {error, Reason} + end. + +get_trace_size() -> + TraceDir = emqx_trace:trace_dir(), + Node = node(), + case file:list_dir(TraceDir) of + {ok, AllFiles} -> + lists:foldl(fun(File, Acc) -> + FullFileName = filename:join(TraceDir, File), + Acc#{{Node, File} => filelib:file_size(FullFileName)} + end, #{}, lists:delete("zip", AllFiles)); + _ -> #{} end. %% this is an rpc call for stream_log_file/2 @@ -134,7 +173,6 @@ read_trace_file(Name, Position, Limit) -> [] -> {error, not_found} end. --dialyzer({nowarn_function, read_file/3}). read_file(Path, Offset, Bytes) -> {ok, IoDevice} = file:open(Path, [read, raw, binary]), try @@ -146,9 +184,27 @@ read_file(Path, Offset, Bytes) -> {ok, Bin} -> {ok, Bin}; {error, Reason} -> {error, Reason}; eof -> - #file_info{size = Size} = file:read_file_info(IoDevice), + {ok, #file_info{size = Size}} = file:read_file_info(IoDevice), {eof, Size} end after file:close(IoDevice) end. + +to_node(Node) -> + try {ok, binary_to_existing_atom(Node)} + catch _:_ -> + {error, "node not found"} + end. + +collect_file_size(Nodes, FileName, AllFiles) -> + lists:foldl(fun(Node, Acc) -> + Size = maps:get({Node, FileName}, AllFiles, 0), + Acc#{Node => Size} + end, #{}, Nodes). + +%% status(false, _Start, End, Now) when End > Now -> <<"stopped">>; +status(false, _Start, _End, _Now) -> <<"stopped">>; +status(true, Start, _End, Now) when Now < Start -> <<"waiting">>; +status(true, _Start, End, Now) when Now >= End -> <<"stopped">>; +status(true, _Start, _End, _Now) -> <<"running">>. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index 169fd50bc..ffa2bc1fb 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -23,21 +23,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). --record(emqx_trace, { - name, - type, - topic, - clientid, - packets = [], - enable = true, - start_at, - end_at, - log_size = #{} - }). - --define(PACKETS, ['CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL' - , 'PUBCOMP', 'SUBSCRIBE', 'SUBACK', 'UNSUBSCRIBE', 'UNSUBACK' - , 'PINGREQ', 'PINGRESP', 'DISCONNECT', 'AUTH']). +-record(emqx_trace, {name, type, filter, enable = true, start_at, end_at}). %%-------------------------------------------------------------------- %% Setups @@ -61,16 +47,14 @@ t_base_create_delete(_Config) -> End = to_rfc3339(Now + 30 * 60), Name = <<"name1">>, ClientId = <<"test-device">>, - Packets = [atom_to_binary(E) || E <- ?PACKETS], - Trace = [ - {<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, - {<<"clientid">>, ClientId}, - {<<"packets">>, Packets}, - {<<"start_at">>, Start}, - {<<"end_at">>, End} - ], - AnotherTrace = lists:keyreplace(<<"name">>, 1, Trace, {<<"name">>, <<"AnotherName">>}), + Trace = #{ + name => Name, + type => <<"clientid">>, + clientid => ClientId, + start_at => Start, + end_at => End + }, + AnotherTrace = Trace#{name => <<"anotherTrace">>}, ok = emqx_trace:create(Trace), ?assertEqual({error, {already_existed, Name}}, emqx_trace:create(Trace)), ?assertEqual({error, {duplicate_condition, Name}}, emqx_trace:create(AnotherTrace)), @@ -78,24 +62,19 @@ t_base_create_delete(_Config) -> Expect = #emqx_trace{ name = Name, type = clientid, - topic = undefined, - clientid = ClientId, - packets = ?PACKETS, + filter = ClientId, start_at = Now, end_at = Now + 30 * 60 }, ?assertEqual(Expect, TraceRec), ExpectFormat = [ #{ - clientid => <<"test-device">>, + filter => <<"test-device">>, enable => true, type => clientid, - packets => ?PACKETS, name => <<"name1">>, - start_at => Start, - end_at => End, - log_size => #{}, - topic => undefined + start_at => Now, + end_at => Now + 30 * 60 } ], ?assertEqual(ExpectFormat, emqx_trace:format([TraceRec])), @@ -108,13 +87,12 @@ t_create_size_max(_Config) -> emqx_trace:clear(), lists:map(fun(Seq) -> Name = list_to_binary("name" ++ integer_to_list(Seq)), - Trace = [{<<"name">>, Name}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, - {<<"topic">>, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], + Trace = [{name, Name}, {type, <<"topic">>}, + {topic, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], ok = emqx_trace:create(Trace) end, lists:seq(1, 30)), - Trace31 = [{<<"name">>, <<"name31">>}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/31">>}], + Trace31 = [{<<"name">>, <<"name31">>}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/31">>}], {error, _} = emqx_trace:create(Trace31), ok = emqx_trace:delete(<<"name30">>), ok = emqx_trace:create(Trace31), @@ -125,11 +103,11 @@ t_create_failed(_Config) -> ok = emqx_trace:clear(), UnknownField = [{<<"unknown">>, 12}], {error, Reason1} = emqx_trace:create(UnknownField), - ?assertEqual(<<"unknown field: {<<\"unknown\">>,12}">>, iolist_to_binary(Reason1)), + ?assertEqual(<<"unknown field: {unknown,12}">>, iolist_to_binary(Reason1)), InvalidTopic = [{<<"topic">>, "#/#//"}], {error, Reason2} = emqx_trace:create(InvalidTopic), - ?assertEqual(<<"#/#// invalid by function_clause">>, iolist_to_binary(Reason2)), + ?assertEqual(<<"topic: #/#// invalid by function_clause">>, iolist_to_binary(Reason2)), InvalidStart = [{<<"start_at">>, <<"2021-12-3:12">>}], {error, Reason3} = emqx_trace:create(InvalidStart), @@ -141,41 +119,34 @@ t_create_failed(_Config) -> ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, iolist_to_binary(Reason4)), - InvalidPackets = [{<<"packets">>, [<<"publish">>]}], - {error, Reason5} = emqx_trace:create(InvalidPackets), - ?assertEqual(<<"unsupport packets: [publish]">>, iolist_to_binary(Reason5)), - - InvalidPackets2 = [{<<"packets">>, <<"publish">>}], - {error, Reason6} = emqx_trace:create(InvalidPackets2), - ?assertEqual(<<"unsupport packets: <<\"publish\">>">>, iolist_to_binary(Reason6)), - {error, Reason7} = emqx_trace:create([{<<"name">>, <<"test">>}, {<<"type">>, <<"clientid">>}]), - ?assertEqual(<<"topic/clientid cannot be both empty">>, iolist_to_binary(Reason7)), + ?assertEqual(<<"topic/clientid/ip_address filter required">>, iolist_to_binary(Reason7)), InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, {<<"type">>, <<"clientid">>}], {error, Reason9} = emqx_trace:create(InvalidPackets4), ?assertEqual(<<"name cannot contain /">>, iolist_to_binary(Reason9)), - ?assertEqual({error, "type required"}, emqx_trace:create([{<<"name">>, <<"test-name">>}, - {<<"packets">>, []}, {<<"clientid">>, <<"good">>}])), + ?assertEqual({error, "type=[topic,clientid,ip_address] required"}, + emqx_trace:create([{<<"name">>, <<"test-name">>}, {<<"clientid">>, <<"good">>}])), - ?assertEqual({error, "incorrect type: only support clientid/topic"}, + ?assertEqual({error, "incorrect type: only support clientid/topic/ip_address"}, emqx_trace:create([{<<"name">>, <<"test-name">>}, - {<<"packets">>, []}, {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), + {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), + + ?assertEqual({error, "ip address: einval"}, + emqx_trace:create([{<<"ip_address">>, <<"test-name">>}])), ok. t_create_default(_Config) -> ok = emqx_trace:clear(), {error, "name required"} = emqx_trace:create([]), ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, - {<<"type">>, <<"clientid">>}, {<<"packets">>, []}, {<<"clientid">>, <<"good">>}]), - [#emqx_trace{packets = Packets}] = emqx_trace:list(), - ?assertEqual([], Packets), + {<<"type">>, <<"clientid">>}, {<<"clientid">>, <<"good">>}]), + [#emqx_trace{name = <<"test-name">>}] = emqx_trace:list(), ok = emqx_trace:clear(), Trace = [ {<<"name">>, <<"test-name">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}, {<<"start_at">>, <<"2021-10-28T10:54:47+08:00">>}, @@ -186,15 +157,13 @@ t_create_default(_Config) -> Trace2 = [ {<<"name">>, <<"test-name">>}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>}, {<<"start_at">>, to_rfc3339(Now + 10)}, {<<"end_at">>, to_rfc3339(Now + 3)} ], {error, "failed by start_at >= end_at"} = emqx_trace:create(Trace2), ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, - {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>}]), + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}]), [#emqx_trace{start_at = Start, end_at = End}] = emqx_trace:list(), ?assertEqual(10 * 60, End - Start), ?assertEqual(true, Start - erlang:system_time(second) < 5), @@ -205,8 +174,8 @@ t_update_enable(_Config) -> Name = <<"test-name">>, Now = erlang:system_time(second), End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), - ok = emqx_trace:create([{<<"name">>, Name}, {<<"packets">>, [<<"PUBLISH">>]}, - {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), + ok = emqx_trace:create([{<<"name">>, Name}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), [#emqx_trace{enable = Enable}] = emqx_trace:list(), ?assertEqual(Enable, true), ok = emqx_trace:update(Name, false), @@ -293,13 +262,10 @@ t_get_log_filename(_Config) -> Start = calendar:system_time_to_rfc3339(Now), End = calendar:system_time_to_rfc3339(Now + 2), Name = <<"name1">>, - ClientId = <<"test-device">>, - Packets = [atom_to_binary(E) || E <- ?PACKETS], Trace = [ {<<"name">>, Name}, - {<<"type">>, <<"clientid">>}, - {<<"clientid">>, ClientId}, - {<<"packets">>, Packets}, + {<<"type">>, <<"ip_address">>}, + {<<"ip_address">>, <<"127.0.0.1">>}, {<<"start_at">>, list_to_binary(Start)}, {<<"end_at">>, list_to_binary(End)} ], diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl index 1daab1520..0b2963af6 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -19,7 +19,7 @@ %% API -export([ list_trace/2 , create_trace/2 - , update_trace/2 + , disable_trace/2 , delete_trace/2 , clear_traces/2 , download_zip_log/2 @@ -52,11 +52,11 @@ func => clear_traces, descr => "clear all traces"}). --rest_api(#{name => update_trace, +-rest_api(#{name => disable_trace, method => 'PUT', - path => "/trace/:bin:name/:atom:operation", - func => update_trace, - descr => "diable/enable trace"}). + path => "/trace/:bin:name/stop", + func => disable_trace, + descr => "stop trace"}). -rest_api(#{name => download_zip_log, method => 'GET', @@ -82,8 +82,8 @@ delete_trace(Path, Params) -> clear_traces(Path, Params) -> return(emqx_trace_api:clear_traces(Path, Params)). -update_trace(Path, Params) -> - return(emqx_trace_api:update_trace(Path, Params)). +disable_trace(#{name := Name}, Params) -> + return(emqx_trace_api:update_trace(#{name => Name, operation => disable}, Params)). download_zip_log(Path, Params) -> case emqx_trace_api:download_zip_log(Path, Params) of diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl index fc786dbd0..36ceb8c49 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl @@ -52,14 +52,13 @@ t_http_test(_Config) -> %% create ErrorTrace = #{}, {ok, Error} = request_api(post, api_path("trace"), Header, ErrorTrace), - ?assertEqual(#{<<"message">> => <<"unknown field: {}">>, + ?assertEqual(#{<<"message">> => <<"name required">>, <<"code">> => <<"INCORRECT_PARAMS">>}, json(Error)), Name = <<"test-name">>, Trace = [ {<<"name">>, Name}, {<<"type">>, <<"topic">>}, - {<<"packets">>, [<<"PUBLISH">>]}, {<<"topic">>, <<"/x/y/z">>} ], @@ -71,14 +70,23 @@ t_http_test(_Config) -> ?assertEqual(Name, maps:get(<<"name">>, Data)), %% update - {ok, Update} = request_api(put, api_path("trace/test-name/disable"), Header, #{}), + {ok, Update} = request_api(put, api_path("trace/test-name/stop"), Header, #{}), ?assertEqual(#{<<"code">> => 0, <<"data">> => #{<<"enable">> => false, <<"name">> => <<"test-name">>}}, json(Update)), {ok, List1} = request_api(get, api_path("trace"), Header), #{<<"code">> := 0, <<"data">> := [Data1]} = json(List1), - ?assertEqual(false, maps:get(<<"enable">>, Data1)), + Node = atom_to_binary(node()), + ?assertMatch(#{ + <<"status">> := <<"stopped">>, + <<"name">> := <<"test-name">>, + <<"log_size">> := #{Node := _}, + <<"start_at">> := _, + <<"end_at">> := _, + <<"type">> := <<"topic">>, + <<"topic">> := <<"/x/y/z">> + }, Data1), %% delete {ok, Delete} = request_api(delete, api_path("trace/test-name"), Header), @@ -86,7 +94,7 @@ t_http_test(_Config) -> {ok, DeleteNotFound} = request_api(delete, api_path("trace/test-name"), Header), ?assertEqual(#{<<"code">> => <<"NOT_FOUND">>, - <<"message">> => <<"test-nameNOT FOUND">>}, json(DeleteNotFound)), + <<"message">> => <<"test-name NOT FOUND">>}, json(DeleteNotFound)), {ok, List2} = request_api(get, api_path("trace"), Header), ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(List2)), diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index 14617e2e1..bcf343432 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -126,8 +126,9 @@ subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> -spec(subscribe(emqx_topic:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok). subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) -> SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0), - _ = emqx_tracer:trace_subscribe(Topic, SubId, SubOpts), - case ets:member(?SUBOPTION, {SubPid = self(), Topic}) of + _ = emqx_trace:subscribe(Topic, SubId, SubOpts), + SubPid = self(), + case ets:member(?SUBOPTION, {SubPid, Topic}) of false -> %% New ok = emqx_broker_helper:register_sub(SubPid, SubId), do_subscribe(Topic, SubPid, with_subid(SubId, SubOpts)); @@ -172,7 +173,7 @@ unsubscribe(Topic) when is_binary(Topic) -> SubPid = self(), case ets:lookup(?SUBOPTION, {SubPid, Topic}) of [{_, SubOpts}] -> - emqx_tracer:trace_unsubscribe(Topic, SubOpts), + emqx_trace:unsubscribe(Topic, SubOpts), _ = emqx_broker_helper:reclaim_seq(Topic), do_unsubscribe(Topic, SubPid, SubOpts); [] -> ok @@ -195,7 +196,7 @@ do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> -spec(publish(emqx_types:message()) -> emqx_types:publish_result()). publish(Msg) when is_record(Msg, message) -> - _ = emqx_tracer:trace_publish(Msg), + _ = emqx_trace:publish(Msg), emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of #message{headers = #{allow_publish := false}} -> diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 4c49a1cb3..bad053845 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -275,7 +275,7 @@ take_ws_cookie(ClientInfo, ConnInfo) -> handle_in(?CONNECT_PACKET(), Channel = #channel{conn_state = connected}) -> handle_out(disconnect, ?RC_PROTOCOL_ERROR, Channel); -handle_in(?CONNECT_PACKET(ConnPkt), Channel) -> +handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) -> case pipeline([fun enrich_conninfo/2, fun run_conn_hooks/2, fun check_connect/2, @@ -285,6 +285,7 @@ handle_in(?CONNECT_PACKET(ConnPkt), Channel) -> fun auth_connect/2 ], ConnPkt, Channel#channel{conn_state = connecting}) of {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} -> + ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]), NChannel1 = NChannel#channel{ will_msg = emqx_packet:will_msg(NConnPkt), alias_maximum = init_alias_maximum(NConnPkt, ClientInfo) diff --git a/src/emqx_logger.erl b/src/emqx_logger.erl index a733b0d3a..9cf7050ea 100644 --- a/src/emqx_logger.erl +++ b/src/emqx_logger.erl @@ -18,6 +18,8 @@ -compile({no_auto_import, [error/1]}). +-elvis([{elvis_style, god_modules, disable}]). + %% Logs -export([ debug/1 , debug/2 @@ -64,10 +66,11 @@ id := logger:handler_id(), level := logger:level(), dst := logger_dst(), + filters := [{logger:filter_id(), logger:filter()}], status := started | stopped }). --define(stopped_handlers, {?MODULE, stopped_handlers}). +-define(STOPPED_HANDLERS, {?MODULE, stopped_handlers}). %%-------------------------------------------------------------------- %% APIs @@ -171,19 +174,19 @@ get_log_handlers() -> -spec(get_log_handlers(started | stopped) -> [logger_handler_info()]). get_log_handlers(started) -> - [log_hanlder_info(Conf, started) || Conf <- logger:get_handler_config()]; + [log_handler_info(Conf, started) || Conf <- logger:get_handler_config()]; get_log_handlers(stopped) -> - [log_hanlder_info(Conf, stopped) || Conf <- list_stopped_handler_config()]. + [log_handler_info(Conf, stopped) || Conf <- list_stopped_handler_config()]. -spec(get_log_handler(logger:handler_id()) -> logger_handler_info()). get_log_handler(HandlerId) -> case logger:get_handler_config(HandlerId) of {ok, Conf} -> - log_hanlder_info(Conf, started); + log_handler_info(Conf, started); {error, _} -> case read_stopped_handler_config(HandlerId) of error -> {error, {not_found, HandlerId}}; - {ok, Conf} -> log_hanlder_info(Conf, stopped) + {ok, Conf} -> log_handler_info(Conf, stopped) end end. @@ -245,21 +248,21 @@ parse_transform(AST, _Opts) -> %% Internal Functions %%-------------------------------------------------------------------- -log_hanlder_info(#{id := Id, level := Level, module := logger_std_h, - config := #{type := Type}}, Status) when +log_handler_info(#{id := Id, level := Level, module := logger_std_h, + filters := Filters, config := #{type := Type}}, Status) when Type =:= standard_io; Type =:= standard_error -> - #{id => Id, level => Level, dst => console, status => Status}; -log_hanlder_info(#{id := Id, level := Level, module := logger_std_h, - config := Config = #{type := file}}, Status) -> - #{id => Id, level => Level, status => Status, + #{id => Id, level => Level, dst => console, status => Status, filters => Filters}; +log_handler_info(#{id := Id, level := Level, module := logger_std_h, + filters := Filters, config := Config = #{type := file}}, Status) -> + #{id => Id, level => Level, status => Status, filters => Filters, dst => maps:get(file, Config, atom_to_list(Id))}; -log_hanlder_info(#{id := Id, level := Level, module := logger_disk_log_h, - config := #{file := Filename}}, Status) -> - #{id => Id, level => Level, dst => Filename, status => Status}; -log_hanlder_info(#{id := Id, level := Level, module := _OtherModule}, Status) -> - #{id => Id, level => Level, dst => unknown, status => Status}. +log_handler_info(#{id := Id, level := Level, module := logger_disk_log_h, + filters := Filters, config := #{file := Filename}}, Status) -> + #{id => Id, level => Level, dst => Filename, status => Status, filters => Filters}; +log_handler_info(#{id := Id, level := Level, filters := Filters}, Status) -> + #{id => Id, level => Level, dst => unknown, status => Status, filters => Filters}. %% set level for all log handlers in one command set_all_log_handlers_level(Level) -> @@ -281,29 +284,29 @@ rollback([{ID, Level} | List]) -> rollback([]) -> ok. save_stopped_handler_config(HandlerId, Config) -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> - persistent_term:put(?stopped_handlers, #{HandlerId => Config}); + persistent_term:put(?STOPPED_HANDLERS, #{HandlerId => Config}); ConfList -> - persistent_term:put(?stopped_handlers, ConfList#{HandlerId => Config}) + persistent_term:put(?STOPPED_HANDLERS, ConfList#{HandlerId => Config}) end. read_stopped_handler_config(HandlerId) -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> error; ConfList -> maps:find(HandlerId, ConfList) end. remove_stopped_handler_config(HandlerId) -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> ok; ConfList -> case maps:find(HandlerId, ConfList) of error -> ok; {ok, _} -> - persistent_term:put(?stopped_handlers, maps:remove(HandlerId, ConfList)) + persistent_term:put(?STOPPED_HANDLERS, maps:remove(HandlerId, ConfList)) end end. list_stopped_handler_config() -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> []; ConfList -> maps:values(ConfList) end. diff --git a/src/emqx_trace_handler.erl b/src/emqx_trace_handler.erl new file mode 100644 index 000000000..e4008405a --- /dev/null +++ b/src/emqx_trace_handler.erl @@ -0,0 +1,206 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_handler). + +-include("emqx.hrl"). +-include("logger.hrl"). + +-logger_header("[Tracer]"). + +%% APIs +-export([ running/0 + , install/3 + , install/4 + , uninstall/1 + , uninstall/2 + ]). + +%% For logger handler filters callbacks +-export([ filter_clientid/2 + , filter_topic/2 + , filter_ip_address/2 + ]). + +-type tracer() :: #{ + name := binary(), + type := clientid | topic | ip_address, + filter := emqx_types:clientid() | emqx_types:topic() | emqx_trace:ip_address() + }. + +-define(FORMAT, + {logger_formatter, #{ + template => [ + time, " [", level, "] ", + {clientid, + [{peername, [clientid, "@", peername, " "], [clientid, " "]}], + [{peername, [peername, " "], []}] + }, + msg, "\n" + ], + single_line => false, + max_size => unlimited, + depth => unlimited + }} +). + +-define(CONFIG(_LogFile_), #{ + type => halt, + file => _LogFile_, + max_no_bytes => 512 * 1024 * 1024, + overload_kill_enable => true, + overload_kill_mem_size => 50 * 1024 * 1024, + overload_kill_qlen => 20000, + %% disable restart + overload_kill_restart_after => infinity + }). + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +-spec install(Name :: binary() | list(), + Type :: clientid | topic | ip_address, + Filter ::emqx_types:clientid() | emqx_types:topic() | string(), + Level :: logger:level() | all, + LogFilePath :: string()) -> ok | {error, term()}. +install(Name, Type, Filter, Level, LogFile) -> + Who = #{type => Type, filter => ensure_bin(Filter), name => ensure_bin(Name)}, + install(Who, Level, LogFile). + +-spec install(Type :: clientid | topic | ip_address, + Filter ::emqx_types:clientid() | emqx_types:topic() | string(), + Level :: logger:level() | all, + LogFilePath :: string()) -> ok | {error, term()}. +install(Type, Filter, Level, LogFile) -> + install(Filter, Type, Filter, Level, LogFile). + +-spec install(tracer(), logger:level() | all, string()) -> ok | {error, term()}. +install(Who, all, LogFile) -> + install(Who, debug, LogFile); +install(Who, Level, LogFile) -> + PrimaryLevel = emqx_logger:get_primary_log_level(), + try logger:compare_levels(Level, PrimaryLevel) of + lt -> + {error, + io_lib:format( + "Cannot trace at a log level (~s) " + "lower than the primary log level (~s)", + [Level, PrimaryLevel] + )}; + _GtOrEq -> + install_handler(Who, Level, LogFile) + catch + error:badarg -> + {error, {invalid_log_level, Level}} + end. + +-spec uninstall(Type :: clientid | topic | ip_address, + Name :: binary() | list()) -> ok | {error, term()}. +uninstall(Type, Name) -> + HandlerId = handler_id(#{type => Type, name => ensure_bin(Name)}), + uninstall(HandlerId). + +-spec uninstall(HandlerId :: atom()) -> ok | {error, term()}. +uninstall(HandlerId) -> + Res = logger:remove_handler(HandlerId), + show_prompts(Res, HandlerId, "Stop trace"), + Res. + +%% @doc Return all running trace handlers information. +-spec running() -> + [ + #{ + name => binary(), + type => topic | clientid | ip_address, + id => atom(), + filter => emqx_types:topic() | emqx_types:clienetid() | emqx_trace:ip_address(), + level => logger:level(), + dst => file:filename() | console | unknown + } + ]. +running() -> + lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). + +-spec filter_clientid(logger:log_event(), string()) -> logger:log_event() | ignore. +filter_clientid(#{meta := #{clientid := ClientId}} = Log, ClientId) -> Log; +filter_clientid(_Log, _ExpectId) -> ignore. + +-spec filter_topic(logger:log_event(), string()) -> logger:log_event() | ignore. +filter_topic(#{meta := #{topic := Topic}} = Log, TopicFilter) -> + case emqx_topic:match(Topic, TopicFilter) of + true -> Log; + false -> ignore + end; +filter_topic(_Log, _ExpectId) -> ignore. + +-spec filter_ip_address(logger:log_event(), string()) -> logger:log_event() | ignore. +filter_ip_address(#{meta := #{peername := Peername}} = Log, IP) -> + case lists:prefix(IP, Peername) of + true -> Log; + false -> ignore + end; +filter_ip_address(_Log, _ExpectId) -> ignore. + +install_handler(Who, Level, LogFile) -> + HandlerId = handler_id(Who), + Config = #{ + level => Level, + formatter => ?FORMAT, + filter_default => stop, + filters => filters(Who), + config => ?CONFIG(LogFile) + }, + Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), + show_prompts(Res, Who, "Start trace"), + Res. + +filters(#{type := clientid, filter := Filter}) -> + [{clientid, {fun ?MODULE:filter_clientid/2, ensure_list(Filter)}}]; +filters(#{type := topic, filter := Filter}) -> + [{topic, {fun ?MODULE:filter_topic/2, ensure_bin(Filter)}}]; +filters(#{type := ip_address, filter := Filter}) -> + [{ip_address, {fun ?MODULE:filter_ip_address/2, ensure_list(Filter)}}]. + +filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> + Init = #{id => Id, level => Level, dst => Dst}, + case Filters of + [{topic, {_FilterFun, Filter}}] -> + <<"trace_topic_", Name/binary>> = atom_to_binary(Id), + [Init#{type => topic, filter => Filter, name => Name} | Acc]; + [{clientid, {_FilterFun, Filter}}] -> + <<"trace_clientid_", Name/binary>> = atom_to_binary(Id), + [Init#{type => clientid, filter => Filter, name => Name} | Acc]; + [{ip_address, {_FilterFun, Filter}}] -> + <<"trace_ip_address_", Name/binary>> = atom_to_binary(Id), + [Init#{type => ip_address, filter => Filter, name => Name} | Acc]; + _ -> + Acc + end. + +handler_id(#{type := Type, name := Name}) -> + binary_to_atom(<<"trace_", (atom_to_binary(Type))/binary, "_", Name/binary>>). + +ensure_bin(List) when is_list(List) -> iolist_to_binary(List); +ensure_bin(Bin) when is_binary(Bin) -> Bin. + +ensure_list(Bin) when is_binary(Bin) -> binary_to_list(Bin); +ensure_list(List) when is_list(List) -> List. + +show_prompts(ok, Who, Msg) -> + ?LOG(info, Msg ++ " ~p " ++ "successfully~n", [Who]); +show_prompts({error, Reason}, Who, Msg) -> + ?LOG(error, Msg ++ " ~p " ++ "failed by ~p~n", [Who, Reason]). diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl deleted file mode 100644 index e563c8eab..000000000 --- a/src/emqx_tracer.erl +++ /dev/null @@ -1,261 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_tracer). - --include("emqx.hrl"). --include("logger.hrl"). - --logger_header("[Tracer]"). - -%% APIs --export([ trace_publish/1 - , trace_subscribe/3 - , trace_unsubscribe/2 - , start_trace/3 - , start_trace/4 - , lookup_traces/0 - , stop_trace/3 - , stop_trace/2 - ]). - --ifdef(TEST). --export([is_match/3]). --endif. - --type(label() :: 'CONNECT' | 'CONNACK' | 'PUBLISH' | 'PUBACK' | 'PUBREC' | - 'PUBREL' | 'PUBCOMP' | 'SUBSCRIBE' | 'SUBACK' | 'UNSUBSCRIBE' | - 'UNSUBACK' | 'PINGREQ' | 'PINGRESP' | 'DISCONNECT' | 'AUTH'). - --type(tracer() :: #{name := binary(), - type := clientid | topic, - clientid => emqx_types:clientid(), - topic => emqx_types:topic(), - labels := [label()]}). - --define(TRACER, ?MODULE). --define(FORMAT, {logger_formatter, - #{template => - [time, " [", level, "] ", - {clientid, - [{peername, - [clientid, "@", peername, " "], - [clientid, " "]}], - [{peername, - [peername, " "], - []}]}, - msg, "\n"], - single_line => false - }}). --define(TOPIC_COMBINATOR, <<"_trace_topic_">>). --define(CLIENTID_COMBINATOR, <<"_trace_clientid_">>). --define(TOPIC_TRACE_ID(T, N), - binary_to_atom(<<(N)/binary, ?TOPIC_COMBINATOR/binary, (T)/binary>>)). --define(CLIENT_TRACE_ID(C, N), - binary_to_atom(<<(N)/binary, ?CLIENTID_COMBINATOR/binary, (C)/binary>>)). --define(TOPIC_TRACE(T, N, M), {topic, T, N, M}). --define(CLIENT_TRACE(C, N, M), {clientid, C, N, M}). --define(TOPIC_TRACE(T, N), {topic, T, N}). --define(CLIENT_TRACE(C, N), {clientid, C, N}). - --define(IS_LOG_LEVEL(L), - L =:= emergency orelse - L =:= alert orelse - L =:= critical orelse - L =:= error orelse - L =:= warning orelse - L =:= notice orelse - L =:= info orelse - L =:= debug). - -%%------------------------------------------------------------------------------ -%% APIs -%%------------------------------------------------------------------------------ -trace_publish(#message{topic = <<"$SYS/", _/binary>>}) -> - %% Do not trace '$SYS' publish - ignore; -trace_publish(#message{from = From, topic = Topic, payload = Payload}) - when is_binary(From); is_atom(From) -> - emqx_logger:info(#{topic => Topic, - mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} }, - "PUBLISH to ~s: ~0p", [Topic, Payload]). - -trace_subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore; -trace_subscribe(Topic, SubId, SubOpts) -> - emqx_logger:info(#{topic => Topic, - mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, - "~ts SUBSCRIBE ~ts: Options: ~0p", [SubId, Topic, SubOpts]). - -trace_unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; -trace_unsubscribe(Topic, SubOpts) -> - emqx_logger:info(#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, - "~ts UNSUBSCRIBE ~ts: Options: ~0p", - [maps:get(subid, SubOpts, ""), Topic, SubOpts]). - --spec(start_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic(), - logger:level() | all, string()) -> ok | {error, term()}). -start_trace(clientid, ClientId0, Level, LogFile) -> - ClientId = ensure_bin(ClientId0), - Who = #{type => clientid, clientid => ClientId, name => ClientId, labels => []}, - start_trace(Who, Level, LogFile); -start_trace(topic, Topic0, Level, LogFile) -> - Topic = ensure_bin(Topic0), - Who = #{type => topic, topic => Topic, name => Topic, labels => []}, - start_trace(Who, Level, LogFile). - -%% @doc Start to trace clientid or topic. --spec(start_trace(tracer(), logger:level() | all, string()) -> ok | {error, term()}). -start_trace(Who, all, LogFile) -> - start_trace(Who, debug, LogFile); -start_trace(Who, Level, LogFile) -> - case ?IS_LOG_LEVEL(Level) of - true -> - PrimaryLevel = emqx_logger:get_primary_log_level(), - try logger:compare_levels(Level, PrimaryLevel) of - lt -> - {error, - io_lib:format("Cannot trace at a log level (~s) " - "lower than the primary log level (~s)", - [Level, PrimaryLevel])}; - _GtOrEq -> - install_trace_handler(Who, Level, LogFile) - catch - _:Error -> - {error, Error} - end; - false -> {error, {invalid_log_level, Level}} - end. - --spec(stop_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic()) -> - ok | {error, term()}). -stop_trace(Type, ClientIdOrTopic) -> - stop_trace(Type, ClientIdOrTopic, ClientIdOrTopic). - -%% @doc Stop tracing clientid or topic. --spec(stop_trace(clientid | topic, emqx_types:clientid() | emqx_types:topic(), binary()) -> - ok | {error, term()}). -stop_trace(clientid, ClientId, Name) -> - Who = #{type => clientid, clientid => ensure_bin(ClientId), name => ensure_bin(Name)}, - uninstall_trance_handler(Who); -stop_trace(topic, Topic, Name) -> - Who = #{type => topic, topic => ensure_bin(Topic), name => ensure_bin(Name)}, - uninstall_trance_handler(Who). - -%% @doc Lookup all traces --spec(lookup_traces() -> [#{ type => topic | clientid, - name => binary(), - topic => emqx_types:topic(), - level => logger:level(), - dst => file:filename() | console | unknown - }]). -lookup_traces() -> - lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). - -install_trace_handler(Who, Level, LogFile) -> - case logger:add_handler(handler_id(Who), logger_disk_log_h, - #{level => Level, - formatter => ?FORMAT, - config => #{type => halt, file => LogFile}, - filter_default => stop, - filters => [{meta_key_filter, - {fun filter_by_meta_key/2, Who}}]}) - of - ok -> - ?LOG(info, "Start trace for ~p", [Who]), - ok; - {error, Reason} -> - ?LOG(error, "Start trace for ~p failed, error: ~p", [Who, Reason]), - {error, Reason} - end. - -uninstall_trance_handler(Who) -> - case logger:remove_handler(handler_id(Who)) of - ok -> - ?LOG(info, "Stop trace for ~p", [Who]), - ok; - {error, Reason} -> - ?LOG(error, "Stop trace for ~p failed, error: ~p", [Who, Reason]), - {error, Reason} - end. - -filter_traces(#{id := Id, level := Level, dst := Dst}, Acc) -> - IdStr = atom_to_binary(Id), - case binary:split(IdStr, [?TOPIC_COMBINATOR]) of - [Name, Topic] -> - [#{ type => topic, - name => Name, - topic => Topic, - level => Level, - dst => Dst} | Acc]; - _ -> - case binary:split(IdStr, [?CLIENTID_COMBINATOR]) of - [Name, ClientId] -> - [#{ type => clientid, - name => Name, - clientid => ClientId, - level => Level, - dst => Dst} | Acc]; - _ -> Acc - end - end. - -%% Plan to support topic_and_client type, so we have type field. -handler_id(#{type := topic, topic := Topic, name := Name}) -> - ?TOPIC_TRACE_ID(format(Topic), format(Name)); -handler_id(#{type := clientid, clientid := ClientId, name := Name}) -> - ?CLIENT_TRACE_ID(format(ClientId), format(Name)). - -filter_by_meta_key(#{meta := Meta, level := Level} = Log, Context) -> - case is_match(Context, Meta, Level) of - true -> Log; - false -> ignore - end. - -%% When the log level is higher than debug and clientid/topic is match, -%% it will be logged without judging the content inside the labels. -%% When the log level is debug, in addition to the matched clientid/topic, -%% you also need to determine whether the label is in the labels -is_match(#{type := clientid, clientid := ExpectId, labels := Labels}, - #{clientid := RealId} = Meta, - Level) -> - is_match(ExpectId =:= iolist_to_binary(RealId), Level, Meta, Labels); -is_match(#{type := topic, topic := TopicFilter, labels := Labels}, - #{topic := Topic} = Meta, Level) -> - is_match(emqx_topic:match(Topic, TopicFilter), Level, Meta, Labels); -is_match(_, _, _) -> - false. - -is_match(true, debug, Meta, Labels) -> is_match_labels(Meta, Labels); -is_match(Boolean, _, _Meta, _Labels) -> Boolean. - -is_match_labels(#{trace_label := 'ALL'}, _Context) -> true; -is_match_labels(_, []) -> true; -is_match_labels(#{trace_label := Packet}, Context) -> - lists:member(Packet, Context); -is_match_labels(_, _) -> false. - -format(List)when is_list(List) -> - format(list_to_binary(List)); -format(Atom)when is_atom(Atom) -> - format(atom_to_list(Atom)); -format(Bin0)when is_binary(Bin0) -> - case byte_size(Bin0) of - Size when Size =< 200 -> Bin0; - _ -> emqx_misc:bin2hexstr_a_f_upper(Bin0) - end. - -ensure_bin(List) when is_list(List) -> iolist_to_binary(List); -ensure_bin(Bin) when is_binary(Bin) -> Bin. diff --git a/test/emqx_trace_handler_SUITE.erl b/test/emqx_trace_handler_SUITE.erl new file mode 100644 index 000000000..5e909c45b --- /dev/null +++ b/test/emqx_trace_handler_SUITE.erl @@ -0,0 +1,191 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_handler_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). +-define(CLIENT, [{host, "localhost"}, + {clientid, <<"client">>}, + {username, <<"testuser">>}, + {password, <<"pass">>} + ]). + +all() -> [t_trace_clientid, t_trace_topic, t_trace_ip_address]. + +init_per_suite(Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([]). + +init_per_testcase(t_trace_clientid, Config) -> + Config; +init_per_testcase(_Case, Config) -> + ok = emqx_logger:set_log_level(debug), + _ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()], + Config. + +end_per_testcase(_Case, _Config) -> + ok = emqx_logger:set_log_level(warning), + ok. + +t_trace_clientid(_Config) -> + %% Start tracing + emqx_logger:set_log_level(error), + {error, _} = emqx_trace_handler:install(clientid, <<"client">>, debug, "tmp/client.log"), + emqx_logger:set_log_level(debug), + %% add list clientid + ok = emqx_trace_handler:install(clientid, "client", debug, "tmp/client.log"), + ok = emqx_trace_handler:install(clientid, <<"client2">>, all, "tmp/client2.log"), + ok = emqx_trace_handler:install(clientid, <<"client3">>, all, "tmp/client3.log"), + {error, {invalid_log_level, bad_level}} = + emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"), + {error, {handler_not_added, {file_error, ".", eisdir}}} = + emqx_trace_handler:install(clientid, <<"client5">>, debug, "."), + ct:sleep(100), + + %% Verify the tracing file exits + ?assert(filelib:is_regular("tmp/client.log")), + ?assert(filelib:is_regular("tmp/client2.log")), + ?assert(filelib:is_regular("tmp/client3.log")), + + %% Get current traces + ?assertEqual([#{type => clientid, filter => "client", name => <<"client">>, + id => trace_clientid_client, level => debug, dst => "tmp/client.log"}, + #{type => clientid, filter => "client2", name => <<"client2">>, + id => trace_clientid_client2, level => debug, dst => "tmp/client2.log"}, + #{type => clientid, filter => "client3", name => <<"client3">>, + id => trace_clientid_client3, level => debug, dst => "tmp/client3.log"} + ], emqx_trace_handler:running()), + + %% Client with clientid = "client" publishes a "hi" message to "a/b/c". + {ok, T} = emqtt:start_link(?CLIENT), + emqtt:connect(T), + emqtt:publish(T, <<"a/b/c">>, <<"hi">>), + emqtt:ping(T), + ct:sleep(200), + + %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". + {ok, Bin} = file:read_file("tmp/client.log"), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"CONNECT">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"CONNACK">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PUBLISH">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PINGREQ">>])), + ?assert(filelib:file_size("tmp/client2.log") == 0), + + %% Stop tracing + ok = emqx_trace_handler:uninstall(clientid, <<"client">>), + ok = emqx_trace_handler:uninstall(clientid, <<"client2">>), + ok = emqx_trace_handler:uninstall(clientid, <<"client3">>), + + emqtt:disconnect(T), + ?assertEqual([], emqx_trace_handler:running()). + +t_trace_topic(_Config) -> + {ok, T} = emqtt:start_link(?CLIENT), + emqtt:connect(T), + + %% Start tracing + emqx_logger:set_log_level(debug), + ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), + ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), + ct:sleep(100), + + %% Verify the tracing file exits + ?assert(filelib:is_regular("tmp/topic_trace_x.log")), + ?assert(filelib:is_regular("tmp/topic_trace_y.log")), + + %% Get current traces + ?assertEqual([#{type => topic, filter => <<"x/#">>, id => 'trace_topic_x/#', + level => debug, dst => "tmp/topic_trace_x.log", name => <<"x/#">>}, + #{type => topic, filter => <<"y/#">>, id => 'trace_topic_y/#', + name => <<"y/#">>, level => debug, dst => "tmp/topic_trace_y.log"} + ], + emqx_trace_handler:running()), + + %% Client with clientid = "client" publishes a "hi" message to "x/y/z". + emqtt:publish(T, <<"x/y/z">>, <<"hi1">>), + emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), + emqtt:subscribe(T, <<"x/y/z">>), + emqtt:unsubscribe(T, <<"x/y/z">>), + ct:sleep(200), + + {ok, Bin} = file:read_file("tmp/topic_trace_x.log"), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi2">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PUBLISH">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"SUBSCRIBE">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"UNSUBSCRIBE">>])), + ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0), + + %% Stop tracing + ok = emqx_trace_handler:uninstall(topic, <<"x/#">>), + ok = emqx_trace_handler:uninstall(topic, <<"y/#">>), + {error, _Reason} = emqx_trace_handler:uninstall(topic, <<"z/#">>), + ?assertEqual([], emqx_trace_handler:running()), + emqtt:disconnect(T). + +t_trace_ip_address(_Config) -> + {ok, T} = emqtt:start_link(?CLIENT), + emqtt:connect(T), + + %% Start tracing + ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"), + ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"), + ct:sleep(100), + + %% Verify the tracing file exits + ?assert(filelib:is_regular("tmp/ip_trace_x.log")), + ?assert(filelib:is_regular("tmp/ip_trace_y.log")), + + %% Get current traces + ?assertEqual([#{type => ip_address, filter => "127.0.0.1", + id => 'trace_ip_address_127.0.0.1', name => <<"127.0.0.1">>, + level => debug, dst => "tmp/ip_trace_x.log"}, + #{type => ip_address, filter => "192.168.1.1", + id => 'trace_ip_address_192.168.1.1', name => <<"192.168.1.1">>, + level => debug, dst => "tmp/ip_trace_y.log"} + ], + emqx_trace_handler:running()), + + %% Client with clientid = "client" publishes a "hi" message to "x/y/z". + emqtt:publish(T, <<"x/y/z">>, <<"hi1">>), + emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), + emqtt:subscribe(T, <<"x/y/z">>), + emqtt:unsubscribe(T, <<"x/y/z">>), + ct:sleep(200), + + {ok, Bin} = file:read_file("tmp/ip_trace_x.log"), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi2">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PUBLISH">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"SUBSCRIBE">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"UNSUBSCRIBE">>])), + ?assert(filelib:file_size("tmp/ip_trace_y.log") =:= 0), + + %% Stop tracing + ok = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.1">>), + ok = emqx_trace_handler:uninstall(ip_address, <<"192.168.1.1">>), + {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>), + emqtt:disconnect(T), + ?assertEqual([], emqx_trace_handler:running()). diff --git a/test/emqx_tracer_SUITE.erl b/test/emqx_tracer_SUITE.erl deleted file mode 100644 index 5f7105774..000000000 --- a/test/emqx_tracer_SUITE.erl +++ /dev/null @@ -1,171 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_tracer_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --include_lib("common_test/include/ct.hrl"). --define(CLIENT, [{host, "localhost"}, - {clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]). - -all() -> [t_trace_clientid, t_trace_topic, t_is_match]. - -init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). - -t_trace_clientid(_Config) -> - {ok, T} = emqtt:start_link(?CLIENT), - emqtt:connect(T), - - %% Start tracing - emqx_logger:set_log_level(error), - {error, _} = emqx_tracer:start_trace(clientid, <<"client">>, debug, "tmp/client.log"), - emqx_logger:set_log_level(debug), - %% add list clientid - ok = emqx_tracer:start_trace(clientid, "client", debug, "tmp/client.log"), - ok = emqx_tracer:start_trace(clientid, <<"client2">>, all, "tmp/client2.log"), - ok = emqx_tracer:start_trace(clientid, <<"client3">>, all, "tmp/client3.log"), - {error, {invalid_log_level, bad_level}} = - emqx_tracer:start_trace(clientid, <<"client4">>, bad_level, "tmp/client4.log"), - {error, {handler_not_added, {file_error, ".", eisdir}}} = - emqx_tracer:start_trace(clientid, <<"client5">>, debug, "."), - ct:sleep(100), - - %% Verify the tracing file exits - ?assert(filelib:is_regular("tmp/client.log")), - ?assert(filelib:is_regular("tmp/client2.log")), - ?assert(filelib:is_regular("tmp/client3.log")), - - %% Get current traces - ?assertEqual([#{type => clientid, clientid => <<"client">>, - name => <<"client">>, level => debug, dst => "tmp/client.log"}, - #{type => clientid, clientid => <<"client2">>, - name => <<"client2">>, level => debug, dst => "tmp/client2.log"}, - #{type => clientid, clientid => <<"client3">>, - name => <<"client3">>, level => debug, dst => "tmp/client3.log"} - ], emqx_tracer:lookup_traces()), - - %% set the overall log level to debug - emqx_logger:set_log_level(debug), - - %% Client with clientid = "client" publishes a "hi" message to "a/b/c". - emqtt:publish(T, <<"a/b/c">>, <<"hi">>), - ct:sleep(200), - - %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". - ?assert(filelib:file_size("tmp/client.log") > 0), - ?assert(filelib:file_size("tmp/client2.log") == 0), - - %% Stop tracing - ok = emqx_tracer:stop_trace(clientid, <<"client">>), - ok = emqx_tracer:stop_trace(clientid, <<"client2">>), - ok = emqx_tracer:stop_trace(clientid, <<"client3">>), - emqtt:disconnect(T), - - emqx_logger:set_log_level(warning). - -t_trace_topic(_Config) -> - {ok, T} = emqtt:start_link(?CLIENT), - emqtt:connect(T), - - %% Start tracing - emqx_logger:set_log_level(debug), - ok = emqx_tracer:start_trace(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), - ok = emqx_tracer:start_trace(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), - ct:sleep(100), - - %% Verify the tracing file exits - ?assert(filelib:is_regular("tmp/topic_trace_x.log")), - ?assert(filelib:is_regular("tmp/topic_trace_y.log")), - - %% Get current traces - ?assertEqual([#{type => topic, topic => <<"x/#">>, name => <<"x/#">>, - level => debug, dst => "tmp/topic_trace_x.log"}, - #{type => topic, topic => <<"y/#">>, name => <<"y/#">>, - level => debug, dst => "tmp/topic_trace_y.log"}], - emqx_tracer:lookup_traces()), - - %% set the overall log level to debug - emqx_logger:set_log_level(debug), - - %% Client with clientid = "client" publishes a "hi" message to "x/y/z". - emqtt:publish(T, <<"x/y/z">>, <<"hi1">>), - emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), - ct:sleep(200), - - ?assert(filelib:file_size("tmp/topic_trace_x.log") > 0), - ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0), - - %% Stop tracing - ok = emqx_tracer:stop_trace(topic, <<"x/#">>), - ok = emqx_tracer:stop_trace(topic, <<"y/#">>), - {error, _Reason} = emqx_tracer:stop_trace(topic, <<"z/#">>), - emqtt:disconnect(T), - - emqx_logger:set_log_level(warning). - -t_is_match(_Config) -> - ClientId = <<"test">>, - ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, - #{clientid => ClientId}, warning)), - ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, - #{clientid => ClientId}, debug)), - ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, - labels => ['PUBLISH']}, #{clientid => ClientId}, debug)), - ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, - #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), - ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['PUBLISH']}, - #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), - ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['SUBACK']}, - #{clientid => ClientId, trace_label => 'PUBLISH'}, debug)), - ?assert(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => ['SUBACK']}, - #{clientid => ClientId, trace_label => 'ALL'}, debug)), - ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, - #{clientid => <<"Bad">>}, warning)), - ?assertNot(emqx_tracer:is_match(#{clientid => ClientId, type => clientid, labels => []}, - #{clientid => <<"Bad">>, trace_label => 'PUBLISH'}, debug)), - - Topic = <<"/test/#">>, - ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, - #{topic => <<"/test/1">>}, warning)), - ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, - #{topic => <<"/test/1/2">>}, debug)), - ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['SUBSCRIBE']}, - #{topic => <<"/test/1/2">>}, debug)), - ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, - #{topic => <<"/test/3">>, trace_label => 'PUBLISH'}, debug)), - ?assert(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['PUBLISH']}, - #{topic => <<"/test/398/">>, trace_label => 'PUBLISH'}, debug)), - ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['SUBACK']}, - #{topic => <<"/test/1/xy/y">>, trace_label => 'PUBLISH'}, debug)), - - ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => ['PUBLISH']}, - #{topic => <<"/t1est/398/">>, trace_label => 'PUBLISH'}, debug)), - ?assertNot(emqx_tracer:is_match(#{type => topic, topic => Topic, labels => []}, - #{topic => <<"/t1est/1/xy/y">>, trace_label => 'PUBLISH'}, debug)), - ok. From a91f975dc2a77d8fc9a37095208cfcba3c284954 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 22 Nov 2021 18:08:48 +0800 Subject: [PATCH 087/363] fix: make sure keepalive only 0~65535 (#6232) --- apps/emqx_management/src/emqx_mgmt.erl | 6 ++++-- apps/emqx_management/src/emqx_mgmt_api_clients.erl | 1 + apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 9 +++++++++ src/emqx_keepalive.erl | 14 +++++++++++++- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 93f24cd37..61c32613b 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -274,8 +274,10 @@ set_ratelimit_policy(ClientId, Policy) -> set_quota_policy(ClientId, Policy) -> call_client(ClientId, {quota, Policy}). -set_keepalive(ClientId, Interval) -> - call_client(ClientId, {keepalive, Interval}). +set_keepalive(ClientId, Interval)when Interval >= 0 andalso Interval =< 65535 -> + call_client(ClientId, {keepalive, Interval}); +set_keepalive(_ClientId, _Interval) -> + {error, ?ERROR2, <<"mqtt3.1.1 specification: keepalive must between 0~65535">>}. %% @private call_client(ClientId, Req) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 5aacc7895..a25cc9ff5 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -261,6 +261,7 @@ set_keepalive(#{clientid := ClientId}, Params) -> case emqx_mgmt:set_keepalive(emqx_mgmt_util:urldecode(ClientId), Interval) of ok -> minirest:return(); {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Code, Reason} -> minirest:return({error, Code, Reason}); {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index f3bc42f3a..a3eea2ab3 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -657,6 +657,15 @@ t_keepalive(_Config) -> {ok, _} = emqtt:connect(C1), {ok, Ok} = request_api(put, Path, "interval=5", AuthHeader, [#{}]), ?assertEqual("{\"code\":0}", Ok), + [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), + #{conninfo := #{keepalive := Keepalive}} = emqx_connection:info(Pid), + ?assertEqual(5, Keepalive), + {ok, Error1} = request_api(put, Path, "interval=-1", AuthHeader, [#{}]), + {ok, Error2} = request_api(put, Path, "interval=65536", AuthHeader, [#{}]), + ErrMsg = #{<<"code">> => 102, + <<"message">> => <<"mqtt3.1.1 specification: keepalive must between 0~65535">>}, + ?assertEqual(ErrMsg, jiffy:decode(Error1, [return_maps])), + ?assertEqual(Error1, Error2), emqtt:disconnect(C1), application:stop(emqx_dashboard), ok. diff --git a/src/emqx_keepalive.erl b/src/emqx_keepalive.erl index 216999acc..bef0714f9 100644 --- a/src/emqx_keepalive.erl +++ b/src/emqx_keepalive.erl @@ -73,7 +73,19 @@ check(NewVal, KeepAlive = #keepalive{statval = OldVal, true -> {error, timeout} end. +%% from mqtt-v3.1.1 specific +%% A Keep Alive value of zero (0) has the effect of turning off the keep alive mechanism. +%% This means that, in this case, the Server is not required +%% to disconnect the Client on the grounds of inactivity. +%% Note that a Server is permitted to disconnect a Client that it determines +%% to be inactive or non-responsive at any time, +%% regardless of the Keep Alive value provided by that Client. +%% Non normative comment +%%The actual value of the Keep Alive is application specific; +%% typically this is a few minutes. +%% The maximum value is (65535s) 18 hours 12 minutes and 15 seconds. + %% @doc Update keepalive's interval -spec(set(interval, non_neg_integer(), keepalive()) -> keepalive()). -set(interval, Interval, KeepAlive) -> +set(interval, Interval, KeepAlive) when Interval >= 0 andalso Interval =< 65535000 -> KeepAlive#keepalive{interval = Interval}. From 30fb9dd7ae8c2684170155295eaf6b00335a353d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 22 Nov 2021 20:40:56 +0800 Subject: [PATCH 088/363] fix: name must be printable unicode and len < 256 --- .../src/emqx_trace/emqx_trace.erl | 28 +++--- src/emqx_trace_handler.erl | 90 +++++++++++-------- test/emqx_trace_handler_SUITE.erl | 32 +++---- 3 files changed, 86 insertions(+), 64 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index eb41703fa..2190e5bab 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -128,7 +128,7 @@ create(Trace) -> {error, Reason} -> {error, Reason} end; false -> - {error, "The number of traces created has reached the maximum" + {error, "The number of traces created has reache the maximum" " please delete the useless ones first"} end. @@ -379,11 +379,16 @@ to_trace(TraceParam) -> {error, "type=[topic,clientid,ip_address] required"}; {ok, #?TRACE{filter = undefined}} -> {error, "topic/clientid/ip_address filter required"}; - {ok, TraceRec0} -> - case fill_default(TraceRec0) of - #?TRACE{start_at = Start, end_at = End} when End =< Start -> - {error, "failed by start_at >= end_at"}; - TraceRec -> {ok, TraceRec} + {ok, TraceRec0 = #?TRACE{name = Name, type = Type}} -> + case emqx_trace_handler:handler_id(Name, Type) of + {ok, _} -> + case fill_default(TraceRec0) of + #?TRACE{start_at = Start, end_at = End} when End =< Start -> + {error, "failed by start_at >= end_at"}; + TraceRec -> {ok, TraceRec} + end; + {error, Reason} -> + {error, Reason} end end. @@ -403,10 +408,13 @@ fill_default(Trace) -> Trace. to_trace([], Rec) -> {ok, Rec}; to_trace([{name, Name} | Trace], Rec) -> - case binary:match(Name, [<<"/">>], []) of - nomatch when byte_size(Name) < 200 -> to_trace(Trace, Rec#?TRACE{name = Name}); - nomatch -> {error, "name(latin1) length must < 200"}; - _ -> {error, "name cannot contain /"} + case io_lib:printable_unicode_list(binary_to_list(Name)) of + true -> + case binary:match(Name, [<<"/">>], []) of + nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); + _ -> {error, "name cannot contain /"} + end; + false -> {error, "name must printable unicode"} end; to_trace([{type, Type} | Trace], Rec) -> case lists:member(Type, [<<"clientid">>, <<"topic">>, <<"ip_address">>]) of diff --git a/src/emqx_trace_handler.erl b/src/emqx_trace_handler.erl index e4008405a..7006fc815 100644 --- a/src/emqx_trace_handler.erl +++ b/src/emqx_trace_handler.erl @@ -35,6 +35,8 @@ , filter_ip_address/2 ]). +-export([handler_id/2]). + -type tracer() :: #{ name := binary(), type := clientid | topic | ip_address, @@ -111,8 +113,10 @@ install(Who, Level, LogFile) -> -spec uninstall(Type :: clientid | topic | ip_address, Name :: binary() | list()) -> ok | {error, term()}. uninstall(Type, Name) -> - HandlerId = handler_id(#{type => Type, name => ensure_bin(Name)}), - uninstall(HandlerId). + case handler_id(ensure_bin(Name), Type) of + {ok, HandlerId} -> uninstall(HandlerId); + {error, Reason} -> {error, Reason} + end. -spec uninstall(HandlerId :: atom()) -> ok | {error, term()}. uninstall(HandlerId) -> @@ -135,64 +139,74 @@ uninstall(HandlerId) -> running() -> lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). --spec filter_clientid(logger:log_event(), string()) -> logger:log_event() | ignore. -filter_clientid(#{meta := #{clientid := ClientId}} = Log, ClientId) -> Log; +-spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log; filter_clientid(_Log, _ExpectId) -> ignore. --spec filter_topic(logger:log_event(), string()) -> logger:log_event() | ignore. -filter_topic(#{meta := #{topic := Topic}} = Log, TopicFilter) -> +-spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) -> case emqx_topic:match(Topic, TopicFilter) of true -> Log; false -> ignore end; filter_topic(_Log, _ExpectId) -> ignore. --spec filter_ip_address(logger:log_event(), string()) -> logger:log_event() | ignore. -filter_ip_address(#{meta := #{peername := Peername}} = Log, IP) -> +-spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) -> case lists:prefix(IP, Peername) of true -> Log; false -> ignore end; filter_ip_address(_Log, _ExpectId) -> ignore. -install_handler(Who, Level, LogFile) -> - HandlerId = handler_id(Who), - Config = #{ - level => Level, - formatter => ?FORMAT, - filter_default => stop, - filters => filters(Who), - config => ?CONFIG(LogFile) - }, - Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), - show_prompts(Res, Who, "Start trace"), - Res. +install_handler(Who = #{name := Name, type := Type}, Level, LogFile) -> + case handler_id(Name, Type) of + {ok, HandlerId} -> + Config = #{ + level => Level, + formatter => ?FORMAT, + filter_default => stop, + filters => filters(Who), + config => ?CONFIG(LogFile) + }, + Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), + show_prompts(Res, Who, "Start trace"), + Res; + {error, _Reason} = Error -> + show_prompts(Error, Who, "Start trace"), + Error + end. -filters(#{type := clientid, filter := Filter}) -> - [{clientid, {fun ?MODULE:filter_clientid/2, ensure_list(Filter)}}]; -filters(#{type := topic, filter := Filter}) -> - [{topic, {fun ?MODULE:filter_topic/2, ensure_bin(Filter)}}]; -filters(#{type := ip_address, filter := Filter}) -> - [{ip_address, {fun ?MODULE:filter_ip_address/2, ensure_list(Filter)}}]. +filters(#{type := clientid, filter := Filter, name := Name}) -> + [{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}]; +filters(#{type := topic, filter := Filter, name := Name}) -> + [{topic, {fun ?MODULE:filter_topic/2, {ensure_bin(Filter), Name}}}]; +filters(#{type := ip_address, filter := Filter, name := Name}) -> + [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}]. filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> Init = #{id => Id, level => Level, dst => Dst}, case Filters of - [{topic, {_FilterFun, Filter}}] -> - <<"trace_topic_", Name/binary>> = atom_to_binary(Id), - [Init#{type => topic, filter => Filter, name => Name} | Acc]; - [{clientid, {_FilterFun, Filter}}] -> - <<"trace_clientid_", Name/binary>> = atom_to_binary(Id), - [Init#{type => clientid, filter => Filter, name => Name} | Acc]; - [{ip_address, {_FilterFun, Filter}}] -> - <<"trace_ip_address_", Name/binary>> = atom_to_binary(Id), - [Init#{type => ip_address, filter => Filter, name => Name} | Acc]; + [{Type, {_FilterFun, {Filter, Name}}}] when + Type =:= topic orelse + Type =:= clientid orelse + Type =:= ip_address -> + [Init#{type => Type, filter => Filter, name => Name} | Acc]; _ -> Acc end. -handler_id(#{type := Type, name := Name}) -> - binary_to_atom(<<"trace_", (atom_to_binary(Type))/binary, "_", Name/binary>>). +handler_id(Name, Type) -> + TypeBin = atom_to_binary(Type), + case io_lib:printable_unicode_list(binary_to_list(Name)) of + true when byte_size(Name) < 256 -> + NameHash = base64:encode(crypto:hash(sha, Name)), + {ok, binary_to_atom(<<"trace_", TypeBin/binary, "_", NameHash/binary>>)}; + true -> + {error, <<"Name length must < 256">>}; + false -> + {error, <<"Name must printable unicode">>} + end. ensure_bin(List) when is_list(List) -> iolist_to_binary(List); ensure_bin(Bin) when is_binary(Bin) -> Bin. @@ -203,4 +217,4 @@ ensure_list(List) when is_list(List) -> List. show_prompts(ok, Who, Msg) -> ?LOG(info, Msg ++ " ~p " ++ "successfully~n", [Who]); show_prompts({error, Reason}, Who, Msg) -> - ?LOG(error, Msg ++ " ~p " ++ "failed by ~p~n", [Who, Reason]). + ?LOG(error, Msg ++ " ~p " ++ "failed with ~p~n", [Who, Reason]). diff --git a/test/emqx_trace_handler_SUITE.erl b/test/emqx_trace_handler_SUITE.erl index 5e909c45b..e314c7994 100644 --- a/test/emqx_trace_handler_SUITE.erl +++ b/test/emqx_trace_handler_SUITE.erl @@ -70,12 +70,12 @@ t_trace_clientid(_Config) -> ?assert(filelib:is_regular("tmp/client3.log")), %% Get current traces - ?assertEqual([#{type => clientid, filter => "client", name => <<"client">>, - id => trace_clientid_client, level => debug, dst => "tmp/client.log"}, - #{type => clientid, filter => "client2", name => <<"client2">>, - id => trace_clientid_client2, level => debug, dst => "tmp/client2.log"}, - #{type => clientid, filter => "client3", name => <<"client3">>, - id => trace_clientid_client3, level => debug, dst => "tmp/client3.log"} + ?assertMatch([#{type := clientid, filter := "client", name := <<"client">>, + level := debug, dst := "tmp/client.log"}, + #{type := clientid, filter := "client2", name := <<"client2">> + , level := debug, dst := "tmp/client2.log"}, + #{type := clientid, filter := "client3", name := <<"client3">>, + level := debug, dst := "tmp/client3.log"} ], emqx_trace_handler:running()), %% Client with clientid = "client" publishes a "hi" message to "a/b/c". @@ -116,10 +116,10 @@ t_trace_topic(_Config) -> ?assert(filelib:is_regular("tmp/topic_trace_y.log")), %% Get current traces - ?assertEqual([#{type => topic, filter => <<"x/#">>, id => 'trace_topic_x/#', - level => debug, dst => "tmp/topic_trace_x.log", name => <<"x/#">>}, - #{type => topic, filter => <<"y/#">>, id => 'trace_topic_y/#', - name => <<"y/#">>, level => debug, dst => "tmp/topic_trace_y.log"} + ?assertMatch([#{type := topic, filter := <<"x/#">>, + level := debug, dst := "tmp/topic_trace_x.log", name := <<"x/#">>}, + #{type := topic, filter := <<"y/#">>, + name := <<"y/#">>, level := debug, dst := "tmp/topic_trace_y.log"} ], emqx_trace_handler:running()), @@ -159,12 +159,12 @@ t_trace_ip_address(_Config) -> ?assert(filelib:is_regular("tmp/ip_trace_y.log")), %% Get current traces - ?assertEqual([#{type => ip_address, filter => "127.0.0.1", - id => 'trace_ip_address_127.0.0.1', name => <<"127.0.0.1">>, - level => debug, dst => "tmp/ip_trace_x.log"}, - #{type => ip_address, filter => "192.168.1.1", - id => 'trace_ip_address_192.168.1.1', name => <<"192.168.1.1">>, - level => debug, dst => "tmp/ip_trace_y.log"} + ?assertMatch([#{type := ip_address, filter := "127.0.0.1", + name := <<"127.0.0.1">>, + level := debug, dst := "tmp/ip_trace_x.log"}, + #{type := ip_address, filter := "192.168.1.1", + name := <<"192.168.1.1">>, + level := debug, dst := "tmp/ip_trace_y.log"} ], emqx_trace_handler:running()), From 0f5282487287715e2c7d3397d6413a27ae15724d Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 22 Nov 2021 16:49:58 +0100 Subject: [PATCH 089/363] refactor(trace): hash non-printable or too long names --- src/emqx_trace_handler.erl | 42 ++++++++++++++--------------- test/emqx_trace_handler_tests.erl | 44 +++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 21 deletions(-) create mode 100644 test/emqx_trace_handler_tests.erl diff --git a/src/emqx_trace_handler.erl b/src/emqx_trace_handler.erl index 7006fc815..328d734f0 100644 --- a/src/emqx_trace_handler.erl +++ b/src/emqx_trace_handler.erl @@ -160,22 +160,16 @@ filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) -> filter_ip_address(_Log, _ExpectId) -> ignore. install_handler(Who = #{name := Name, type := Type}, Level, LogFile) -> - case handler_id(Name, Type) of - {ok, HandlerId} -> - Config = #{ - level => Level, + HandlerId = handler_id(Name, Type), + Config = #{ level => Level, formatter => ?FORMAT, filter_default => stop, filters => filters(Who), config => ?CONFIG(LogFile) - }, - Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), - show_prompts(Res, Who, "Start trace"), - Res; - {error, _Reason} = Error -> - show_prompts(Error, Who, "Start trace"), - Error - end. + }, + Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), + show_prompts(Res, Who, "Start trace"), + Res. filters(#{type := clientid, filter := Filter, name := Name}) -> [{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}]; @@ -197,17 +191,23 @@ filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) end. handler_id(Name, Type) -> - TypeBin = atom_to_binary(Type), - case io_lib:printable_unicode_list(binary_to_list(Name)) of - true when byte_size(Name) < 256 -> - NameHash = base64:encode(crypto:hash(sha, Name)), - {ok, binary_to_atom(<<"trace_", TypeBin/binary, "_", NameHash/binary>>)}; - true -> - {error, <<"Name length must < 256">>}; - false -> - {error, <<"Name must printable unicode">>} + try + do_handler_id(Name, Type) + catch + _ : _ -> + Hash = emqx_misc:bin2hexstr_a_f_lower(crypto:hash(md5, Name)), + do_handler_id(Hash, Type) end. +%% Handler ID must be an atom. +do_handler_id(Name, Type) -> + TypeStr = atom_to_list(Type), + NameStr = unicode:characters_to_list(Name, utf8), + FullNameStr = "trace_" ++ TypeStr ++ "_" ++ NameStr, + true = io_lib:printable_unicode_list(FullNameStr), + FullNameBin = unicode:characters_to_binary(FullNameStr, utf8), + binary_to_atom(FullNameBin, utf8). + ensure_bin(List) when is_list(List) -> iolist_to_binary(List); ensure_bin(Bin) when is_binary(Bin) -> Bin. diff --git a/test/emqx_trace_handler_tests.erl b/test/emqx_trace_handler_tests.erl new file mode 100644 index 000000000..40586810e --- /dev/null +++ b/test/emqx_trace_handler_tests.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_handler_tests). + +-include_lib("eunit/include/eunit.hrl"). + +handler_id_test_() -> + [{"normal_printable", + fun() -> + ?assertEqual(trace_topic_t1, emqx_trace_handler:handler_id("t1", topic)) + end}, + {"normal_unicode", + fun() -> + ?assertEqual('trace_topic_主题', emqx_trace_handler:handler_id("主题", topic)) + end + }, + {"not_printable", + fun() -> + ?assertEqual('trace_topic_93b885adfe0da089cdf634904fd59f71', + emqx_trace_handler:handler_id("\0", topic)) + end + }, + {"too_long", + fun() -> + T = lists:duplicate(250, $a), + ?assertEqual('trace_topic_1bdbdf1c9087c796394bcda5789f7206', + emqx_trace_handler:handler_id(T, topic)) + end + } + ]. From 2514f474b0290a3417758b4e16442935bef56c21 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 23 Nov 2021 02:33:20 +0100 Subject: [PATCH 090/363] Sync v4.3 to v4.4 (#6262) * fix(http): fix duplicate http headers * chore(appup): add appup.src * fix(appup): fix multiply defined module in appup * chore(appup): fix wrong version * chore(ekka): Bump version to 0.8.1.5 * fix(update_appup): Fix warnings, add support for external repos * build: use find command's -delete option * ci: do not sync master branch * build: ensure openssl11 * build: copy only libcrypto and libtinfo * fix(trace): handler_id now always return atom Co-authored-by: zhouzb Co-authored-by: k32 <10274441+k32@users.noreply.github.com> --- .ci/build_packages/tests.sh | 6 ++ .github/workflows/git_sync.yaml | 7 +-- Makefile | 2 +- .../emqx_auth_http/src/emqx_auth_http.app.src | 2 +- .../src/emqx_auth_http.appup.src | 10 ++++ .../emqx_auth_http/src/emqx_auth_http_app.erl | 2 +- .../emqx_auth_http/src/emqx_auth_http_cli.erl | 2 +- .../src/emqx_trace/emqx_trace.erl | 18 +++--- apps/emqx_web_hook/src/emqx_web_hook.app.src | 2 +- .../emqx_web_hook/src/emqx_web_hook.appup.src | 12 +++- .../src/emqx_web_hook_actions.erl | 11 ++-- apps/emqx_web_hook/src/emqx_web_hook_app.erl | 4 +- bin/emqx | 7 ++- build | 6 +- deploy/packages/rpm/emqx.spec | 6 ++ scripts/update_appup.escript | 40 +++++-------- src/emqx.app.src | 2 +- src/emqx.appup.src | 56 +++++++++++++------ src/emqx_http_lib.erl | 4 +- src/emqx_trace_handler.erl | 6 +- test/emqx_http_lib_tests.erl | 2 +- 21 files changed, 117 insertions(+), 90 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 68d8f2df2..4211d1d8d 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -88,6 +88,12 @@ emqx_test(){ ;; "rpm") packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm) + + if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; then + # EMQX OTP requires openssl11 to have TLS1.3 support + yum install -y openssl11 + fi + rpm -ivh "${PACKAGE_PATH}/${packagename}" if ! rpm -q emqx | grep -q emqx; then echo "package install error" diff --git a/.github/workflows/git_sync.yaml b/.github/workflows/git_sync.yaml index 50fa8c364..2941f5173 100644 --- a/.github/workflows/git_sync.yaml +++ b/.github/workflows/git_sync.yaml @@ -3,7 +3,6 @@ name: Sync to enterprise on: push: branches: - - master - main-v* jobs: @@ -23,11 +22,7 @@ jobs: id: create_pull_request run: | set -euo pipefail - if [ "$GITHUB_REF" = "refs/heads/master" ]; then - EE_REF="refs/heads/enterprise" - else - EE_REF="${GITHUB_REF}-enterprise" - fi + EE_REF="${GITHUB_REF}-enterprise" R=$(curl --silent --show-error \ -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ diff --git a/Makefile b/Makefile index 34f41c34c..9a0d55511 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ $(PROFILES:%=clean-%): rm rebar.lock \ rm -rf _build/$(@:clean-%=%)/rel; \ find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ - find _build/$(@:clean-%=%) -type l | xargs rm -i -f ; \ + find _build/$(@:clean-%=%) -type l -delete; \ fi .PHONY: clean-all diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index 51ddcc69b..c4695348b 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_http, [{description, "EMQ X Authentication/ACL with HTTP API"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_http_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.appup.src b/apps/emqx_auth_http/src/emqx_auth_http.appup.src index d1b1061d4..8ebc195dd 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.appup.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.appup.src @@ -2,12 +2,22 @@ {VSN, [ + {"4.3.2", [ + {apply, {application, stop, [emqx_auth_http]}}, + {load_module, emqx_auth_http_app, brutal_purge, soft_purge,[]}, + {load_module, emqx_auth_http_cli, brutal_purge, soft_purge,[]} + ]}, {<<"4.3.[0-1]">>, [ {restart_application, emqx_auth_http} ]}, {<<".*">>, []} ], [ + {"4.3.2", [ + {apply, {application, stop, [emqx_auth_http]}}, + {load_module, emqx_auth_http_app, brutal_purge, soft_purge,[]}, + {load_module, emqx_auth_http_cli, brutal_purge, soft_purge,[]} + ]}, {<<"4.3.[0-1]">>, [ {restart_application, emqx_auth_http} ]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index 2958b93ea..20f2c381e 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -150,7 +150,7 @@ ensure_content_type_header(Method, Headers) when Method =:= post orelse Method =:= put -> Headers; ensure_content_type_header(_Method, Headers) -> - lists:keydelete("content-type", 1, Headers). + lists:keydelete(<<"content-type">>, 1, Headers). path(#{path := "", 'query' := Query}) -> "?" ++ Query; diff --git a/apps/emqx_auth_http/src/emqx_auth_http_cli.erl b/apps/emqx_auth_http/src/emqx_auth_http_cli.erl index 5ef222ae5..d01f571e5 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_cli.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_cli.erl @@ -32,7 +32,7 @@ request(PoolName, get, Path, Headers, Params, Timeout) -> reply(ehttpc:request(PoolName, get, {NewPath, Headers}, Timeout)); request(PoolName, post, Path, Headers, Params, Timeout) -> - Body = case proplists:get_value("content-type", Headers) of + Body = case proplists:get_value(<<"content-type">>, Headers) of "application/x-www-form-urlencoded" -> cow_qs:qs(bin_kw(Params)); "application/json" -> diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index 2190e5bab..c71a457a6 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -379,16 +379,12 @@ to_trace(TraceParam) -> {error, "type=[topic,clientid,ip_address] required"}; {ok, #?TRACE{filter = undefined}} -> {error, "topic/clientid/ip_address filter required"}; - {ok, TraceRec0 = #?TRACE{name = Name, type = Type}} -> - case emqx_trace_handler:handler_id(Name, Type) of - {ok, _} -> - case fill_default(TraceRec0) of - #?TRACE{start_at = Start, end_at = End} when End =< Start -> - {error, "failed by start_at >= end_at"}; - TraceRec -> {ok, TraceRec} - end; - {error, Reason} -> - {error, Reason} + {ok, TraceRec0 = #?TRACE{}} -> + case fill_default(TraceRec0) of + #?TRACE{start_at = Start, end_at = End} when End =< Start -> + {error, "failed by start_at >= end_at"}; + TraceRec -> + {ok, TraceRec} end end. @@ -408,7 +404,7 @@ fill_default(Trace) -> Trace. to_trace([], Rec) -> {ok, Rec}; to_trace([{name, Name} | Trace], Rec) -> - case io_lib:printable_unicode_list(binary_to_list(Name)) of + case io_lib:printable_unicode_list(unicode:characters_to_list(Name, utf8)) of true -> case binary:match(Name, [<<"/">>], []) of nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index e0632625f..0a99a2e0a 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.7"}, % strict semver, bump manually! + {vsn, "4.3.8"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index ec716f45c..40bdfbdbf 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -1,6 +1,10 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + [{"4.3.7", + [{apply,{application,stop,[emqx_web_hook]}}, + {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, @@ -9,7 +13,11 @@ {<<"4.3.[3-4]">>, [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + [{"4.3.7", + [{apply,{application,stop,[emqx_web_hook]}}, + {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.5",[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index 79aefdb85..68e3e85d4 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -295,7 +295,7 @@ create_req(_, Path, Headers, Body) -> parse_action_params(Params = #{<<"url">> := URL}) -> {ok, #{path := CommonPath}} = emqx_http_lib:uri_parse(URL), Method = method(maps:get(<<"method">>, Params, <<"POST">>)), - Headers = headers(maps:get(<<"headers">>, Params, undefined)), + Headers = headers(maps:get(<<"headers">>, Params, #{})), NHeaders = ensure_content_type_header(Headers, Method), #{method => Method, path => merge_path(CommonPath, maps:get(<<"path">>, Params, <<>>)), @@ -307,7 +307,7 @@ parse_action_params(Params = #{<<"url">> := URL}) -> ensure_content_type_header(Headers, Method) when Method =:= post orelse Method =:= put -> Headers; ensure_content_type_header(Headers, _Method) -> - lists:keydelete("content-type", 1, Headers). + lists:keydelete(<<"content-type">>, 1, Headers). merge_path(CommonPath, <<>>) -> l2b(CommonPath); @@ -326,11 +326,8 @@ method(POST) when POST == <<"POST">>; POST == <<"post">> -> post; method(PUT) when PUT == <<"PUT">>; PUT == <<"put">> -> put; method(DEL) when DEL == <<"DELETE">>; DEL == <<"delete">> -> delete. -headers(undefined) -> []; -headers(Headers) when is_map(Headers) -> - headers(maps:to_list(Headers)); -headers(Headers) when is_list(Headers) -> - [{string:to_lower(str(K)), str(V)} || {K, V} <- Headers]. +headers(Headers) -> + emqx_http_lib:normalise_headers(maps:to_list(Headers)). str(Str) when is_list(Str) -> Str; str(Atom) when is_atom(Atom) -> atom_to_list(Atom); diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl index 8265fa664..e728aca9d 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_app.erl @@ -87,7 +87,7 @@ translate_env() -> application:set_env(?APP, path, Path), application:set_env(?APP, pool_opts, PoolOpts), Headers = application:get_env(?APP, headers, []), - NHeaders = set_content_type(Headers), + NHeaders = set_content_type(emqx_http_lib:normalise_headers(Headers)), application:set_env(?APP, headers, NHeaders). path(#{path := "", 'query' := Query}) -> @@ -100,5 +100,5 @@ path(#{path := Path}) -> Path. set_content_type(Headers) -> - NHeaders = proplists:delete(<<"Content-Type">>, proplists:delete(<<"content-type">>, Headers)), + NHeaders = proplists:delete(<<"content-type">>, Headers), [{<<"content-type">>, <<"application/json">>} | NHeaders]. diff --git a/bin/emqx b/bin/emqx index e68c1f8c2..6a80f4d8d 100755 --- a/bin/emqx +++ b/bin/emqx @@ -41,9 +41,10 @@ if ! check_eralng_start >/dev/null 2>&1; then export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH" if ! check_eralng_start; then ## it's hopeless - echoerr "FATAL: Unable to start Erlang (with libcrypto)." - echoerr "Please make sure it's running on the correct platform with all required dependencies." - echoerr "This EMQ X release is built for $BUILT_ON" + echoerr "FATAL: Unable to start Erlang." + echoerr "Please make sure openssl-1.1.1 (libcrypto) and libncurses are installed." + echoerr "Also ensure it's running on the correct platform," + echoerr "this EMQ X release is built for $BUILT_ON" exit 1 fi echoerr "WARNING: There seem to be missing dynamic libs from the OS. Using libs from ${DYNLIBS_DIR}" diff --git a/build b/build index 50997f76c..7cad5296b 100755 --- a/build +++ b/build @@ -96,7 +96,11 @@ cp_dyn_libs() { mkdir -p "$target_dir" while read -r so_file; do cp -L "$so_file" "$target_dir/" - done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 | xargs -0 ldd | grep -E '^\s+.*=>\s(/lib|/usr)' | awk '{print $3}') + done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ + | xargs -0 ldd \ + | grep -E '(libcrypto)|(libtinfo)' \ + | awk '{print $3}' \ + | sort -u) } ## make_zip turns .tar.gz into a .zip with a slightly different name. diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index ef63b936c..c6eb56a6f 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -19,6 +19,12 @@ BuildRoot: %{_tmppath}/%{_name}-%{_version}-root Provides: %{_name} AutoReq: 0 +%if "%{_arch} %{?rhel}" == "amd64 7" +Requires: openssl11 libatomic +%else +Requires: libatomic +%endif + %description EMQX, a distributed, massively scalable, highly extensible MQTT message broker written in Erlang/OTP. diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 8eb1c7967..39a71b492 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -163,18 +163,6 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) - find_upstream_repo(Remote) -> string:trim(os:cmd("git remote get-url " ++ Remote)). -find_prev_tag(CurrentRelease) -> - case getopt(prev_tag) of - undefined -> - {Maj, Min, Patch} = parse_semver(CurrentRelease), - case Patch of - 0 -> undefined; - _ -> {ok, semver(Maj, Min, Patch - 1)} - end; - Tag -> - {ok, Tag} - end. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Appup action creation and updating %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -208,10 +196,21 @@ find_old_appup_actions(App, PrevVersion) -> {Upgrade0, Downgrade0} = case locate(ebin_current, App, ".appup") of {ok, AppupFile} -> + log("Found the previous appup file: ~s~n", [AppupFile]), {_, U, D} = read_appup(AppupFile), {U, D}; undefined -> - {[], []} + %% Fallback to the app.src file, in case the + %% application doesn't have a release (useful for the + %% apps that live outside the EMQX monorepo): + case locate(src, App, ".appup.src") of + {ok, AppupSrcFile} -> + log("Using ~s as a source of previous update actions~n", [AppupSrcFile]), + {_, U, D} = read_appup(AppupSrcFile), + {U, D}; + undefined -> + {[], []} + end end, {ensure_version(PrevVersion, Upgrade0), ensure_version(PrevVersion, Downgrade0)}. @@ -390,17 +389,6 @@ is_valid() -> %% Utility functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -parse_semver(Version) -> - case re(Version, "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.[0-9]+)?$") of - {match, [Maj, Min, Patch|_]} -> - {list_to_integer(Maj), list_to_integer(Min), list_to_integer(Patch)}; - _ -> - error({not_a_semver, Version}) - end. - -semver(Maj, Min, Patch) -> - lists:flatten(io_lib:format("~p.~p.~p", [Maj, Min, Patch])). - %% Locate a file in a specified application locate(ebin_current, App, Suffix) -> ReleaseDir = getopt(beams_dir), @@ -425,6 +413,7 @@ bash(Script) -> bash(Script, []). bash(Script, Env) -> + log("+ ~s~n+ Env: ~p~n", [Script, Env]), case cmd("bash", #{args => ["-c", Script], env => Env}) of 0 -> true; _ -> fail("Failed to run command: ~s", [Script]) @@ -456,9 +445,6 @@ fail(Str, Args) -> log(Str ++ "~n", Args), halt(1). -re(Subject, RE) -> - re:run(Subject, RE, [{capture, all_but_first, list}]). - log(Msg) -> log(Msg, []). diff --git a/src/emqx.app.src b/src/emqx.app.src index a107c7ed4..b0e1664cc 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -1,7 +1,7 @@ {application, emqx, [{id, "emqx"}, {description, "EMQ X"}, - {vsn, "4.3.11"}, % strict semver, bump manually! + {vsn, "4.3.12"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index d40db1093..653249ccb 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,10 +1,14 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.10", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{"4.3.11", + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, + {"4.3.10", + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -14,7 +18,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -24,7 +29,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -36,7 +42,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -49,7 +56,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, @@ -63,7 +71,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, @@ -78,7 +87,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -157,11 +167,15 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.10", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{"4.3.11", + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, + {"4.3.10", + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -171,7 +185,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -181,7 +196,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -193,7 +209,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -206,7 +223,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, @@ -220,7 +238,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, @@ -235,7 +254,8 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_http_lib.erl b/src/emqx_http_lib.erl index 893c260ee..bf83eaea6 100644 --- a/src/emqx_http_lib.erl +++ b/src/emqx_http_lib.erl @@ -96,11 +96,11 @@ do_parse(URI) -> %% underscores replaced with hyphens %% NOTE: assuming the input Headers list is a proplists, %% that is, when a key is duplicated, list header overrides tail -%% e.g. [{"Content_Type", "applicaiton/binary"}, {"content-type", "applicaiton/json"}] +%% e.g. [{"Content_Type", "applicaiton/binary"}, {<<"content-type">>, "applicaiton/json"}] %% results in: [{"content-type", "applicaiton/binary"}] normalise_headers(Headers0) -> F = fun({K0, V}) -> - K = re:replace(K0, "_", "-", [{return,list}]), + K = re:replace(K0, "_", "-", [{return,binary}]), {string:lowercase(K), V} end, Headers = lists:map(F, Headers0), diff --git a/src/emqx_trace_handler.erl b/src/emqx_trace_handler.erl index 328d734f0..ffe581300 100644 --- a/src/emqx_trace_handler.erl +++ b/src/emqx_trace_handler.erl @@ -113,10 +113,8 @@ install(Who, Level, LogFile) -> -spec uninstall(Type :: clientid | topic | ip_address, Name :: binary() | list()) -> ok | {error, term()}. uninstall(Type, Name) -> - case handler_id(ensure_bin(Name), Type) of - {ok, HandlerId} -> uninstall(HandlerId); - {error, Reason} -> {error, Reason} - end. + HandlerId = handler_id(ensure_bin(Name), Type), + uninstall(HandlerId). -spec uninstall(HandlerId :: atom()) -> ok | {error, term()}. uninstall(HandlerId) -> diff --git a/test/emqx_http_lib_tests.erl b/test/emqx_http_lib_tests.erl index 7bcb7d056..54dd85d94 100644 --- a/test/emqx_http_lib_tests.erl +++ b/test/emqx_http_lib_tests.erl @@ -89,6 +89,6 @@ uri_parse_test_() -> ]. normalise_headers_test() -> - ?assertEqual([{"content-type", "applicaiton/binary"}], + ?assertEqual([{<<"content-type">>, "applicaiton/binary"}], emqx_http_lib:normalise_headers([{"Content_Type", "applicaiton/binary"}, {"content-type", "applicaiton/json"}])). From afd55b31e85cb9c80c022569ebe894193530aba6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 23 Nov 2021 02:34:56 +0100 Subject: [PATCH 091/363] build: define default builder docker image tag (#6245) prior to this change, the OTP_VSN varaible was taken from the docker host's OTP version which may differ from the desired OTP version for the docker builder image. --- Makefile | 3 +++ build | 7 +++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9a0d55511..4ace92341 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ REBAR_VERSION = 3.14.3-emqx-8 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 +export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X @@ -96,6 +98,7 @@ $(PROFILES:%=clean-%): .PHONY: clean-all clean-all: + @rm -f rebar.lock @rm -rf _build .PHONY: deps-all diff --git a/build b/build index 7cad5296b..2fca5bdf2 100755 --- a/build +++ b/build @@ -130,13 +130,12 @@ make_zip() { ## This function builds the default docker image based on alpine:3.14 (by default) make_docker() { - EMQX_RUNNER_IMAGE="${EMQX_RUNNER_IMAGE:-alpine:3.14}" - EMQX_RUNNER_IMAGE_COMPACT="$(echo "$EMQX_RUNNER_IMAGE" | tr -d ':')" - EMQX_BUILDER="${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/4.4-2:${OTP_VSN}-${EMQX_RUNNER_IMAGE_COMPACT}}" + EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}" + EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}" set -x docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ - --build-arg RUN_FROM="${EMQX_RUNNER_IMAGE}" \ + --build-arg RUN_FROM="${EMQX_RUNNER}" \ --build-arg EMQX_NAME="$PROFILE" \ --tag "emqx/$PROFILE:${PKG_VSN}" \ -f "${DOCKERFILE}" . From b87ed0666c7920f4facfa4b785d84042d9cc44a6 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 23 Nov 2021 17:54:17 +0800 Subject: [PATCH 092/363] ci: add new steps for push ecr image when release --- .github/workflows/build_packages.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index b2104e618..ca963da68 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -391,6 +391,20 @@ jobs: EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile context: source + - uses: aws-actions/configure-aws-credentials@v1 + if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Push image to aws ecr + if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' + run: | + version=${GITHUB_REF##*/} + docker pull emqx/emqx:${version#v} + docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + docker push public.ecr.aws/emqx/emqx:${version#v} delete-artifact: runs-on: ubuntu-20.04 From fef3fc27cb8165943514bea0b2e8e163bb42021e Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 26 Nov 2021 10:42:15 +0800 Subject: [PATCH 093/363] refactor(emqx_slow_subs): refactor use moving average (#6287) * refactor(emqx_slow_subs): refactor use moving average * fix(emqx_slow_subs): change elapsed to latency, and fix some error * fix(emqx_slow_subs): fix emqx_mgmt_api.erl indent * fix(emqx_slow_subs): change api name * fix(emqx_slow_subs): fix and improve some code * fix(emqx_slow_subs): move clienid filed from latency_stats to session --- apps/emqx_management/src/emqx_mgmt_api.erl | 33 +- ...x_st_statistics.hrl => emqx_slow_subs.hrl} | 15 +- .../src/emqx_slow_subs/emqx_slow_subs.erl | 319 +++++++++++++++ .../src/emqx_slow_subs/emqx_slow_subs_api.erl | 57 +++ .../emqx_st_statistics/emqx_st_statistics.erl | 379 ------------------ .../emqx_st_statistics_api.erl | 83 ---- etc/emqx.conf | 41 +- ..._statistics.erl => emqx_mod_slow_subs.erl} | 8 +- lib-ce/emqx_modules/src/emqx_mod_sup.erl | 1 + ...ics_SUITE.erl => emqx_slow_subs_SUITE.erl} | 86 ++-- ...SUITE.erl => emqx_slow_subs_api_SUITE.erl} | 97 ++--- priv/emqx.schema | 49 ++- rebar.config.erl | 2 +- src/emqx_session.erl | 105 +++-- .../emqx_message_latency_stats.erl | 105 +++++ src/emqx_slow_subs/emqx_moving_average.erl | 90 +++++ test/emqx_session_SUITE.erl | 16 +- 17 files changed, 826 insertions(+), 660 deletions(-) rename apps/emqx_plugin_libs/include/{emqx_st_statistics.hrl => emqx_slow_subs.hrl} (68%) create mode 100644 apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl create mode 100644 apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl delete mode 100644 apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl delete mode 100644 apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl rename lib-ce/emqx_modules/src/{emqx_mod_st_statistics.erl => emqx_mod_slow_subs.erl} (90%) rename lib-ce/emqx_modules/test/{emqx_st_statistics_SUITE.erl => emqx_slow_subs_SUITE.erl} (53%) rename lib-ce/emqx_modules/test/{emqx_st_statistics_api_SUITE.erl => emqx_slow_subs_api_SUITE.erl} (59%) create mode 100644 src/emqx_slow_subs/emqx_message_latency_stats.erl create mode 100644 src/emqx_slow_subs/emqx_moving_average.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index e068c5384..d42d2a486 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -49,17 +49,46 @@ paginate(Tables, Params, RowFun) -> query_handle(Table) when is_atom(Table) -> qlc:q([R|| R <- ets:table(Table)]); + +query_handle({Table, Opts}) when is_atom(Table) -> + qlc:q([R|| R <- ets:table(Table, Opts)]); + query_handle([Table]) when is_atom(Table) -> qlc:q([R|| R <- ets:table(Table)]); + +query_handle([{Table, Opts}]) when is_atom(Table) -> + qlc:q([R|| R <- ets:table(Table, Opts)]); + query_handle(Tables) -> - qlc:append([qlc:q([E || E <- ets:table(T)]) || T <- Tables]). + Fold = fun({Table, Opts}, Acc) -> + Handle = qlc:q([R|| R <- ets:table(Table, Opts)]), + [Handle | Acc]; + (Table, Acc) -> + Handle = qlc:q([R|| R <- ets:table(Table)]), + [Handle | Acc] + end, + Handles = lists:foldl(Fold, [], Tables), + qlc:append(lists:reverse(Handles)). count(Table) when is_atom(Table) -> ets:info(Table, size); + +count({Table, _Opts}) when is_atom(Table) -> + ets:info(Table, size); + count([Table]) when is_atom(Table) -> ets:info(Table, size); + +count([{Table, _Opts}]) when is_atom(Table) -> + ets:info(Table, size); + count(Tables) -> - lists:sum([count(T) || T <- Tables]). + Fold = fun({Table, _Opts}, Acc) -> + count(Table) ++ Acc; + (Table, Acc) -> + count(Table) ++ Acc + end, + lists:foldl(Fold, 0, Tables). count(Table, Nodes) -> lists:sum([rpc_call(Node, ets, info, [Table, size], 5000) || Node <- Nodes]). diff --git a/apps/emqx_plugin_libs/include/emqx_st_statistics.hrl b/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl similarity index 68% rename from apps/emqx_plugin_libs/include/emqx_st_statistics.hrl rename to apps/emqx_plugin_libs/include/emqx_slow_subs.hrl index 9184c3194..0b5e3a035 100644 --- a/apps/emqx_plugin_libs/include/emqx_st_statistics.hrl +++ b/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl @@ -14,12 +14,15 @@ %% limitations under the License. %%-------------------------------------------------------------------- --define(LOG_TAB, emqx_st_statistics_log). --define(TOPK_TAB, emqx_st_statistics_topk). +-define(TOPK_TAB, emqx_slow_subs_topk). --record(top_k, { rank :: pos_integer() - , topic :: emqx_types:topic() - , average_count :: number() - , average_elapsed :: number()}). +-define(INDEX(Latency, ClientId), {Latency, ClientId}). + +-record(top_k, { index :: index() + , type :: emqx_message_latency_stats:latency_type() + , last_update_time :: pos_integer() + , extra = [] + }). -type top_k() :: #top_k{}. +-type index() :: ?INDEX(non_neg_integer(), emqx_types:clientid()). diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl new file mode 100644 index 000000000..a50b5eb0e --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -0,0 +1,319 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs). + +-behaviour(gen_server). + +-include_lib("include/emqx.hrl"). +-include_lib("include/logger.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_slow_subs.hrl"). + +-logger_header("[SLOW Subs]"). + +-export([ start_link/1, on_stats_update/2, enable/0 + , disable/0, clear_history/0, init_topk_tab/0 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-compile(nowarn_unused_type). + +-type state() :: #{ config := proplist:proplist() + , enable := boolean() + , last_tick_at := pos_integer() + }. + +-type log() :: #{ rank := pos_integer() + , clientid := emqx_types:clientid() + , latency := non_neg_integer() + , type := emqx_message_latency_stats:latency_type() + }. + +-type window_log() :: #{ last_tick_at := pos_integer() + , logs := [log()] + }. + +-type message() :: #message{}. + +-import(proplists, [get_value/2]). + +-type stats_update_args() :: #{ clientid := emqx_types:clientid() + , latency := non_neg_integer() + , type := emqx_message_latency_stats:latency_type() + , last_insert_value := non_neg_integer() + , update_time := timer:time() + }. + +-type stats_update_env() :: #{max_size := pos_integer()}. + +-ifdef(TEST). +-define(EXPIRE_CHECK_INTERVAL, timer:seconds(1)). +-else. +-define(EXPIRE_CHECK_INTERVAL, timer:seconds(10)). +-endif. + +-define(NOW, erlang:system_time(millisecond)). +-define(NOTICE_TOPIC_NAME, "slow_subs"). +-define(DEF_CALL_TIMEOUT, timer:seconds(10)). + +%% erlang term order +%% number < atom < reference < fun < port < pid < tuple < list < bit string + +%% ets ordered_set is ascending by term order + +%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- +%% @doc Start the st_statistics +-spec(start_link(Env :: list()) -> emqx_types:startlink_ret()). +start_link(Env) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). + +%% XXX NOTE:pay attention to the performance here +-spec on_stats_update(stats_update_args(), stats_update_env()) -> true. +on_stats_update(#{clientid := ClientId, + latency := Latency, + type := Type, + last_insert_value := LIV, + update_time := Ts}, + #{max_size := MaxSize}) -> + + LastIndex = ?INDEX(LIV, ClientId), + Index = ?INDEX(Latency, ClientId), + + %% check whether the client is in the table + case ets:lookup(?TOPK_TAB, LastIndex) of + [#top_k{index = Index}] -> + %% if last value == the new value, return + true; + [_] -> + %% if Latency > minimum value, we should update it + %% if Latency < minimum value, maybe it can replace the minimum value + %% so alwyas update at here + %% do we need check if Latency == minimum ??? + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}), + ets:delete(?TOPK_TAB, LastIndex); + [] -> + %% try to insert + try_insert_to_topk(MaxSize, Index, Latency, Type, Ts) + end. + +clear_history() -> + gen_server:call(?MODULE, ?FUNCTION_NAME, ?DEF_CALL_TIMEOUT). + +enable() -> + gen_server:call(?MODULE, {enable, true}, ?DEF_CALL_TIMEOUT). + +disable() -> + gen_server:call(?MODULE, {enable, false}, ?DEF_CALL_TIMEOUT). + +init_topk_tab() -> + case ets:whereis(?TOPK_TAB) of + undefined -> + ?TOPK_TAB = ets:new(?TOPK_TAB, + [ ordered_set, public, named_table + , {keypos, #top_k.index}, {write_concurrency, true} + , {read_concurrency, true} + ]); + _ -> + ?TOPK_TAB + end. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Conf]) -> + notice_tick(Conf), + expire_tick(Conf), + MaxSize = get_value(top_k_num, Conf), + load(MaxSize), + {ok, #{config => Conf, + last_tick_at => ?NOW, + enable => true}}. + +handle_call({enable, Enable}, _From, + #{config := Cfg, enable := IsEnable} = State) -> + State2 = case Enable of + IsEnable -> + State; + true -> + MaxSize = get_value(max_topk_num, Cfg), + load(MaxSize), + State#{enable := true}; + _ -> + unload(), + State#{enable := false} + end, + {reply, ok, State2}; + +handle_call(clear_history, _, State) -> + ets:delete_all_objects(?TOPK_TAB), + {reply, ok, State}; + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(expire_tick, #{config := Cfg} = State) -> + expire_tick(Cfg), + Logs = ets:tab2list(?TOPK_TAB), + do_clear(Cfg, Logs), + {noreply, State}; + +handle_info(notice_tick, #{config := Cfg} = State) -> + notice_tick(Cfg), + Logs = ets:tab2list(?TOPK_TAB), + do_notification(Logs, State), + {noreply, State#{last_tick_at := ?NOW}}; + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _) -> + unload(), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +expire_tick(_) -> + erlang:send_after(?EXPIRE_CHECK_INTERVAL, self(), ?FUNCTION_NAME). + +notice_tick(Cfg) -> + case get_value(notice_interval, Cfg) of + 0 -> ok; + Interval -> + erlang:send_after(Interval, self(), ?FUNCTION_NAME), + ok + end. + +-spec do_notification(list(), state()) -> ok. +do_notification([], _) -> + ok; + +do_notification(Logs, #{last_tick_at := LastTickTime, config := Cfg}) -> + start_publish(Logs, LastTickTime, Cfg), + ok. + +start_publish(Logs, TickTime, Cfg) -> + emqx_pool:async_submit({fun do_publish/4, [Logs, erlang:length(Logs), TickTime, Cfg]}). + +do_publish([], _, _, _) -> + ok; + +do_publish(Logs, Rank, TickTime, Cfg) -> + BatchSize = get_value(notice_batch_size, Cfg), + do_publish(Logs, BatchSize, Rank, TickTime, Cfg, []). + +do_publish([Log | T], Size, Rank, TickTime, Cfg, Cache) when Size > 0 -> + Cache2 = [convert_to_notice(Rank, Log) | Cache], + do_publish(T, Size - 1, Rank - 1, TickTime, Cfg, Cache2); + +do_publish(Logs, Size, Rank, TickTime, Cfg, Cache) when Size =:= 0 -> + publish(TickTime, Cfg, Cache), + do_publish(Logs, Rank, TickTime, Cfg); + +do_publish([], _, _Rank, TickTime, Cfg, Cache) -> + publish(TickTime, Cfg, Cache), + ok. + +convert_to_notice(Rank, #top_k{index = ?INDEX(Latency, ClientId), + type = Type, + last_update_time = Ts}) -> + #{rank => Rank, + clientid => ClientId, + latency => Latency, + type => Type, + timestamp => Ts}. + +publish(TickTime, Cfg, Notices) -> + WindowLog = #{last_tick_at => TickTime, + logs => lists:reverse(Notices)}, + Payload = emqx_json:encode(WindowLog), + Msg = #message{ id = emqx_guid:gen() + , qos = get_value(notice_qos, Cfg) + , from = ?MODULE + , topic = emqx_topic:systop(?NOTICE_TOPIC_NAME) + , payload = Payload + , timestamp = ?NOW + }, + _ = emqx_broker:safe_publish(Msg), + ok. + +load(MaxSize) -> + _ = emqx:hook('message.slow_subs_stats', + fun ?MODULE:on_stats_update/2, + [#{max_size => MaxSize}]), + ok. + +unload() -> + emqx:unhook('message.slow_subs_stats', fun ?MODULE:on_stats_update/2). + +do_clear(Cfg, Logs) -> + Now = ?NOW, + Interval = get_value(expire_interval, Cfg), + Each = fun(#top_k{index = Index, last_update_time = Ts}) -> + case Now - Ts >= Interval of + true -> + ets:delete(?TOPK_TAB, Index); + _ -> + true + end + end, + lists:foreach(Each, Logs). + +try_insert_to_topk(MaxSize, Index, Latency, Type, Ts) -> + case ets:info(?TOPK_TAB, size) of + Size when Size < MaxSize -> + %% if the size is under limit, insert it directly + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}); + _Size -> + %% find the minimum value + ?INDEX(Min, _) = First = + case ets:first(?TOPK_TAB) of + ?INDEX(_, _) = I -> I; + _ -> ?INDEX(Latency - 1, <<>>) + end, + + case Latency =< Min of + true -> true; + _ -> + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}), + + ets:delete(?TOPK_TAB, First) + end + end. diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl new file mode 100644 index 000000000..a7c9fc050 --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl @@ -0,0 +1,57 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs_api). + +-rest_api(#{name => clear_history, + method => 'DELETE', + path => "/slow_subscriptions", + func => clear_history, + descr => "Clear current data and re count slow topic"}). + +-rest_api(#{name => get_history, + method => 'GET', + path => "/slow_subscriptions", + func => get_history, + descr => "Get slow topics statistics record data"}). + +-export([ clear_history/2 + , get_history/2 + ]). + +-include("include/emqx_slow_subs.hrl"). + +-import(minirest, [return/1]). + +%%-------------------------------------------------------------------- +%% HTTP API +%%-------------------------------------------------------------------- + +clear_history(_Bindings, _Params) -> + ok = emqx_slow_subs:clear_history(), + return(ok). + +get_history(_Bindings, Params) -> + RowFun = fun(#top_k{index = ?INDEX(Latency, ClientId), + type = Type, + last_update_time = Ts}) -> + [{clientid, ClientId}, + {latency, Latency}, + {type, Type}, + {last_update_time, Ts}] + end, + Return = emqx_mgmt_api:paginate({?TOPK_TAB, [{traverse, last_prev}]}, Params, RowFun), + return({ok, Return}). diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl deleted file mode 100644 index 668cb3926..000000000 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics.erl +++ /dev/null @@ -1,379 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_st_statistics). - --behaviour(gen_server). - --include_lib("include/emqx.hrl"). --include_lib("include/logger.hrl"). --include_lib("emqx_plugin_libs/include/emqx_st_statistics.hrl"). - --logger_header("[SLOW TOPICS]"). - --export([ start_link/1, on_publish_done/3, enable/0 - , disable/0, clear_history/0 - ]). - -%% gen_server callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --compile(nowarn_unused_type). - --type state() :: #{ config := proplist:proplist() - , period := pos_integer() - , last_tick_at := pos_integer() - , counter := counters:counters_ref() - , enable := boolean() - }. - --type log() :: #{ topic := emqx_types:topic() - , count := pos_integer() - , average := float() - }. - --type window_log() :: #{ last_tick_at := pos_integer() - , logs := [log()] - }. - --record(slow_log, { topic :: emqx_types:topic() - , count :: integer() %% 0 will be used in initial value - , elapsed :: integer() - }). - --type message() :: #message{}. - --import(proplists, [get_value/2]). - --define(NOW, erlang:system_time(millisecond)). --define(QUOTA_IDX, 1). - --type slow_log() :: #slow_log{}. --type top_k_map() :: #{emqx_types:topic() => top_k()}. - --type publish_done_env() :: #{ ignore_before_create := boolean() - , threshold := pos_integer() - , counter := counters:counters_ref() - }. - --type publish_done_args() :: #{session_rebirth_time => pos_integer()}. - --ifdef(TEST). --define(TOPK_ACCESS, public). --else. --define(TOPK_ACCESS, protected). --endif. - -%% erlang term order -%% number < atom < reference < fun < port < pid < tuple < list < bit string - -%% ets ordered_set is ascending by term order - -%%-------------------------------------------------------------------- -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- -%% @doc Start the st_statistics --spec(start_link(Env :: list()) -> emqx_types:startlink_ret()). -start_link(Env) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). - --spec on_publish_done(message(), publish_done_args(), publish_done_env()) -> ok. -on_publish_done(#message{timestamp = Timestamp}, - #{session_rebirth_time := Created}, - #{ignore_before_create := IgnoreBeforeCreate}) - when IgnoreBeforeCreate, Timestamp < Created -> - ok; - -on_publish_done(#message{timestamp = Timestamp} = Msg, - _, - #{threshold := Threshold, counter := Counter}) -> - case ?NOW - Timestamp of - Elapsed when Elapsed > Threshold -> - case get_log_quota(Counter) of - true -> - update_log(Msg, Elapsed); - _ -> - ok - end; - _ -> - ok - end. - -clear_history() -> - gen_server:call(?MODULE, ?FUNCTION_NAME). - -enable() -> - gen_server:call(?MODULE, {enable, true}). - -disable() -> - gen_server:call(?MODULE, {enable, false}). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([Env]) -> - erlang:process_flag(trap_exit, true), - init_log_tab(Env), - init_topk_tab(Env), - notification_tick(Env), - Counter = counters:new(1, [write_concurrency]), - set_log_quota(Env, Counter), - Threshold = get_value(threshold_time, Env), - IgnoreBeforeCreate = get_value(ignore_before_create, Env), - load(IgnoreBeforeCreate, Threshold, Counter), - {ok, #{config => Env, - period => 1, - last_tick_at => ?NOW, - counter => Counter, - enable => true}}. - -handle_call({enable, Enable}, _From, - #{config := Cfg, counter := Counter, enable := IsEnable} = State) -> - State2 = case Enable of - IsEnable -> - State; - true -> - Threshold = get_value(threshold_time, Cfg), - IgnoreBeforeCreate = get_value(ignore_before_create, Cfg), - load(IgnoreBeforeCreate, Threshold, Counter), - State#{enable := true}; - _ -> - unload(), - State#{enable := false} - end, - {reply, ok, State2}; - -handle_call(clear_history, _, State) -> - ets:delete_all_objects(?TOPK_TAB), - {reply, ok, State}; - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), - {noreply, State}. - -handle_info(notification_tick, #{config := Cfg, period := Period} = State) -> - notification_tick(Cfg), - do_notification(State), - {noreply, State#{last_tick_at := ?NOW, - period := Period + 1}}; - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, _) -> - unload(), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- -notification_tick(Env) -> - TimeWindow = get_value(time_window, Env), - erlang:send_after(TimeWindow, self(), ?FUNCTION_NAME). - -init_log_tab(_) -> - ?LOG_TAB = ets:new(?LOG_TAB, [ set, public, named_table - , {keypos, #slow_log.topic}, {write_concurrency, true} - , {read_concurrency, true} - ]). - -init_topk_tab(_) -> - ?TOPK_TAB = ets:new(?TOPK_TAB, [ set, ?TOPK_ACCESS, named_table - , {keypos, #top_k.rank}, {write_concurrency, false} - , {read_concurrency, true} - ]). - --spec get_log_quota(counters:counters_ref()) -> boolean(). -get_log_quota(Counter) -> - case counters:get(Counter, ?QUOTA_IDX) of - Quota when Quota > 0 -> - counters:sub(Counter, ?QUOTA_IDX, 1), - true; - _ -> - false - end. - --spec set_log_quota(proplists:proplist(), counters:counters_ref()) -> ok. -set_log_quota(Cfg, Counter) -> - MaxLogNum = get_value(max_log_num, Cfg), - counters:put(Counter, ?QUOTA_IDX, MaxLogNum). - --spec update_log(message(), pos_integer()) -> ok. -update_log(#message{topic = Topic}, Elapsed) -> - _ = ets:update_counter(?LOG_TAB, - Topic, - [{#slow_log.count, 1}, {#slow_log.elapsed, Elapsed}], - #slow_log{topic = Topic, - count = 0, - elapsed = 0}), - ok. - --spec do_notification(state()) -> true. -do_notification(#{last_tick_at := TickTime, - config := Cfg, - period := Period, - counter := Counter}) -> - Logs = ets:tab2list(?LOG_TAB), - ets:delete_all_objects(?LOG_TAB), - start_publish(Logs, TickTime, Cfg), - set_log_quota(Cfg, Counter), - MaxRecord = get_value(top_k_num, Cfg), - update_topk(Logs, MaxRecord, Period). - --spec update_topk(list(slow_log()), pos_integer(), pos_integer()) -> true. -update_topk(Logs, MaxRecord, Period) -> - TopkMap = get_topk_map(Period), - TopkMap2 = update_topk_map(Logs, Period, TopkMap), - SortFun = fun(A, B) -> - A#top_k.average_count > B#top_k.average_count - end, - TopkL = lists:sort(SortFun, maps:values(TopkMap2)), - TopkL2 = lists:sublist(TopkL, 1, MaxRecord), - update_topk_tab(TopkL2). - --spec update_topk_map(list(slow_log()), pos_integer(), top_k_map()) -> top_k_map(). -update_topk_map([#slow_log{topic = Topic, - count = LogTimes, - elapsed = LogElapsed} | T], Period, TopkMap) -> - case maps:get(Topic, TopkMap, undefined) of - undefined -> - Record = #top_k{rank = 1, - topic = Topic, - average_count = LogTimes, - average_elapsed = LogElapsed}, - TopkMap2 = TopkMap#{Topic => Record}, - update_topk_map(T, Period, TopkMap2); - #top_k{average_count = AvgCount, - average_elapsed = AvgElapsed} = Record -> - NewPeriod = Period + 1, - %% (a + b) / c = a / c + b / c - %% average_count(elapsed) dived NewPeriod in function get_topk_maps - AvgCount2 = AvgCount + LogTimes / NewPeriod, - AvgElapsed2 = AvgElapsed + LogElapsed / NewPeriod, - Record2 = Record#top_k{average_count = AvgCount2, - average_elapsed = AvgElapsed2}, - update_topk_map(T, Period, TopkMap#{Topic := Record2}) - end; - -update_topk_map([], _, TopkMap) -> - TopkMap. - --spec update_topk_tab(list(top_k())) -> true. -update_topk_tab(Records) -> - Zip = fun(Rank, Item) -> Item#top_k{rank = Rank} end, - Len = erlang:length(Records), - RankedTopics = lists:zipwith(Zip, lists:seq(1, Len), Records), - ets:insert(?TOPK_TAB, RankedTopics). - -start_publish(Logs, TickTime, Cfg) -> - emqx_pool:async_submit({fun do_publish/3, [Logs, TickTime, Cfg]}). - -do_publish([], _, _) -> - ok; - -do_publish(Logs, TickTime, Cfg) -> - BatchSize = get_value(notice_batch_size, Cfg), - do_publish(Logs, BatchSize, TickTime, Cfg, []). - -do_publish([Log | T], Size, TickTime, Cfg, Cache) when Size > 0 -> - Cache2 = [convert_to_notice(Log) | Cache], - do_publish(T, Size - 1, TickTime, Cfg, Cache2); - -do_publish(Logs, Size, TickTime, Cfg, Cache) when Size =:= 0 -> - publish(TickTime, Cfg, Cache), - do_publish(Logs, TickTime, Cfg); - -do_publish([], _, TickTime, Cfg, Cache) -> - publish(TickTime, Cfg, Cache), - ok. - -convert_to_notice(#slow_log{topic = Topic, - count = Count, - elapsed = Elapsed}) -> - #{topic => Topic, - count => Count, - average => Elapsed / Count}. - -publish(TickTime, Cfg, Notices) -> - WindowLog = #{last_tick_at => TickTime, - logs => Notices}, - Payload = emqx_json:encode(WindowLog), - _ = emqx:publish(#message{ id = emqx_guid:gen() - , qos = get_value(notice_qos, Cfg) - , from = ?MODULE - , topic = get_topic(Cfg) - , payload = Payload - , timestamp = ?NOW - }), - ok. - -load(IgnoreBeforeCreate, Threshold, Counter) -> - _ = emqx:hook('message.publish_done', - fun ?MODULE:on_publish_done/3, - [#{ignore_before_create => IgnoreBeforeCreate, - threshold => Threshold, - counter => Counter} - ]), - ok. - -unload() -> - emqx:unhook('message.publish_done', fun ?MODULE:on_publish_done/3). - --spec get_topic(proplists:proplist()) -> binary(). -get_topic(Cfg) -> - case get_value(notice_topic, Cfg) of - Topic when is_binary(Topic) -> - Topic; - Topic -> - erlang:list_to_binary(Topic) - end. - --spec get_topk_map(pos_integer()) -> top_k_map(). -get_topk_map(Period) -> - Size = ets:info(?TOPK_TAB, size), - get_topk_map(1, Size, Period, #{}). - --spec get_topk_map(pos_integer(), - non_neg_integer(), pos_integer(), top_k_map()) -> top_k_map(). -get_topk_map(Index, Size, _, TopkMap) when Index > Size -> - TopkMap; -get_topk_map(Index, Size, Period, TopkMap) -> - [#top_k{topic = Topic, - average_count = AvgCount, - average_elapsed = AvgElapsed} = R] = ets:lookup(?TOPK_TAB, Index), - NewPeriod = Period + 1, - TotalTimes = AvgCount * Period, - AvgCount2 = TotalTimes / NewPeriod, - AvgElapsed2 = TotalTimes * AvgElapsed / NewPeriod, - TopkMap2 = TopkMap#{Topic => R#top_k{average_count = AvgCount2, - average_elapsed = AvgElapsed2}}, - get_topk_map(Index + 1, Size, Period, TopkMap2). diff --git a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl b/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl deleted file mode 100644 index dc74dcf36..000000000 --- a/apps/emqx_plugin_libs/src/emqx_st_statistics/emqx_st_statistics_api.erl +++ /dev/null @@ -1,83 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_st_statistics_api). - --rest_api(#{name => clear_history, - method => 'DELETE', - path => "/slow_topic", - func => clear_history, - descr => "Clear current data and re count slow topic"}). - --rest_api(#{name => get_history, - method => 'GET', - path => "/slow_topic", - func => get_history, - descr => "Get slow topics statistics record data"}). - --export([ clear_history/2 - , get_history/2 - ]). - --include("include/emqx_st_statistics.hrl"). - --import(minirest, [return/1]). - -%%-------------------------------------------------------------------- -%% HTTP API -%%-------------------------------------------------------------------- - -clear_history(_Bindings, _Params) -> - ok = emqx_st_statistics:clear_history(), - return(ok). - -get_history(_Bindings, Params) -> - PageT = proplists:get_value(<<"_page">>, Params), - LimitT = proplists:get_value(<<"_limit">>, Params), - Page = erlang:binary_to_integer(PageT), - Limit = erlang:binary_to_integer(LimitT), - Start = (Page - 1) * Limit + 1, - Size = ets:info(?TOPK_TAB, size), - End = Start + Limit - 1, - {HasNext, Count, Infos} = get_history(Start, End, Size), - return({ok, #{meta => #{page => Page, - limit => Limit, - hasnext => HasNext, - count => Count}, - data => Infos}}). - - -get_history(Start, _End, Size) when Start > Size -> - {false, 0, []}; - -get_history(Start, End, Size) when End > Size -> - get_history(Start, Size, Size); - -get_history(Start, End, Size) -> - Fold = fun(Rank, Acc) -> - [#top_k{topic = Topic - , average_count = Count - , average_elapsed = Elapsed}] = ets:lookup(?TOPK_TAB, Rank), - - Info = [ {rank, Rank} - , {topic, Topic} - , {count, Count} - , {elapsed, Elapsed}], - - [Info | Acc] - end, - Infos = lists:foldl(Fold, [], lists:seq(Start, End)), - {End < Size, End - Start + 1, Infos}. diff --git a/etc/emqx.conf b/etc/emqx.conf index 78d383930..de5c062a4 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2217,47 +2217,34 @@ module.presence.qos = 1 ## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 ##-------------------------------------------------------------------- -## Slow Topic Statistics Module +## Slow Subscribers Statistics Module -## Threshold time of slow topics statistics -## -## Default: 10 seconds -#module.st_statistics.threshold_time = 10S - -## ignore the messages that before than session created -## -## Default: true -#module.st_statistics.ignore_before_create = true - -## Time window of slow topics statistics +## the expire time of the record which in topk ## ## Value: 5 minutes -#module.st_statistics.time_window = 5M +#module.slow_subs.expire_interval = 5m -## Maximum of slow topics log, log will clear when enter new time window +## maximum number of Top-K record ## -## Value: 500 -#module.st_statistics.max_log_num = 500 +## Value: 10 +#module.slow_subs.top_k_num = 10 -## Top-K record for slow topics, update from logs +## enable notification +## publish topk list to $SYS/brokers/${node}/slow_subs per notice_interval +## publish is disabled if set to 0s. ## -## Value: 500 -#module.st_statistics.top_k_num = 500 - -## Topic of notification -## -## Defaut: $slow_topics -#module.st_statistics.notice_topic = $slow_topics +## Defaut: 0s +#module.slow_subs.notice_interval = 0s ## QoS of notification message in notice topic ## ## Defaut: 0 -#module.st_statistics.notice_qos = 0 +#module.slow_subs.notice_qos = 0 ## Maximum information number in one notification ## -## Default: 500 -#module.st_statistics.notice_batch_size = 500 +## Default: 100 +#module.slow_subs.notice_batch_size = 100 ## CONFIG_SECTION_END=modules ================================================== diff --git a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl similarity index 90% rename from lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl rename to lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl index d6796122c..b9117fe8b 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_st_statistics.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl @@ -14,14 +14,14 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mod_st_statistics). +-module(emqx_mod_slow_subs). -behaviour(emqx_gen_mod). -include_lib("include/emqx.hrl"). -include_lib("include/logger.hrl"). --logger_header("[SLOW TOPICS]"). +-logger_header("[SLOW Subs]"). %% emqx_gen_mod callbacks -export([ load/1 @@ -29,7 +29,7 @@ , description/0 ]). --define(LIB, emqx_st_statistics). +-define(LIB, emqx_slow_subs). %%-------------------------------------------------------------------- %% Load/Unload @@ -46,4 +46,4 @@ unload(_Env) -> ok. description() -> - "EMQ X Slow Topic Statistics Module". + "EMQ X Slow Subscribers Statistics Module". diff --git a/lib-ce/emqx_modules/src/emqx_mod_sup.erl b/lib-ce/emqx_modules/src/emqx_mod_sup.erl index 109564f65..4c8f54679 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_sup.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_sup.erl @@ -69,6 +69,7 @@ stop_child(ChildId) -> init([]) -> ok = emqx_tables:new(emqx_modules, [set, public, {write_concurrency, true}]), + emqx_slow_subs:init_topk_tab(), {ok, {{one_for_one, 10, 100}, []}}. %%-------------------------------------------------------------------- diff --git a/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl similarity index 53% rename from lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl rename to lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl index 8df07f8b8..46f144a57 100644 --- a/lib-ce/emqx_modules/test/emqx_st_statistics_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl @@ -14,18 +14,18 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_st_statistics_SUITE). +-module(emqx_slow_subs_SUITE). -compile(export_all). -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include("include/emqx_mqtt.hrl"). -include_lib("include/emqx.hrl"). %-define(LOGT(Format, Args), ct:pal(Format, Args)). --define(LOG_TAB, emqx_st_statistics_log). --define(TOPK_TAB, emqx_st_statistics_topk). +-define(TOPK_TAB, emqx_slow_subs_topk). -define(NOW, erlang:system_time(millisecond)). all() -> emqx_ct:all(?MODULE). @@ -39,11 +39,11 @@ end_per_suite(Config) -> Config. init_per_testcase(_, Config) -> - emqx_mod_st_statistics:load(base_conf()), + emqx_mod_slow_subs:load(base_conf()), Config. end_per_testcase(_, _) -> - emqx_mod_st_statistics:unload(undefined), + emqx_mod_slow_subs:unload([]), ok. %%-------------------------------------------------------------------- @@ -51,62 +51,74 @@ end_per_testcase(_, _) -> %%-------------------------------------------------------------------- t_log_and_pub(_) -> %% Sub topic first - SubBase = "/test", - emqx:subscribe("$slow_topics"), - Clients = start_client(SubBase), + Subs = [{<<"/test1/+">>, ?QOS_1}, {<<"/test2/+">>, ?QOS_2}], + Clients = start_client(Subs), + emqx:subscribe("$SYS/brokers/+/slow_subs"), timer:sleep(1000), Now = ?NOW, %% publish - ?assert(ets:info(?LOG_TAB, size) =:= 0), + lists:foreach(fun(I) -> - Topic = list_to_binary(io_lib:format("~s~p", [SubBase, I])), - Msg = emqx_message:make(Topic, <<"Hello">>), - emqx:publish(Msg#message{timestamp = Now - 1000}) + Topic = list_to_binary(io_lib:format("/test1/~p", [I])), + Msg = emqx_message:make(undefined, ?QOS_1, Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 500}) end, lists:seq(1, 10)), - timer:sleep(2400), + lists:foreach(fun(I) -> + Topic = list_to_binary(io_lib:format("/test2/~p", [I])), + Msg = emqx_message:make(undefined, ?QOS_2, Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 500}) + end, + lists:seq(1, 10)), - ?assert(ets:info(?LOG_TAB, size) =:= 0), - ?assert(ets:info(?TOPK_TAB, size) =:= 3), - try_receive(3), - try_receive(2), + timer:sleep(1000), + Size = ets:info(?TOPK_TAB, size), + %% some time record maybe delete due to it expired + ?assert(Size =< 6 andalso Size >= 4), + + timer:sleep(1500), + Recs = try_receive([]), + RecSum = lists:sum(Recs), + ?assert(RecSum >= 5), + ?assert(lists:all(fun(E) -> E =< 3 end, Recs)), + + timer:sleep(2000), + ?assert(ets:info(?TOPK_TAB, size) =:= 0), [Client ! stop || Client <- Clients], ok. - base_conf() -> - [{top_k_num, 3}, - {threshold_time, 10}, - {notice_qos, 0}, - {notice_batch_size, 3}, - {notice_topic,"$slow_topics"}, - {time_window, 2000}, - {max_log_num, 5}]. + [ {top_k_num, 5} + , {expire_interval, timer:seconds(3)} + , {notice_interval, 1500} + , {notice_qos, 0} + , {notice_batch_size, 3} + ]. -start_client(Base) -> - [spawn(fun() -> - Topic = list_to_binary(io_lib:format("~s~p", [Base, I])), - client(Topic) - end) - || I <- lists:seq(1, 10)]. +start_client(Subs) -> + [spawn(fun() -> client(I, Subs) end) || I <- lists:seq(1, 10)]. -client(Topic) -> +client(I, Subs) -> {ok, C} = emqtt:start_link([{host, "localhost"}, - {clientid, Topic}, + {clientid, io_lib:format("slow_subs_~p", [I])}, {username, <<"plain">>}, {password, <<"plain">>}]), {ok, _} = emqtt:connect(C), - {ok, _, _} = emqtt:subscribe(C, Topic), + + Len = erlang:length(Subs), + Sub = lists:nth(I rem Len + 1, Subs), + _ = emqtt:subscribe(C, Sub), + receive stop -> ok end. -try_receive(L) -> +try_receive(Acc) -> receive {deliver, _, #message{payload = Payload}} -> #{<<"logs">> := Logs} = emqx_json:decode(Payload, [return_maps]), - ?assertEqual(length(Logs), L) + try_receive([length(Logs) | Acc]) after 500 -> - ?assert(false) + Acc end. diff --git a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl similarity index 59% rename from lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl rename to lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl index 0fb9e904b..2efc7230a 100644 --- a/lib-ce/emqx_modules/test/emqx_st_statistics_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl @@ -12,9 +12,9 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- +%%--------------------------------------------------------------------n --module(emqx_st_statistics_api_SUITE). +-module(emqx_slow_subs_api_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -24,7 +24,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_management/include/emqx_mgmt.hrl"). --include_lib("emqx_plugin_libs/include/emqx_st_statistics.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_slow_subs.hrl"). -define(CONTENT_TYPE, "application/x-www-form-urlencoded"). @@ -33,6 +33,7 @@ -define(API_VERSION, "v4"). -define(BASE_PATH, "api"). +-define(NOW, erlang:system_time(millisecond)). all() -> emqx_ct:all(?MODULE). @@ -48,78 +49,52 @@ end_per_suite(Config) -> Config. init_per_testcase(_, Config) -> - emqx_mod_st_statistics:load(emqx_st_statistics_SUITE:base_conf()), + emqx_mod_slow_subs:load(base_conf()), Config. end_per_testcase(_, Config) -> - emqx_mod_st_statistics:unload(undefined), + emqx_mod_slow_subs:unload([]), Config. +base_conf() -> + [ {top_k_num, 5} + , {expire_interval, timer:seconds(60)} + , {notice_interval, 0} + , {notice_qos, 0} + , {notice_batch_size, 3} + ]. + t_get_history(_) -> - ets:insert(?TOPK_TAB, #top_k{rank = 1, - topic = <<"test">>, - average_count = 12, - average_elapsed = 1500}), + Now = ?NOW, + Each = fun(I) -> + ClientId = erlang:list_to_binary(io_lib:format("test_~p", [I])), + ets:insert(?TOPK_TAB, #top_k{index = ?INDEX(I, ClientId), + type = average, + last_update_time = Now}) + end, - {ok, Data} = request_api(get, api_path(["slow_topic"]), "_page=1&_limit=10", + lists:foreach(Each, lists:seq(1, 5)), + + {ok, Data} = request_api(get, api_path(["slow_subscriptions"]), "_page=1&_limit=10", auth_header_()), + #{meta := Meta, data := [First | _]} = decode(Data), - ShouldRet = #{meta => #{page => 1, - limit => 10, - hasnext => false, - count => 1}, - data => [#{topic => <<"test">>, - rank => 1, - elapsed => 1500, - count => 12}], - code => 0}, + RMeta = #{page => 1, limit => 10, count => 5}, + ?assertEqual(RMeta, Meta), - Ret = decode(Data), + RFirst = #{clientid => <<"test_5">>, + latency => 5, + type => <<"average">>, + last_update_time => Now}, - ?assertEqual(ShouldRet, Ret). - -t_rank_range(_) -> - Insert = fun(Rank) -> - ets:insert(?TOPK_TAB, - #top_k{rank = Rank, - topic = <<"test">>, - average_count = 12, - average_elapsed = 1500}) - end, - lists:foreach(Insert, lists:seq(1, 15)), - - timer:sleep(100), - - {ok, Data} = request_api(get, api_path(["slow_topic"]), "_page=1&_limit=10", - auth_header_()), - - Meta1 = #{page => 1, limit => 10, hasnext => true, count => 10}, - Ret1 = decode(Data), - ?assertEqual(Meta1, maps:get(meta, Ret1)), - - %% End > Size - {ok, Data2} = request_api(get, api_path(["slow_topic"]), "_page=2&_limit=10", - auth_header_()), - - Meta2 = #{page => 2, limit => 10, hasnext => false, count => 5}, - Ret2 = decode(Data2), - ?assertEqual(Meta2, maps:get(meta, Ret2)), - - %% Start > Size - {ok, Data3} = request_api(get, api_path(["slow_topic"]), "_page=3&_limit=10", - auth_header_()), - - Meta3 = #{page => 3, limit => 10, hasnext => false, count => 0}, - Ret3 = decode(Data3), - ?assertEqual(Meta3, maps:get(meta, Ret3)). + ?assertEqual(RFirst, First). t_clear(_) -> - ets:insert(?TOPK_TAB, #top_k{rank = 1, - topic = <<"test">>, - average_count = 12, - average_elapsed = 1500}), + ets:insert(?TOPK_TAB, #top_k{index = ?INDEX(1, <<"test">>), + type = average, + last_update_time = ?NOW}), - {ok, _} = request_api(delete, api_path(["slow_topic"]), [], + {ok, _} = request_api(delete, api_path(["slow_subscriptions"]), [], auth_header_()), ?assertEqual(0, ets:info(?TOPK_TAB, size)). diff --git a/priv/emqx.schema b/priv/emqx.schema index 15f0324cd..61a98f824 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1007,6 +1007,18 @@ end}. {datatype, {duration, s}} ]}. +%% @doc the number of smaples for calculate the average latency of delivery +{mapping, "zone.$name.latency_samples", "emqx.zones", [ + {default, 10}, + {datatype, integer} +]}. + +%% @doc Threshold for slow subscription statistics +{mapping, "zone.$name.latency_stats_threshold", "emqx.zones", [ + {default, "100ms"}, + {datatype, {duration, ms}} +]}. + %% @doc Max Packets that Awaiting PUBREL, 0 means no limit {mapping, "zone.$name.max_awaiting_rel", "emqx.zones", [ {default, 0}, @@ -2218,43 +2230,28 @@ end}. {datatype, string} ]}. -{mapping, "module.st_statistics.threshold_time", "emqx.modules", [ - {default, "10s"}, +{mapping, "module.slow_subs.expire_interval", "emqx.modules", [ + {default, "5m"}, {datatype, {duration, ms}} ]}. -{mapping, "module.st_statistics.ignore_before_create", "emqx.modules", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "module.st_statistics.time_window", "emqx.modules", [ - {default, "5M"}, - {datatype, {duration, ms}} -]}. - -{mapping, "module.st_statistics.max_log_num", "emqx.modules", [ +{mapping, "module.slow_subs.top_k_num", "emqx.modules", [ {default, 500}, {datatype, integer} ]}. -{mapping, "module.st_statistics.top_k_num", "emqx.modules", [ - {default, 500}, - {datatype, integer} +{mapping, "module.slow_subs.notice_interval", "emqx.modules", [ + {default, "0s"}, + {datatype, {duration, ms}} ]}. -{mapping, "module.st_statistics.notice_topic", "emqx.modules", [ - {default, "$slow_topics"}, - {datatype, string} -]}. - -{mapping, "module.st_statistics.notice_qos", "emqx.modules", [ +{mapping, "module.slow_subs.notice_qos", "emqx.modules", [ {default, 0}, {datatype, integer}, {validators, ["range:0-1"]} ]}. -{mapping, "module.st_statistics.notice_batch_size", "emqx.modules", [ +{mapping, "module.slow_subs.notice_batch_size", "emqx.modules", [ {default, 500}, {datatype, integer} ]}. @@ -2283,8 +2280,8 @@ end}. end, TotalRules) end, - SlowTopic = fun() -> - List = cuttlefish_variable:filter_by_prefix("module.st_statistics", Conf), + SlowSubs = fun() -> + List = cuttlefish_variable:filter_by_prefix("module.slow_subs", Conf), [{erlang:list_to_atom(Key), Value} || {[_, _, Key], Value} <- List] end, @@ -2295,7 +2292,7 @@ end}. [{emqx_mod_topic_metrics, []}], [{emqx_mod_delayed, []}], [{emqx_mod_trace, []}], - [{emqx_mod_st_statistics, SlowTopic()}], + [{emqx_mod_slow_subs, SlowSubs()}], [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] ]) end}. diff --git a/rebar.config.erl b/rebar.config.erl index 1000a2c92..6c14b6ae8 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -106,7 +106,7 @@ test_plugins() -> test_deps() -> [ {bbmustache, "1.10.0"} - , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.9"}}} + , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.11"}}} , meck ]. diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 6122982ae..3e1e48a13 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -95,6 +95,8 @@ -import(emqx_zone, [get_env/3]). -record(session, { + %% Client's id + clientid :: emqx_types:clientid(), %% Client’s Subscriptions. subscriptions :: map(), %% Max subscriptions allowed @@ -121,8 +123,15 @@ %% Awaiting PUBREL Timeout (Unit: millsecond) await_rel_timeout :: timeout(), %% Created at - created_at :: pos_integer() - }). + created_at :: pos_integer(), + %% Message deliver latency stats + latency_stats :: emqx_message_latency_stats:stats() + }). + +%% in the previous code, we will replace the message record with the pubrel atom +%% in the pubrec function, this will lose the creation time of the message, +%% but now we need this time to calculate latency, so now pubrel atom is changed to this record +-record(pubrel_await, {timestamp :: non_neg_integer()}). -type(session() :: #session{}). @@ -148,19 +157,26 @@ mqueue_dropped, next_pkt_id, awaiting_rel_cnt, - awaiting_rel_max + awaiting_rel_max, + latency_stats ]). -define(DEFAULT_BATCH_N, 1000). +-ifdef(TEST). +-define(GET_CLIENT_ID(C), maps:get(clientid, C, <<>>)). +-else. +-define(GET_CLIENT_ID(C), maps:get(clientid, C)). +-endif. %%-------------------------------------------------------------------- %% Init a Session %%-------------------------------------------------------------------- -spec(init(emqx_types:clientinfo(), emqx_types:conninfo()) -> session()). -init(#{zone := Zone}, #{receive_maximum := MaxInflight}) -> - #session{max_subscriptions = get_env(Zone, max_subscriptions, 0), +init(#{zone := Zone} = CInfo, #{receive_maximum := MaxInflight}) -> + #session{clientid = ?GET_CLIENT_ID(CInfo), + max_subscriptions = get_env(Zone, max_subscriptions, 0), subscriptions = #{}, upgrade_qos = get_env(Zone, upgrade_qos, false), inflight = emqx_inflight:new(MaxInflight), @@ -170,7 +186,8 @@ init(#{zone := Zone}, #{receive_maximum := MaxInflight}) -> awaiting_rel = #{}, max_awaiting_rel = get_env(Zone, max_awaiting_rel, 100), await_rel_timeout = timer:seconds(get_env(Zone, await_rel_timeout, 300)), - created_at = erlang:system_time(millisecond) + created_at = erlang:system_time(millisecond), + latency_stats = emqx_message_latency_stats:new(Zone) }. %% @private init mq @@ -227,7 +244,9 @@ info(awaiting_rel_max, #session{max_awaiting_rel = Max}) -> info(await_rel_timeout, #session{await_rel_timeout = Timeout}) -> Timeout div 1000; info(created_at, #session{created_at = CreatedAt}) -> - CreatedAt. + CreatedAt; +info(latency_stats, #session{latency_stats = Stats}) -> + emqx_message_latency_stats:latency(Stats). %% @doc Get stats of the session. -spec(stats(session()) -> emqx_types:stats()). @@ -317,13 +336,12 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, -> {ok, emqx_types:message(), session()} | {ok, emqx_types:message(), replies(), session()} | {error, emqx_types:reason_code()}). -puback(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt}) -> +puback(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - emqx:run_hook('message.publish_done', - [Msg, #{session_rebirth_time => CreatedAt}]), Inflight1 = emqx_inflight:delete(PacketId, Inflight), - return_with(Msg, dequeue(Session#session{inflight = Inflight1})); + Session2 = update_latency(Msg, Session), + return_with(Msg, dequeue(Session2#session{inflight = Inflight1})); {value, {_Pubrel, _Ts}} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> @@ -343,15 +361,13 @@ return_with(Msg, {ok, Publishes, Session}) -> -spec(pubrec(emqx_types:packet_id(), session()) -> {ok, emqx_types:message(), session()} | {error, emqx_types:reason_code()}). -pubrec(PacketId, Session = #session{inflight = Inflight, created_at = CreatedAt}) -> +pubrec(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - %% execute hook here, because message record will be replaced by pubrel - emqx:run_hook('message.publish_done', - [Msg, #{session_rebirth_time => CreatedAt}]), - Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight), + Update = with_ts(#pubrel_await{timestamp = Msg#message.timestamp}), + Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; - {value, {pubrel, _Ts}} -> + {value, {_PUBREL, _Ts}} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} @@ -380,9 +396,10 @@ pubrel(PacketId, Session = #session{awaiting_rel = AwaitingRel}) -> | {error, emqx_types:reason_code()}). pubcomp(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of - {value, {pubrel, _Ts}} -> + {value, {Pubrel, _Ts}} when is_record(Pubrel, pubrel_await) -> + Session2 = update_latency(Pubrel, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), - dequeue(Session#session{inflight = Inflight1}); + dequeue(Session2#session{inflight = Inflight1}); {value, _Other} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> @@ -550,11 +567,16 @@ await(PacketId, Msg, Session = #session{inflight = Inflight}) -> %%-------------------------------------------------------------------- -spec(retry(session()) -> {ok, session()} | {ok, replies(), timeout(), session()}). -retry(Session = #session{inflight = Inflight}) -> +retry(Session = #session{inflight = Inflight, retry_interval = RetryInterval}) -> case emqx_inflight:is_empty(Inflight) of true -> {ok, Session}; - false -> retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight), - [], erlang:system_time(millisecond), Session) + false -> + Now = erlang:system_time(millisecond), + Session2 = check_expire_latency(Now, RetryInterval, Session), + retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight), + [], + Now, + Session2) end. retry_delivery([], Acc, _Now, Session = #session{retry_interval = Interval}) -> @@ -581,8 +603,8 @@ retry_delivery(PacketId, Msg, Now, Acc, Inflight) when is_record(Msg, message) - {[{PacketId, Msg1} | Acc], Inflight1} end; -retry_delivery(PacketId, pubrel, Now, Acc, Inflight) -> - Inflight1 = emqx_inflight:update(PacketId, {pubrel, Now}, Inflight), +retry_delivery(PacketId, Pubrel, Now, Acc, Inflight) -> + Inflight1 = emqx_inflight:update(PacketId, {Pubrel, Now}, Inflight), {[{pubrel, PacketId} | Acc], Inflight1}. %%-------------------------------------------------------------------- @@ -626,10 +648,10 @@ resume(ClientInfo = #{clientid := ClientId}, Session = #session{subscriptions = -spec(replay(session()) -> {ok, replies(), session()}). replay(Session = #session{inflight = Inflight}) -> - Pubs = lists:map(fun({PacketId, {pubrel, _Ts}}) -> - {pubrel, PacketId}; + Pubs = lists:map(fun({PacketId, {Pubrel, _Ts}}) when is_record(Pubrel, pubrel_await) -> + {pubrel, PacketId}; ({PacketId, {Msg, _Ts}}) -> - {PacketId, emqx_message:set_flag(dup, true, Msg)} + {PacketId, emqx_message:set_flag(dup, true, Msg)} end, emqx_inflight:to_list(Inflight)), case dequeue(Session) of {ok, NSession} -> {ok, Pubs, NSession}; @@ -677,6 +699,35 @@ next_pkt_id(Session = #session{next_pkt_id = ?MAX_PACKET_ID}) -> next_pkt_id(Session = #session{next_pkt_id = Id}) -> Session#session{next_pkt_id = Id + 1}. +%%-------------------------------------------------------------------- +%% Message Latency Stats +%%-------------------------------------------------------------------- +update_latency(Msg, + #session{clientid = ClientId, + latency_stats = Stats, + created_at = CreateAt} = S) -> + case get_birth_timestamp(Msg, CreateAt) of + 0 -> S; + Ts -> + Latency = erlang:system_time(millisecond) - Ts, + Stats2 = emqx_message_latency_stats:update(ClientId, Latency, Stats), + S#session{latency_stats = Stats2} + end. + +check_expire_latency(Now, Interval, + #session{clientid = ClientId, latency_stats = Stats} = S) -> + Stats2 = emqx_message_latency_stats:check_expire(ClientId, Now, Interval, Stats), + S#session{latency_stats = Stats2}. + +get_birth_timestamp(#message{timestamp = Ts}, CreateAt) when CreateAt =< Ts -> + Ts; + +get_birth_timestamp(#pubrel_await{timestamp = Ts}, CreateAt) when CreateAt =< Ts -> + Ts; + +get_birth_timestamp(_, _) -> + 0. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl new file mode 100644 index 000000000..f9661ab91 --- /dev/null +++ b/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -0,0 +1,105 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_message_latency_stats). + +%% API +-export([ new/1, new/2, update/3 + , check_expire/4, latency/1]). + +-define(NOW, erlang:system_time(millisecond)). +-define(MINIMUM_INSERT_INTERVAL, 1000). +-define(MINIMUM_THRESHOLD, 100). + +-opaque stats() :: #{ threshold := number() + , ema := emqx_moving_average:ema() + , last_update_time := timestamp() + , last_access_time := timestamp() %% timestamp of last access top-k + , last_insert_value := non_neg_integer() + }. + +-type timestamp() :: non_neg_integer(). +-type timespan() :: number(). + +-type latency_type() :: average + | expire. + +-import(emqx_zone, [get_env/3]). + +-export_type([stats/0, latency_type/0]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new(emqx_types:zone()) -> stats(). +new(Zone) -> + Samples = get_env(Zone, latency_samples, 1), + Threshold = get_env(Zone, latency_stats_threshold, ?MINIMUM_THRESHOLD), + new(Samples, Threshold). + +-spec new(non_neg_integer(), number()) -> stats(). +new(SamplesT, ThresholdT) -> + Samples = erlang:max(1, SamplesT), + Threshold = erlang:max(?MINIMUM_THRESHOLD, ThresholdT), + #{ ema => emqx_moving_average:new(exponential, #{period => Samples}) + , threshold => Threshold + , last_update_time => 0 + , last_access_time => 0 + , last_insert_value => 0 + }. + +-spec update(emqx_types:clientid(), number(), stats()) -> stats(). +update(ClientId, Val, #{ema := EMA} = Stats) -> + Now = ?NOW, + #{average := Latency} = EMA2 = emqx_moving_average:update(Val, EMA), + Stats2 = call_hook(ClientId, Now, average, Latency, Stats), + Stats2#{ ema := EMA2 + , last_update_time := ?NOW}. + +-spec check_expire(emqx_types:clientid(), timestamp(), timespan(), stats()) -> stats(). +check_expire(_, Now, Interval, #{last_update_time := LUT} = S) + when LUT >= Now - Interval -> + S; + +check_expire(ClientId, Now, _Interval, #{last_update_time := LUT} = S) -> + Latency = Now - LUT, + call_hook(ClientId, Now, expire, Latency, S). + +-spec latency(stats()) -> number(). +latency(#{ema := #{average := Average}}) -> + Average. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +-spec call_hook(emqx_types:clientid(), timestamp(), latency_type(), timespan(), stats()) -> stats(). +call_hook(_, Now, _, _, #{last_access_time := LIT} = S) + when LIT >= Now - ?MINIMUM_INSERT_INTERVAL -> + S; + +call_hook(_, _, _, Latency, #{threshold := Threshold} = S) + when Latency =< Threshold -> + S; + +call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> + Arg = #{clientid => ClientId, + latency => erlang:floor(Latency), + type => Type, + last_insert_value => LIV, + update_time => Now}, + emqx:run_hook('message.slow_subs_stats', [Arg]), + Stats#{last_insert_value := Latency, + last_access_time := Now}. diff --git a/src/emqx_slow_subs/emqx_moving_average.erl b/src/emqx_slow_subs/emqx_moving_average.erl new file mode 100644 index 000000000..64c73f987 --- /dev/null +++ b/src/emqx_slow_subs/emqx_moving_average.erl @@ -0,0 +1,90 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @see https://en.wikipedia.org/wiki/Moving_average + +-module(emqx_moving_average). + +%% API +-export([new/0, new/1, new/2, update/2]). + +-type type() :: cumulative + | exponential. + +-type ema() :: #{ type := exponential + , average := 0 | float() + , coefficient := float() + }. + +-type cma() :: #{ type := cumulative + , average := 0 | float() + , count := non_neg_integer() + }. + +-type moving_average() :: ema() + | cma(). + +-define(DEF_EMA_ARG, #{period => 10}). +-define(DEF_AVG_TYPE, exponential). + +-export_type([type/0, moving_average/0, ema/0, cma/0]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new() -> moving_average(). +new() -> + new(?DEF_AVG_TYPE, #{}). + +-spec new(type()) -> moving_average(). +new(Type) -> + new(Type, #{}). + +-spec new(type(), Args :: map()) -> moving_average(). +new(cumulative, _) -> + #{ type => cumulative + , average => 0 + , count => 0 + }; + +new(exponential, Arg) -> + #{period := Period} = maps:merge(?DEF_EMA_ARG, Arg), + #{ type => exponential + , average => 0 + %% coefficient = 2/(N+1) is a common convention, see the wiki link for details + , coefficient => 2 / (Period + 1) + }. + +-spec update(number(), moving_average()) -> moving_average(). + +update(Val, #{average := 0} = Avg) -> + Avg#{average := Val}; + +update(Val, #{ type := cumulative + , average := Average + , count := Count} = CMA) -> + NewCount = Count + 1, + CMA#{average := (Count * Average + Val) / NewCount, + count := NewCount}; + +update(Val, #{ type := exponential + , average := Average + , coefficient := Coefficient} = EMA) -> + EMA#{average := Coefficient * Val + (1 - Coefficient) * Average}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- diff --git a/test/emqx_session_SUITE.erl b/test/emqx_session_SUITE.erl index cb7c10cae..f838d052a 100644 --- a/test/emqx_session_SUITE.erl +++ b/test/emqx_session_SUITE.erl @@ -180,7 +180,8 @@ t_puback_with_dequeue(_) -> ?assertEqual(<<"t2">>, emqx_message:topic(Msg3)). t_puback_error_packet_id_in_use(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Now = ts(millisecond), + Inflight = emqx_inflight:insert(1, {{pubrel_await, Now}, Now}, emqx_inflight:new()), {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:puback(1, session(#{inflight => Inflight})). @@ -192,10 +193,11 @@ t_pubrec(_) -> Inflight = emqx_inflight:insert(2, {Msg, ts(millisecond)}, emqx_inflight:new()), Session = session(#{inflight => Inflight}), {ok, Msg, Session1} = emqx_session:pubrec(2, Session), - ?assertMatch([{pubrel, _}], emqx_inflight:values(emqx_session:info(inflight, Session1))). + ?assertMatch([{{pubrel_await, _}, _}], emqx_inflight:values(emqx_session:info(inflight, Session1))). t_pubrec_packet_id_in_use_error(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Now = ts(millisecond), + Inflight = emqx_inflight:insert(1, {{pubrel_await, Now}, Now}, emqx_inflight:new()), {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:pubrec(1, session(#{inflight => Inflight})). @@ -211,7 +213,8 @@ t_pubrel_error_packetid_not_found(_) -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubrel(1, session()). t_pubcomp(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Now = ts(millisecond), + Inflight = emqx_inflight:insert(1, {{pubrel_await, Now}, Now}, emqx_inflight:new()), Session = session(#{inflight => Inflight}), {ok, Session1} = emqx_session:pubcomp(1, Session), ?assertEqual(0, emqx_session:info(inflight_cnt, Session1)). @@ -260,7 +263,7 @@ t_deliver_qos0(_) -> t_deliver_qos1(_) -> ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), {ok, Session} = emqx_session:subscribe( - clientinfo(), <<"t1">>, subopts(#{qos => ?QOS_1}), session()), + clientinfo(), <<"t1">>, subopts(#{qos => ?QOS_1}), session()), Delivers = [delivery(?QOS_1, T) || T <- [<<"t1">>, <<"t2">>]], {ok, [{1, Msg1}, {2, Msg2}], Session1} = emqx_session:deliver(Delivers, Session), ?assertEqual(2, emqx_session:info(inflight_cnt, Session1)), @@ -373,7 +376,7 @@ mqueue(Opts) -> session() -> session(#{}). session(InitFields) when is_map(InitFields) -> maps:fold(fun(Field, Value, Session) -> - emqx_session:set_field(Field, Value, Session) + emqx_session:set_field(Field, Value, Session) end, emqx_session:init(#{zone => channel}, #{receive_maximum => 0}), InitFields). @@ -396,4 +399,3 @@ ts(second) -> erlang:system_time(second); ts(millisecond) -> erlang:system_time(millisecond). - From 8dd4d88d5b1a75b63ccf7778a47a78adee2c6e4c Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 26 Nov 2021 14:57:25 +0800 Subject: [PATCH 094/363] fix(emx_slow_updates): fix the error of topk update (#6312) --- apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl | 6 ++++-- src/emqx_slow_subs/emqx_message_latency_stats.erl | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index a50b5eb0e..e96c2a907 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -106,8 +106,10 @@ on_stats_update(#{clientid := ClientId, %% check whether the client is in the table case ets:lookup(?TOPK_TAB, LastIndex) of [#top_k{index = Index}] -> - %% if last value == the new value, return - true; + %% if last value == the new value, update the type and last_update_time + %% XXX for clients whose latency are stable for a long time, is it possible to reduce updates? + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}); [_] -> %% if Latency > minimum value, we should update it %% if Latency < minimum value, maybe it can replace the minimum value diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl index f9661ab91..dfeba1c68 100644 --- a/src/emqx_slow_subs/emqx_message_latency_stats.erl +++ b/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -95,11 +95,12 @@ call_hook(_, _, _, Latency, #{threshold := Threshold} = S) S; call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> + ToInsert = erlang:floor(Latency), Arg = #{clientid => ClientId, - latency => erlang:floor(Latency), + latency => ToInsert, type => Type, last_insert_value => LIV, update_time => Now}, emqx:run_hook('message.slow_subs_stats', [Arg]), - Stats#{last_insert_value := Latency, + Stats#{last_insert_value := ToInsert, last_access_time := Now}. From 1b14b792865355000f7d4818ce0e4b398b2d124f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 29 Nov 2021 15:18:09 +0800 Subject: [PATCH 095/363] fix: create trace schema at runtime --- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 2 +- .../src/emqx_trace/emqx_trace.erl | 28 ++++++++----------- .../test/emqx_trace_SUITE.erl | 27 ++++++------------ 3 files changed, 22 insertions(+), 35 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 434e96e21..f21a0f66c 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -240,7 +240,7 @@ t_trace_cmd(_) -> logger:set_primary_config(level, error). t_traces_cmd(_) -> - emqx_trace:mnesia(boot), + emqx_trace:create_table(), Count1 = emqx_mgmt_cli:traces(["list"]), ?assertEqual(0, Count1), Error1 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index c71a457a6..135a1f53b 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -22,12 +22,6 @@ -logger_header("[Tracer]"). -%% Mnesia bootstrap --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). - -export([ publish/1 , subscribe/3 , unsubscribe/2 @@ -57,7 +51,9 @@ -define(MAX_SIZE, 30). -ifdef(TEST). --export([log_file/2]). +-export([ log_file/2 + , create_table/0 + ]). -endif. -export_type([ip_address/0]). @@ -72,15 +68,6 @@ , end_at :: integer() | undefined | '_' }). -mnesia(boot) -> - ok = ekka_mnesia:create_table(?TRACE, [ - {type, set}, - {disc_copies, [node()]}, - {record_name, ?TRACE}, - {attributes, record_info(fields, ?TRACE)}]); -mnesia(copy) -> - ok = ekka_mnesia:copy_table(?TRACE, disc_copies). - publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore; publish(#message{from = From, topic = Topic, payload = Payload}) when is_binary(From); is_atom(From) -> @@ -198,6 +185,7 @@ format(Traces) -> end, Traces). init([]) -> + ok = create_table(), erlang:process_flag(trap_exit, true), OriginLogLevel = emqx_logger:get_primary_log_level(), ok = filelib:ensure_dir(trace_dir()), @@ -208,6 +196,14 @@ init([]) -> TRef = update_trace(Traces), {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. +create_table() -> + ok = ekka_mnesia:create_table(?TRACE, [ + {type, set}, + {disc_copies, [node()]}, + {record_name, ?TRACE}, + {attributes, record_info(fields, ?TRACE)}]), + ok = ekka_mnesia:copy_table(?TRACE, disc_copies). + handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ok, State}. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index ffa2bc1fb..c08211e02 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -33,15 +33,22 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - application:load(emqx_plugin_libs), emqx_ct_helpers:start_apps([]), Config. end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). -t_base_create_delete(_Config) -> +init_per_testcase(_, Config) -> + load(), ok = emqx_trace:clear(), + Config. + +end_per_testcase(_) -> + unload(), + ok. + +t_base_create_delete(_Config) -> Now = erlang:system_time(second), Start = to_rfc3339(Now), End = to_rfc3339(Now + 30 * 60), @@ -84,7 +91,6 @@ t_base_create_delete(_Config) -> ok. t_create_size_max(_Config) -> - emqx_trace:clear(), lists:map(fun(Seq) -> Name = list_to_binary("name" ++ integer_to_list(Seq)), Trace = [{name, Name}, {type, <<"topic">>}, @@ -100,7 +106,6 @@ t_create_size_max(_Config) -> ok. t_create_failed(_Config) -> - ok = emqx_trace:clear(), UnknownField = [{<<"unknown">>, 12}], {error, Reason1} = emqx_trace:create(UnknownField), ?assertEqual(<<"unknown field: {unknown,12}">>, iolist_to_binary(Reason1)), @@ -139,7 +144,6 @@ t_create_failed(_Config) -> ok. t_create_default(_Config) -> - ok = emqx_trace:clear(), {error, "name required"} = emqx_trace:create([]), ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, {<<"type">>, <<"clientid">>}, {<<"clientid">>, <<"good">>}]), @@ -170,7 +174,6 @@ t_create_default(_Config) -> ok. t_update_enable(_Config) -> - ok = emqx_trace:clear(), Name = <<"test-name">>, Now = erlang:system_time(second), End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), @@ -192,8 +195,6 @@ t_update_enable(_Config) -> ok. t_load_state(_Config) -> - emqx_trace:clear(), - load(), Now = erlang:system_time(second), Running = [{<<"name">>, <<"Running">>}, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/1">>}, {<<"start_at">>, to_rfc3339(Now - 1)}, @@ -218,14 +219,11 @@ t_load_state(_Config) -> Enables2 = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces2), ExpectEnables2 = [{<<"Running">>, false}, {<<"Waiting">>, true}], ?assertEqual(ExpectEnables2, lists:sort(Enables2)), - unload(), ok. t_client_event(_Config) -> application:set_env(emqx, allow_anonymous, true), - emqx_trace:clear(), ClientId = <<"client-test">>, - load(), Now = erlang:system_time(second), Start = to_rfc3339(Now), Name = <<"test_client_id_event">>, @@ -252,12 +250,9 @@ t_client_event(_Config) -> ?assert(erlang:byte_size(Bin) > 0), ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)), ?assert(erlang:byte_size(Bin3) > 0), - unload(), ok. t_get_log_filename(_Config) -> - ok = emqx_trace:clear(), - load(), Now = erlang:system_time(second), Start = calendar:system_time_to_rfc3339(Now), End = calendar:system_time_to_rfc3339(Now + 2), @@ -274,7 +269,6 @@ t_get_log_filename(_Config) -> ?assertEqual(ok, element(1, emqx_trace:get_trace_filename(Name))), ct:sleep(3000), ?assertEqual(ok, element(1, emqx_trace:get_trace_filename(Name))), - unload(), ok. t_trace_file(_Config) -> @@ -290,8 +284,6 @@ t_trace_file(_Config) -> ok. t_download_log(_Config) -> - emqx_trace:clear(), - load(), ClientId = <<"client-test">>, Now = erlang:system_time(second), Start = to_rfc3339(Now), @@ -305,7 +297,6 @@ t_download_log(_Config) -> {ok, ZipFile} = emqx_trace_api:download_zip_log(#{name => Name}, []), ?assert(filelib:file_size(ZipFile) > 0), ok = emqtt:disconnect(Client), - unload(), ok. to_rfc3339(Second) -> From 0d218df14d19632eac72aa075d365dbdd978db07 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 29 Nov 2021 23:08:56 +0800 Subject: [PATCH 096/363] fix: replace ct:sleep/1 by filesync/2 --- .../src/emqx_trace/emqx_trace_api.erl | 6 ++- .../test/emqx_trace_SUITE.erl | 11 +++--- test/emqx_trace_handler_SUITE.erl | 37 ++++++++++++++++--- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 7d982c00d..8e052ca66 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -107,8 +107,10 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> case Res of {ok, Node, Bin} -> ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, - ok = file:write_file(ZipName, Bin), - [Node ++ "-" ++ TraceLog | Acc]; + case file:write_file(ZipName, Bin) of + ok -> [Node ++ "-" ++ TraceLog | Acc]; + _ -> Acc + end; {error, Node, Reason} -> ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), Acc diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index c08211e02..94ab71270 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -229,21 +229,22 @@ t_client_event(_Config) -> Name = <<"test_client_id_event">>, ok = emqx_trace:create([{<<"name">>, Name}, {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), - ct:sleep(200), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), {ok, _} = emqtt:connect(Client), emqtt:ping(Client), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]), - ct:sleep(200), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), ok = emqx_trace:create([{<<"name">>, <<"test_topic">>}, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]), - ct:sleep(200), + ok = emqx_trace_handler_SUITE:filesync(<<"test_topic">>, topic), {ok, Bin} = file:read_file(emqx_trace:log_file(Name, Now)), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]), ok = emqtt:disconnect(Client), - ct:sleep(200), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), + ok = emqx_trace_handler_SUITE:filesync(<<"test_topic">>, topic), {ok, Bin2} = file:read_file(emqx_trace:log_file(Name, Now)), {ok, Bin3} = file:read_file(emqx_trace:log_file(<<"test_topic">>, Now)), ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), @@ -293,7 +294,7 @@ t_download_log(_Config) -> {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), {ok, _} = emqtt:connect(Client), [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], - ct:sleep(100), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), {ok, ZipFile} = emqx_trace_api:download_zip_log(#{name => Name}, []), ?assert(filelib:file_size(ZipFile) > 0), ok = emqtt:disconnect(Client), diff --git a/test/emqx_trace_handler_SUITE.erl b/test/emqx_trace_handler_SUITE.erl index e314c7994..b42b1a591 100644 --- a/test/emqx_trace_handler_SUITE.erl +++ b/test/emqx_trace_handler_SUITE.erl @@ -62,7 +62,9 @@ t_trace_clientid(_Config) -> emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"), {error, {handler_not_added, {file_error, ".", eisdir}}} = emqx_trace_handler:install(clientid, <<"client5">>, debug, "."), - ct:sleep(100), + ok = filesync(<<"client">>, clientid), + ok = filesync(<<"client2">>, clientid), + ok = filesync(<<"client3">>, clientid), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/client.log")), @@ -83,7 +85,9 @@ t_trace_clientid(_Config) -> emqtt:connect(T), emqtt:publish(T, <<"a/b/c">>, <<"hi">>), emqtt:ping(T), - ct:sleep(200), + ok = filesync(<<"client">>, clientid), + ok = filesync(<<"client2">>, clientid), + ok = filesync(<<"client3">>, clientid), %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". {ok, Bin} = file:read_file("tmp/client.log"), @@ -109,7 +113,8 @@ t_trace_topic(_Config) -> emqx_logger:set_log_level(debug), ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), - ct:sleep(100), + ok = filesync(<<"x/#">>, topic), + ok = filesync(<<"y/#">>, topic), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/topic_trace_x.log")), @@ -128,7 +133,8 @@ t_trace_topic(_Config) -> emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), emqtt:subscribe(T, <<"x/y/z">>), emqtt:unsubscribe(T, <<"x/y/z">>), - ct:sleep(200), + ok = filesync(<<"x/#">>, topic), + ok = filesync(<<"y/#">>, topic), {ok, Bin} = file:read_file("tmp/topic_trace_x.log"), ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), @@ -152,7 +158,8 @@ t_trace_ip_address(_Config) -> %% Start tracing ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"), ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"), - ct:sleep(100), + ok = filesync(<<"127.0.0.1">>, ip_address), + ok = filesync(<<"192.168.1.1">>, ip_address), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/ip_trace_x.log")), @@ -173,7 +180,8 @@ t_trace_ip_address(_Config) -> emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), emqtt:subscribe(T, <<"x/y/z">>), emqtt:unsubscribe(T, <<"x/y/z">>), - ct:sleep(200), + ok = filesync(<<"127.0.0.1">>, ip_address), + ok = filesync(<<"192.168.1.1">>, ip_address), {ok, Bin} = file:read_file("tmp/ip_trace_x.log"), ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), @@ -189,3 +197,20 @@ t_trace_ip_address(_Config) -> {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>), emqtt:disconnect(T), ?assertEqual([], emqx_trace_handler:running()). + + +filesync(Name, Type) -> + filesync(Name, Type, 3). + +%% sometime the handler process is not started yet. +filesync(_Name, _Type, 0) -> ok; +filesync(Name, Type, Retry) -> + try + Handler = binary_to_atom(<<"trace_", + (atom_to_binary(Type))/binary, "_", Name/binary>>), + ok = logger_disk_log_h:filesync(Handler) + catch E:R -> + ct:pal("Filesync error:~p ~p~n", [{Name, Type, Retry}, {E, R}]), + ct:sleep(100), + filesync(Name, Type, Retry - 1) + end. From 28e76e498c3ab979a6b44c433b4c1f2a1c9b1d3b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 3 Dec 2021 10:24:36 +0800 Subject: [PATCH 097/363] fix: trace len > 1 return 500 --- apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 8e052ca66..d243235a4 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -38,7 +38,8 @@ list_trace(_, _Params) -> case emqx_trace:list() of [] -> {ok, []}; List0 -> - List = lists:sort(fun(#{start_at := A}, #{start_at := B}) -> A > B end, List0), + List = lists:sort(fun(#{start_at := A}, #{start_at := B}) -> A > B end, + emqx_trace:format(List0)), Nodes = ekka_mnesia:running_nodes(), TraceSize = cluster_call(?MODULE, get_trace_size, [], 30000), AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize), @@ -50,12 +51,12 @@ list_trace(_, _Params) -> LogSize = collect_file_size(Nodes, FileName, AllFileSize), Trace0 = maps:without([enable, filter], Trace), Trace0#{ log_size => LogSize - , Type => Filter + , Type => iolist_to_binary(Filter) , start_at => list_to_binary(calendar:system_time_to_rfc3339(Start)) , end_at => list_to_binary(calendar:system_time_to_rfc3339(End)) , status => status(Enable, Start, End, Now) } - end, emqx_trace:format(List)), + end, List), {ok, Traces} end. From 48f8c735ea93dbf57dfe9534e5231c63c9fb0f67 Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 3 Dec 2021 18:00:22 +0800 Subject: [PATCH 098/363] feat(emqx_slow_subs): add dyanamic threshold --- .../emqx_modules/src/emqx_mod_slow_subs.erl | 2 + .../test/emqx_slow_subs_SUITE.erl | 3 +- .../test/emqx_slow_subs_api_SUITE.erl | 3 +- priv/emqx.schema | 11 ++- .../emqx_message_latency_stats.erl | 70 +++++++++++-------- 5 files changed, 50 insertions(+), 39 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl index b9117fe8b..1f5ebebf6 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl @@ -37,6 +37,8 @@ -spec(load(list()) -> ok). load(Env) -> + Threshold = proplists:get_value(threshold, Env), + _ = emqx_message_latency_stats:update_threshold(Threshold), emqx_mod_sup:start_child(?LIB, worker, [Env]), ok. diff --git a/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl index 46f144a57..f2763ae6c 100644 --- a/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl @@ -88,7 +88,8 @@ t_log_and_pub(_) -> [Client ! stop || Client <- Clients], ok. base_conf() -> - [ {top_k_num, 5} + [ {threshold, 500} + , {top_k_num, 5} , {expire_interval, timer:seconds(3)} , {notice_interval, 1500} , {notice_qos, 0} diff --git a/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl index 2efc7230a..c8f6b9cfc 100644 --- a/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl @@ -57,7 +57,8 @@ end_per_testcase(_, Config) -> Config. base_conf() -> - [ {top_k_num, 5} + [ {threshold, 500} + , {top_k_num, 5} , {expire_interval, timer:seconds(60)} , {notice_interval, 0} , {notice_qos, 0} diff --git a/priv/emqx.schema b/priv/emqx.schema index 61a98f824..facd86455 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1013,12 +1013,6 @@ end}. {datatype, integer} ]}. -%% @doc Threshold for slow subscription statistics -{mapping, "zone.$name.latency_stats_threshold", "emqx.zones", [ - {default, "100ms"}, - {datatype, {duration, ms}} -]}. - %% @doc Max Packets that Awaiting PUBREL, 0 means no limit {mapping, "zone.$name.max_awaiting_rel", "emqx.zones", [ {default, 0}, @@ -2230,6 +2224,11 @@ end}. {datatype, string} ]}. +{mapping, "module.slow_subs.threshold", "emqx.modules", [ + {default, "500ms"}, + {datatype, {duration, ms}} +]}. + {mapping, "module.slow_subs.expire_interval", "emqx.modules", [ {default, "5m"}, {datatype, {duration, ms}} diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl index dfeba1c68..54fe04760 100644 --- a/src/emqx_slow_subs/emqx_message_latency_stats.erl +++ b/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -17,17 +17,18 @@ -module(emqx_message_latency_stats). %% API --export([ new/1, new/2, update/3 - , check_expire/4, latency/1]). +-export([ new/1, update/3, check_expire/4, latency/1]). + +-export([get_threshold/0, update_threshold/1]). -define(NOW, erlang:system_time(millisecond)). -define(MINIMUM_INSERT_INTERVAL, 1000). --define(MINIMUM_THRESHOLD, 100). +-define(MINIMUM_THRESHOLD, 500). +-define(THRESHOLD_KEY, {?MODULE, threshold}). --opaque stats() :: #{ threshold := number() - , ema := emqx_moving_average:ema() +-opaque stats() :: #{ ema := emqx_moving_average:ema() , last_update_time := timestamp() - , last_access_time := timestamp() %% timestamp of last access top-k + , last_access_time := timestamp() %% timestamp of last try to call hook , last_insert_value := non_neg_integer() }. @@ -44,22 +45,19 @@ %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- --spec new(emqx_types:zone()) -> stats(). -new(Zone) -> - Samples = get_env(Zone, latency_samples, 1), - Threshold = get_env(Zone, latency_stats_threshold, ?MINIMUM_THRESHOLD), - new(Samples, Threshold). --spec new(non_neg_integer(), number()) -> stats(). -new(SamplesT, ThresholdT) -> +-spec new(non_neg_integer() | emqx_types:zone()) -> stats(). +new(SamplesT) when is_integer(SamplesT) -> Samples = erlang:max(1, SamplesT), - Threshold = erlang:max(?MINIMUM_THRESHOLD, ThresholdT), #{ ema => emqx_moving_average:new(exponential, #{period => Samples}) - , threshold => Threshold , last_update_time => 0 , last_access_time => 0 , last_insert_value => 0 - }. + }; + +new(Zone) -> + Samples = get_env(Zone, latency_samples, 1), + new(Samples). -spec update(emqx_types:clientid(), number(), stats()) -> stats(). update(ClientId, Val, #{ema := EMA} = Stats) -> @@ -82,25 +80,35 @@ check_expire(ClientId, Now, _Interval, #{last_update_time := LUT} = S) -> latency(#{ema := #{average := Average}}) -> Average. +-spec update_threshold(pos_integer()) -> pos_integer(). +update_threshold(Threshold) -> + Val = erlang:max(Threshold, ?MINIMUM_THRESHOLD), + persistent_term:put(?THRESHOLD_KEY, Val), + Val. + +get_threshold() -> + persistent_term:get(?THRESHOLD_KEY, ?MINIMUM_THRESHOLD). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- -spec call_hook(emqx_types:clientid(), timestamp(), latency_type(), timespan(), stats()) -> stats(). -call_hook(_, Now, _, _, #{last_access_time := LIT} = S) - when LIT >= Now - ?MINIMUM_INSERT_INTERVAL -> - S; - -call_hook(_, _, _, Latency, #{threshold := Threshold} = S) - when Latency =< Threshold -> +call_hook(_, _, _, Latency, S) + when Latency =< ?MINIMUM_THRESHOLD -> S; call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> - ToInsert = erlang:floor(Latency), - Arg = #{clientid => ClientId, - latency => ToInsert, - type => Type, - last_insert_value => LIV, - update_time => Now}, - emqx:run_hook('message.slow_subs_stats', [Arg]), - Stats#{last_insert_value := ToInsert, - last_access_time := Now}. + case get_threshold() >= Latency of + true -> + Stats#{last_access_time := Now}; + _ -> + ToInsert = erlang:floor(Latency), + Arg = #{clientid => ClientId, + latency => ToInsert, + type => Type, + last_insert_value => LIV, + update_time => Now}, + emqx:run_hook('message.slow_subs_stats', [Arg]), + Stats#{last_insert_value := ToInsert, + last_access_time := Now} + end. From 3f49e3186cfda9252d7cf34172a23fe48a86fdee Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 6 Dec 2021 17:22:36 +0800 Subject: [PATCH 099/363] fix(emqx_slow_subs): add default threshold macro --- src/emqx_slow_subs/emqx_message_latency_stats.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl index 54fe04760..49f0c8f1e 100644 --- a/src/emqx_slow_subs/emqx_message_latency_stats.erl +++ b/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -23,7 +23,8 @@ -define(NOW, erlang:system_time(millisecond)). -define(MINIMUM_INSERT_INTERVAL, 1000). --define(MINIMUM_THRESHOLD, 500). +-define(MINIMUM_THRESHOLD, 100). +-define(DEFAULT_THRESHOLD, 500). -define(THRESHOLD_KEY, {?MODULE, threshold}). -opaque stats() :: #{ ema := emqx_moving_average:ema() @@ -87,7 +88,7 @@ update_threshold(Threshold) -> Val. get_threshold() -> - persistent_term:get(?THRESHOLD_KEY, ?MINIMUM_THRESHOLD). + persistent_term:get(?THRESHOLD_KEY, ?DEFAULT_THRESHOLD). %%-------------------------------------------------------------------- %% Internal functions @@ -98,7 +99,7 @@ call_hook(_, _, _, Latency, S) S; call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> - case get_threshold() >= Latency of + case Latency =< get_threshold() of true -> Stats#{last_access_time := Now}; _ -> From d4dd4a124c5c08e728e6c40b89c3ead2eafef034 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 6 Dec 2021 18:24:47 +0800 Subject: [PATCH 100/363] fix: trace_name format [A-Za-z0-9-_];waiting status if create time to closed" --- .../src/emqx_trace/emqx_trace.erl | 15 +++++++------ .../src/emqx_trace/emqx_trace_api.erl | 21 ++++++++----------- .../test/emqx_trace_SUITE.erl | 2 +- test/emqx_trace_handler_SUITE.erl | 1 + 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index 135a1f53b..252da8c0f 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -398,15 +398,13 @@ fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60}); fill_default(Trace) -> Trace. +-define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$"). + to_trace([], Rec) -> {ok, Rec}; to_trace([{name, Name} | Trace], Rec) -> - case io_lib:printable_unicode_list(unicode:characters_to_list(Name, utf8)) of - true -> - case binary:match(Name, [<<"/">>], []) of - nomatch -> to_trace(Trace, Rec#?TRACE{name = Name}); - _ -> {error, "name cannot contain /"} - end; - false -> {error, "name must printable unicode"} + case re:run(Name, ?NAME_RE) of + nomatch -> {error, "Name should be " ?NAME_RE}; + _ -> to_trace(Trace, Rec#?TRACE{name = Name}) end; to_trace([{type, Type} | Trace], Rec) -> case lists:member(Type, [<<"clientid">>, <<"topic">>, <<"ip_address">>]) of @@ -453,7 +451,8 @@ validate_topic(TopicName) -> to_system_second(At) -> try Sec = calendar:rfc3339_to_system_time(binary_to_list(At), [{unit, second}]), - {ok, Sec} + Now = erlang:system_time(second), + {ok, erlang:max(Now, Sec)} catch error: {badmatch, _} -> {error, ["The rfc3339 specification not satisfied: ", At]} end. diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index d243235a4..d2bca542b 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -135,7 +135,7 @@ stream_log_file(#{name := Name}, Params) -> {ok, Node} -> Position = binary_to_integer(Position0), Bytes = binary_to_integer(Bytes0), - case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of {ok, Bin} -> Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => Bin}}; @@ -143,7 +143,7 @@ stream_log_file(#{name := Name}, Params) -> Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => <<"">>}}; {error, Reason} -> - logger:log(error, "read_file_failed by ~p", [{Name, Reason, Position, Bytes}]), + logger:log(error, "read_file_failed by ~p", [{Node, Name, Reason, Position, Bytes}]), {error, Reason}; {badrpc, nodedown} -> {error, "BadRpc node down"} @@ -165,15 +165,12 @@ get_trace_size() -> %% this is an rpc call for stream_log_file/2 read_trace_file(Name, Position, Limit) -> - TraceDir = emqx_trace:trace_dir(), - {ok, AllFiles} = file:list_dir(TraceDir), - TracePrefix = "trace_" ++ binary_to_list(Name) ++ "_", - Filter = fun(FileName) -> nomatch =/= string:prefix(FileName, TracePrefix) end, - case lists:filter(Filter, AllFiles) of - [TraceFile] -> + case emqx_trace:get_trace_filename(Name) of + {error, _} = Error -> Error; + {ok, TraceFile} -> + TraceDir = emqx_trace:trace_dir(), TracePath = filename:join([TraceDir, TraceFile]), - read_file(TracePath, Position, Limit); - [] -> {error, not_found} + read_file(TracePath, Position, Limit) end. read_file(Path, Offset, Bytes) -> @@ -206,8 +203,8 @@ collect_file_size(Nodes, FileName, AllFiles) -> Acc#{Node => Size} end, #{}, Nodes). -%% status(false, _Start, End, Now) when End > Now -> <<"stopped">>; status(false, _Start, _End, _Now) -> <<"stopped">>; -status(true, Start, _End, Now) when Now < Start -> <<"waiting">>; +%% asynchronously create trace, we should wait 1 seconds +status(true, Start, _End, Now) when Now < Start + 2 -> <<"waiting">>; status(true, _Start, End, Now) when Now >= End -> <<"stopped">>; status(true, _Start, _End, _Now) -> <<"running">>. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index 94ab71270..4f33e5b7f 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -130,7 +130,7 @@ t_create_failed(_Config) -> InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, {<<"type">>, <<"clientid">>}], {error, Reason9} = emqx_trace:create(InvalidPackets4), - ?assertEqual(<<"name cannot contain /">>, iolist_to_binary(Reason9)), + ?assertEqual(<<"Name should be ^[A-Za-z]+[A-Za-z0-9-_]*$">>, iolist_to_binary(Reason9)), ?assertEqual({error, "type=[topic,clientid,ip_address] required"}, emqx_trace:create([{<<"name">>, <<"test-name">>}, {<<"clientid">>, <<"good">>}])), diff --git a/test/emqx_trace_handler_SUITE.erl b/test/emqx_trace_handler_SUITE.erl index b42b1a591..cbf455d2d 100644 --- a/test/emqx_trace_handler_SUITE.erl +++ b/test/emqx_trace_handler_SUITE.erl @@ -200,6 +200,7 @@ t_trace_ip_address(_Config) -> filesync(Name, Type) -> + ct:sleep(50), filesync(Name, Type, 3). %% sometime the handler process is not started yet. From e7c765aaa3f5276601e3040f78ed2bef0719d0b7 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 7 Dec 2021 14:30:16 +0800 Subject: [PATCH 101/363] chore(helm): support cluster discovery by dns --- .github/workflows/run_fvt_tests.yaml | 53 ++++++++++++++++--- deploy/charts/emqx/templates/StatefulSet.yaml | 13 +---- deploy/charts/emqx/templates/configmap.yaml | 2 +- deploy/charts/emqx/templates/rbac.yaml | 4 +- deploy/charts/emqx/values.yaml | 13 +++++ deploy/docker/docker-entrypoint.sh | 20 ++++--- 6 files changed, 80 insertions(+), 25 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index a16586bd9..772fd4fa2 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -65,6 +65,12 @@ jobs: helm_test: runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + discovery: + - k8s + - dns steps: - uses: actions/checkout@v1 @@ -104,17 +110,18 @@ jobs: sudo chmod 700 get_helm.sh sudo ./get_helm.sh helm version - - name: run emqx on chart - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - timeout-minutes: 5 + - name: setup emqx chart run: | sudo docker save ${TARGET}:${EMQX_TAG} -o emqx.tar.gz sudo k3s ctr image import emqx.tar.gz sed -i -r "s/^appVersion: .*$/appVersion: \"${EMQX_TAG}\"/g" deploy/charts/emqx/Chart.yaml sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml - + - name: run emqx on chart + if: matrix.discovery == 'k8s' + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | helm install emqx \ --set image.repository=${TARGET} \ --set image.pullPolicy=Never \ @@ -124,7 +131,29 @@ jobs: --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ deploy/charts/emqx \ --debug - + - name: run emqx on chart + if: matrix.discovery == 'dns' + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | + helm install emqx \ + --set emqxConfig.EMQX_CLUSTER__DISCOVERY="dns" \ + --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \ + --set emqxConfig.EMQX_CLUSTER__DNS__APP="emqx" \ + --set emqxConfig.EMQX_CLUSTER__DNS__TYPE="srv" \ + --set image.repository=${TARGET} \ + --set image.pullPolicy=Never \ + --set emqxAclConfig="" \ + --set image.pullPolicy=Never \ + --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ + deploy/charts/emqx \ + --debug + - name: waiting emqx started + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + timeout-minutes: 5 + run: | while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \ != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do echo "=============================="; @@ -133,6 +162,18 @@ jobs: echo "waiting emqx started"; sleep 10; done + - name: Check ${{ matrix.kind[0]}} cluster + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + timeout-minutes: 10 + run: | + while + nodes="$(kubectl exec -i emqx-0 -- curl --silent --basic -u admin:public -X GET http://localhost:8081/api/v4/brokers | jq '.data|length')"; + [ "$nodes" != "3" ]; + do + echo "waiting emqx cluster scale" + sleep 1 + done - name: get emqx-0 pods log if: failure() env: diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index dcc03b996..7ad870b6d 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -87,7 +87,9 @@ spec: secret: secretName: {{ .Values.emqxLicneseSecretName }} {{- end }} + {{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s" }} serviceAccountName: {{ include "emqx.fullname" . }} + {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} @@ -134,17 +136,6 @@ spec: envFrom: - configMapRef: name: {{ include "emqx.fullname" . }}-env - env: - - name: EMQX_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__K8S__APP_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__DISCOVERY - value: k8s - - name: EMQX_CLUSTER__K8S__SERVICE_NAME - value: {{ include "emqx.fullname" . }}-headless - - name: EMQX_CLUSTER__K8S__NAMESPACE - value: {{ .Release.Namespace }} resources: {{ toYaml .Values.resources | indent 12 }} volumeMounts: diff --git a/deploy/charts/emqx/templates/configmap.yaml b/deploy/charts/emqx/templates/configmap.yaml index c9c4b4770..328df2000 100644 --- a/deploy/charts/emqx/templates/configmap.yaml +++ b/deploy/charts/emqx/templates/configmap.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} data: {{- range $index, $value := .Values.emqxConfig}} - {{$index}}: "{{ $value }}" + {{$index}}: "{{ tpl (printf "%v" $value) $ }}" {{- end}} --- diff --git a/deploy/charts/emqx/templates/rbac.yaml b/deploy/charts/emqx/templates/rbac.yaml index 87cd18178..79b431442 100644 --- a/deploy/charts/emqx/templates/rbac.yaml +++ b/deploy/charts/emqx/templates/rbac.yaml @@ -1,3 +1,4 @@ +{{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s" }} apiVersion: v1 kind: ServiceAccount metadata: @@ -39,4 +40,5 @@ subjects: roleRef: kind: Role name: {{ include "emqx.fullname" . }} - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 62cf779ea..373ba0849 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -50,7 +50,20 @@ initContainers: {} ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) emqxConfig: + EMQX_NAME: "{{ .Release.Name }}" + + ## Cluster discovery by dns + # EMQX_CLUSTER__DISCOVERY: "dns" + # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local" + # EMQX_CLUSTER__DNS__APP: "{{ .Release.Name }}" + # EMQX_CLUSTER__DNS__TYPE: "srv" + + ## Cluster discovery by k8s + EMQX_CLUSTER__DISCOVERY: "k8s" + EMQX_CLUSTER__K8S__APP_NAME: "{{ .Release.Name }}" EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" + EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless" + EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}" ## The address type is used to extract host from k8s service. ## Value: ip | dns | hostname ## Note：Hostname is only supported after v4.0-rc.2 diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index 16b6cb077..1abef430b 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -28,12 +28,20 @@ if [[ -z "$EMQX_NAME" ]]; then fi if [[ -z "$EMQX_HOST" ]]; then - if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then - EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"} - EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX" - elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then - EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'} - EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX") + if [[ "$EMQX_CLUSTER__DISCOVERY" == "dns" ]] && \ + [[ "$EMQX_CLUSTER__DNS__TYPE" == "srv" ]] && \ + grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then + EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME" + elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \ + [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \ + [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then + EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"} + EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX" + elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \ + [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \ + [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then + EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'} + EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX") else EMQX_HOST="$LOCAL_IP" fi From be6160f5bd81aa1c4db2c04882c16b2e883286ad Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 7 Dec 2021 20:24:56 +0800 Subject: [PATCH 102/363] fix: trace handler start time not correct --- .../src/emqx_trace/emqx_trace.erl | 13 +++--- .../src/emqx_trace/emqx_trace_api.erl | 40 ++++++++++--------- .../test/emqx_trace_SUITE.erl | 30 ++++++++++++-- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index 252da8c0f..ad7112707 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -53,6 +53,7 @@ -ifdef(TEST). -export([ log_file/2 , create_table/0 + , find_closest_time/2 ]). -endif. @@ -288,15 +289,15 @@ get_enable_trace() -> find_closest_time(Traces, Now) -> Sec = lists:foldl( - fun(#?TRACE{start_at = Start, end_at = End}, Closest) - when Start >= Now andalso Now < End -> %% running - min(End - Now, Closest); - (#?TRACE{start_at = Start}, Closest) when Start < Now -> %% waiting - min(Now - Start, Closest); - (_, Closest) -> Closest %% finished + fun(#?TRACE{start_at = Start, end_at = End, enable = true}, Closest) -> + min(closest(End, Now, Closest), closest(Start, Now, Closest)); + (_, Closest) -> Closest end, 60 * 15, Traces), timer:seconds(Sec). +closest(Time, Now, Closest) when Now >= Time -> Closest; +closest(Time, Now, Closest) -> min(Time - Now, Closest). + disable_finished([]) -> ok; disable_finished(Traces) -> transaction(fun() -> diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index d2bca542b..0e298698e 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -135,7 +135,7 @@ stream_log_file(#{name := Name}, Params) -> {ok, Node} -> Position = binary_to_integer(Position0), Bytes = binary_to_integer(Bytes0), - case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of {ok, Bin} -> Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => Bin}}; @@ -143,7 +143,7 @@ stream_log_file(#{name := Name}, Params) -> Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => <<"">>}}; {error, Reason} -> - logger:log(error, "read_file_failed by ~p", [{Node, Name, Reason, Position, Bytes}]), + logger:log(error, "read_file_failed ~p", [{Node, Name, Reason, Position, Bytes}]), {error, Reason}; {badrpc, nodedown} -> {error, "BadRpc node down"} @@ -174,21 +174,24 @@ read_trace_file(Name, Position, Limit) -> end. read_file(Path, Offset, Bytes) -> - {ok, IoDevice} = file:open(Path, [read, raw, binary]), - try - _ = case Offset of - 0 -> ok; - _ -> file:position(IoDevice, {bof, Offset}) - end, - case file:read(IoDevice, Bytes) of - {ok, Bin} -> {ok, Bin}; - {error, Reason} -> {error, Reason}; - eof -> - {ok, #file_info{size = Size}} = file:read_file_info(IoDevice), - {eof, Size} - end - after - file:close(IoDevice) + case file:open(Path, [read, raw, binary]) of + {ok, IoDevice} -> + try + _ = case Offset of + 0 -> ok; + _ -> file:position(IoDevice, {bof, Offset}) + end, + case file:read(IoDevice, Bytes) of + {ok, Bin} -> {ok, Bin}; + {error, Reason} -> {error, Reason}; + eof -> + {ok, #file_info{size = Size}} = file:read_file_info(IoDevice), + {eof, Size} + end + after + file:close(IoDevice) + end; + {error, Reason} -> {error, Reason} end. to_node(Node) -> @@ -204,7 +207,6 @@ collect_file_size(Nodes, FileName, AllFiles) -> end, #{}, Nodes). status(false, _Start, _End, _Now) -> <<"stopped">>; -%% asynchronously create trace, we should wait 1 seconds -status(true, Start, _End, Now) when Now < Start + 2 -> <<"waiting">>; +status(true, Start, _End, Now) when Now < Start -> <<"waiting">>; status(true, _Start, End, Now) when Now >= End -> <<"stopped">>; status(true, _Start, _End, _Now) -> <<"running">>. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index 4f33e5b7f..e20fb6957 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -196,9 +196,9 @@ t_update_enable(_Config) -> t_load_state(_Config) -> Now = erlang:system_time(second), - Running = [{<<"name">>, <<"Running">>}, {<<"type">>, <<"topic">>}, - {<<"topic">>, <<"/x/y/1">>}, {<<"start_at">>, to_rfc3339(Now - 1)}, - {<<"end_at">>, to_rfc3339(Now + 2)}], + Running = #{name => <<"Running">>, type => <<"topic">>, + topic => <<"/x/y/1">>, start_at => to_rfc3339(Now - 1), + end_at => to_rfc3339(Now + 2)}, Waiting = [{<<"name">>, <<"Waiting">>}, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/2">>}, {<<"start_at">>, to_rfc3339(Now + 3)}, {<<"end_at">>, to_rfc3339(Now + 8)}], @@ -300,6 +300,30 @@ t_download_log(_Config) -> ok = emqtt:disconnect(Client), ok. +t_find_closed_time(_Config) -> + DefaultMs = 60 * 15000, + Now = erlang:system_time(second), + Traces2 = [], + ?assertEqual(DefaultMs, emqx_trace:find_closest_time(Traces2, Now)), + Traces3 = [#emqx_trace{name = <<"disable">>, start_at = Now + 1, + end_at = Now + 2, enable = false}], + ?assertEqual(DefaultMs, emqx_trace:find_closest_time(Traces3, Now)), + Traces4 = [#emqx_trace{name = <<"running">>, start_at = Now, end_at = Now + 10, enable = true}], + ?assertEqual(10000, emqx_trace:find_closest_time(Traces4, Now)), + Traces5 = [#emqx_trace{name = <<"waiting">>, start_at = Now + 2, + end_at = Now + 10, enable = true}], + ?assertEqual(2000, emqx_trace:find_closest_time(Traces5, Now)), + Traces = [ + #emqx_trace{name = <<"waiting">>, start_at = Now + 1, end_at = Now + 2, enable = true}, + #emqx_trace{name = <<"running0">>, start_at = Now, end_at = Now + 5, enable = true}, + #emqx_trace{name = <<"running1">>, start_at = Now - 1, end_at = Now + 1, enable = true}, + #emqx_trace{name = <<"finished">>, start_at = Now - 2, end_at = Now - 1, enable = true}, + #emqx_trace{name = <<"waiting">>, start_at = Now + 1, end_at = Now + 1, enable = true}, + #emqx_trace{name = <<"stopped">>, start_at = Now, end_at = Now + 10, enable = false} + ], + ?assertEqual(1000, emqx_trace:find_closest_time(Traces, Now)), + ok. + to_rfc3339(Second) -> list_to_binary(calendar:system_time_to_rfc3339(Second)). From e651becd99fc008762a32734981c9ca2e6555f8e Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 8 Dec 2021 15:05:39 +0800 Subject: [PATCH 103/363] fix(emqx_slow_subs): fix threshold related bugs 1. limit the interval between calling hooks 2. improve the code of update threshold --- .../src/emqx_slow_subs/emqx_slow_subs.erl | 16 +++++++++++----- lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl | 2 -- .../emqx_message_latency_stats.erl | 4 ++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index e96c2a907..11d25380e 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -151,8 +151,8 @@ init_topk_tab() -> init([Conf]) -> notice_tick(Conf), expire_tick(Conf), - MaxSize = get_value(top_k_num, Conf), - load(MaxSize), + update_threshold(Conf), + load(Conf), {ok, #{config => Conf, last_tick_at => ?NOW, enable => true}}. @@ -163,8 +163,8 @@ handle_call({enable, Enable}, _From, IsEnable -> State; true -> - MaxSize = get_value(max_topk_num, Cfg), - load(MaxSize), + update_threshold(Cfg), + load(Cfg), State#{enable := true}; _ -> unload(), @@ -274,7 +274,8 @@ publish(TickTime, Cfg, Notices) -> _ = emqx_broker:safe_publish(Msg), ok. -load(MaxSize) -> +load(Cfg) -> + MaxSize = get_value(top_k_num, Cfg), _ = emqx:hook('message.slow_subs_stats', fun ?MODULE:on_stats_update/2, [#{max_size => MaxSize}]), @@ -319,3 +320,8 @@ try_insert_to_topk(MaxSize, Index, Latency, Type, Ts) -> ets:delete(?TOPK_TAB, First) end end. + +update_threshold(Conf) -> + Threshold = proplists:get_value(threshold, Conf), + _ = emqx_message_latency_stats:update_threshold(Threshold), + ok. diff --git a/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl index 1f5ebebf6..b9117fe8b 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl @@ -37,8 +37,6 @@ -spec(load(list()) -> ok). load(Env) -> - Threshold = proplists:get_value(threshold, Env), - _ = emqx_message_latency_stats:update_threshold(Threshold), emqx_mod_sup:start_child(?LIB, worker, [Env]), ok. diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl index 49f0c8f1e..1d6158d59 100644 --- a/src/emqx_slow_subs/emqx_message_latency_stats.erl +++ b/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -98,6 +98,10 @@ call_hook(_, _, _, Latency, S) when Latency =< ?MINIMUM_THRESHOLD -> S; +call_hook(_, Now, _, _, #{last_access_time := LIT} = S) + when Now =< LIT + ?MINIMUM_INSERT_INTERVAL -> + S; + call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> case Latency =< get_threshold() of true -> From d435f1211ee08e8b2083e7ab3699ff3384917e04 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 8 Dec 2021 10:33:57 -0300 Subject: [PATCH 104/363] fix(live_conn): fix live connection count on race condition When multiple clients try to connect concurrently using the same client ID, they all call `emqx_channel:ensure_connected`, increasing the live connection count, but only one will successfully acquire the lock for that client ID. This means that all other clients that increased the live connection count will not get to call neither `emqx_channel:ensure_disconnected` nor be monitored for `DOWN` messages, effectively causing a count leak. By moving the increment to `emqx_cm:register_channel`, which is only called inside the lock, we can remove this leakage. Also, during the handling of `DOWN` messages, we now iterate over all channel PIDs returned by `eqmx_misc:drain_down`, since it could be that one or more PIDs are not contained in the `pmon` state. --- src/emqx_channel.erl | 2 - src/emqx_cm.erl | 7 +--- test/emqx_broker_SUITE.erl | 75 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index bad053845..5524a391d 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1548,8 +1548,6 @@ ensure_connected(Channel = #channel{conninfo = ConnInfo, clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.connected', [ClientInfo, NConnInfo]), - ChanPid = self(), - emqx_cm:mark_channel_connected(ChanPid), Channel#channel{conninfo = NConnInfo, conn_state = connected }. diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index c54089e41..1c6d4080a 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -137,6 +137,7 @@ register_channel(ClientId, ChanPid, #{conn_mod := ConnMod}) when is_pid(ChanPid) true = ets:insert(?CHAN_TAB, Chan), true = ets:insert(?CHAN_CONN_TAB, {Chan, ConnMod}), ok = emqx_cm_registry:register_channel(Chan), + mark_channel_connected(ChanPid), cast({registered, Chan}). %% @doc Unregister a channel. @@ -465,11 +466,7 @@ handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon} ?tp(emqx_cm_process_down, #{pid => Pid, reason => _Reason}), ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), - lists:foreach( - fun({ChanPid, _ClientID}) -> - mark_channel_disconnected(ChanPid) - end, - Items), + lists:foreach(fun mark_channel_disconnected/1, ChanPids), ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]), {noreply, State#{chan_pmon := PMon1}}; handle_info(Info, State) -> diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl index 57f0d6acf..d44bb3412 100644 --- a/test/emqx_broker_SUITE.erl +++ b/test/emqx_broker_SUITE.erl @@ -37,6 +37,7 @@ groups() -> TCs = emqx_ct:all(?MODULE), ConnClientTCs = [ t_connected_client_count_persistent , t_connected_client_count_anonymous + , t_connected_client_count_transient_takeover , t_connected_client_stats ], OtherTCs = TCs -- ConnClientTCs, @@ -451,6 +452,80 @@ t_connected_client_count_anonymous({'end', _Config}) -> snabbkaffe:stop(), ok. +t_connected_client_count_transient_takeover({init, Config}) -> + ok = snabbkaffe:start_trace(), + process_flag(trap_exit, true), + Config; +t_connected_client_count_transient_takeover(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ClientID = <<"clientid">>, + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% we spawn several clients simultaneously to cause the race + %% condition for the client id lock + NumClients = 20, + {ok, {ok, [_, _]}} = + wait_for_events( + fun() -> + lists:foreach( + fun(_) -> + spawn( + fun() -> + {ok, ConnPid} = + emqtt:start_link([ {clean_start, true} + , {clientid, ClientID} + | Config]), + %% don't assert the result: most of them fail + %% during the race + emqtt:ConnFun(ConnPid), + ok + end), + ok + end, + lists:seq(1, NumClients)) + end, + %% there can be only one channel that wins the race for the + %% lock for this client id. we also expect a decrement + %% event because the client dies along with the ephemeral + %% process. + [ emqx_cm_connected_client_count_inc + , emqx_cm_connected_client_count_dec + ], + 1000), + %% Since more than one pair of inc/dec may be emitted, we need to + %% wait for full stabilization + timer:sleep(100), + %% It must be 0 again because we spawn-linked the clients in + %% ephemeral processes above, and all should be dead now. + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% connecting again + {ok, ConnPid1} = emqtt:start_link([ {clean_start, true} + , {clientid, ClientID} + | Config + ]), + {{ok, _}, {ok, [_]}} = + wait_for_events( + fun() -> emqtt:ConnFun(ConnPid1) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% abnormal exit of channel process + [ChanPid] = emqx_cm:all_channels(), + {ok, {ok, [_, _]}} = + wait_for_events( + fun() -> + exit(ChanPid, kill), + ok + end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ok; +t_connected_client_count_transient_takeover({'end', _Config}) -> + snabbkaffe:stop(), + ok. + t_connected_client_stats({init, Config}) -> ok = supervisor:terminate_child(emqx_kernel_sup, emqx_stats), {ok, _} = supervisor:restart_child(emqx_kernel_sup, emqx_stats), From c410571ee22aa8e1e6bdce7e436e9977724d62c1 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 10 Dec 2021 17:39:18 +0300 Subject: [PATCH 105/363] fix(wss): update cowboy & ranch for OTP24 compatibility --- rebar.config | 2 +- test/emqx_listeners_SUITE.erl | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 8a1adaef3..1e3c758e6 100644 --- a/rebar.config +++ b/rebar.config @@ -41,7 +41,7 @@ , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.5"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} - , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} + , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.0"}}} diff --git a/test/emqx_listeners_SUITE.erl b/test/emqx_listeners_SUITE.erl index f49d33004..96a2d8bb8 100644 --- a/test/emqx_listeners_SUITE.erl +++ b/test/emqx_listeners_SUITE.erl @@ -47,6 +47,12 @@ t_restart_listeners(_) -> ok = emqx_listeners:restart(), ok = emqx_listeners:stop(). +t_wss_conn(_) -> + ok = emqx_listeners:start(), + {ok, Socket} = ssl:connect({127, 0, 0, 1}, 8084, [{verify, verify_none}], 1000), + ok = ssl:close(Socket), + ok = emqx_listeners:stop(). + render_config_file() -> Path = local_path(["..", "..", "..", "..", "etc", "emqx.conf"]), {ok, Temp} = file:read_file(Path), @@ -91,4 +97,3 @@ get_base_dir(Module) -> get_base_dir() -> get_base_dir(?MODULE). - From e59c5cc3f34af9a3307bdbc557b00d0da4f1b06d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 10 Dec 2021 19:16:16 +0800 Subject: [PATCH 106/363] fix(trace): create trace validate type and filter more strictly --- .../src/emqx_trace/emqx_trace.erl | 65 ++++++++++--------- .../test/emqx_trace_SUITE.erl | 38 +++++++---- 2 files changed, 60 insertions(+), 43 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index ad7112707..f074055f9 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -368,14 +368,12 @@ classify_by_time([Trace | Traces], Now, Wait, Run, Finish) -> classify_by_time(Traces, Now, Wait, [Trace | Run], Finish). to_trace(TraceParam) -> - case to_trace(ensure_proplists(TraceParam), #?TRACE{}) of + case to_trace(ensure_map(TraceParam), #?TRACE{}) of {error, Reason} -> {error, Reason}; {ok, #?TRACE{name = undefined}} -> {error, "name required"}; {ok, #?TRACE{type = undefined}} -> {error, "type=[topic,clientid,ip_address] required"}; - {ok, #?TRACE{filter = undefined}} -> - {error, "topic/clientid/ip_address filter required"}; {ok, TraceRec0 = #?TRACE{}} -> case fill_default(TraceRec0) of #?TRACE{start_at = Start, end_at = End} when End =< Start -> @@ -385,13 +383,13 @@ to_trace(TraceParam) -> end end. -ensure_proplists(#{} = Trace) -> maps:to_list(Trace); -ensure_proplists(Trace) when is_list(Trace) -> +ensure_map(#{} = Trace) -> Trace; +ensure_map(Trace) when is_list(Trace) -> lists:foldl( - fun({K, V}, Acc) when is_binary(K) -> [{binary_to_existing_atom(K), V} | Acc]; - ({K, V}, Acc) when is_atom(K) -> [{K, V} | Acc]; + fun({K, V}, Acc) when is_binary(K) -> Acc#{binary_to_existing_atom(K) => V}; + ({K, V}, Acc) when is_atom(K) -> Acc#{K => V}; (_, Acc) -> Acc - end, [], Trace). + end, #{}, Trace). fill_default(Trace = #?TRACE{start_at = undefined}) -> fill_default(Trace#?TRACE{start_at = erlang:system_time(second)}); @@ -401,45 +399,45 @@ fill_default(Trace) -> Trace. -define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$"). -to_trace([], Rec) -> {ok, Rec}; -to_trace([{name, Name} | Trace], Rec) -> +to_trace(#{name := Name} = Trace, Rec) -> case re:run(Name, ?NAME_RE) of nomatch -> {error, "Name should be " ?NAME_RE}; - _ -> to_trace(Trace, Rec#?TRACE{name = Name}) + _ -> to_trace(maps:remove(name, Trace), Rec#?TRACE{name = Name}) end; -to_trace([{type, Type} | Trace], Rec) -> - case lists:member(Type, [<<"clientid">>, <<"topic">>, <<"ip_address">>]) of - true -> to_trace(Trace, Rec#?TRACE{type = binary_to_existing_atom(Type)}); - false -> {error, "incorrect type: only support clientid/topic/ip_address"} +to_trace(#{type := <<"clientid">>, clientid := Filter} = Trace, Rec) -> + Trace0 = maps:without([type, clientid], Trace), + to_trace(Trace0, Rec#?TRACE{type = clientid, filter = Filter}); +to_trace(#{type := <<"topic">>, topic := Filter} = Trace, Rec) -> + case validate_topic(Filter) of + ok -> + Trace0 = maps:without([type, topic], Trace), + to_trace(Trace0, Rec#?TRACE{type = topic, filter = Filter}); + Error -> Error end; -to_trace([{topic, Topic} | Trace], Rec) -> - case validate_topic(Topic) of - ok -> to_trace(Trace, Rec#?TRACE{filter = Topic}); - {error, Reason} -> {error, Reason} +to_trace(#{type := <<"ip_address">>, ip_address := Filter} = Trace, Rec) -> + case validate_ip_address(Filter) of + ok -> + Trace0 = maps:without([type, ip_address], Trace), + to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = Filter}); + Error -> Error end; -to_trace([{clientid, ClientId} | Trace], Rec) -> - to_trace(Trace, Rec#?TRACE{filter = ClientId}); -to_trace([{ip_address, IP} | Trace], Rec) -> - case inet:parse_address(binary_to_list(IP)) of - {ok, _} -> to_trace(Trace, Rec#?TRACE{filter = binary_to_list(IP)}); - {error, Reason} -> {error, lists:flatten(io_lib:format("ip address: ~p", [Reason]))} - end; -to_trace([{start_at, StartAt} | Trace], Rec) -> +to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])}; +to_trace(#{start_at := StartAt} = Trace, Rec) -> case to_system_second(StartAt) of - {ok, Sec} -> to_trace(Trace, Rec#?TRACE{start_at = Sec}); + {ok, Sec} -> to_trace(maps:remove(start_at, Trace), Rec#?TRACE{start_at = Sec}); {error, Reason} -> {error, Reason} end; -to_trace([{end_at, EndAt} | Trace], Rec) -> +to_trace(#{end_at := EndAt} = Trace, Rec) -> Now = erlang:system_time(second), case to_system_second(EndAt) of {ok, Sec} when Sec > Now -> - to_trace(Trace, Rec#?TRACE{end_at = Sec}); + to_trace(maps:remove(end_at, Trace), Rec#?TRACE{end_at = Sec}); {ok, _Sec} -> {error, "end_at time has already passed"}; {error, Reason} -> {error, Reason} end; -to_trace([Unknown | _Trace], _Rec) -> {error, io_lib:format("unknown field: ~p", [Unknown])}. +to_trace(_, Rec) -> {ok, Rec}. validate_topic(TopicName) -> try emqx_topic:validate(filter, TopicName) of @@ -448,6 +446,11 @@ validate_topic(TopicName) -> error:Error -> {error, io_lib:format("topic: ~s invalid by ~p", [TopicName, Error])} end. +validate_ip_address(IP) -> + case inet:parse_address(binary_to_list(IP)) of + {ok, _} -> ok; + {error, Reason} -> {error, lists:flatten(io_lib:format("ip address: ~p", [Reason]))} + end. to_system_second(At) -> try diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index e20fb6957..56b81424e 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -106,26 +106,29 @@ t_create_size_max(_Config) -> ok. t_create_failed(_Config) -> - UnknownField = [{<<"unknown">>, 12}], + Name = {<<"name">>, <<"test">>}, + UnknownField = [Name, {<<"unknown">>, 12}], {error, Reason1} = emqx_trace:create(UnknownField), - ?assertEqual(<<"unknown field: {unknown,12}">>, iolist_to_binary(Reason1)), + ?assertEqual(<<"type=[topic,clientid,ip_address] required">>, iolist_to_binary(Reason1)), - InvalidTopic = [{<<"topic">>, "#/#//"}], + InvalidTopic = [Name, {<<"topic">>, "#/#//"}, {<<"type">>, <<"topic">>}], {error, Reason2} = emqx_trace:create(InvalidTopic), ?assertEqual(<<"topic: #/#// invalid by function_clause">>, iolist_to_binary(Reason2)), - InvalidStart = [{<<"start_at">>, <<"2021-12-3:12">>}], + InvalidStart = [Name, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/sys/">>}, + {<<"start_at">>, <<"2021-12-3:12">>}], {error, Reason3} = emqx_trace:create(InvalidStart), ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, iolist_to_binary(Reason3)), - InvalidEnd = [{<<"end_at">>, <<"2021-12-3:12">>}], + InvalidEnd = [Name, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/sys/">>}, + {<<"end_at">>, <<"2021-12-3:12">>}], {error, Reason4} = emqx_trace:create(InvalidEnd), ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, iolist_to_binary(Reason4)), - {error, Reason7} = emqx_trace:create([{<<"name">>, <<"test">>}, {<<"type">>, <<"clientid">>}]), - ?assertEqual(<<"topic/clientid/ip_address filter required">>, iolist_to_binary(Reason7)), + {error, Reason7} = emqx_trace:create([Name, {<<"type">>, <<"clientid">>}]), + ?assertEqual(<<"required clientid field">>, iolist_to_binary(Reason7)), InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, {<<"type">>, <<"clientid">>}], @@ -135,12 +138,9 @@ t_create_failed(_Config) -> ?assertEqual({error, "type=[topic,clientid,ip_address] required"}, emqx_trace:create([{<<"name">>, <<"test-name">>}, {<<"clientid">>, <<"good">>}])), - ?assertEqual({error, "incorrect type: only support clientid/topic/ip_address"}, - emqx_trace:create([{<<"name">>, <<"test-name">>}, - {<<"clientid">>, <<"good">>}, {<<"type">>, <<"typeerror">> }])), - ?assertEqual({error, "ip address: einval"}, - emqx_trace:create([{<<"ip_address">>, <<"test-name">>}])), + emqx_trace:create([Name, {<<"type">>, <<"ip_address">>}, + {<<"ip_address">>, <<"test-name">>}])), ok. t_create_default(_Config) -> @@ -173,6 +173,20 @@ t_create_default(_Config) -> ?assertEqual(true, Start - erlang:system_time(second) < 5), ok. +t_create_with_extra_fields(_Config) -> + ok = emqx_trace:clear(), + Trace = [ + {<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"clientid">>, <<"dev001">>}, + {<<"ip_address">>, <<"127.0.0.1">>} + ], + ok = emqx_trace:create(Trace), + ?assertMatch([#emqx_trace{name = <<"test-name">>, filter = <<"/x/y/z">>, type = topic}], + emqx_trace:list()), + ok. + t_update_enable(_Config) -> Name = <<"test-name">>, Now = erlang:system_time(second), From a93c63f168656ef4b1d4cd11a0f7848127393b4f Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 13 Dec 2021 16:04:15 +0800 Subject: [PATCH 107/363] chore(otp): be compatibility with crypto in otp 24 --- apps/emqx_auth_jwt/rebar.config | 2 +- apps/emqx_auth_pgsql/rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_jwt/rebar.config b/apps/emqx_auth_jwt/rebar.config index 5e7575881..b0a07eb8c 100644 --- a/apps/emqx_auth_jwt/rebar.config +++ b/apps/emqx_auth_jwt/rebar.config @@ -1,6 +1,6 @@ {deps, [ - {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} + {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_auth_pgsql/rebar.config b/apps/emqx_auth_pgsql/rebar.config index e1a1c752c..7a6aaf411 100644 --- a/apps/emqx_auth_pgsql/rebar.config +++ b/apps/emqx_auth_pgsql/rebar.config @@ -1,5 +1,5 @@ {deps, - [{epgsql, {git, "https://github.com/epgsql/epgsql.git", {tag, "4.4.0"}}} + [{epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} ]}. {erl_opts, [warn_unused_vars, From 9d14604935c1b441ae439fbf0b7e52ca3fdbabe7 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 13 Dec 2021 16:23:09 +0800 Subject: [PATCH 108/363] chore: bump versions of jwt and pgsql authn to 4.4.0 --- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 8db4ffe84..7d784e3b2 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src index f70612262..e97487e21 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_pgsql, [{description, "EMQ X Authentication/ACL with PostgreSQL"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_pgsql_sup]}, {applications, [kernel,stdlib,epgsql,ecpool]}, From 396d328a702698b57ceeaa2225fc8fc83f6d9195 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 13 Dec 2021 18:29:05 +0800 Subject: [PATCH 109/363] chore: upgrade coap&lwm2m tag for otp24 --- apps/emqx_coap/rebar.config | 2 +- apps/emqx_lwm2m/rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_coap/rebar.config index 0f8759b8a..e1da2c197 100644 --- a/apps/emqx_coap/rebar.config +++ b/apps/emqx_coap/rebar.config @@ -1,4 +1,4 @@ {deps, [ - {gen_coap, {git, "https://github.com/emqx/gen_coap", {tag, "v0.3.2"}}} + {gen_coap, {git, "https://github.com/emqx/gen_coap", {tag, "v0.4.2"}}} ]}. diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config index f190fa55e..ce4388e23 100644 --- a/apps/emqx_lwm2m/rebar.config +++ b/apps/emqx_lwm2m/rebar.config @@ -1,5 +1,5 @@ {deps, - [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v1.1.5"}}} + [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v2.0.1"}}} ]}. {profiles, From e9b8fc8606d4adc0a908c87700d46a9a4b463e21 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 13 Dec 2021 20:10:02 +0800 Subject: [PATCH 110/363] fix(OTP): OTP 24 warnings for ssl:ssh_accept/1,2 --- apps/emqx_lwm2m/rebar.config | 2 +- rebar.config.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config index f190fa55e..3cbd76c05 100644 --- a/apps/emqx_lwm2m/rebar.config +++ b/apps/emqx_lwm2m/rebar.config @@ -4,7 +4,7 @@ {profiles, [{test, - [{deps, [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}, + [{deps, [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.4"}}}, {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.0"}}} ]} diff --git a/rebar.config.erl b/rebar.config.erl index 6c14b6ae8..ce8933570 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -89,7 +89,7 @@ project_app_dirs() -> plugins(HasElixir) -> [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}} - , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}} + , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.4"}}} %% emqx main project does not require port-compiler %% pin at root level for deterministic , {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}} From ce3006c916f872b5d9db69a82bf720c9ee3d7cc7 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 14 Dec 2021 10:11:19 +0800 Subject: [PATCH 111/363] chore: clean up emqx_modules's appup.src for 4.4.0 --- .../emqx_modules/src/emqx_modules.appup.src | 44 +++---------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index aed228213..1170e952c 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,39 +1,9 @@ %% -*-: erlang -*- {VSN, - [ - {"4.3.3", [ - {load_module, emqx_mod_st_statistics, brutal_purge, soft_purge, []} - ]}, - {"4.3.2", [ - {load_module, emqx_mod_presence, brutal_purge, soft_purge, []} - ]}, - {"4.3.1", [ - {load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} - ]}, - {"4.3.0", [ - {update, emqx_mod_delayed, {advanced, []}}, - {load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], - [ - {"4.3.3", [ - {load_module, emqx_mod_st_statistics, brutal_purge, soft_purge, []} - ]}, - {"4.3.2", [ - {load_module, emqx_mod_presence, brutal_purge, soft_purge, []} - ]}, - {"4.3.1", [ - {load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} - ]}, - {"4.3.0", [ - {update, emqx_mod_delayed, {advanced, []}}, - {load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. + [ + {<<".*">>, []} + ], + [ + {<<".*">>, []} + ] +}. \ No newline at end of file From baf8d7d91c3242301b86380ff29ada5e2e994df7 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 10 Dec 2021 09:58:36 +0800 Subject: [PATCH 112/363] ci: bump otp version to 24.1.5-3 --- .ci/build_packages/Dockerfile | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .ci/fvt_tests/local_relup_test_run.sh | 4 ++-- .github/workflows/build_packages.yaml | 23 ++++++++++++------- .github/workflows/build_slim_packages.yaml | 5 ++-- .github/workflows/check_deps_integrity.yaml | 2 +- .../workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_automate_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 12 +++++----- .github/workflows/run_test_cases.yaml | 4 ++-- .tool-versions | 2 +- Makefile | 2 +- build | 2 +- deploy/docker/Dockerfile | 2 +- 14 files changed, 37 insertions(+), 29 deletions(-) diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index 1063a2c13..b1edd8409 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 8bd6455e9..bd4f5f391 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh index 575fcf4f0..649af9587 100755 --- a/.ci/fvt_tests/local_relup_test_run.sh +++ b/.ci/fvt_tests/local_relup_test_run.sh @@ -15,8 +15,8 @@ PROFILE="$1" VSN="$2" OLD_VSN="$3" PACKAGE_PATH="$4" -FROM_OTP_VSN="${5:-23.3.4.9-3}" -TO_OTP_VSN="${6:-23.3.4.9-3}" +FROM_OTP_VSN="${5:-24.1.5-3}" +TO_OTP_VSN="${6:-24.1.5-3}" TEMPDIR=$(mktemp -d) trap '{ rm -rf -- "$TEMPDIR"; }' EXIT diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index a34c3d85c..d68bcee18 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -16,7 +16,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles }} @@ -141,7 +141,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} erl_otp: - - 23.3.4.9-3 + - 24.1.5-3 macos: - macos-11 - macos-10.15 @@ -224,8 +224,12 @@ jobs: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + package: + - zip + - pkg otp: - 23.3.4.9-3 + - 24.1.5-3 arch: - amd64 - arm64 @@ -242,6 +246,8 @@ jobs: - raspbian10 # - raspbian9 exclude: + - package: pkg + otp: 23.3.4.9-3 - os: centos6 arch: arm64 - os: raspbian9 @@ -274,6 +280,7 @@ jobs: - name: unzip source code run: unzip -q source.zip - name: downloads old emqx zip packages + if: matrix.package == 'zip' env: OTP_VSN: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} @@ -306,6 +313,7 @@ jobs: env: OTP: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} + PACKAGE: ${{ matrix.package}} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} working-directory: source @@ -314,9 +322,8 @@ jobs: -v $(pwd):/emqx \ --workdir /emqx \ --platform linux/$ARCH \ - ghcr.io/emqx/emqx-builder/4.4-2:$OTP-$SYSTEM \ - bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ - make $PROFILE-pkg || cat rebar3.crashdump; \ + ghcr.io/emqx/emqx-builder/4.4-4:$OTP-$SYSTEM \ + bash -euc "make ${PROFILE}-${PACKAGE} || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" - name: create sha256 working-directory: source @@ -345,7 +352,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.9-3 + - 24.1.5-3 steps: - uses: actions/download-artifact@v2 @@ -386,7 +393,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -425,7 +432,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.9-3 + - 24.1.5-3 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 23d0e3e53..90a3db420 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -22,11 +22,12 @@ jobs: matrix: erl_otp: - 23.3.4.9-3 + - 24.1.5-3 os: - ubuntu20.04 - centos7 - container: ghcr.io/emqx/emqx-builder/4.4-2:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 @@ -64,7 +65,7 @@ jobs: fail-fast: false matrix: erl_otp: - - 23.3.4.9-3 + - 24.1.5-3 macos: - macos-11 - macos-10.15 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index b8c4a5c18..d09270e65 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 5f452194c..1ecbdead1 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index ca7a84612..67269e068 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -33,7 +33,7 @@ jobs: fi - name: build docker image env: - OTP_VSN: 23.3.4.9-3 + OTP_VSN: 24.1.5-3 run: | make ${{ steps.prepare.outputs.imgname }}-docker docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 772fd4fa2..f314770fc 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -31,7 +31,7 @@ jobs: fi - name: make emqx image env: - OTP_VSN: 23.3.4.9-3 + OTP_VSN: 24.1.5-3 run: make ${PROFILE}-docker - name: run emqx timeout-minutes: 5 @@ -92,7 +92,7 @@ jobs: fi - name: make emqx image env: - OTP_VSN: 23.3.4.9-3 + OTP_VSN: 24.1.5-3 run: make ${PROFILE}-docker - name: install k3s env: @@ -224,7 +224,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -273,7 +273,7 @@ jobs: relup_test_build: needs: relup_test_plan runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 defaults: run: shell: bash @@ -360,8 +360,8 @@ jobs: --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ - --var FROM_OTP_VSN="23.3.4.9-3" \ - --var TO_OTP_VSN="23.3.4.9-3" \ + --var FROM_OTP_VSN="24.1.5-3" \ + --var TO_OTP_VSN="24.1.5-3" \ emqx_built/.ci/fvt_tests/relup.lux - uses: actions/upload-artifact@v2 name: Save debug data diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 2f5570e28..a6ccf5dad 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_static_analysis: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 @@ -27,7 +27,7 @@ jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.tool-versions b/.tool-versions index 757309f18..a6568713b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 23.3.4.9-3 +erlang 24.1.5-3 diff --git a/Makefile b/Makefile index 4ace92341..d6b4e9b33 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REBAR_VERSION = 3.14.3-emqx-8 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-alpine3.14 export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) diff --git a/build b/build index 2fca5bdf2..8f43934fe 100755 --- a/build +++ b/build @@ -150,7 +150,7 @@ make_docker() { ## Name Default Example ## --------------------------------------------------------------------- ## EMQX_BASE_IMAGE current os centos:7 -## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.zip +## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp24.1.5-3-centos7-amd64.zip ## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag ## make_docker_testing() { diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 88f80bdf1..cb8d83309 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder From b5925653631cc7e41b1317d01a1d67b595b5d8fc Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 15 Dec 2021 09:53:46 +0800 Subject: [PATCH 113/363] fix: trace not work if client is utf8 --- src/emqx_trace_handler.erl | 45 +++++++++++++++++++++++++++++-- test/emqx_trace_handler_SUITE.erl | 18 ++++++++++++- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/emqx_trace_handler.erl b/src/emqx_trace_handler.erl index ffe581300..49f3157cc 100644 --- a/src/emqx_trace_handler.erl +++ b/src/emqx_trace_handler.erl @@ -59,6 +59,22 @@ }} ). +-define(CLIENT_FORMAT, + {logger_formatter, #{ + template => [ + time, " [", level, "] ", + {clientid, + [{peername, [clientid, "@", peername, " "], [clientid, " "]}], + [{peername, [peername, " "], []}] + }, + msg, "\n" + ], + single_line => false, + max_size => unlimited, + depth => unlimited + }} +). + -define(CONFIG(_LogFile_), #{ type => halt, file => _LogFile_, @@ -160,7 +176,7 @@ filter_ip_address(_Log, _ExpectId) -> ignore. install_handler(Who = #{name := Name, type := Type}, Level, LogFile) -> HandlerId = handler_id(Name, Type), Config = #{ level => Level, - formatter => ?FORMAT, + formatter => formatter(Who), filter_default => stop, filters => filters(Who), config => ?CONFIG(LogFile) @@ -176,6 +192,31 @@ filters(#{type := topic, filter := Filter, name := Name}) -> filters(#{type := ip_address, filter := Filter, name := Name}) -> [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}]. +formatter(#{type := Type}) -> + {logger_formatter, + #{ + template => template(Type), + single_line => false, + max_size => unlimited, + depth => unlimited + } + }. + +%% Don't log clientid since clientid only supports exact match, all client ids are the same. +%% if clientid is not latin characters. the logger_formatter restricts the output must be `~tp` +%% (actually should use `~ts`), the utf8 characters clientid will become very difficult to read. +template(clientid) -> + [time, " [", level, "] ", {peername, [peername, " "], []}, msg, "\n"]; +%% TODO better format when clientid is utf8. +template(_) -> + [ time, " [", level, "] ", + {clientid, + [{peername, [clientid, "@", peername, " "], [clientid, " "]}], + [{peername, [peername, " "], []}] + }, + msg, "\n" + ]. + filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> Init = #{id => Id, level => Level, dst => Dst}, case Filters of @@ -209,7 +250,7 @@ do_handler_id(Name, Type) -> ensure_bin(List) when is_list(List) -> iolist_to_binary(List); ensure_bin(Bin) when is_binary(Bin) -> Bin. -ensure_list(Bin) when is_binary(Bin) -> binary_to_list(Bin); +ensure_list(Bin) when is_binary(Bin) -> unicode:characters_to_list(Bin, utf8); ensure_list(List) when is_list(List) -> List. show_prompts(ok, Who, Msg) -> diff --git a/test/emqx_trace_handler_SUITE.erl b/test/emqx_trace_handler_SUITE.erl index cbf455d2d..d46c18a70 100644 --- a/test/emqx_trace_handler_SUITE.erl +++ b/test/emqx_trace_handler_SUITE.erl @@ -28,7 +28,7 @@ {password, <<"pass">>} ]). -all() -> [t_trace_clientid, t_trace_topic, t_trace_ip_address]. +all() -> [t_trace_clientid, t_trace_topic, t_trace_ip_address, t_trace_clientid_utf8]. init_per_suite(Config) -> emqx_ct_helpers:boot_modules(all), @@ -105,6 +105,22 @@ t_trace_clientid(_Config) -> emqtt:disconnect(T), ?assertEqual([], emqx_trace_handler:running()). +t_trace_clientid_utf8(_) -> + emqx_logger:set_log_level(debug), + + Utf8Id = <<"client 漢字編碼"/utf8>>, + ok = emqx_trace_handler:install(clientid, Utf8Id, debug, "tmp/client-utf8.log"), + {ok, T} = emqtt:start_link([{clientid, Utf8Id}]), + emqtt:connect(T), + [begin emqtt:publish(T, <<"a/b/c">>, <<"hi">>) end|| _ <- lists:seq(1, 10)], + emqtt:ping(T), + + ok = filesync(Utf8Id, clientid), + ok = emqx_trace_handler:uninstall(clientid, Utf8Id), + emqtt:disconnect(T), + ?assertEqual([], emqx_trace_handler:running()), + ok. + t_trace_topic(_Config) -> {ok, T} = emqtt:start_link(?CLIENT), emqtt:connect(T), From a1a9d002e3ad446dee2f94bd49a6929bb619921b Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 16 Dec 2021 16:34:36 +0800 Subject: [PATCH 114/363] fix(rules): add SQL test examples for user properties --- .../emqx_rule_engine/src/emqx_rule_events.erl | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 72f8345bf..3472b0b86 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -454,9 +454,8 @@ columns_with_exam('message.publish') -> , {<<"topic">>, <<"t/a">>} , {<<"qos">>, 1} , {<<"flags">>, #{}} - , {<<"headers">>, #{<<"properties">> => #{<<"User-Property">> => - #{'prop_key' => <<"prop_val">>}}}} , {<<"publish_received_at">>, erlang:system_time(millisecond)} + , columns_example_props(pub_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -473,6 +472,7 @@ columns_with_exam('message.delivered') -> , {<<"qos">>, 1} , {<<"flags">>, #{}} , {<<"publish_received_at">>, erlang:system_time(millisecond)} + , columns_example_props(pub_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -489,6 +489,8 @@ columns_with_exam('message.acked') -> , {<<"qos">>, 1} , {<<"flags">>, #{}} , {<<"publish_received_at">>, erlang:system_time(millisecond)} + , columns_example_props(pub_props) + , columns_example_props(puback_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -504,6 +506,7 @@ columns_with_exam('message.dropped') -> , {<<"qos">>, 1} , {<<"flags">>, #{}} , {<<"publish_received_at">>, erlang:system_time(millisecond)} + , columns_example_props(pub_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -521,6 +524,7 @@ columns_with_exam('client.connected') -> , {<<"expiry_interval">>, 3600} , {<<"is_bridge">>, false} , {<<"connected_at">>, erlang:system_time(millisecond)} + , columns_example_props(conn_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -532,6 +536,7 @@ columns_with_exam('client.disconnected') -> , {<<"peername">>, <<"192.168.0.10:56431">>} , {<<"sockname">>, <<"0.0.0.0:1883">>} , {<<"disconnected_at">>, erlang:system_time(millisecond)} + , columns_example_props(disconn_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -542,6 +547,7 @@ columns_with_exam('session.subscribed') -> , {<<"peerhost">>, <<"192.168.0.10">>} , {<<"topic">>, <<"t/a">>} , {<<"qos">>, 1} + , columns_example_props(sub_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -552,10 +558,42 @@ columns_with_exam('session.unsubscribed') -> , {<<"peerhost">>, <<"192.168.0.10">>} , {<<"topic">>, <<"t/a">>} , {<<"qos">>, 1} + , columns_example_props(unsub_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]. +columns_example_props(PropType) -> + Props = columns_example_props_specific(PropType), + UserProps = #{ + 'User-Property' => #{<<"foo">> => <<"bar">>}, + 'User-Property-Pairs' => [ + #{key => <<"foo">>}, #{value => <<"bar">>} + ] + }, + {PropType, maps:merge(Props, UserProps)}. + +columns_example_props_specific(pub_props) -> + #{ 'Payload-Format-Indicator' => 0 + , 'Message-Expiry-Interval' => 30 + }; +columns_example_props_specific(puback_props) -> + #{ 'Reason-String' => <<"OK">> + }; +columns_example_props_specific(conn_props) -> + #{ 'Session-Expiry-Interval' => 7200 + , 'Receive-Maximum' => 32 + }; +columns_example_props_specific(disconn_props) -> + #{ 'Session-Expiry-Interval' => 7200 + , 'Reason-String' => <<"Redirect to another server">> + , 'Server Reference' => <<"192.168.22.129">> + }; +columns_example_props_specific(sub_props) -> + #{}; +columns_example_props_specific(unsub_props) -> + #{}. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- @@ -613,6 +651,10 @@ printable_maps(Headers) -> AccIn#{K => ntoa(V0)}; ('User-Property', V0, AccIn) when is_list(V0) -> AccIn#{ + %% The 'User-Property' field is for the convenience of querying properties + %% using the '.' syntax, e.g. "SELECT 'User-Property'.foo as foo" + %% However, this does not allow duplicate property keys. To allow + %% duplicate keys, we have to use the 'User-Property-Pairs' field instead. 'User-Property' => maps:from_list(V0), 'User-Property-Pairs' => [#{ key => Key, From 66e848b771f22103c667a841ec966b745c368835 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 17 Dec 2021 15:30:08 +0800 Subject: [PATCH 115/363] fix(code_port): improve node stop wait loop --- bin/emqx | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/bin/emqx b/bin/emqx index 6a80f4d8d..be18a85e7 100755 --- a/bin/emqx +++ b/bin/emqx @@ -4,6 +4,11 @@ set -e +DEBUG="${DEBUG:-0}" +if [ "$DEBUG" -eq 1 ]; then + set -x +fi + ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 . "$ROOT_DIR"/releases/emqx_vars @@ -299,6 +304,43 @@ generate_config() { fi } +# check if a PID is down +is_down() { + PID="$1" + if ps -p "$PID" >/dev/null; then + # still around + # shellcheck disable=SC2009 # this grep pattern is not a part of the progra names + if ps -p "$PID" | grep -q 'defunct'; then + # zombie state, print parent pid + parent="$(ps -o ppid= -p "$PID" | tr -d ' ')" + echo "WARN: $PID is marked , parent:" + ps -p "$parent" + return 0 + fi + return 1 + fi + # it's gone + return 0 +} + +wait_for() { + local WAIT_TIME + local CMD + WAIT_TIME="$1" + shift + CMD="$*" + while true; do + if $CMD >/dev/null 2>&1; then + return 0 + fi + if [ "$WAIT_TIME" -le 0 ]; then + return 1 + fi + WAIT_TIME=$((WAIT_TIME - 1)) + sleep 1 + done +} + # Call bootstrapd for daemon commands like start/stop/console bootstrapd() { if [ -e "$RUNNER_DATA_DIR/.erlang.cookie" ]; then @@ -485,11 +527,21 @@ case "$1" in # Wait for the node to completely stop... PID="$(relx_get_pid)" if ! relx_nodetool "stop"; then + echoerr "Graceful shutdown failed PID=[$PID]" exit 1 fi - while kill -s 0 "$PID" 2>/dev/null; do - sleep 1 - done + WAIT_TIME="${WAIT_FOR_ERLANG_STOP:-60}" + if ! wait_for "$WAIT_TIME" 'is_down' "$PID"; then + msg="dangling after ${WAIT_TIME} seconds" + # also log to syslog + logger -t "${REL_NAME}[${PID}]" "STOP: $msg" + # log to user console + echoerr "stop failed, $msg" + echo "ERROR: $PID is still around" + ps -p "$PID" + exit 1 + fi + logger -t "${REL_NAME}[${PID}]" "STOP: OK" ;; restart|reboot) From 88e7c4042316afc89b6309c076393d4ea5e31265 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 20 Dec 2021 14:19:45 +0800 Subject: [PATCH 116/363] chore(ekka): update ekka vsn to 0.8.1.7 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 1e3c758e6..f1dc55512 100644 --- a/rebar.config +++ b/rebar.config @@ -43,7 +43,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.2"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.7"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} From 799aabdd04dfbc7a66bfa73965fcada711988817 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 20 Dec 2021 15:12:09 +0800 Subject: [PATCH 117/363] chore(emqx): bump emqx to 4.4.0 --- src/emqx.app.src | 2 +- src/emqx.appup.src | 388 +-------------------------------------------- 2 files changed, 4 insertions(+), 386 deletions(-) diff --git a/src/emqx.app.src b/src/emqx.app.src index b0e1664cc..fa501f461 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -1,7 +1,7 @@ {application, emqx, [{id, "emqx"}, {description, "EMQ X"}, - {vsn, "4.3.12"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index cea35e90d..109eeef80 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,387 +1,5 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.11", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, - {"4.3.10", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, - {"4.3.9", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.8", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.7", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.6", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.5", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.4", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.3", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.2", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.1", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_trie,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}], - [{"4.3.11", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, - {"4.3.10", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, - {"4.3.9", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.8", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.7", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.6", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.5", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.4", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.3", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.2", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.1", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_trie,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}]}. + [{<<".*">>,[]}], + [{<<".*">>,[]}] +}. From c74dc11c920c9e257264ea9274f7af19d2042f83 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 21 Dec 2021 13:42:49 +0800 Subject: [PATCH 118/363] docs(change log): add change log for 4.4-beta.1 --- CHANGES-4.4.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 CHANGES-4.4.md diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md new file mode 100644 index 000000000..bd098daaa --- /dev/null +++ b/CHANGES-4.4.md @@ -0,0 +1,49 @@ +# EMQ X 4.4 Changes + +## 4.4-beta.1 + +### Important changes + +- **For Debian/Ubuntu users**, Debian/Ubuntu package (deb) installed EMQ X now now run on systemd. This is to use systemd's supervision functionality to ensure that EMQ X service restarts after a crash. The package installation service upgrade from init.d to systemd has been verified, but it is still recommended that you verify and confirm again before deploying to the production environment, at least to ensure that systemd is available in your system + +- MongoDB authentication supports DNS SRV and TXT Records resolution, which can seamlessly connect with MongoDB Altas + +- Support dynamic modification of MQTT Keep Alive to adapt to different energy consumption strategies + +### Minor changes + +- Dashboard supports relative paths and custom access paths + +- Supports configuring whether to forward retained messages with empty payload to suit users who are still using MQTT v3.1. The relevant configurable item is `retainer.stop_publish_clear_msg` + +- Multi-language hook extension (ExHook) supports dynamic cancellation of subsequent forwarding of client messages + +- Rule engine SQL supports the use of single quotes in FROM clause, for example: `SELECT * FROM't/#'` + +- Change the default value of the `max_topic_levels` configurable item to 128. Previously, it had no limit (configured to 0), which may be a potential DoS threat + +- Improve the error log content when the Proxy Protocol message is received but the `proxy_protocol` configuration is not turned on + +- Add additional message attributes to the message reported by the gateway. Messages from gateways such as CoAP, LwM2M, Stomp, ExProto, etc., when converted to EMQ X messages, add fields such as protocol name, protocol version, user name, client IP, etc., which can be used for multi-language hook extension (ExHook) + +- HTTP client performance improvement + +- Add openssl-1.1 to RPM dependency + +### Bug fixes + +- Fix the issue that the client process becomes unresponsive due to the blockage of RPC calls between nodes + +- Fix the issue that the lock management process `ekka_locker` crashes after killing the suspended lock owner + +- Fix the issue that the Path parameter of WebHook action in rule engine cannot use the rule engine variable + +- Fix MongoDB authentication module cannot use Replica Set mode and other issues + +- Fix the issue of out-of-sequence message forwarding between clusters. The relevant configurable item is `rpc.tcp_client_num` + +- Fix the issue of incorrect calculation of memory usage + +- Fix MQTT bridge malfunction when remote host is unreachable (hangs the connection) + +- Fix the issue that HTTP headers may be duplicated From e768c601ab64622a092c7e7a2df8037cb470d145 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 21 Dec 2021 17:05:22 +0800 Subject: [PATCH 119/363] chore(release): update emqx release version --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 3e96f1aa7..534a887c4 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4-alpha.2"}). +-define(EMQX_RELEASE, {opensource, "4.4-beta.1"}). -else. From c1480ab52c37f8dc111c12e510859bcef4209348 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 22 Dec 2021 09:40:09 +0800 Subject: [PATCH 120/363] ci(cts): use makefile target instead of the rebar command --- .github/workflows/run_cts_tests.yaml | 20 +++++--------------- Makefile | 2 +- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 53dd74b6a..fd074a18f 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -48,9 +48,7 @@ jobs: run: | export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env - docker exec -i erlang sh -c "make ensure-rebar3" - docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap" - docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap" + docker exec --env-file .env -i erlang sh -c ".make apps/emqx_auth_ldap-ct" - uses: actions/upload-artifact@v1 if: failure() with: @@ -117,9 +115,7 @@ jobs: run: | export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env - docker exec -i erlang sh -c "make ensure-rebar3" - docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo" - docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo" + docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_mongo-ct" - uses: actions/upload-artifact@v1 if: failure() with: @@ -199,9 +195,7 @@ jobs: run: | export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env - docker exec -i erlang sh -c "make ensure-rebar3" - docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql" - docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql" + docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_mysql-ct" - uses: actions/upload-artifact@v1 if: failure() with: @@ -273,9 +267,7 @@ jobs: EMQX_AUTH__PGSQL__DATABASE=mqtt \ CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env - docker exec -i erlang sh -c "make ensure-rebar3" - docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql" - docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql" + docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_pgsql-ct" - uses: actions/upload-artifact@v1 if: failure() with: @@ -394,9 +386,7 @@ jobs: export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ export EMQX_AUTH__REIDS__PASSWORD=public printenv > .env - docker exec -i erlang sh -c "make ensure-rebar3" - docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis" - docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis" + docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_redis-ct" - uses: actions/upload-artifact@v1 if: failure() with: diff --git a/Makefile b/Makefile index d6b4e9b33..61bd06ba2 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh) ## app/name-ct targets are intended for local tests hence cover is not enabled .PHONY: $(APPS:%=%-ct) define gen-app-ct-target -$1-ct: +$1-ct: $(REBAR) $(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) From f5ec6b730c9df002040ab240809e4183775ee790 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 22 Dec 2021 10:22:56 +0800 Subject: [PATCH 121/363] ci(cts): fix spell error --- .github/workflows/run_cts_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index fd074a18f..514162b04 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -48,7 +48,7 @@ jobs: run: | export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env - docker exec --env-file .env -i erlang sh -c ".make apps/emqx_auth_ldap-ct" + docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_ldap-ct" - uses: actions/upload-artifact@v1 if: failure() with: From e15eadde7232c3a813def83c40790322b417eeb1 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 22 Dec 2021 12:00:45 +0800 Subject: [PATCH 122/363] fix(test): revert the changes on ldap ct --- .github/workflows/run_cts_tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 514162b04..eb79f8842 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -48,7 +48,8 @@ jobs: run: | export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ printenv > .env - docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_ldap-ct" + docker exec -i erlang sh -c "make ensure-rebar3" + docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap" - uses: actions/upload-artifact@v1 if: failure() with: From 15f40889c64032ed6c088813e931abada12d8774 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 22 Dec 2021 16:01:58 +0800 Subject: [PATCH 123/363] ci(cts): fix env error --- .github/workflows/run_cts_tests.yaml | 43 +++++++++++++++++----------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index eb79f8842..a069e143d 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -3,6 +3,7 @@ name: Compatibility Test Suite on: schedule: - cron: '0 */6 * * *' + pull_request: push: tags: - v* @@ -46,10 +47,12 @@ jobs: fi - name: run test cases run: | - export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ - printenv > .env docker exec -i erlang sh -c "make ensure-rebar3" - docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap" + printenv | grep "^EMQX_" > .env + docker exec -i \ + -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \ + --env-file .env \ + erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap" - uses: actions/upload-artifact@v1 if: failure() with: @@ -114,9 +117,11 @@ jobs: fi - name: run test cases run: | - export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ - printenv > .env - docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_mongo-ct" + printenv | grep "^EMQX_" > .env + docker exec -i \ + -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \ + --env-file .env \ + erlang sh -c "make apps/emqx_auth_mongo-ct" - uses: actions/upload-artifact@v1 if: failure() with: @@ -194,9 +199,11 @@ jobs: fi - name: run test cases run: | - export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ - printenv > .env - docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_mysql-ct" + printenv | grep "^EMQX_" > .env + docker exec -i \ + -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \ + --env-file .env \ + erlang sh -c "make apps/emqx_auth_mysql-ct" - uses: actions/upload-artifact@v1 if: failure() with: @@ -265,10 +272,12 @@ jobs: run: | export EMQX_AUTH__PGSQL__USERNAME=root \ EMQX_AUTH__PGSQL__PASSWORD=public \ - EMQX_AUTH__PGSQL__DATABASE=mqtt \ - CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ - printenv > .env - docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_pgsql-ct" + EMQX_AUTH__PGSQL__DATABASE=mqtt + printenv | grep "^EMQX_" > .env + docker exec -i \ + -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \ + --env-file .env \ + erlang sh -c "make apps/emqx_auth_pgsql-ct" - uses: actions/upload-artifact@v1 if: failure() with: @@ -384,10 +393,12 @@ jobs: fi - name: run test cases run: | - export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ export EMQX_AUTH__REIDS__PASSWORD=public - printenv > .env - docker exec --env-file .env -i erlang sh -c "make apps/emqx_auth_redis-ct" + printenv | grep "^EMQX_" > .env + docker exec -i \ + -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \ + --env-file .env \ + erlang sh -c "make apps/emqx_auth_redis-ct" - uses: actions/upload-artifact@v1 if: failure() with: From 67f4bba6940c81a063c9827d4a6730d5766a6f75 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 22 Dec 2021 20:26:06 +0800 Subject: [PATCH 124/363] ci(build_packages): delete needless packages --- .github/workflows/build_packages.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index d68bcee18..cc6769eaa 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -502,13 +502,6 @@ jobs: echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin sudo TARGET=emqx/${{ matrix.profile }} make docker-push sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list - - name: push docker image to aws ecr - if: github.event_name == 'release' - run: | - set -e -x -u - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} - docker push public.ecr.aws/emqx/emqx:${version#v} - name: update repo.emqx.io if: github.event_name == 'release' && matrix.profile == 'emqx-ee' run: | From 1136b1628152e1c3936f9d46de2bffcfe3e590a0 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 22 Dec 2021 20:39:32 +0800 Subject: [PATCH 125/363] ci(build_packages): fix docker tag error --- .github/workflows/build_packages.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index cc6769eaa..e1f8a43e2 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -375,8 +375,7 @@ jobs: tags: | type=ref,event=branch type=ref,event=pr - type=ref,event=tag - type=semver,pattern={{version}} + type=match,pattern=[v|e](.*),group=1 labels: org.opencontainers.image.otp.version=${{ matrix.otp }} - uses: docker/login-action@v1 From 499c1ab2f48030f235aa733e31064e3c7a62409e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 22 Dec 2021 23:54:54 +0800 Subject: [PATCH 126/363] ci(build_packages): fix upload error for mac --- .github/workflows/build_packages.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e1f8a43e2..0d83a6710 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -140,7 +140,7 @@ jobs: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} - erl_otp: + otp: - 24.1.5-3 macos: - macos-11 @@ -165,7 +165,7 @@ jobs: id: cache with: path: ~/.kerl - key: erl${{ matrix.erl_otp }}-${{ matrix.macos }} + key: erl${{ matrix.otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 @@ -174,12 +174,12 @@ jobs: OTP_GITHUB_URL: https://github.com/emqx/otp run: | kerl update releases - kerl build ${{ matrix.erl_otp }} - kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} + kerl build ${{ matrix.otp }} + kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} - name: build working-directory: source run: | - . $HOME/.kerl/${{ matrix.erl_otp }}/activate + . $HOME/.kerl/${{ matrix.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 rm -rf _build/${{ matrix.profile }}/lib @@ -431,6 +431,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: + - 23.3.4.9-3 - 24.1.5-3 steps: From 26c0f2c96ee09886bc8798b46d1f97d8dd14700a Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 27 Dec 2021 11:33:33 +0800 Subject: [PATCH 127/363] fix(CI): show outputs of erlang console if ./bin/emqx start failed --- .ci/build_packages/tests.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 3b97c01b8..e7169b8d9 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -42,7 +42,11 @@ emqx_test(){ sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins echo "running ${packagename} start" - "${PACKAGE_PATH}"/emqx/bin/emqx start || ( tail "${PACKAGE_PATH}"/emqx/log/emqx.log.1 && exit 1 ) + if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then + cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true + cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true + exit 1 + fi IDLE_TIME=0 while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' do @@ -120,7 +124,11 @@ running_test(){ EMQX_MQTT__MAX_TOPIC_ALIAS=10 sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins - emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 ) + if ! emqx start; then + cat /var/log/emqx/erlang.log.1 || true + cat /var/log/emqx/emqx.log.1 || true + exit 1 + fi IDLE_TIME=0 while ! emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' do @@ -146,7 +154,11 @@ relup_test(){ while read -r pkg; do packagename=$(basename "${pkg}") unzip "$packagename" - ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 ) + if ! ./emqx/bin/emqx start; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + exit 1 + fi ./emqx/bin/emqx_ctl status ./emqx/bin/emqx versions cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases From a21dade9272427e8247b529cdc7c5800e4aa8bc0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 29 Dec 2021 15:06:57 +0100 Subject: [PATCH 128/363] build: add scripts/buildx --- scripts/buildx.sh | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100755 scripts/buildx.sh diff --git a/scripts/buildx.sh b/scripts/buildx.sh new file mode 100755 index 000000000..c0406d5ce --- /dev/null +++ b/scripts/buildx.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +## This script helps to run docker buildx to build cross-arch/platform packages (linux only) +## It mounts (not copy) host directory to a cross-arch/platform builder container +## Make sure the source dir (specified by --src_dir option) is clean before running this script + +## NOTE: it requires $USER in docker group +## i.e. will not work if docker command has to be executed with sudo + +## example: +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 --arch arm64 + +set -euo pipefail + +help() { + echo + echo "-h|--help: To display this usage information" + echo "--pkgtype zip|pkg: Specify to build zip or deb|rpm package" + echo "--arch amd64|arm64: Target arch to build the EMQ X package for." + echo "--src_dir : EMQ X source ode in this dir, default to PWD" + echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" + echo "--builder : Builder image to pull." + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" +} + +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + --src_dir) + SRC_DIR="$2" + shift 2 + ;; + --profile) + PROFILE="$2" + shift 2 + ;; + --pkgtype) + PKGTYPE="$2" + shift 2 + ;; + --builder) + BUILDER="$2" + shift 2 + ;; + --arch) + ARCH="$2" + shift 2 + ;; + *) + echo "WARN: Unknown arg (ignored): $1" + shift + continue + ;; + esac +done + +if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ -z "${ARCH:-}" ]; then + help + exit 1 +fi + +docker run --rm --privileged tonistiigi/binfmt:latest --install ${ARCH} + +cd "${SRC_DIR:-.}" + +docker run -i --rm \ + -v "$(pwd)":/emqx \ + --workdir /emqx \ + --platform="linux/$ARCH" \ + -e EMQX_NAME="$PROFILE" \ + "$BUILDER" \ + bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh" From 8f92d286e35bce36382b16848ba101cc7734eae6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Dec 2021 00:28:50 +0100 Subject: [PATCH 129/363] fix(bin/emqx): bump WAIT_FOR_ERLANG default to 150 seconds --- bin/emqx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/emqx b/bin/emqx index be18a85e7..6882a3ba0 100755 --- a/bin/emqx +++ b/bin/emqx @@ -504,7 +504,7 @@ case "$1" in "$BINDIR/run_erl" -daemon "$PIPE_DIR" "$RUNNER_LOG_DIR" \ "$(relx_start_command)" - WAIT_TIME=${WAIT_FOR_ERLANG:-15} + WAIT_TIME=${WAIT_FOR_ERLANG:-150} while [ "$WAIT_TIME" -gt 0 ]; do if ! relx_nodetool "ping" >/dev/null 2>&1; then WAIT_TIME=$((WAIT_TIME - 1)) @@ -516,7 +516,7 @@ case "$1" in echo "$EMQX_DESCRIPTION $REL_VSN is started successfully!" exit 0 fi - done && echo "$EMQX_DESCRIPTION $REL_VSN failed to start within ${WAIT_FOR_ERLANG:-15} seconds," + done && echo "$EMQX_DESCRIPTION $REL_VSN failed to start within ${WAIT_FOR_ERLANG:-150} seconds," echo "see the output of '$0 console' for more information." echo "If you want to wait longer, set the environment variable" echo "WAIT_FOR_ERLANG to the number of seconds to wait." @@ -530,7 +530,7 @@ case "$1" in echoerr "Graceful shutdown failed PID=[$PID]" exit 1 fi - WAIT_TIME="${WAIT_FOR_ERLANG_STOP:-60}" + WAIT_TIME="${WAIT_FOR_ERLANG_STOP:-120}" if ! wait_for "$WAIT_TIME" 'is_down' "$PID"; then msg="dangling after ${WAIT_TIME} seconds" # also log to syslog From bc79e136fff5bbdfe3ecf1a362af87459ba9a685 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Dec 2021 00:35:52 +0100 Subject: [PATCH 130/363] docs: update CHANGES-4.4 --- CHANGES-4.4.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index bd098daaa..e26663ea7 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -4,27 +4,39 @@ ### Important changes -- **For Debian/Ubuntu users**, Debian/Ubuntu package (deb) installed EMQ X now now run on systemd. This is to use systemd's supervision functionality to ensure that EMQ X service restarts after a crash. The package installation service upgrade from init.d to systemd has been verified, but it is still recommended that you verify and confirm again before deploying to the production environment, at least to ensure that systemd is available in your system +- **For Debian/Ubuntu users**, Debian/Ubuntu package (deb) installed EMQ X is now started from systemd. + This is to use systemd's supervision functionality to ensure that EMQ X service restarts after a crash. + The package installation service upgrade from init.d to systemd has been verified, + it is still recommended that you verify and confirm again before deploying to the production environment, + at least to ensure that systemd is available in your system - MongoDB authentication supports DNS SRV and TXT Records resolution, which can seamlessly connect with MongoDB Altas -- Support dynamic modification of MQTT Keep Alive to adapt to different energy consumption strategies +- Support dynamic modification of MQTT Keep Alive to adapt to different energy consumption strategies. ### Minor changes +- Bumpped default boot wait time from 15 seconds to 150 seconds + because in some simulated environments it may take up to 70 seconds to boot in build CI + - Dashboard supports relative paths and custom access paths -- Supports configuring whether to forward retained messages with empty payload to suit users who are still using MQTT v3.1. The relevant configurable item is `retainer.stop_publish_clear_msg` +- Supports configuring whether to forward retained messages with empty payload to suit users + who are still using MQTT v3.1. The relevant configurable item is `retainer.stop_publish_clear_msg` - Multi-language hook extension (ExHook) supports dynamic cancellation of subsequent forwarding of client messages -- Rule engine SQL supports the use of single quotes in FROM clause, for example: `SELECT * FROM't/#'` +- Rule engine SQL supports the use of single quotes in `FROM` clauses, for example: `SELECT * FROM 't/#'` -- Change the default value of the `max_topic_levels` configurable item to 128. Previously, it had no limit (configured to 0), which may be a potential DoS threat +- Change the default value of the `max_topic_levels` configurable item to 128. + Previously, it had no limit (configured to 0), which may be a potential DoS threat -- Improve the error log content when the Proxy Protocol message is received but the `proxy_protocol` configuration is not turned on +- Improve the error log content when the Proxy Protocol message is received without `proxy_protocol` configured. -- Add additional message attributes to the message reported by the gateway. Messages from gateways such as CoAP, LwM2M, Stomp, ExProto, etc., when converted to EMQ X messages, add fields such as protocol name, protocol version, user name, client IP, etc., which can be used for multi-language hook extension (ExHook) +- Add additional message attributes to the message reported by the gateway. + Messages from gateways such as CoAP, LwM2M, Stomp, ExProto, etc., when converted to EMQ X messages, + add fields such as protocol name, protocol version, user name, client IP, etc., + which can be used for multi-language hook extension (ExHook) - HTTP client performance improvement From feecaa6c98be57558fd6fe5e47a97097e48f4d11 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Dec 2021 13:37:10 +0100 Subject: [PATCH 131/363] build: ci tests.sh test specific package --- .ci/build_packages/tests.sh | 50 ++++++++++++++++++---- .github/workflows/build_packages.yaml | 12 +++--- .github/workflows/build_slim_packages.yaml | 7 ++- scripts/buildx.sh | 31 +++++++++----- 4 files changed, 71 insertions(+), 29 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index e7169b8d9..4c82b3d9e 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -1,5 +1,23 @@ -#!/bin/bash +#!/usr/bin/env bash + +## This script tests built package start/stop +## Accept 2 args PACKAGE_NAME and PACKAGE_TYPE + set -x -e -u + +if [ -z "${1:-}" ]; then + echo "Usage $0 zip|pkg" + exit 1 +fi + +if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then + echo "Usage $0 zip|pkg" + exit 1 +fi + +PACKAGE_NAME="${1}" +PACKAGE_TYPE="${2}" + export CODE_PATH=${CODE_PATH:-"/emqx"} export EMQX_NAME=${EMQX_NAME:-"emqx"} export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" @@ -7,6 +25,26 @@ export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" # export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1" # export EMQX_NODE_COOKIE=$(date +%s%N) +if [ "$PACKAGE_TYPE" = 'zip' ]; then + PKG_SUFFIX="zip" +else + SYSTEM="$($CODE_PATH/scripts/get-distro.sh)" + case "${SYSTEM:-}" in + ubuntu*|debian*|raspbian*) + PKG_SUFFIX='deb' + ;; + *) + PKG_SUFFIX='rpm' + ;; + esac +fi +PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" + +PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}.${PKG_SUFFIX}" +if ! [ -f "$PACKAGE_FILE" ]; then + echo "$PACKAGE_FILE is not a file" +fi + case "$(uname -m)" in x86_64) ARCH='amd64' @@ -22,7 +60,6 @@ export ARCH emqx_prepare(){ mkdir -p "${PACKAGE_PATH}" - if [ ! -d "/paho-mqtt-testing" ]; then git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho-mqtt-testing fi @@ -31,11 +68,10 @@ emqx_prepare(){ emqx_test(){ cd "${PACKAGE_PATH}" + local packagename="${PACKAGE_FILE_NAME}" - for var in "$PACKAGE_PATH"/"${EMQX_NAME}"-*;do - case ${var##*.} in + case ${packagename##*.} in "zip") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip) unzip -q "${PACKAGE_PATH}/${packagename}" export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \ EMQX_MQTT__MAX_TOPIC_ALIAS=10 @@ -64,7 +100,6 @@ emqx_test(){ rm -rf "${PACKAGE_PATH}"/emqx ;; "deb") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.deb) dpkg -i "${PACKAGE_PATH}/${packagename}" if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ] then @@ -91,8 +126,6 @@ emqx_test(){ fi ;; "rpm") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm) - if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; then # EMQX OTP requires openssl11 to have TLS1.3 support yum install -y openssl11 @@ -116,7 +149,6 @@ emqx_test(){ ;; esac - done } running_test(){ diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e1f8a43e2..696aedd15 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -318,13 +318,11 @@ jobs: SYSTEM: ${{ matrix.os }} working-directory: source run: | - docker run -i --rm \ - -v $(pwd):/emqx \ - --workdir /emqx \ - --platform linux/$ARCH \ - ghcr.io/emqx/emqx-builder/4.4-4:$OTP-$SYSTEM \ - bash -euc "make ${PROFILE}-${PACKAGE} || cat rebar3.crashdump; \ - EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" + ./scripts/buildx.sh \ + --profile "${PROFILE}" \ + --pkgtype "${PACKAGE}" \ + --arch "${ARCH}" \ + --builder "ghcr.io/emqx/emqx-builder/4.4-4:${OTP}-${SYSTEM}" - name: create sha256 working-directory: source env: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 90a3db420..e9b62a166 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -52,8 +52,11 @@ jobs: path: ./rebar3.crashdump - name: packages test run: | - export CODE_PATH=$GITHUB_WORKSPACE - .ci/build_packages/tests.sh + PKG_VSN="$(./pkg-vsn.sh)" + PKG_NAME="${EMQX_NAME}-${PKG_VSN}-otp${{ matrix.erl_otp }}-${{ matrix.os }}-amd64" + export CODE_PATH="$GITHUB_WORKSPACE" + .ci/build_packages/tests.sh "$PKG_NAME" zip + .ci/build_packages/tests.sh "$PKG_NAME" pkg - uses: actions/upload-artifact@v2 with: name: ${{ matrix.os }} diff --git a/scripts/buildx.sh b/scripts/buildx.sh index c0406d5ce..6e3ef8160 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -14,19 +14,19 @@ set -euo pipefail help() { echo - echo "-h|--help: To display this usage information" - echo "--pkgtype zip|pkg: Specify to build zip or deb|rpm package" - echo "--arch amd64|arm64: Target arch to build the EMQ X package for." - echo "--src_dir : EMQ X source ode in this dir, default to PWD" - echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" - echo "--builder : Builder image to pull." - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" + echo "-h|--help: To display this usage information" + echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" + echo "--pkgtype zip|pkg: Specify which package to build, zip for .zip and pkg for .rpm or .deb" + echo "--arch amd64|arm64: Target arch to build the EMQ X package for" + echo "--src_dir : EMQ X source ode in this dir, default to PWD" + echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" } while [ "$#" -gt 0 ]; do case $1 in - -h|--help) - help + -h|--help) + help exit 0 ;; --src_dir) @@ -62,14 +62,23 @@ if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ exit 1 fi -docker run --rm --privileged tonistiigi/binfmt:latest --install ${ARCH} +if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then + echo "Bad --pkgtype option, should be zip or pkg" + exit 1 +fi cd "${SRC_DIR:-.}" +PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" +OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2) +PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}" + +docker info +docker run --rm --privileged tonistiigi/binfmt:latest --install ${ARCH} docker run -i --rm \ -v "$(pwd)":/emqx \ --workdir /emqx \ --platform="linux/$ARCH" \ -e EMQX_NAME="$PROFILE" \ "$BUILDER" \ - bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh" + bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PKG_NAME $PKGTYPE" From d69ba4b7752549adc741f2f293c586653f9754e0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 31 Dec 2021 14:30:30 +0100 Subject: [PATCH 132/363] ci: refactor tests package test script Check suffix directly, and fix indentation --- .ci/build_packages/tests.sh | 138 ++++++++++++++++++------------------ 1 file changed, 68 insertions(+), 70 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 4c82b3d9e..9ff693ed1 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -69,86 +69,84 @@ emqx_prepare(){ emqx_test(){ cd "${PACKAGE_PATH}" local packagename="${PACKAGE_FILE_NAME}" + case "$PKG_SUFFIX" in + "zip") + unzip -q "${PACKAGE_PATH}/${packagename}" + export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \ + EMQX_MQTT__MAX_TOPIC_ALIAS=10 + sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins - case ${packagename##*.} in - "zip") - unzip -q "${PACKAGE_PATH}/${packagename}" - export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \ - EMQX_MQTT__MAX_TOPIC_ALIAS=10 - sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins - - echo "running ${packagename} start" - if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then - cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true - cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true - exit 1 - fi - IDLE_TIME=0 - while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' - do - if [ $IDLE_TIME -gt 10 ] - then - echo "emqx running error" - exit 1 - fi - sleep 10 - IDLE_TIME=$((IDLE_TIME+1)) - done - pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic - "${PACKAGE_PATH}"/emqx/bin/emqx stop - echo "running ${packagename} stop" - rm -rf "${PACKAGE_PATH}"/emqx - ;; - "deb") - dpkg -i "${PACKAGE_PATH}/${packagename}" - if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ] + echo "running ${packagename} start" + if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then + cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true + cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true + exit 1 + fi + IDLE_TIME=0 + while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' + do + if [ $IDLE_TIME -gt 10 ] then - echo "package install error" + echo "emqx running error" exit 1 fi + sleep 10 + IDLE_TIME=$((IDLE_TIME+1)) + done + pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic + "${PACKAGE_PATH}"/emqx/bin/emqx stop + echo "running ${packagename} stop" + rm -rf "${PACKAGE_PATH}"/emqx + ;; + "deb") + dpkg -i "${PACKAGE_PATH}/${packagename}" + if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ] + then + echo "package install error" + exit 1 + fi - echo "running ${packagename} start" - running_test - echo "running ${packagename} stop" + echo "running ${packagename} start" + running_test + echo "running ${packagename} stop" - dpkg -r "${EMQX_NAME}" - if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ] - then - echo "package remove error" - exit 1 - fi + dpkg -r "${EMQX_NAME}" + if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ] + then + echo "package remove error" + exit 1 + fi - dpkg -P "${EMQX_NAME}" - if dpkg -l |grep -q emqx - then - echo "package uninstall error" - exit 1 - fi - ;; - "rpm") - if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; then - # EMQX OTP requires openssl11 to have TLS1.3 support - yum install -y openssl11 - fi + dpkg -P "${EMQX_NAME}" + if dpkg -l |grep -q emqx + then + echo "package uninstall error" + exit 1 + fi + ;; + "rpm") + if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; then + # EMQX OTP requires openssl11 to have TLS1.3 support + yum install -y openssl11 + fi - rpm -ivh "${PACKAGE_PATH}/${packagename}" - if ! rpm -q emqx | grep -q emqx; then - echo "package install error" - exit 1 - fi + rpm -ivh "${PACKAGE_PATH}/${packagename}" + if ! rpm -q emqx | grep -q emqx; then + echo "package install error" + exit 1 + fi - echo "running ${packagename} start" - running_test - echo "running ${packagename} stop" + echo "running ${packagename} start" + running_test + echo "running ${packagename} stop" - rpm -e "${EMQX_NAME}" - if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then - echo "package uninstall error" - exit 1 - fi - ;; - - esac + rpm -e "${EMQX_NAME}" + if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then + echo "package uninstall error" + exit 1 + fi + ;; + esac } running_test(){ From 02d347d4eaefdbb3945a3d98bc0f250e09cf06e1 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 4 Jan 2022 10:49:00 +0800 Subject: [PATCH 133/363] ci(build_packages): delete needless steps --- .github/workflows/build_packages.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 696aedd15..d71996639 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -490,15 +490,6 @@ jobs: -X POST \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - - name: push docker image to docker hub - if: github.event_name == 'release' - run: | - set -e -x -u - sudo make docker-prepare - cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd - - echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin - sudo TARGET=emqx/${{ matrix.profile }} make docker-push - sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list - name: update repo.emqx.io if: github.event_name == 'release' && matrix.profile == 'emqx-ee' run: | From 93abd69db16648e47d6047cb82616c2a89b98cc7 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 4 Jan 2022 10:08:02 +0800 Subject: [PATCH 134/363] chore(helm): fix spell errors --- deploy/charts/emqx/templates/StatefulSet.yaml | 15 +++++++++++++-- deploy/charts/emqx/values.yaml | 4 ++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 7ad870b6d..c6e225e3a 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -82,7 +82,12 @@ spec: claimName: {{ tpl . $ }} {{- end }} {{- end }} - {{- if .Values.emqxLicneseSecretName }} + {{- if .Values.emqxLicenseSecretName }} + - name: emqx-license + secret: + secretName: {{ .Values.emqxLicenseSecretName }} + ## Compatible with previous misspellings + {{- else if .Values.emqxLicneseSecretName }} - name: emqx-license secret: secretName: {{ .Values.emqxLicneseSecretName }} @@ -150,7 +155,13 @@ spec: - name: emqx-loaded-modules mountPath: "/opt/emqx/data/loaded_modules" subPath: "loaded_modules" - {{ if .Values.emqxLicneseSecretName }} + {{ if .Values.emqxLicenseSecretName }} + - name: emqx-license + mountPath: "/opt/emqx/etc/emqx.lic" + subPath: "emqx.lic" + readOnly: true + ## Compatible with previous misspellings + {{ else if .Values.emqxLicneseSecretName }} - name: emqx-license mountPath: "/opt/emqx/etc/emqx.lic" subPath: "emqx.lic" diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 373ba0849..1b129cf75 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -114,10 +114,10 @@ emqxLoadedModules: > {emqx_mod_subscription, false}. {emqx_mod_topic_metrics, false}. -## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicneseSecretName" +## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicenseSecretName" ## Example: ## kubectl create secret generic emqx-license-secret-name --from-file=/path/to/emqx.lic -emqxLicneseSecretName: +emqxLicenseSecretName: service: ## Service type From 8e88f8b9f9cca2ec75014816165ebdfd62b1172b Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 4 Jan 2022 16:32:32 +0800 Subject: [PATCH 135/363] ci(build_packages): if it's enterprise, use Dockerfile.enterprise Signed-off-by: zhanghongtong --- .github/workflows/build_packages.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index d71996639..6fea32eac 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -382,6 +382,7 @@ jobs: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/build-push-action@v2 + if: matrix.profile != 'emqx-ee' with: push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} pull: true @@ -395,6 +396,21 @@ jobs: EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile context: source + - uses: docker/build-push-action@v2 + if: matrix.profile == 'emqx-ee' + with: + push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} + pull: true + no-cache: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.otp }}-alpine3.14 + RUN_FROM=alpine:3.14 + EMQX_NAME=${{ matrix.profile }} + file: source/deploy/docker/Dockerfile.enterprise + context: source - uses: aws-actions/configure-aws-credentials@v1 if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' with: From 03c8d35480d11d780d8bcc9ab0d3085752e470df Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 4 Jan 2022 17:40:00 +0100 Subject: [PATCH 136/363] ci: test rpm install with yum --- .ci/build_packages/tests.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 9ff693ed1..670a92d76 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -125,13 +125,8 @@ emqx_test(){ fi ;; "rpm") - if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; then - # EMQX OTP requires openssl11 to have TLS1.3 support - yum install -y openssl11 - fi - - rpm -ivh "${PACKAGE_PATH}/${packagename}" - if ! rpm -q emqx | grep -q emqx; then + yum install -y "${PACKAGE_PATH}/${packagename}" + if ! rpm -q ${EMQX_NAME} | grep -q "${EMQX_NAME}"; then echo "package install error" exit 1 fi From cf57b77f032308955da7bfec839e3b056d78dfd0 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 5 Jan 2022 11:09:45 +0800 Subject: [PATCH 137/363] ci(static checks): do not run static check --- .github/workflows/run_test_cases.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index a6ccf5dad..c32c13531 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -8,23 +8,6 @@ on: pull_request: jobs: - run_static_analysis: - runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 - - steps: - - uses: actions/checkout@v2 - - name: set git credentials - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - fi - - name: xref - run: make xref - - name: dialyzer - run: make dialyzer - run_proper_test: runs-on: ubuntu-20.04 container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 From 5424588a72c4c7659b525ee988814b2c18878551 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 Jan 2022 11:11:30 +0800 Subject: [PATCH 138/363] chore: remove unwanted appup commands that came with merge --- .../src/emqx_management.appup.src | 16 +--- .../emqx_retainer/src/emqx_retainer.appup.src | 15 +-- .../src/emqx_rule_engine.appup.src | 92 +------------------ 3 files changed, 7 insertions(+), 116 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src index 1463334b4..109eeef80 100644 --- a/apps/emqx_management/src/emqx_management.appup.src +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -1,17 +1,5 @@ %% -*- mode: erlang -*- {VSN, - [ {<<"4\\.3\\.[0-9]+">>, - [ {apply,{minirest,stop_http,['http:management']}}, - {apply,{minirest,stop_http,['https:management']}}, - {restart_application, emqx_management} - ]}, - {<<".*">>, []} - ], - [ {<<"4\\.3\\.[0-9]+">>, - [ {apply,{minirest,stop_http,['http:management']}}, - {apply,{minirest,stop_http,['https:management']}}, - {restart_application, emqx_management} - ]}, - {<<".*">>, []} - ] + [{<<".*">>,[]}], + [{<<".*">>,[]}] }. diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src index 45ec6420c..109eeef80 100644 --- a/apps/emqx_retainer/src/emqx_retainer.appup.src +++ b/apps/emqx_retainer/src/emqx_retainer.appup.src @@ -1,14 +1,5 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.2", - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-1]">>, - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}], - [{"4.3.2", - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-1]">>, - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}]}. + [{<<".*">>,[]}], + [{<<".*">>,[]}] +}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index d97c6542f..109eeef80 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,93 +1,5 @@ %% -*- mode: erlang -*- {VSN, - [ - {"4.3.6", - [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - ]}, - {"4.3.5", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - ]}, - {"4.3.4", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - ]}, - {"4.3.3", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - ]}, - {"4.3.2", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - ]}, - {"4.3.1", - [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} - , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - ]}, - {"4.3.0", - [ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} - , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - ]}, - {<<".*">>, []} - ], - [ - {"4.3.6", - [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - ]}, - {"4.3.5", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - ]}, - {"4.3.4", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - ]}, - {"4.3.3", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - ]}, - {"4.3.2", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - ]}, - {"4.3.1", - [ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} - , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - ]}, - {"4.3.0", - [ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {apply,{emqx_stats,cancel_update,[rule_registery_stats]}} - , {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - ]}, - {<<".*">>, []} - ] + [{<<".*">>,[]}], + [{<<".*">>,[]}] }. From 80eed7f2d7323ff32f85c10248686b94e7e217c4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 12 Jan 2022 11:54:03 +0100 Subject: [PATCH 139/363] ci: fix a typo in tests.sh NOTE: this typo did not cause anything other than a false message about a missing file. --- .ci/build_packages/tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 670a92d76..76fbe394b 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -40,9 +40,10 @@ else fi PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" -PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}.${PKG_SUFFIX}" +PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" if ! [ -f "$PACKAGE_FILE" ]; then echo "$PACKAGE_FILE is not a file" + exit 1 fi case "$(uname -m)" in From b3862fb28352fdfefe6ff7aad5867260210129c1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 13 Jan 2022 17:30:20 +0800 Subject: [PATCH 140/363] chore: fix the app vsn and appup.src --- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_management.appup.src | 10 ++++++++-- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- apps/emqx_retainer/src/emqx_retainer.appup.src | 6 ++++-- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 14 ++++++++++++-- apps/emqx_rule_engine/src/emqx_rule_metrics.erl | 8 ++++---- src/emqx.app.src | 2 +- src/emqx.appup.src | 8 ++++++-- 9 files changed, 38 insertions(+), 16 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index fe68fef44..1efd30dcc 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "4.4.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src index 109eeef80..5121efb88 100644 --- a/apps/emqx_management/src/emqx_management.appup.src +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -1,5 +1,11 @@ %% -*- mode: erlang -*- {VSN, - [{<<".*">>,[]}], - [{<<".*">>,[]}] + [{<<".*">>, + [{apply,{minirest,stop_http,['http:management']}}, + {apply,{minirest,stop_http,['https:management']}}, + {restart_application, emqx_management}]}], + [{<<".*">>, + [{apply,{minirest,stop_http,['http:management']}}, + {apply,{minirest,stop_http,['https:management']}}, + {restart_application, emqx_management}]}] }. diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index a423fb9b7..4ef423b78 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "4.4.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src index 109eeef80..82f353e6e 100644 --- a/apps/emqx_retainer/src/emqx_retainer.appup.src +++ b/apps/emqx_retainer/src/emqx_retainer.appup.src @@ -1,5 +1,7 @@ %% -*- mode: erlang -*- {VSN, - [{<<".*">>,[]}], - [{<<".*">>,[]}] + [{"4.4.0",[{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.4.0",[{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}] }. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 98e5487e2..eaeded042 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "4.4.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 109eeef80..1e146ebd9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,5 +1,15 @@ %% -*- mode: erlang -*- {VSN, - [{<<".*">>,[]}], - [{<<".*">>,[]}] + [{"4.4.0", + [ {update, emqx_rule_metrics, {advanced, ["4.4.0"]}} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {<<".*">>,[]}], + [{"4.4.0", + [ {update, emqx_rule_metrics, {advanced, ["4.4.0"]}} + , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} + ]}, + {<<".*">>,[]}] }. diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index 8532ec4f7..0da6b1197 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -328,9 +328,9 @@ handle_info(_Info, State) -> code_change({down, _Vsn}, State = #state{metric_ids = MIDs}, [Vsn]) -> case string:tokens(Vsn, ".") of - ["4", "3", SVal] -> + ["4", "4", SVal] -> {Val, []} = string:to_integer(SVal), - case Val =< 7 of + case Val == 0 of true -> [begin Passed = get_rules_passed(Id), @@ -356,9 +356,9 @@ code_change({down, _Vsn}, State = #state{metric_ids = MIDs}, [Vsn]) -> code_change(_Vsn, State = #state{metric_ids = MIDs}, [Vsn]) -> case string:tokens(Vsn, ".") of - ["4", "3", SVal] -> + ["4", "4", SVal] -> {Val, []} = string:to_integer(SVal), - case Val =< 7 of + case Val == 0 of true -> [begin Matched = get_rules_matched(Id), diff --git a/src/emqx.app.src b/src/emqx.app.src index fa501f461..f5b739e48 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -1,7 +1,7 @@ {application, emqx, [{id, "emqx"}, {description, "EMQ X"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "4.4.1"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 109eeef80..15f962297 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,5 +1,9 @@ %% -*- mode: erlang -*- {VSN, - [{<<".*">>,[]}], - [{<<".*">>,[]}] + [{"4.4.0", + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.4.0", + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}] }. From 1daf2e4fc97df0f667227fa96a5a27bf9ad1bf7d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 14 Jan 2022 15:57:50 +0800 Subject: [PATCH 141/363] fix(appup): add emqx_alarm into appup file --- src/emqx.appup.src | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index e250d2153..8e1016cc4 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -4,12 +4,14 @@ [ {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} + , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} ]}, {<<".*">>,[]}], [{"4.4.0", [ {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} + , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} ]}, {<<".*">>,[]}] }. From 1ce77de080e4f5875c99033e7cd6b8a036112ae8 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 17 Jan 2022 14:51:46 +0800 Subject: [PATCH 142/363] feat(metrics): client metrics --- src/emqx_connection.erl | 53 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 235d92783..5eeb323b0 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -108,9 +108,39 @@ -type(state() :: #state{}). -define(ACTIVE_N, 100). --define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]). --define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). --define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). + +-define(INFO_KEYS, [ socktype + , peername + , sockname + , sockstate + , active_n + ]). + +-define(CONN_STATS, [ recv_pkt + , recv_msg + , 'recv_msg.qos0' + , 'recv_msg.qos1' + , 'recv_msg.qos2' + , 'recv_msg.dropped' + , 'recv_msg.dropped.expired' + + , send_pkt + , send_msg + , 'send_msg.qos0' + , 'send_msg.qos1' + , 'send_msg.qos2' + , 'send_msg.dropped' + , 'send_msg.dropped.expired' + , 'send_msg.dropped.queue_full' + , 'send_msg.dropped.too_large' + ]). + +-define(SOCK_STATS, [ recv_oct + , recv_cnt + , send_oct + , send_cnt + , send_pend + ]). -define(ENABLED(X), (X =/= undefined)). @@ -691,6 +721,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> [emqx_packet:format(Packet)]), ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), + ok = inc_outgoing_stats({error, message_too_large}), <<>>; Data -> ?LOG(debug, "SEND ~s", [emqx_packet:format(Packet)]), ok = inc_outgoing_stats(Packet), @@ -835,17 +866,31 @@ inc_incoming_stats(Packet = ?PACKET(Type)) -> emqx_metrics:inc_recv(Packet). -compile({inline, [inc_outgoing_stats/1]}). +inc_outgoing_stats({error, message_too_large}) -> + inc_counter('send_msg.dropped', 1), + inc_counter('send_msg.dropped.too_large', 1); inc_outgoing_stats(Packet = ?PACKET(Type)) -> inc_counter(send_pkt, 1), case Type =:= ?PUBLISH of true -> inc_counter(send_msg, 1), - inc_counter(outgoing_pubs, 1); + inc_counter(outgoing_pubs, 1), + inc_qos_stats(send_msg, Packet); false -> ok end, emqx_metrics:inc_sent(Packet). +inc_qos_stats(Type, #mqtt_packet{header = #mqtt_packet_header{qos = QoS}}) -> + inc_counter(inc_qos_stats_key(Type, QoS), 1). + +inc_qos_stats_key(send_msg, ?QOS_0) -> 'send_msg.qos0'; +inc_qos_stats_key(send_msg, ?QOS_1) -> 'send_msg.qos1'; +inc_qos_stats_key(send_msg, ?QOS_2) -> 'send_msg.qos2'; + +inc_qos_stats_key(recv_msg, ?QOS_0) -> 'recv_msg.qos0'; +inc_qos_stats_key(recv_msg, ?QOS_1) -> 'recv_msg.qos1'; +inc_qos_stats_key(recv_msg, ?QOS_2) -> 'recv_msg.qos2'. %%-------------------------------------------------------------------- %% Helper functions From 5397d80680d7e6c0902206bc59c64db2bf5a4c74 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 17 Jan 2022 15:12:08 +0800 Subject: [PATCH 143/363] feat(metrics): session metrics & api format --- .../src/emqx_mgmt_api_clients.erl | 12 +++++++++--- src/emqx.appup.src | 2 ++ src/emqx_connection.erl | 10 +++++----- src/emqx_session.erl | 18 ++++++++++++++++-- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 71e94f841..de1fbabde 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -313,8 +313,14 @@ format_channel_info({_Key, Info, Stats0}) -> inflight, max_inflight, awaiting_rel, max_awaiting_rel, mqueue_len, mqueue_dropped, max_mqueue, heap_size, reductions, mailbox_len, - recv_cnt, recv_msg, recv_oct, recv_pkt, send_cnt, - send_msg, send_oct, send_pkt], NStats), + recv_cnt, + recv_msg, 'recv_msg.qos0', 'recv_msg.qos1', 'recv_msg.qos2', + 'recv_msg.dropped', 'recv_msg.dropped.expired', + recv_oct, recv_pkt, send_cnt, + send_msg, 'send_msg.qos0', 'send_msg.qos1', 'send_msg.qos2', + 'send_msg.dropped', 'send_msg.dropped.expired', + 'send_msg.dropped.queue_full', 'send_msg.dropped.too_large', + send_oct, send_pkt], NStats), maps:with([clientid, username, mountpoint, is_bridge, zone], ClientInfo), maps:with([clean_start, keepalive, expiry_interval, proto_name, proto_ver, peername, connected_at, disconnected_at], ConnInfo), @@ -373,7 +379,7 @@ match_fun(Ms, Fuzzy) -> run_fuzzy_match(_, []) -> true; -run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr}|Fuzzy]) -> +run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr} | Fuzzy]) -> Val = case maps:get(Key, ClientInfo, undefined) of undefined -> <<>>; V -> V diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 8e1016cc4..2e300b85e 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,6 +2,7 @@ {VSN, [{"4.4.0", [ {load_module,emqx_channel,brutal_purge,soft_purge,[]} + , {load_module,emqx_connection,brutal_purge,soft_purge,[]} , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} @@ -9,6 +10,7 @@ {<<".*">>,[]}], [{"4.4.0", [ {load_module,emqx_channel,brutal_purge,soft_purge,[]} + , {load_module,emqx_connection,brutal_purge,soft_purge,[]} , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 5eeb323b0..2c2cc472d 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -176,7 +176,7 @@ start_link(Transport, Socket, Options) -> %%-------------------------------------------------------------------- %% @doc Get infos of the connection/channel. --spec(info(pid()|state()) -> emqx_types:infos()). +-spec(info(pid() | state()) -> emqx_types:infos()). info(CPid) when is_pid(CPid) -> call(CPid, info); info(State = #state{channel = Channel}) -> @@ -205,7 +205,7 @@ info(limiter, #state{limiter = Limiter}) -> maybe_apply(fun emqx_limiter:info/1, Limiter). %% @doc Get stats of the connection/channel. --spec(stats(pid()|state()) -> emqx_types:stats()). +-spec(stats(pid() | state()) -> emqx_types:stats()). stats(CPid) when is_pid(CPid) -> call(CPid, stats); stats(#state{transport = Transport, @@ -389,7 +389,7 @@ cancel_stats_timer(State) -> State. process_msg([], State) -> {ok, State}; -process_msg([Msg|More], State) -> +process_msg([Msg | More], State) -> try case handle_msg(Msg, State) of ok -> @@ -483,7 +483,7 @@ handle_msg({Passive, _Sock}, State) handle_msg(Deliver = {deliver, _Topic, _Msg}, #state{active_n = ActiveN} = State) -> - Delivers = [Deliver|emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent @@ -657,7 +657,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> {Packets, State#state{parse_state = NParseState}}; {ok, Packet, Rest, NParseState} -> NState = State#state{parse_state = NParseState}, - parse_incoming(Rest, [Packet|Packets], NState) + parse_incoming(Rest, [Packet | Packets], NState) catch error:proxy_protocol_config_disabled:_Stk -> ?LOG(error, diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 6d4b2af34..9648083d9 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -427,8 +427,8 @@ dequeue(ClientInfo, Cnt, Msgs, Q) -> {{value, Msg}, Q1} -> case emqx_message:is_expired(Msg) of true -> - ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, expired]), ok = inc_delivery_expired_cnt(), + ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, expired]), dequeue(ClientInfo, Cnt, Msgs, Q1); false -> dequeue(ClientInfo, acc_cnt(Msg, Cnt), [Msg | Msgs], Q1) end @@ -503,11 +503,14 @@ log_dropped(ClientInfo, Msg = #message{qos = QoS}, #session{mqueue = Q}) -> ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, qos0_msg]), ok = emqx_metrics:inc('delivery.dropped'), ok = emqx_metrics:inc('delivery.dropped.qos0_msg'), + ok = inc_pd('send_msg.dropped'), ?LOG(warning, "Dropped qos0 msg: ~s", [emqx_message:format(Msg)]); false -> ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, queue_full]), ok = emqx_metrics:inc('delivery.dropped'), ok = emqx_metrics:inc('delivery.dropped.queue_full'), + ok = inc_pd('send_msg.dropped'), + ok = inc_pd('send_msg.dropped.queue_full'), ?LOG(warning, "Dropped msg due to mqueue is full: ~s", [emqx_message:format(Msg)]) end. @@ -568,7 +571,8 @@ await(PacketId, Msg, Session = #session{inflight = Inflight}) -> %% Retry Delivery %%-------------------------------------------------------------------- --spec(retry(emqx_types:clientinfo(), session()) -> {ok, session()} | {ok, replies(), timeout(), session()}). +-spec(retry(emqx_types:clientinfo(), session()) -> + {ok, session()} | {ok, replies(), timeout(), session()}). retry(ClientInfo, Session = #session{inflight = Inflight, retry_interval = RetryInterval}) -> case emqx_inflight:is_empty(Inflight) of true -> {ok, Session}; @@ -680,13 +684,23 @@ inc_delivery_expired_cnt() -> inc_delivery_expired_cnt(1). inc_delivery_expired_cnt(N) -> + ok = inc_pd('send_msg.dropped', N), + ok = inc_pd('send_msg.dropped.expired', N), ok = emqx_metrics:inc('delivery.dropped', N), emqx_metrics:inc('delivery.dropped.expired', N). inc_await_pubrel_timeout(N) -> + ok = inc_pd('recv_msg.dropped', N), + ok = inc_pd('recv_msg.dropped.expired', N), ok = emqx_metrics:inc('messages.dropped', N), emqx_metrics:inc('messages.dropped.await_pubrel_timeout', N). +inc_pd(Key) -> + inc_pd(Key, 1). +inc_pd(Key, Inc) -> + _ = emqx_pd:inc_counter(Key, Inc), + ok. + %%-------------------------------------------------------------------- %% Next Packet Id %%-------------------------------------------------------------------- From 6278951d578b3a1e15c8174052a58b4ff2d55bcd Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 17 Jan 2022 15:42:34 +0800 Subject: [PATCH 144/363] fix: code style --- src/emqx_connection.erl | 1 - src/emqx_session.erl | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 2c2cc472d..bd9a9fbe4 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -123,7 +123,6 @@ , 'recv_msg.qos2' , 'recv_msg.dropped' , 'recv_msg.dropped.expired' - , send_pkt , send_msg , 'send_msg.qos0' diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 9648083d9..3ba286f54 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -571,8 +571,9 @@ await(PacketId, Msg, Session = #session{inflight = Inflight}) -> %% Retry Delivery %%-------------------------------------------------------------------- --spec(retry(emqx_types:clientinfo(), session()) -> - {ok, session()} | {ok, replies(), timeout(), session()}). +-spec(retry(emqx_types:clientinfo(), session()) + -> {ok, session()} + | {ok, replies(), timeout(), session()}). retry(ClientInfo, Session = #session{inflight = Inflight, retry_interval = RetryInterval}) -> case emqx_inflight:is_empty(Inflight) of true -> {ok, Session}; From f4c9738b99ae7927024d91961311007f31ec1b11 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 17 Jan 2022 19:31:03 +0800 Subject: [PATCH 145/363] fix(build): make emqx-zip failed on making relup file --- build | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build b/build index 2a9e6a4a6..7b5e6645e 100755 --- a/build +++ b/build @@ -65,18 +65,18 @@ make_relup() { if [ -d "$releases_dir" ]; then while read -r zip; do local base_vsn - base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-f]{8})?")" + base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-f]{8})?" | head -1)" if [ ! -d "$releases_dir/$base_vsn" ]; then local tmp_dir tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir" unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir" - cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" - cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" + cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true + cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true rm -rf "$tmp_dir" fi releases+=( "$base_vsn" ) - done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f) + done < <(find _upgrade_base -maxdepth 1 -name "${PROFILE}-*-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" -type f) fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" From 1e3429986ac34bd11b149b939150ba00fabce2a2 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 18 Jan 2022 10:19:02 +0800 Subject: [PATCH 146/363] fix(ci): download previous zip packages failed --- .github/workflows/run_fvt_tests.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index de2b65ca9..c0f0bee94 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -274,6 +274,11 @@ jobs: relup_test_build: needs: relup_test_plan + strategy: + fail-fast: false + matrix: + otp: + - 24.1.5-3 runs-on: ubuntu-20.04 container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 defaults: @@ -302,7 +307,7 @@ jobs: cd emqx/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for old_vsn in ${old_vsns[@]}; do - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${{ matrix.otp }}-ubuntu20.04-amd64.zip done - name: Build emqx run: make -C emqx ${PROFILE}-zip @@ -324,6 +329,8 @@ jobs: fail-fast: false matrix: old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }} + otp: + - 24.1.5-3 env: OLD_VSN: "${{ matrix.old_vsn }}" PROFILE: "${{ needs.relup_test_plan.outputs.profile }}" @@ -350,7 +357,7 @@ jobs: mkdir -p packages cp emqx_built/_packages/*/*.zip packages cd packages - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-ubuntu20.04-${OLD_VSN#[e|v]}-amd64.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${OLD_VSN#[e|v]}-otp${{ matrix.otp }}-ubuntu20.04-amd64.zip - name: Run relup test scenario timeout-minutes: 5 run: | From 578199ad3a6b3ad2c4e54f9d49e9af07941413c7 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 18 Jan 2022 10:38:06 +0800 Subject: [PATCH 147/363] fix: client metrics count with qos --- src/emqx_connection.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index bd9a9fbe4..abba2bcdd 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -858,6 +858,7 @@ inc_incoming_stats(Packet = ?PACKET(Type)) -> case Type =:= ?PUBLISH of true -> inc_counter(recv_msg, 1), + inc_qos_stats(recv_msg, Packet), inc_counter(incoming_pubs, 1); false -> ok From deaea12aca873febc43e9f6d82eac1356585ae04 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 13 Jan 2022 11:41:45 +0100 Subject: [PATCH 148/363] ci(mac): cache otp install only only cache otp installation instead of the entire kerl dir to save cache spaces --- .github/workflows/build_packages.yaml | 4 ++-- .github/workflows/build_slim_packages.yaml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 78e48f3df..8575a5391 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -164,8 +164,8 @@ jobs: - uses: actions/cache@v2 id: cache with: - path: ~/.kerl - key: erl${{ matrix.otp }}-${{ matrix.macos }} + path: ~/.kerl/${{ matrix.otp }} + key: otp-install-${{ matrix.otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index e9b62a166..df02bc625 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -67,7 +67,7 @@ jobs: strategy: fail-fast: false matrix: - erl_otp: + otp: - 24.1.5-3 macos: - macos-11 @@ -96,8 +96,8 @@ jobs: - uses: actions/cache@v2 id: cache with: - path: ~/.kerl - key: otp-${{ matrix.erl_otp }}-${{ matrix.macos }} + path: ~/.kerl/${{ matrix.otp }} + key: otp-install-${{ matrix.otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 @@ -106,11 +106,11 @@ jobs: OTP_GITHUB_URL: https://github.com/emqx/otp run: | kerl update releases - kerl build ${{ matrix.erl_otp }} - kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} + kerl build ${{ matrix.otp }} + kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} - name: build run: | - . $HOME/.kerl/${{ matrix.erl_otp }}/activate + . $HOME/.kerl/${{ matrix.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 make ${EMQX_NAME}-zip From 040d04c9fb42aadd47688ccfdc9962d7e80ce39d Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 18 Jan 2022 18:02:13 +0800 Subject: [PATCH 149/363] fix(test): for paho qos3 --- src/emqx_connection.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index abba2bcdd..ccf4430a9 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -881,8 +881,9 @@ inc_outgoing_stats(Packet = ?PACKET(Type)) -> end, emqx_metrics:inc_sent(Packet). -inc_qos_stats(Type, #mqtt_packet{header = #mqtt_packet_header{qos = QoS}}) -> - inc_counter(inc_qos_stats_key(Type, QoS), 1). +inc_qos_stats(Type, #mqtt_packet{header = #mqtt_packet_header{qos = QoS}}) when ?IS_QOS(QoS) -> + inc_counter(inc_qos_stats_key(Type, QoS), 1); +inc_qos_stats(_, _) -> ok. inc_qos_stats_key(send_msg, ?QOS_0) -> 'send_msg.qos0'; inc_qos_stats_key(send_msg, ?QOS_1) -> 'send_msg.qos1'; From 44fe882f14393725ff0a445c70d504e00fa6034a Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 7 Jan 2022 17:21:30 +0800 Subject: [PATCH 150/363] refactor(emqx_slow_subs): refactor slow subs --- .../include/emqx_slow_subs.hrl | 18 +- .../src/emqx_slow_subs/emqx_slow_subs.erl | 290 ++++++++---------- .../src/emqx_slow_subs/emqx_slow_subs_api.erl | 67 +++- etc/emqx.conf | 21 +- lib-ce/emqx_modules/src/emqx_mod_sup.erl | 2 +- .../test/emqx_slow_subs_SUITE.erl | 56 ++-- .../test/emqx_slow_subs_api_SUITE.erl | 23 +- priv/emqx.schema | 16 +- src/emqx_session.erl | 75 ++--- .../emqx_message_latency_stats.erl | 119 ------- test/emqx_session_SUITE.erl | 24 +- 11 files changed, 300 insertions(+), 411 deletions(-) delete mode 100644 src/emqx_slow_subs/emqx_message_latency_stats.erl diff --git a/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl b/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl index 0b5e3a035..2bb3f9b16 100644 --- a/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl +++ b/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl @@ -15,14 +15,24 @@ %%-------------------------------------------------------------------- -define(TOPK_TAB, emqx_slow_subs_topk). +-define(INDEX_TAB, emqx_slow_subs_index). --define(INDEX(Latency, ClientId), {Latency, ClientId}). +-define(ID(ClientId, Topic), {ClientId, Topic}). +-define(INDEX(TimeSpan, Id), {Id, TimeSpan}). +-define(TOPK_INDEX(TimeSpan, Id), {TimeSpan, Id}). --record(top_k, { index :: index() - , type :: emqx_message_latency_stats:latency_type() +-define(MAX_SIZE, 1000). + +-record(top_k, { index :: topk_index() , last_update_time :: pos_integer() , extra = [] }). +-record(index_tab, { index :: index()}). + -type top_k() :: #top_k{}. --type index() :: ?INDEX(non_neg_integer(), emqx_types:clientid()). +-type index_tab() :: #index_tab{}. + +-type id() :: {emqx_types:clientid(), emqx_types:topic()}. +-type index() :: ?INDEX(non_neg_integer(), id()). +-type topk_index() :: ?TOPK_INDEX(non_neg_integer(), id()). diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index 11d25380e..254d902bd 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -24,8 +24,8 @@ -logger_header("[SLOW Subs]"). --export([ start_link/1, on_stats_update/2, enable/0 - , disable/0, clear_history/0, init_topk_tab/0 +-export([ start_link/1, on_publish_completed/3, enable/0 + , disable/0, clear_history/0, init_tab/0 ]). %% gen_server callbacks @@ -44,28 +44,19 @@ , last_tick_at := pos_integer() }. --type log() :: #{ rank := pos_integer() - , clientid := emqx_types:clientid() - , latency := non_neg_integer() - , type := emqx_message_latency_stats:latency_type() - }. - --type window_log() :: #{ last_tick_at := pos_integer() - , logs := [log()] - }. - -type message() :: #message{}. -import(proplists, [get_value/2]). --type stats_update_args() :: #{ clientid := emqx_types:clientid() - , latency := non_neg_integer() - , type := emqx_message_latency_stats:latency_type() - , last_insert_value := non_neg_integer() - , update_time := timer:time() - }. +-type stats_type() :: whole %% whole = internal + response + | internal %% timespan from message in to deliver + | response. %% timespan from delivery to client response --type stats_update_env() :: #{max_size := pos_integer()}. +-type stats_update_args() :: #{ clientid := emqx_types:clientid()}. + +-type stats_update_env() :: #{ threshold := non_neg_integer() + , stats_type := stats_type() + , max_size := pos_integer()}. -ifdef(TEST). -define(EXPIRE_CHECK_INTERVAL, timer:seconds(1)). @@ -74,7 +65,6 @@ -endif. -define(NOW, erlang:system_time(millisecond)). --define(NOTICE_TOPIC_NAME, "slow_subs"). -define(DEF_CALL_TIMEOUT, timer:seconds(10)). %% erlang term order @@ -91,36 +81,29 @@ start_link(Env) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). -%% XXX NOTE:pay attention to the performance here --spec on_stats_update(stats_update_args(), stats_update_env()) -> true. -on_stats_update(#{clientid := ClientId, - latency := Latency, - type := Type, - last_insert_value := LIV, - update_time := Ts}, - #{max_size := MaxSize}) -> +on_publish_completed(#message{timestamp = Ts}, #{session_birth_time := BirthTime}, _Cfg) + when Ts =< BirthTime -> + ok; - LastIndex = ?INDEX(LIV, ClientId), - Index = ?INDEX(Latency, ClientId), +on_publish_completed(Msg, Env, Cfg) -> + on_publish_completed(Msg, Env, erlang:system_time(millisecond), Cfg). - %% check whether the client is in the table - case ets:lookup(?TOPK_TAB, LastIndex) of - [#top_k{index = Index}] -> - %% if last value == the new value, update the type and last_update_time - %% XXX for clients whose latency are stable for a long time, is it possible to reduce updates? - ets:insert(?TOPK_TAB, - #top_k{index = Index, type = Type, last_update_time = Ts}); - [_] -> - %% if Latency > minimum value, we should update it - %% if Latency < minimum value, maybe it can replace the minimum value - %% so alwyas update at here - %% do we need check if Latency == minimum ??? - ets:insert(?TOPK_TAB, - #top_k{index = Index, type = Type, last_update_time = Ts}), - ets:delete(?TOPK_TAB, LastIndex); - [] -> - %% try to insert - try_insert_to_topk(MaxSize, Index, Latency, Type, Ts) +on_publish_completed(#message{topic = Topic} = Msg, + #{clientid := ClientId}, + Now, #{threshold := Threshold, + stats_type := StatsType, + max_size := MaxSize}) -> + TimeSpan = calc_timespan(StatsType, Msg, Now), + case TimeSpan =< Threshold of + true -> ok; + _ -> + Id = ?ID(ClientId, Topic), + LastUpdateValue = find_last_update_value(Id), + case TimeSpan =< LastUpdateValue of + true -> ok; + _ -> + try_insert_to_topk(MaxSize, Now, LastUpdateValue, TimeSpan, Id) + end end. clear_history() -> @@ -132,26 +115,23 @@ enable() -> disable() -> gen_server:call(?MODULE, {enable, false}, ?DEF_CALL_TIMEOUT). -init_topk_tab() -> - case ets:whereis(?TOPK_TAB) of - undefined -> - ?TOPK_TAB = ets:new(?TOPK_TAB, - [ ordered_set, public, named_table - , {keypos, #top_k.index}, {write_concurrency, true} +init_tab() -> + safe_create_tab(?TOPK_TAB, [ ordered_set, public, named_table + , {keypos, #top_k.index}, {write_concurrency, true} + , {read_concurrency, true} + ]), + + safe_create_tab(?INDEX_TAB, [ ordered_set, public, named_table + , {keypos, #index_tab.index}, {write_concurrency, true} , {read_concurrency, true} - ]); - _ -> - ?TOPK_TAB - end. + ]). %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- init([Conf]) -> - notice_tick(Conf), expire_tick(Conf), - update_threshold(Conf), load(Conf), {ok, #{config => Conf, last_tick_at => ?NOW, @@ -163,7 +143,6 @@ handle_call({enable, Enable}, _From, IsEnable -> State; true -> - update_threshold(Cfg), load(Cfg), State#{enable := true}; _ -> @@ -173,7 +152,7 @@ handle_call({enable, Enable}, _From, {reply, ok, State2}; handle_call(clear_history, _, State) -> - ets:delete_all_objects(?TOPK_TAB), + do_clear_history(), {reply, ok, State}; handle_call(Req, _From, State) -> @@ -190,12 +169,6 @@ handle_info(expire_tick, #{config := Cfg} = State) -> do_clear(Cfg, Logs), {noreply, State}; -handle_info(notice_tick, #{config := Cfg} = State) -> - notice_tick(Cfg), - Logs = ets:tab2list(?TOPK_TAB), - do_notification(Logs, State), - {noreply, State#{last_tick_at := ?NOW}}; - handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), {noreply, State}. @@ -213,115 +186,116 @@ code_change(_OldVsn, State, _Extra) -> expire_tick(_) -> erlang:send_after(?EXPIRE_CHECK_INTERVAL, self(), ?FUNCTION_NAME). -notice_tick(Cfg) -> - case get_value(notice_interval, Cfg) of - 0 -> ok; - Interval -> - erlang:send_after(Interval, self(), ?FUNCTION_NAME), - ok - end. - --spec do_notification(list(), state()) -> ok. -do_notification([], _) -> - ok; - -do_notification(Logs, #{last_tick_at := LastTickTime, config := Cfg}) -> - start_publish(Logs, LastTickTime, Cfg), - ok. - -start_publish(Logs, TickTime, Cfg) -> - emqx_pool:async_submit({fun do_publish/4, [Logs, erlang:length(Logs), TickTime, Cfg]}). - -do_publish([], _, _, _) -> - ok; - -do_publish(Logs, Rank, TickTime, Cfg) -> - BatchSize = get_value(notice_batch_size, Cfg), - do_publish(Logs, BatchSize, Rank, TickTime, Cfg, []). - -do_publish([Log | T], Size, Rank, TickTime, Cfg, Cache) when Size > 0 -> - Cache2 = [convert_to_notice(Rank, Log) | Cache], - do_publish(T, Size - 1, Rank - 1, TickTime, Cfg, Cache2); - -do_publish(Logs, Size, Rank, TickTime, Cfg, Cache) when Size =:= 0 -> - publish(TickTime, Cfg, Cache), - do_publish(Logs, Rank, TickTime, Cfg); - -do_publish([], _, _Rank, TickTime, Cfg, Cache) -> - publish(TickTime, Cfg, Cache), - ok. - -convert_to_notice(Rank, #top_k{index = ?INDEX(Latency, ClientId), - type = Type, - last_update_time = Ts}) -> - #{rank => Rank, - clientid => ClientId, - latency => Latency, - type => Type, - timestamp => Ts}. - -publish(TickTime, Cfg, Notices) -> - WindowLog = #{last_tick_at => TickTime, - logs => lists:reverse(Notices)}, - Payload = emqx_json:encode(WindowLog), - Msg = #message{ id = emqx_guid:gen() - , qos = get_value(notice_qos, Cfg) - , from = ?MODULE - , topic = emqx_topic:systop(?NOTICE_TOPIC_NAME) - , payload = Payload - , timestamp = ?NOW - }, - _ = emqx_broker:safe_publish(Msg), - ok. - load(Cfg) -> MaxSize = get_value(top_k_num, Cfg), - _ = emqx:hook('message.slow_subs_stats', - fun ?MODULE:on_stats_update/2, - [#{max_size => MaxSize}]), + StatsType = get_value(stats_type, Cfg), + Threshold = get_value(threshold, Cfg), + _ = emqx:hook('message.publish_completed', + fun ?MODULE:on_publish_completed/3, + [#{max_size => MaxSize, + stats_type => StatsType, + threshold => Threshold + }]), ok. unload() -> - emqx:unhook('message.slow_subs_stats', fun ?MODULE:on_stats_update/2). + emqx:unhook('message.publish_completed', fun ?MODULE:on_publish_completed/3 ), + do_clear_history(). do_clear(Cfg, Logs) -> Now = ?NOW, Interval = get_value(expire_interval, Cfg), - Each = fun(#top_k{index = Index, last_update_time = Ts}) -> + Each = fun(#top_k{index = ?TOPK_INDEX(TimeSpan, Id), last_update_time = Ts}) -> case Now - Ts >= Interval of true -> - ets:delete(?TOPK_TAB, Index); + delete_with_index(TimeSpan, Id); _ -> true - end + end end, lists:foreach(Each, Logs). -try_insert_to_topk(MaxSize, Index, Latency, Type, Ts) -> +-spec calc_timespan(stats_type(), emqx_types:message(), non_neg_integer()) -> non_neg_integer(). +calc_timespan(whole, #message{timestamp = Ts}, Now) -> + Now - Ts; + +calc_timespan(internal, #message{timestamp = Ts} = Msg, Now) -> + End = emqx_message:get_header(deliver_begin_at, Msg, Now), + End - Ts; + +calc_timespan(response, Msg, Now) -> + Begin = emqx_message:get_header(deliver_begin_at, Msg, Now), + Now - Begin. + +%% update_topk is safe, because each process has a unique clientid +%% insert or delete are bind to this clientid, so there is no race condition +%% +%% but, the delete_with_index in L249 may have a race condition +%% because the data belong to other clientid will be deleted here (deleted the data written by other processes).%% so it may appear that: +%% when deleting a record, the other process is performing an update operation on this recrod +%% in order to solve this race condition problem, the index table also uses the ordered_set type, +%% so that even if the above situation occurs, it will only cause the old data to be deleted twice +%% and the correctness of the data will not be affected + +try_insert_to_topk(MaxSize, Now, LastUpdateValue, TimeSpan, Id) -> case ets:info(?TOPK_TAB, size) of Size when Size < MaxSize -> - %% if the size is under limit, insert it directly - ets:insert(?TOPK_TAB, - #top_k{index = Index, type = Type, last_update_time = Ts}); + update_topk(Now, LastUpdateValue, TimeSpan, Id); _Size -> - %% find the minimum value - ?INDEX(Min, _) = First = - case ets:first(?TOPK_TAB) of - ?INDEX(_, _) = I -> I; - _ -> ?INDEX(Latency - 1, <<>>) - end, - - case Latency =< Min of - true -> true; - _ -> - ets:insert(?TOPK_TAB, - #top_k{index = Index, type = Type, last_update_time = Ts}), - - ets:delete(?TOPK_TAB, First) + case ets:first(?TOPK_TAB) of + '$end_of_table' -> + update_topk(Now, LastUpdateValue, TimeSpan, Id); + ?TOPK_INDEX(_, Id) -> + update_topk(Now, LastUpdateValue, TimeSpan, Id); + ?TOPK_INDEX(Min, MinId) -> + case TimeSpan =< Min of + true -> false; + _ -> + update_topk(Now, LastUpdateValue, TimeSpan, Id), + delete_with_index(Min, MinId) + end end end. -update_threshold(Conf) -> - Threshold = proplists:get_value(threshold, Conf), - _ = emqx_message_latency_stats:update_threshold(Threshold), - ok. +-spec find_last_update_value(id()) -> non_neg_integer(). +find_last_update_value(Id) -> + case ets:next(?INDEX_TAB, ?INDEX(0, Id)) of + #index_tab{index = ?INDEX(LastUpdateValue, Id)} -> + LastUpdateValue; + _ -> + 0 + end. + +-spec update_topk(non_neg_integer(), non_neg_integer(), non_neg_integer(), integer()) -> true. +update_topk(Now, LastUpdateValue, TimeSpan, Id) -> + %% update record + ets:insert(?TOPK_TAB, #top_k{index = ?TOPK_INDEX(TimeSpan, Id), + last_update_time = Now, + extra = [] + }), + + %% update index + ets:insert(?INDEX_TAB, #index_tab{index = ?INDEX(TimeSpan, Id)}), + + %% delete the old record & index + delete_with_index(LastUpdateValue, Id). + +-spec delete_with_index(non_neg_integer(), id()) -> true. +delete_with_index(0, _) -> + true; + +delete_with_index(TimeSpan, Id) -> + ets:delete(?INDEX_TAB, ?INDEX(TimeSpan, Id)), + ets:delete(?TOPK_TAB, ?TOPK_INDEX(TimeSpan, Id)). + +safe_create_tab(Name, Opts) -> + case ets:whereis(Name) of + undefined -> + Name = ets:new(Name, Opts); + _ -> + Name + end. + +do_clear_history() -> + ets:delete_all_objects(?INDEX_TAB), + ets:delete_all_objects(?TOPK_TAB). diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl index a7c9fc050..23d39273d 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl @@ -30,9 +30,12 @@ -export([ clear_history/2 , get_history/2 + , get_history/0 ]). --include("include/emqx_slow_subs.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_slow_subs.hrl"). + +-define(DEFAULT_RPC_TIMEOUT, timer:seconds(5)). -import(minirest, [return/1]). @@ -41,17 +44,55 @@ %%-------------------------------------------------------------------- clear_history(_Bindings, _Params) -> - ok = emqx_slow_subs:clear_history(), + Nodes = ekka_mnesia:running_nodes(), + _ = [rpc_call(Node, emqx_slow_subs, clear_history, [], ok, ?DEFAULT_RPC_TIMEOUT) + || Node <- Nodes], return(ok). -get_history(_Bindings, Params) -> - RowFun = fun(#top_k{index = ?INDEX(Latency, ClientId), - type = Type, - last_update_time = Ts}) -> - [{clientid, ClientId}, - {latency, Latency}, - {type, Type}, - {last_update_time, Ts}] - end, - Return = emqx_mgmt_api:paginate({?TOPK_TAB, [{traverse, last_prev}]}, Params, RowFun), - return({ok, Return}). +get_history(_Bindings, _Params) -> + Nodes = ekka_mnesia:running_nodes(), + Fun = fun(Node, Acc) -> + NodeRankL = rpc_call(Node, + ?MODULE, + ?FUNCTION_NAME, + [], + [], + ?DEFAULT_RPC_TIMEOUT), + NodeRankL ++ Acc + end, + + RankL = lists:foldl(Fun, [], Nodes), + + SortFun = fun(#{timespan := A}, #{timespan := B}) -> + A > B + end, + + SortedL = lists:sort(SortFun, RankL), + SortedL2 = lists:sublist(SortedL, ?MAX_SIZE), + + return({ok, SortedL2}). + +get_history() -> + Node = node(), + RankL = ets:tab2list(?TOPK_TAB), + ConvFun = fun(#top_k{index = ?TOPK_INDEX(TimeSpan, ?ID(ClientId, Topic)), + last_update_time = LastUpdateTime + }) -> + #{ clientid => ClientId + , node => Node + , topic => Topic + , timespan => TimeSpan + , last_update_time => LastUpdateTime + } + end, + + lists:map(ConvFun, RankL). + +rpc_call(Node, M, F, A, _ErrorR, _T) when Node =:= node() -> + erlang:apply(M, F, A); + +rpc_call(Node, M, F, A, ErrorR, T) -> + case rpc:call(Node, M, F, A, T) of + {badrpc, _} -> ErrorR; + Res -> Res + end. diff --git a/etc/emqx.conf b/etc/emqx.conf index 4e8aba756..19fdbc656 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2228,25 +2228,18 @@ module.presence.qos = 1 ## maximum number of Top-K record ## -## Value: 10 +## Defalut: 10 #module.slow_subs.top_k_num = 10 -## enable notification -## publish topk list to $SYS/brokers/${node}/slow_subs per notice_interval -## publish is disabled if set to 0s. +## Stats Type ## -## Defaut: 0s -#module.slow_subs.notice_interval = 0s +## Default: whole +#module.slow_subs.stats_type = whole -## QoS of notification message in notice topic +## Stats Threshold ## -## Defaut: 0 -#module.slow_subs.notice_qos = 0 - -## Maximum information number in one notification -## -## Default: 100 -#module.slow_subs.notice_batch_size = 100 +## Default: 500ms +#module.slow_subs.threshold = 500ms ## CONFIG_SECTION_END=modules ================================================== diff --git a/lib-ce/emqx_modules/src/emqx_mod_sup.erl b/lib-ce/emqx_modules/src/emqx_mod_sup.erl index 4c8f54679..28f62168c 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_sup.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_sup.erl @@ -69,7 +69,7 @@ stop_child(ChildId) -> init([]) -> ok = emqx_tables:new(emqx_modules, [set, public, {write_concurrency, true}]), - emqx_slow_subs:init_topk_tab(), + emqx_slow_subs:init_tab(), {ok, {{one_for_one, 10, 100}, []}}. %%-------------------------------------------------------------------- diff --git a/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl index f2763ae6c..6cfbdea23 100644 --- a/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl @@ -53,47 +53,39 @@ t_log_and_pub(_) -> %% Sub topic first Subs = [{<<"/test1/+">>, ?QOS_1}, {<<"/test2/+">>, ?QOS_2}], Clients = start_client(Subs), - emqx:subscribe("$SYS/brokers/+/slow_subs"), - timer:sleep(1000), - Now = ?NOW, - %% publish - - lists:foreach(fun(I) -> - Topic = list_to_binary(io_lib:format("/test1/~p", [I])), - Msg = emqx_message:make(undefined, ?QOS_1, Topic, <<"Hello">>), - emqx:publish(Msg#message{timestamp = Now - 500}) - end, - lists:seq(1, 10)), - - lists:foreach(fun(I) -> - Topic = list_to_binary(io_lib:format("/test2/~p", [I])), - Msg = emqx_message:make(undefined, ?QOS_2, Topic, <<"Hello">>), - emqx:publish(Msg#message{timestamp = Now - 500}) - end, - lists:seq(1, 10)), - - timer:sleep(1000), - Size = ets:info(?TOPK_TAB, size), - %% some time record maybe delete due to it expired - ?assert(Size =< 6 andalso Size >= 4), - timer:sleep(1500), - Recs = try_receive([]), - RecSum = lists:sum(Recs), - ?assert(RecSum >= 5), - ?assert(lists:all(fun(E) -> E =< 3 end, Recs)), + Now = ?NOW, + + %% publish + lists:foreach(fun(I) -> + Topic = list_to_binary(io_lib:format("/test1/~p", [I])), + Msg = emqx_message:make(undefined, ?QOS_1, Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 500}) + end, + lists:seq(1, 10)), + + lists:foreach(fun(I) -> + Topic = list_to_binary(io_lib:format("/test2/~p", [I])), + Msg = emqx_message:make(undefined, ?QOS_2, Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 500}) + end, + lists:seq(1, 10)), timer:sleep(2000), + Size = ets:info(?TOPK_TAB, size), + %% some time record maybe delete due to it expired + ?assert(Size =< 6 andalso Size >= 4, + unicode:characters_to_binary(io_lib:format("size is :~p~n", [Size]))), + + timer:sleep(3000), ?assert(ets:info(?TOPK_TAB, size) =:= 0), [Client ! stop || Client <- Clients], ok. base_conf() -> - [ {threshold, 500} + [ {threshold, 300} , {top_k_num, 5} , {expire_interval, timer:seconds(3)} - , {notice_interval, 1500} - , {notice_qos, 0} - , {notice_batch_size, 3} + , {stats_type, whole} ]. start_client(Subs) -> diff --git a/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl index c8f6b9cfc..36ed45f01 100644 --- a/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl @@ -68,31 +68,28 @@ base_conf() -> t_get_history(_) -> Now = ?NOW, Each = fun(I) -> - ClientId = erlang:list_to_binary(io_lib:format("test_~p", [I])), - ets:insert(?TOPK_TAB, #top_k{index = ?INDEX(I, ClientId), - type = average, - last_update_time = Now}) + ClientId = erlang:list_to_binary(io_lib:format("test_~p", [I])), + Topic = erlang:list_to_binary(io_lib:format("topic/~p", [I])), + ets:insert(?TOPK_TAB, #top_k{index = ?TOPK_INDEX(I, ?ID(ClientId, Topic)), + last_update_time = Now}) end, lists:foreach(Each, lists:seq(1, 5)), - {ok, Data} = request_api(get, api_path(["slow_subscriptions"]), "_page=1&_limit=10", + {ok, Data} = request_api(get, api_path(["slow_subscriptions"]), "", auth_header_()), - #{meta := Meta, data := [First | _]} = decode(Data), - - RMeta = #{page => 1, limit => 10, count => 5}, - ?assertEqual(RMeta, Meta), + #{data := [First | _]} = decode(Data), RFirst = #{clientid => <<"test_5">>, - latency => 5, - type => <<"average">>, + topic => <<"topic/5">>, + timespan => 5, + node => erlang:atom_to_binary(node()), last_update_time => Now}, ?assertEqual(RFirst, First). t_clear(_) -> - ets:insert(?TOPK_TAB, #top_k{index = ?INDEX(1, <<"test">>), - type = average, + ets:insert(?TOPK_TAB, #top_k{index = ?TOPK_INDEX(1, ?ID(<<"test">>, <<"test">>)), last_update_time = ?NOW}), {ok, _} = request_api(delete, api_path(["slow_subscriptions"]), [], diff --git a/priv/emqx.schema b/priv/emqx.schema index 0aa77865c..cc616f2e8 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1008,6 +1008,7 @@ end}. ]}. %% @doc the number of smaples for calculate the average latency of delivery +%% @deprecated This is a obsoleted configuration, kept here only for compatibility {mapping, "zone.$name.latency_samples", "emqx.zones", [ {default, 10}, {datatype, integer} @@ -2230,26 +2231,35 @@ end}. ]}. {mapping, "module.slow_subs.expire_interval", "emqx.modules", [ - {default, "5m"}, + {default, "300s"}, {datatype, {duration, ms}} ]}. {mapping, "module.slow_subs.top_k_num", "emqx.modules", [ - {default, 500}, - {datatype, integer} + {default, 10}, + {datatype, integer}, + {validators, ["range:0-1000"]} ]}. +{mapping, "module.slow_subs.stats_type", "emqx.modules", [ + {default, whole}, + {datatype, {enum, [whole, internal, response]}} +]}. + +%% @deprecated This is a obsoleted configuration, kept here only for compatibility {mapping, "module.slow_subs.notice_interval", "emqx.modules", [ {default, "0s"}, {datatype, {duration, ms}} ]}. +%% @deprecated This is a obsoleted configuration, kept here only for compatibility {mapping, "module.slow_subs.notice_qos", "emqx.modules", [ {default, 0}, {datatype, integer}, {validators, ["range:0-1"]} ]}. +%% @deprecated This is a obsoleted configuration, kept here only for compatibility {mapping, "module.slow_subs.notice_batch_size", "emqx.modules", [ {default, 500}, {datatype, integer} diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 3ba286f54..e4487234c 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -124,14 +124,13 @@ await_rel_timeout :: timeout(), %% Created at created_at :: pos_integer(), - %% Message deliver latency stats - latency_stats :: emqx_message_latency_stats:stats() + + extras :: map() }). %% in the previous code, we will replace the message record with the pubrel atom %% in the pubrec function, this will lose the creation time of the message, -%% but now we need this time to calculate latency, so now pubrel atom is changed to this record --record(pubrel_await, {timestamp :: non_neg_integer()}). +-record(pubrel_await, {message :: emqx_types:message()}). -type(session() :: #session{}). @@ -157,8 +156,7 @@ mqueue_dropped, next_pkt_id, awaiting_rel_cnt, - awaiting_rel_max, - latency_stats + awaiting_rel_max ]). -define(DEFAULT_BATCH_N, 1000). @@ -187,7 +185,7 @@ init(#{zone := Zone} = CInfo, #{receive_maximum := MaxInflight}) -> max_awaiting_rel = get_env(Zone, max_awaiting_rel, 100), await_rel_timeout = timer:seconds(get_env(Zone, await_rel_timeout, 300)), created_at = erlang:system_time(millisecond), - latency_stats = emqx_message_latency_stats:new(Zone) + extras = #{} }. %% @private init mq @@ -244,9 +242,7 @@ info(awaiting_rel_max, #session{max_awaiting_rel = Max}) -> info(await_rel_timeout, #session{await_rel_timeout = Timeout}) -> Timeout div 1000; info(created_at, #session{created_at = CreatedAt}) -> - CreatedAt; -info(latency_stats, #session{latency_stats = Stats}) -> - emqx_message_latency_stats:latency(Stats). + CreatedAt. %% @doc Get stats of the session. -spec(stats(session()) -> emqx_types:stats()). @@ -339,10 +335,10 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, puback(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> + on_publish_completed(Msg, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), - Session2 = update_latency(Msg, Session), - return_with(Msg, dequeue(ClientInfo, Session2#session{inflight = Inflight1})); - {value, {_Pubrel, _Ts}} -> + return_with(Msg, dequeue(ClientInfo, Session#session{inflight = Inflight1})); + {value, _Other} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} @@ -364,10 +360,10 @@ return_with(Msg, {ok, Publishes, Session}) -> pubrec(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - Update = with_ts(#pubrel_await{timestamp = Msg#message.timestamp}), + Update = with_ts(#pubrel_await{message = Msg}), Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; - {value, {_PUBREL, _Ts}} -> + {value, _Other} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} @@ -396,10 +392,10 @@ pubrel(PacketId, Session = #session{awaiting_rel = AwaitingRel}) -> | {error, emqx_types:reason_code()}). pubcomp(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of - {value, {Pubrel, _Ts}} when is_record(Pubrel, pubrel_await) -> - Session2 = update_latency(Pubrel, Session), + {value, {Pubrel, Msg}} when is_record(Pubrel, pubrel_await) -> + on_publish_completed(Msg, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), - dequeue(ClientInfo, Session2#session{inflight = Inflight1}); + dequeue(ClientInfo, Session#session{inflight = Inflight1}); {value, _Other} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> @@ -464,8 +460,7 @@ do_deliver(ClientInfo, [Msg | More], Acc, Session) -> end. deliver_msg(_ClientInfo, Msg = #message{qos = ?QOS_0}, Session) -> - emqx:run_hook('message.publish_done', - [Msg, #{session_rebirth_time => Session#session.created_at}]), + on_publish_completed(Msg, Session), {ok, [{undefined, maybe_ack(Msg)}], Session}; deliver_msg(ClientInfo, Msg = #message{qos = QoS}, Session = @@ -480,7 +475,8 @@ deliver_msg(ClientInfo, Msg = #message{qos = QoS}, Session = {ok, Session1}; false -> Publish = {PacketId, maybe_ack(Msg)}, - Session1 = await(PacketId, Msg, Session), + Msg2 = mark_begin_deliver(Msg), + Session1 = await(PacketId, Msg2, Session), {ok, [Publish], next_pkt_id(Session1)} end. @@ -574,14 +570,13 @@ await(PacketId, Msg, Session = #session{inflight = Inflight}) -> -spec(retry(emqx_types:clientinfo(), session()) -> {ok, session()} | {ok, replies(), timeout(), session()}). -retry(ClientInfo, Session = #session{inflight = Inflight, retry_interval = RetryInterval}) -> +retry(ClientInfo, Session = #session{inflight = Inflight}) -> case emqx_inflight:is_empty(Inflight) of true -> {ok, Session}; false -> Now = erlang:system_time(millisecond), - Session2 = check_expire_latency(Now, RetryInterval, Session), retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight), - [], Now, Session2, ClientInfo) + [], Now, Session, ClientInfo) end. retry_delivery([], Acc, _Now, Session = #session{retry_interval = Interval}, _ClientInfo) -> @@ -717,31 +712,15 @@ next_pkt_id(Session = #session{next_pkt_id = Id}) -> %%-------------------------------------------------------------------- %% Message Latency Stats %%-------------------------------------------------------------------- -update_latency(Msg, - #session{clientid = ClientId, - latency_stats = Stats, - created_at = CreateAt} = S) -> - case get_birth_timestamp(Msg, CreateAt) of - 0 -> S; - Ts -> - Latency = erlang:system_time(millisecond) - Ts, - Stats2 = emqx_message_latency_stats:update(ClientId, Latency, Stats), - S#session{latency_stats = Stats2} - end. +on_publish_completed(Msg, + #session{clientid = ClientId, created_at = CreateAt}) -> + emqx:run_hook('message.publish_completed', + [Msg, #{ session_birth_time => CreateAt + , clientid => ClientId + }]). -check_expire_latency(Now, Interval, - #session{clientid = ClientId, latency_stats = Stats} = S) -> - Stats2 = emqx_message_latency_stats:check_expire(ClientId, Now, Interval, Stats), - S#session{latency_stats = Stats2}. - -get_birth_timestamp(#message{timestamp = Ts}, CreateAt) when CreateAt =< Ts -> - Ts; - -get_birth_timestamp(#pubrel_await{timestamp = Ts}, CreateAt) when CreateAt =< Ts -> - Ts; - -get_birth_timestamp(_, _) -> - 0. +mark_begin_deliver(Msg) -> + emqx_message:set_header(deliver_begin_at, erlang:system_time(second), Msg). %%-------------------------------------------------------------------- %% Helper functions diff --git a/src/emqx_slow_subs/emqx_message_latency_stats.erl b/src/emqx_slow_subs/emqx_message_latency_stats.erl deleted file mode 100644 index 1d6158d59..000000000 --- a/src/emqx_slow_subs/emqx_message_latency_stats.erl +++ /dev/null @@ -1,119 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_message_latency_stats). - -%% API --export([ new/1, update/3, check_expire/4, latency/1]). - --export([get_threshold/0, update_threshold/1]). - --define(NOW, erlang:system_time(millisecond)). --define(MINIMUM_INSERT_INTERVAL, 1000). --define(MINIMUM_THRESHOLD, 100). --define(DEFAULT_THRESHOLD, 500). --define(THRESHOLD_KEY, {?MODULE, threshold}). - --opaque stats() :: #{ ema := emqx_moving_average:ema() - , last_update_time := timestamp() - , last_access_time := timestamp() %% timestamp of last try to call hook - , last_insert_value := non_neg_integer() - }. - --type timestamp() :: non_neg_integer(). --type timespan() :: number(). - --type latency_type() :: average - | expire. - --import(emqx_zone, [get_env/3]). - --export_type([stats/0, latency_type/0]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - --spec new(non_neg_integer() | emqx_types:zone()) -> stats(). -new(SamplesT) when is_integer(SamplesT) -> - Samples = erlang:max(1, SamplesT), - #{ ema => emqx_moving_average:new(exponential, #{period => Samples}) - , last_update_time => 0 - , last_access_time => 0 - , last_insert_value => 0 - }; - -new(Zone) -> - Samples = get_env(Zone, latency_samples, 1), - new(Samples). - --spec update(emqx_types:clientid(), number(), stats()) -> stats(). -update(ClientId, Val, #{ema := EMA} = Stats) -> - Now = ?NOW, - #{average := Latency} = EMA2 = emqx_moving_average:update(Val, EMA), - Stats2 = call_hook(ClientId, Now, average, Latency, Stats), - Stats2#{ ema := EMA2 - , last_update_time := ?NOW}. - --spec check_expire(emqx_types:clientid(), timestamp(), timespan(), stats()) -> stats(). -check_expire(_, Now, Interval, #{last_update_time := LUT} = S) - when LUT >= Now - Interval -> - S; - -check_expire(ClientId, Now, _Interval, #{last_update_time := LUT} = S) -> - Latency = Now - LUT, - call_hook(ClientId, Now, expire, Latency, S). - --spec latency(stats()) -> number(). -latency(#{ema := #{average := Average}}) -> - Average. - --spec update_threshold(pos_integer()) -> pos_integer(). -update_threshold(Threshold) -> - Val = erlang:max(Threshold, ?MINIMUM_THRESHOLD), - persistent_term:put(?THRESHOLD_KEY, Val), - Val. - -get_threshold() -> - persistent_term:get(?THRESHOLD_KEY, ?DEFAULT_THRESHOLD). - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- --spec call_hook(emqx_types:clientid(), timestamp(), latency_type(), timespan(), stats()) -> stats(). -call_hook(_, _, _, Latency, S) - when Latency =< ?MINIMUM_THRESHOLD -> - S; - -call_hook(_, Now, _, _, #{last_access_time := LIT} = S) - when Now =< LIT + ?MINIMUM_INSERT_INTERVAL -> - S; - -call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> - case Latency =< get_threshold() of - true -> - Stats#{last_access_time := Now}; - _ -> - ToInsert = erlang:floor(Latency), - Arg = #{clientid => ClientId, - latency => ToInsert, - type => Type, - last_insert_value => LIV, - update_time => Now}, - emqx:run_hook('message.slow_subs_stats', [Arg]), - Stats#{last_insert_value := ToInsert, - last_access_time := Now} - end. diff --git a/test/emqx_session_SUITE.erl b/test/emqx_session_SUITE.erl index a1d24402c..cfcf0f132 100644 --- a/test/emqx_session_SUITE.erl +++ b/test/emqx_session_SUITE.erl @@ -192,7 +192,8 @@ t_pubrec(_) -> Msg = emqx_message:make(test, ?QOS_2, <<"t">>, <<>>), Inflight = emqx_inflight:insert(2, {Msg, ts(millisecond)}, emqx_inflight:new()), Session = session(#{inflight => Inflight}), - {ok, Msg, Session1} = emqx_session:pubrec(2, Session), + {ok, MsgWithTime, Session1} = emqx_session:pubrec(2, Session), + ?assertEqual(Msg, emqx_message:remove_header(deliver_begin_at, MsgWithTime)), ?assertMatch([{{pubrel_await, _}, _}], emqx_inflight:values(emqx_session:info(inflight, Session1))). t_pubrec_packet_id_in_use_error(_) -> @@ -214,7 +215,7 @@ t_pubrel_error_packetid_not_found(_) -> t_pubcomp(_) -> Now = ts(millisecond), - Inflight = emqx_inflight:insert(1, {{pubrel_await, Now}, Now}, emqx_inflight:new()), + Inflight = emqx_inflight:insert(1, {{pubrel_await, undefined}, Now}, emqx_inflight:new()), Session = session(#{inflight => Inflight}), {ok, Session1} = emqx_session:pubcomp(clientinfo(), 1, Session), ?assertEqual(0, emqx_session:info(inflight_cnt, Session1)). @@ -271,9 +272,13 @@ t_deliver_qos1(_) -> ?assertEqual(2, emqx_session:info(inflight_cnt, Session1)), ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)), ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)), - {ok, Msg1, Session2} = emqx_session:puback(clientinfo(), 1, Session1), + + {ok, Msg1WithTime, Session2} = emqx_session:puback(clientinfo(), 1, Session1), + ?assertEqual(Msg1, emqx_message:remove_header(deliver_begin_at, Msg1WithTime)), ?assertEqual(1, emqx_session:info(inflight_cnt, Session2)), - {ok, Msg2, Session3} = emqx_session:puback(clientinfo(), 2, Session2), + + {ok, Msg2WithTime, Session3} = emqx_session:puback(clientinfo(), 2, Session2), + ?assertEqual(Msg2, emqx_message:remove_header(deliver_begin_at, Msg2WithTime)), ?assertEqual(0, emqx_session:info(inflight_cnt, Session3)). t_deliver_qos2(_) -> @@ -317,7 +322,11 @@ t_retry(_) -> {ok, Pubs, Session1} = emqx_session:deliver(clientinfo(), Delivers, Session), ok = timer:sleep(200), Msgs1 = [{I, emqx_message:set_flag(dup, Msg)} || {I, Msg} <- Pubs], - {ok, Msgs1, 100, Session2} = emqx_session:retry(clientinfo(), Session1), + {ok, Msgs1WithTime, 100, Session2} = emqx_session:retry(clientinfo(), Session1), + ?assertEqual(Msgs1, + lists:map(fun({Id, M}) -> + {Id, emqx_message:remove_header(deliver_begin_at, M)} + end, Msgs1WithTime)), ?assertEqual(2, emqx_session:info(inflight_cnt, Session2)). %%-------------------------------------------------------------------- @@ -341,7 +350,10 @@ t_replay(_) -> Session2 = emqx_session:enqueue(clientinfo(), Msg, Session1), Pubs1 = [{I, emqx_message:set_flag(dup, M)} || {I, M} <- Pubs], {ok, ReplayPubs, Session3} = emqx_session:replay(clientinfo(), Session2), - ?assertEqual(Pubs1 ++ [{3, Msg}], ReplayPubs), + ?assertEqual(Pubs1 ++ [{3, Msg}], + lists:map(fun({Id, M}) -> + {Id, emqx_message:remove_header(deliver_begin_at, M)} + end, ReplayPubs)), ?assertEqual(3, emqx_session:info(inflight_cnt, Session3)). t_expire_awaiting_rel(_) -> From 0a85e71e09b005ab2852c48641a73951d4198fbc Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 14 Jan 2022 17:57:22 +0800 Subject: [PATCH 151/363] fix(appup): add slow subs into appup file --- src/emqx.appup.src | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 2e300b85e..6147c0d51 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -6,6 +6,10 @@ , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} + , {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} + , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} + , {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]} + , {load_module,emqx_session,brutal_purge,soft_purge,[]} ]}, {<<".*">>,[]}], [{"4.4.0", @@ -14,6 +18,10 @@ , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} + , {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} + , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} + , {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]} + , {load_module,emqx_session,brutal_purge,soft_purge,[]} ]}, {<<".*">>,[]}] }. From 696acbfc5cc99a4755471cff93c110e6c0aaef4c Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 17 Jan 2022 16:15:16 +0800 Subject: [PATCH 152/363] fix(emqx_slow_subs): change on_publish_completed to on_delivery_completed --- .../src/emqx_slow_subs/emqx_slow_subs.erl | 24 +++++++++---------- src/emqx_session.erl | 12 +++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index 254d902bd..50769a8fc 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -24,7 +24,7 @@ -logger_header("[SLOW Subs]"). --export([ start_link/1, on_publish_completed/3, enable/0 +-export([ start_link/1, on_delivery_completed/3, enable/0 , disable/0, clear_history/0, init_tab/0 ]). @@ -81,18 +81,18 @@ start_link(Env) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). -on_publish_completed(#message{timestamp = Ts}, #{session_birth_time := BirthTime}, _Cfg) +on_delivery_completed(#message{timestamp = Ts}, #{session_birth_time := BirthTime}, _Cfg) when Ts =< BirthTime -> ok; -on_publish_completed(Msg, Env, Cfg) -> - on_publish_completed(Msg, Env, erlang:system_time(millisecond), Cfg). +on_delivery_completed(Msg, Env, Cfg) -> + on_delivery_completed(Msg, Env, erlang:system_time(millisecond), Cfg). -on_publish_completed(#message{topic = Topic} = Msg, - #{clientid := ClientId}, - Now, #{threshold := Threshold, - stats_type := StatsType, - max_size := MaxSize}) -> +on_delivery_completed(#message{topic = Topic} = Msg, + #{clientid := ClientId}, + Now, #{threshold := Threshold, + stats_type := StatsType, + max_size := MaxSize}) -> TimeSpan = calc_timespan(StatsType, Msg, Now), case TimeSpan =< Threshold of true -> ok; @@ -190,8 +190,8 @@ load(Cfg) -> MaxSize = get_value(top_k_num, Cfg), StatsType = get_value(stats_type, Cfg), Threshold = get_value(threshold, Cfg), - _ = emqx:hook('message.publish_completed', - fun ?MODULE:on_publish_completed/3, + _ = emqx:hook('delivery.completed', + fun ?MODULE:on_delivery_completed/3, [#{max_size => MaxSize, stats_type => StatsType, threshold => Threshold @@ -199,7 +199,7 @@ load(Cfg) -> ok. unload() -> - emqx:unhook('message.publish_completed', fun ?MODULE:on_publish_completed/3 ), + emqx:unhook('delivery.completed', fun ?MODULE:on_delivery_completed/3 ), do_clear_history(). do_clear(Cfg, Logs) -> diff --git a/src/emqx_session.erl b/src/emqx_session.erl index e4487234c..ecd5dbcda 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -335,7 +335,7 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, puback(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - on_publish_completed(Msg, Session), + on_delivery_completed(Msg, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), return_with(Msg, dequeue(ClientInfo, Session#session{inflight = Inflight1})); {value, _Other} -> @@ -393,7 +393,7 @@ pubrel(PacketId, Session = #session{awaiting_rel = AwaitingRel}) -> pubcomp(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Pubrel, Msg}} when is_record(Pubrel, pubrel_await) -> - on_publish_completed(Msg, Session), + on_delivery_completed(Msg, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), dequeue(ClientInfo, Session#session{inflight = Inflight1}); {value, _Other} -> @@ -460,7 +460,7 @@ do_deliver(ClientInfo, [Msg | More], Acc, Session) -> end. deliver_msg(_ClientInfo, Msg = #message{qos = ?QOS_0}, Session) -> - on_publish_completed(Msg, Session), + on_delivery_completed(Msg, Session), {ok, [{undefined, maybe_ack(Msg)}], Session}; deliver_msg(ClientInfo, Msg = #message{qos = QoS}, Session = @@ -712,9 +712,9 @@ next_pkt_id(Session = #session{next_pkt_id = Id}) -> %%-------------------------------------------------------------------- %% Message Latency Stats %%-------------------------------------------------------------------- -on_publish_completed(Msg, - #session{clientid = ClientId, created_at = CreateAt}) -> - emqx:run_hook('message.publish_completed', +on_delivery_completed(Msg, + #session{clientid = ClientId, created_at = CreateAt}) -> + emqx:run_hook('delivery.completed', [Msg, #{ session_birth_time => CreateAt , clientid => ClientId }]). From 6414f7e55a6ceb10f246250b380372aacadbc70b Mon Sep 17 00:00:00 2001 From: lafirest Date: Tue, 18 Jan 2022 10:19:23 +0800 Subject: [PATCH 153/363] fix(emqx_slow_subs): add compatibility for old code --- src/emqx_session.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index ecd5dbcda..7d903d77d 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -650,7 +650,7 @@ resume(ClientInfo = #{clientid := ClientId}, Session = #session{subscriptions = -spec(replay(emqx_types:clientinfo(), session()) -> {ok, replies(), session()}). replay(ClientInfo, Session = #session{inflight = Inflight}) -> - Pubs = lists:map(fun({PacketId, {Pubrel, _Ts}}) when is_record(Pubrel, pubrel_await) -> + Pubs = lists:map(fun({PacketId, {Pubrel, _Msg}}) when is_record(Pubrel, pubrel_await) -> {pubrel, PacketId}; ({PacketId, {Msg, _Ts}}) -> {PacketId, emqx_message:set_flag(dup, true, Msg)} @@ -713,11 +713,16 @@ next_pkt_id(Session = #session{next_pkt_id = Id}) -> %% Message Latency Stats %%-------------------------------------------------------------------- on_delivery_completed(Msg, - #session{clientid = ClientId, created_at = CreateAt}) -> + #session{clientid = ClientId, + created_at = CreateAt}) when is_record(Msg, message) -> emqx:run_hook('delivery.completed', [Msg, #{ session_birth_time => CreateAt , clientid => ClientId - }]). + }]); + +%% in 4.4.0, timestamp are stored in pubrel_await, not message +on_delivery_completed(_Ts, _Session) -> + ok. mark_begin_deliver(Msg) -> emqx_message:set_header(deliver_begin_at, erlang:system_time(second), Msg). From 50606a7eab3e50656e2dc4c260ce9f89d09aa6dd Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 19 Jan 2022 11:55:39 +0800 Subject: [PATCH 154/363] fix(data_import): support v4.4 --- apps/emqx_management/src/emqx_mgmt_data_backup.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 7cf72886d..07bd3d27c 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -733,6 +733,8 @@ is_version_supported2("4.1") -> true; is_version_supported2("4.3") -> true; +is_version_supported2("4.4") -> + true; is_version_supported2(Version) -> case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of match -> From 46f86204c00665de8dd6bd7ceeb14e78c1538312 Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 19 Jan 2022 14:02:30 +0800 Subject: [PATCH 155/363] fix(emqx_slow_subs): add ClientInfo into the args of the delivery.completed hook --- .../src/emqx_slow_subs/emqx_slow_subs.erl | 28 ++++++++++--------- .../src/emqx_slow_subs/emqx_slow_subs_api.erl | 4 +-- src/emqx_session.erl | 24 ++++++++-------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index 50769a8fc..e8d18fca7 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -24,7 +24,7 @@ -logger_header("[SLOW Subs]"). --export([ start_link/1, on_delivery_completed/3, enable/0 +-export([ start_link/1, on_delivery_completed/4, enable/0 , disable/0, clear_history/0, init_tab/0 ]). @@ -52,7 +52,7 @@ | internal %% timespan from message in to deliver | response. %% timespan from delivery to client response --type stats_update_args() :: #{ clientid := emqx_types:clientid()}. +-type stats_update_args() :: #{session_birth_time := pos_integer()}. -type stats_update_env() :: #{ threshold := non_neg_integer() , stats_type := stats_type() @@ -81,18 +81,20 @@ start_link(Env) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). -on_delivery_completed(#message{timestamp = Ts}, #{session_birth_time := BirthTime}, _Cfg) +on_delivery_completed(_ClientInfo, #message{timestamp = Ts}, #{session_birth_time := BirthTime}, _Cfg) when Ts =< BirthTime -> ok; -on_delivery_completed(Msg, Env, Cfg) -> - on_delivery_completed(Msg, Env, erlang:system_time(millisecond), Cfg). +on_delivery_completed(ClientInfo, Msg, Env, Cfg) -> + on_delivery_completed(ClientInfo, Msg, Env, erlang:system_time(millisecond), Cfg). -on_delivery_completed(#message{topic = Topic} = Msg, - #{clientid := ClientId}, - Now, #{threshold := Threshold, - stats_type := StatsType, - max_size := MaxSize}) -> +on_delivery_completed(#{clientid := ClientId}, + #message{topic = Topic} = Msg, + _Env, + Now, + #{threshold := Threshold, + stats_type := StatsType, + max_size := MaxSize}) -> TimeSpan = calc_timespan(StatsType, Msg, Now), case TimeSpan =< Threshold of true -> ok; @@ -191,7 +193,7 @@ load(Cfg) -> StatsType = get_value(stats_type, Cfg), Threshold = get_value(threshold, Cfg), _ = emqx:hook('delivery.completed', - fun ?MODULE:on_delivery_completed/3, + fun ?MODULE:on_delivery_completed/4, [#{max_size => MaxSize, stats_type => StatsType, threshold => Threshold @@ -199,7 +201,7 @@ load(Cfg) -> ok. unload() -> - emqx:unhook('delivery.completed', fun ?MODULE:on_delivery_completed/3 ), + emqx:unhook('delivery.completed', fun ?MODULE:on_delivery_completed/4 ), do_clear_history(). do_clear(Cfg, Logs) -> @@ -266,7 +268,7 @@ find_last_update_value(Id) -> 0 end. --spec update_topk(non_neg_integer(), non_neg_integer(), non_neg_integer(), integer()) -> true. +-spec update_topk(pos_integer(), non_neg_integer(), non_neg_integer(), id()) -> true. update_topk(Now, LastUpdateValue, TimeSpan, Id) -> %% update record ets:insert(?TOPK_TAB, #top_k{index = ?TOPK_INDEX(TimeSpan, Id), diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl index 23d39273d..26b70cb71 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl @@ -93,6 +93,6 @@ rpc_call(Node, M, F, A, _ErrorR, _T) when Node =:= node() -> rpc_call(Node, M, F, A, ErrorR, T) -> case rpc:call(Node, M, F, A, T) of - {badrpc, _} -> ErrorR; - Res -> Res + {badrpc, _} -> ErrorR; + Res -> Res end. diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 7d903d77d..2e2a4dafe 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -335,7 +335,7 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, puback(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - on_delivery_completed(Msg, Session), + on_delivery_completed(ClientInfo, Msg, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), return_with(Msg, dequeue(ClientInfo, Session#session{inflight = Inflight1})); {value, _Other} -> @@ -393,7 +393,7 @@ pubrel(PacketId, Session = #session{awaiting_rel = AwaitingRel}) -> pubcomp(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Pubrel, Msg}} when is_record(Pubrel, pubrel_await) -> - on_delivery_completed(Msg, Session), + on_delivery_completed(ClientInfo, Msg, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), dequeue(ClientInfo, Session#session{inflight = Inflight1}); {value, _Other} -> @@ -459,8 +459,8 @@ do_deliver(ClientInfo, [Msg | More], Acc, Session) -> do_deliver(ClientInfo, More, [Publish | Acc], Session1) end. -deliver_msg(_ClientInfo, Msg = #message{qos = ?QOS_0}, Session) -> - on_delivery_completed(Msg, Session), +deliver_msg(ClientInfo, Msg = #message{qos = ?QOS_0}, Session) -> + on_delivery_completed(ClientInfo, Msg, Session), {ok, [{undefined, maybe_ack(Msg)}], Session}; deliver_msg(ClientInfo, Msg = #message{qos = QoS}, Session = @@ -712,16 +712,16 @@ next_pkt_id(Session = #session{next_pkt_id = Id}) -> %%-------------------------------------------------------------------- %% Message Latency Stats %%-------------------------------------------------------------------- -on_delivery_completed(Msg, - #session{clientid = ClientId, - created_at = CreateAt}) when is_record(Msg, message) -> +on_delivery_completed(ClientInfo, + Msg, + #session{created_at = CreateAt}) when is_record(Msg, message) -> emqx:run_hook('delivery.completed', - [Msg, #{ session_birth_time => CreateAt - , clientid => ClientId - }]); + [ClientInfo, Msg, #{session_birth_time => CreateAt}]); -%% in 4.4.0, timestamp are stored in pubrel_await, not message -on_delivery_completed(_Ts, _Session) -> +%% Hot upgrade compatibility clause. +%% In the 4.4.0, timestamp are stored in pubrel_await, not a message record. +%% This clause should be kept in all 4.4.x versions. +on_delivery_completed(_ClientInfo, _Ts, _Session) -> ok. mark_begin_deliver(Msg) -> From 14636a745e02eeb0a06540135409578deb5959c1 Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 19 Jan 2022 15:47:13 +0800 Subject: [PATCH 156/363] fix(emqx_slow_subs_api): return error when the module is not enable --- .../src/emqx_slow_subs/emqx_slow_subs_api.erl | 62 ++++++++++++------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl index 26b70cb71..9c150a9f1 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl @@ -50,27 +50,7 @@ clear_history(_Bindings, _Params) -> return(ok). get_history(_Bindings, _Params) -> - Nodes = ekka_mnesia:running_nodes(), - Fun = fun(Node, Acc) -> - NodeRankL = rpc_call(Node, - ?MODULE, - ?FUNCTION_NAME, - [], - [], - ?DEFAULT_RPC_TIMEOUT), - NodeRankL ++ Acc - end, - - RankL = lists:foldl(Fun, [], Nodes), - - SortFun = fun(#{timespan := A}, #{timespan := B}) -> - A > B - end, - - SortedL = lists:sort(SortFun, RankL), - SortedL2 = lists:sublist(SortedL, ?MAX_SIZE), - - return({ok, SortedL2}). + execute_when_enabled(fun do_get_history/0). get_history() -> Node = node(), @@ -84,10 +64,36 @@ get_history() -> , timespan => TimeSpan , last_update_time => LastUpdateTime } - end, + end, lists:map(ConvFun, RankL). +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +do_get_history() -> + Nodes = ekka_mnesia:running_nodes(), + Fun = fun(Node, Acc) -> + NodeRankL = rpc_call(Node, + ?MODULE, + get_history, + [], + [], + ?DEFAULT_RPC_TIMEOUT), + NodeRankL ++ Acc + end, + + RankL = lists:foldl(Fun, [], Nodes), + + SortFun = fun(#{timespan := A}, #{timespan := B}) -> + A > B + end, + + SortedL = lists:sort(SortFun, RankL), + SortedL2 = lists:sublist(SortedL, ?MAX_SIZE), + + return({ok, SortedL2}). + rpc_call(Node, M, F, A, _ErrorR, _T) when Node =:= node() -> erlang:apply(M, F, A); @@ -96,3 +102,15 @@ rpc_call(Node, M, F, A, ErrorR, T) -> {badrpc, _} -> ErrorR; Res -> Res end. + +-ifdef(EMQX_ENTERPRISE). +execute_when_enabled(Fun) -> + Fun(). +-else. +%% this code from emqx_mod_api_topics_metrics:execute_when_enabled +execute_when_enabled(Fun) -> + case emqx_modules:find_module(emqx_mod_slow_subs) of + [{_, true}] -> Fun(); + _ -> return({error, module_not_loaded}) + end. +-endif. From d02529b1d5ea7f489578f20e531ba4a4bd7f110e Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 19 Jan 2022 16:56:31 +0800 Subject: [PATCH 157/363] fix(emqx_slow_subs_api): fix test case error --- lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl index 36ed45f01..52694c3d9 100644 --- a/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl @@ -39,6 +39,8 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> + ok = meck:new([emqx_modules], [passthrough, no_history, no_link]), + ok = meck:expect(emqx_modules, find_module, fun(_) -> [{true, true}] end), emqx_ct_helpers:boot_modules(all), application:load(emqx_plugin_libs), emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_dashboard]), @@ -46,6 +48,7 @@ init_per_suite(Config) -> end_per_suite(Config) -> emqx_ct_helpers:stop_apps([emqx_management]), + ok = meck:unload(emqx_modules), Config. init_per_testcase(_, Config) -> From 461f856239e6087061a0b28657f0df2db0d159ff Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 20 Jan 2022 14:10:33 +0800 Subject: [PATCH 158/363] fix(plugin-lib): bump vsn to 4.4.1 --- .../src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_plugin_libs.appup.src | 21 ++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 7b7dcbc07..67337af21 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.4.0"}, + {vsn, "4.4.1"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index 9cd66269c..64e2b7d24 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -1,16 +1,13 @@ -%% -*-: erlang -*- - +%% -*- mode: erlang -*- {VSN, - [ - {<<"4.3.0">>, [ - {load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []} + [{"4.4.0", + [ {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} + , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} ]}, - {<<".*">>, []} - ], - [ - {<<"4.3.0">>, [ - {load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []} + {<<".*">>,[]}], + [{"4.4.0", + [ {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} + , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} ]}, - {<<".*">>, []} - ] + {<<".*">>,[]}] }. From 5ec3b6aef7a4e9e9c72f690c96cb32438d051700 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 21 Jan 2022 16:25:06 +0800 Subject: [PATCH 159/363] fix(emqx_slow_subs): add default for stats_type --- apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index e8d18fca7..c2bbc0e6d 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -46,7 +46,7 @@ -type message() :: #message{}. --import(proplists, [get_value/2]). +-import(proplists, [get_value/2, get_value/3]). -type stats_type() :: whole %% whole = internal + response | internal %% timespan from message in to deliver @@ -190,7 +190,7 @@ expire_tick(_) -> load(Cfg) -> MaxSize = get_value(top_k_num, Cfg), - StatsType = get_value(stats_type, Cfg), + StatsType = get_value(stats_type, Cfg, whole), Threshold = get_value(threshold, Cfg), _ = emqx:hook('delivery.completed', fun ?MODULE:on_delivery_completed/4, From 954e85bb73ab625bd107ced0524f6bbb3834a408 Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 24 Jan 2022 17:03:37 +0800 Subject: [PATCH 160/363] fix(emqx_slow_subs): fix qos2 pattern matching error --- src/emqx_session.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 2e2a4dafe..5741b2753 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -392,7 +392,7 @@ pubrel(PacketId, Session = #session{awaiting_rel = AwaitingRel}) -> | {error, emqx_types:reason_code()}). pubcomp(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of - {value, {Pubrel, Msg}} when is_record(Pubrel, pubrel_await) -> + {value, {#pubrel_await{message = Msg}, _Ts}} -> on_delivery_completed(ClientInfo, Msg, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), dequeue(ClientInfo, Session#session{inflight = Inflight1}); From d422e6e7003f82cdb7a28e002e5062679741fcab Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 24 Jan 2022 18:04:20 +0800 Subject: [PATCH 161/363] fix(build): relup for otp24 failed The defination of `#application{}` record in systools.hrl is changed in OTP 24. So we need a rebar3 binary compiled with OTP 24. --- Makefile | 2 +- scripts/ensure-rebar3.sh | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 61bd06ba2..1eb5a05e8 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ $(shell $(CURDIR)/scripts/git-hooks-init.sh) -REBAR_VERSION = 3.14.3-emqx-8 +REBAR_VERSION = 3.18.0-emqx-1 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index e19af1283..71a7f39ea 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -3,6 +3,7 @@ set -euo pipefail VERSION="$1" +REBAR3_FILENAME="${REBAR3_FILENAME:-rebar3}" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." @@ -10,7 +11,11 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." DOWNLOAD_URL='https://github.com/emqx/rebar3/releases/download' download() { - curl -f -L "${DOWNLOAD_URL}/${VERSION}/rebar3" -o ./rebar3 + curl -f -L "${DOWNLOAD_URL}/${VERSION}/${REBAR3_FILENAME}" -o ./rebar3 +} + +version_gt() { + test "$(echo "$@" | tr " " "n" | sort -V | head -n 1)" != "$1"; } # get the version number from the second line of the escript @@ -24,5 +29,10 @@ if [ -f 'rebar3' ] && [ "$(version)" = "$VERSION" ]; then exit 0 fi +if version_gt "${OTP_VSN}" "24.0.0"; then + echo "$OTP_VSN is greater than 24.0.0" + REBAR3_FILENAME="rebar3_otp24.1.5" +fi + download chmod +x ./rebar3 From 1f718fbb94a97f657c3dae9da54994bb457b148a Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 24 Jan 2022 18:16:29 +0800 Subject: [PATCH 162/363] fix(emqx_slow_subs): fix index update error --- apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index c2bbc0e6d..f276d7ba0 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -262,7 +262,7 @@ try_insert_to_topk(MaxSize, Now, LastUpdateValue, TimeSpan, Id) -> -spec find_last_update_value(id()) -> non_neg_integer(). find_last_update_value(Id) -> case ets:next(?INDEX_TAB, ?INDEX(0, Id)) of - #index_tab{index = ?INDEX(LastUpdateValue, Id)} -> + ?INDEX(LastUpdateValue, Id) -> LastUpdateValue; _ -> 0 From 362147924d6351d7fa6cc26d3eb277a0e667a503 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 25 Jan 2022 11:02:39 +0800 Subject: [PATCH 163/363] fix(build): only use rebar3 3.18.0-emqx-1 for OTP 24 --- Makefile | 3 +-- scripts/ensure-rebar3.sh | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 1eb5a05e8..087262459 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ $(shell $(CURDIR)/scripts/git-hooks-init.sh) -REBAR_VERSION = 3.18.0-emqx-1 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts @@ -31,7 +30,7 @@ all: $(REBAR) $(PROFILES) .PHONY: ensure-rebar3 ensure-rebar3: @$(SCRIPTS)/fail-on-old-otp-version.escript - @$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION) + @$(SCRIPTS)/ensure-rebar3.sh $(REBAR): ensure-rebar3 diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index 71a7f39ea..414e9113e 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -2,8 +2,7 @@ set -euo pipefail -VERSION="$1" -REBAR3_FILENAME="${REBAR3_FILENAME:-rebar3}" +VERSION="3.14.3-emqx-8" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." @@ -11,11 +10,12 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." DOWNLOAD_URL='https://github.com/emqx/rebar3/releases/download' download() { - curl -f -L "${DOWNLOAD_URL}/${VERSION}/${REBAR3_FILENAME}" -o ./rebar3 + echo "downloading rebar3 ${VERSION}" + curl -f -L "${DOWNLOAD_URL}/${VERSION}/rebar3" -o ./rebar3 } -version_gt() { - test "$(echo "$@" | tr " " "n" | sort -V | head -n 1)" != "$1"; +version_gte() { + test "$(printf '%s\n' "$1" "$2" | sort -V | head -n 1)" = "$2" } # get the version number from the second line of the escript @@ -25,14 +25,16 @@ version() { head -n 2 ./rebar3 | tail -n 1 | tr ' ' '\n' | grep -E '^.+-emqx-.+' } +if version_gte "${OTP_VSN}" "24.0"; then + ## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5. + ## we have to use an otp24-compiled rebar3 because the defination of record #application{} + ## in systools.hrl is changed in otp24. + VERSION="3.18.0-emqx-1" +fi + if [ -f 'rebar3' ] && [ "$(version)" = "$VERSION" ]; then exit 0 fi -if version_gt "${OTP_VSN}" "24.0.0"; then - echo "$OTP_VSN is greater than 24.0.0" - REBAR3_FILENAME="rebar3_otp24.1.5" -fi - download chmod +x ./rebar3 From cc7770be4587dbcada70d0d500665ba47d71d747 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 25 Jan 2022 14:05:53 +0800 Subject: [PATCH 164/363] chore(dashboard): update dashboard version for v4.4.0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 087262459..9df141302 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X -export EMQX_CE_DASHBOARD_VERSION ?= v4.3.3 +export EMQX_CE_DASHBOARD_VERSION ?= v4.4.0-beta.1 export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) From 669552bd583d01e0dd8e031af2e2559cdad55006 Mon Sep 17 00:00:00 2001 From: lafirest Date: Tue, 25 Jan 2022 17:07:05 +0800 Subject: [PATCH 165/363] fix(emqx_limiter): update esockd version --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 937464813..9c2b72881 100644 --- a/rebar.config +++ b/rebar.config @@ -42,7 +42,7 @@ , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.4"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.7"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} From 2eae1088acd14639c891a9d54b1cd54cf58949b4 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 25 Jan 2022 16:44:09 +0800 Subject: [PATCH 166/363] fix(metrics): client metrics key name --- src/emqx_connection.erl | 2 +- src/emqx_session.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index ccf4430a9..7938ada74 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -122,7 +122,7 @@ , 'recv_msg.qos1' , 'recv_msg.qos2' , 'recv_msg.dropped' - , 'recv_msg.dropped.expired' + , 'recv_msg.dropped.await_pubrel_timeout' , send_pkt , send_msg , 'send_msg.qos0' diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 5741b2753..bb5e84ac4 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -687,7 +687,7 @@ inc_delivery_expired_cnt(N) -> inc_await_pubrel_timeout(N) -> ok = inc_pd('recv_msg.dropped', N), - ok = inc_pd('recv_msg.dropped.expired', N), + ok = inc_pd('recv_msg.dropped.await_pubrel_timeout', N), ok = emqx_metrics:inc('messages.dropped', N), emqx_metrics:inc('messages.dropped.await_pubrel_timeout', N). From 43e528b2ba0caccd545d69b91b37dfde606e805e Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 25 Jan 2022 17:02:06 +0800 Subject: [PATCH 167/363] fix(cli): emqx_auth_mnesia_cli function spec --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl index d206f3e07..b05c3c155 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl @@ -58,7 +58,7 @@ insert_user(User = #emqx_user{login = Login}) -> [_|_] -> mnesia:abort(existed) end. --spec(add_default_user(clientid | username, tuple(), binary()) -> ok | {error, any()}). +-spec(add_default_user(clientid | username, binary(), binary()) -> ok | {error, any()}). add_default_user(Type, Key, Password) -> Login = {Type, Key}, case add_user(Login, Password) of From 301d6bf794b483dd9228b34a850b52eff2477487 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 26 Jan 2022 18:55:17 +0800 Subject: [PATCH 168/363] fix(ci): add some debug print for downloading rebar3 --- .github/workflows/build_packages.yaml | 2 ++ scripts/ensure-rebar3.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index c4d6b005b..24e173511 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -113,6 +113,8 @@ jobs: make ensure-rebar3 copy rebar3 "${{ steps.install_erlang.outputs.erlpath }}\bin" ls "${{ steps.install_erlang.outputs.erlpath }}\bin" + head -2 rebar3 + which rebar3 rebar3 --help make ${{ matrix.profile }} mkdir -p _packages/${{ matrix.profile }} diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index 414e9113e..89058a298 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -25,6 +25,7 @@ version() { head -n 2 ./rebar3 | tail -n 1 | tr ' ' '\n' | grep -E '^.+-emqx-.+' } +echo "OTP_VSN: ${OTP_VSN}" if version_gte "${OTP_VSN}" "24.0"; then ## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5. ## we have to use an otp24-compiled rebar3 because the defination of record #application{} @@ -33,6 +34,7 @@ if version_gte "${OTP_VSN}" "24.0"; then fi if [ -f 'rebar3' ] && [ "$(version)" = "$VERSION" ]; then + echo "rebar3 ${VERSION} already exists" exit 0 fi From 6319c3402f401279b53760205341fc58ffc9ebf9 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 27 Jan 2022 18:01:13 +0800 Subject: [PATCH 169/363] fix(appup): Multiply defined module: emqx_metrics make emqx-zip failed: ``` ===> Error generating relup: Multiply defined module: emqx_metrics ``` --- src/emqx.appup.src | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index e23d72fb2..fa243f7a0 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -6,7 +6,6 @@ , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} , {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} , {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} @@ -20,7 +19,6 @@ , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} , {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} , {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} From d15fd95ad4a0a46ced95fe8ea53b7d82f8ecceec Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 28 Jan 2022 11:54:10 +0800 Subject: [PATCH 170/363] fix(appup): Multiply defined module: emqx_session --- src/emqx.appup.src | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 4643412a0..fc6ccd606 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -6,7 +6,6 @@ , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} , {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} , {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} @@ -25,7 +24,6 @@ , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} , {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} , {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} From 0500c475cf3e0c0f6b631f354ca6d73e664b6d3f Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 28 Jan 2022 12:22:29 +0800 Subject: [PATCH 171/363] fix(appup): Multiply defined module: emqx_slow_subs --- src/emqx.appup.src | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index fc6ccd606..56718e59e 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -7,8 +7,6 @@ , {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_connection,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} - , {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} - , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} , {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_os_mon,brutal_purge,soft_purge,[]} @@ -25,8 +23,6 @@ , {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_connection,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} - , {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} - , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} , {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_os_mon,brutal_purge,soft_purge,[]} From 31a68f4627082033dcea8c1f2d9f9fb82fa18c0d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 28 Jan 2022 14:33:18 +0800 Subject: [PATCH 172/363] fix(appup): No such module: emqx_mod_sup --- src/emqx.appup.src | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 56718e59e..66194a6bb 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -7,7 +7,6 @@ , {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_connection,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} - , {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_os_mon,brutal_purge,soft_purge,[]} , {load_module,emqx,brutal_purge,soft_purge,[]} @@ -23,7 +22,6 @@ , {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_connection,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} - , {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_os_mon,brutal_purge,soft_purge,[]} , {load_module,emqx,brutal_purge,soft_purge,[]} From 1d7a1fde8a146713940f1dd1e3bde7cd5e978190 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 28 Jan 2022 15:43:32 +0800 Subject: [PATCH 173/363] fix(modules): update the appup.src --- lib-ce/emqx_modules/src/emqx_modules.appup.src | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index 97d1fe617..99cb389c8 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,11 +1,13 @@ %% -*-: erlang -*- {VSN, [{"4.4.0", - [{load_module, emqx_mod_presence, brutal_purge, soft_purge, []}]}, + [{load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, + {load_module, emqx_mod_sup, brutal_purge, soft_purge, []}]}, {<<".*">>, []} ], [{"4.4.0", - [{load_module, emqx_mod_presence, brutal_purge, soft_purge, []}]}, + [{load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, + {load_module, emqx_mod_sup, brutal_purge, soft_purge, []}]}, {<<".*">>, []} ] }. From a4f873f7cd596b048bf4287a7551f63cfae30cd9 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 28 Jan 2022 17:07:16 +0800 Subject: [PATCH 174/363] fix(CI): build packages failed on release upgrade testing --- .ci/build_packages/tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 76fbe394b..e7b28057d 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -187,7 +187,7 @@ relup_test(){ fi ./emqx/bin/emqx_ctl status ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases + cp "${PACKAGE_PATH}/${EMQX_NAME}-${TARGET_VERSION}-*-${ARCH}".zip ./emqx/releases ./emqx/bin/emqx install "${TARGET_VERSION}" [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 ./emqx/bin/emqx_ctl status From 633c5c16de95d53a41042623549f7a94fdecf70b Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 28 Jan 2022 17:59:13 +0800 Subject: [PATCH 175/363] fix(CI): build packages failed on release upgrade testing - again --- .ci/build_packages/tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index e7b28057d..36cf801b0 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -187,7 +187,7 @@ relup_test(){ fi ./emqx/bin/emqx_ctl status ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${EMQX_NAME}-${TARGET_VERSION}-*-${ARCH}".zip ./emqx/releases + cp "${PACKAGE_PATH}/${EMQX_NAME}-${TARGET_VERSION}"-*-"${ARCH}".zip ./emqx/releases ./emqx/bin/emqx install "${TARGET_VERSION}" [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 ./emqx/bin/emqx_ctl status From ed98773ca7d5bdc31c53d9d169215eb3c1c82c78 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Wed, 2 Feb 2022 12:42:26 +0100 Subject: [PATCH 176/363] fix(emqx_schema): Allow to set gen_rpc.default_client_driver --- priv/emqx.schema | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/priv/emqx.schema b/priv/emqx.schema index cc616f2e8..832136807 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -373,6 +373,11 @@ end}. , {datatype, {enum, [tcp, ssl]}} ]}. +{mapping, "rpc.default_client_driver", "gen_rpc.default_client_driver", +[ {default, tcp} +, {datatype, {enum, [tcp, ssl]}} +]}. + {mapping, "rpc.tcp_server_port", "gen_rpc.tcp_server_port", [ {default, 5369}, {datatype, integer} From 352635f22741fc588f3a985a412ac85ca499ac32 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 8 Feb 2022 10:40:16 +0800 Subject: [PATCH 177/363] fix(helm): fix deploy error --- .../charts/emqx/templates/configmap.env.yaml | 2 +- deploy/charts/emqx/templates/configmap.yaml | 59 ------------------- 2 files changed, 1 insertion(+), 60 deletions(-) delete mode 100644 deploy/charts/emqx/templates/configmap.yaml diff --git a/deploy/charts/emqx/templates/configmap.env.yaml b/deploy/charts/emqx/templates/configmap.env.yaml index 851a7496b..04a72033c 100644 --- a/deploy/charts/emqx/templates/configmap.env.yaml +++ b/deploy/charts/emqx/templates/configmap.env.yaml @@ -13,7 +13,7 @@ data: {{- range $index, $value := .Values.emqxConfig }} {{- if ne $value nil }} {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }} - {{ print "EMQX_" $key }}: {{ $value | quote }} + {{ print "EMQX_" $key }}: "{{ tpl (printf "%v" $value) $ }}" {{- end }} {{- end}} diff --git a/deploy/charts/emqx/templates/configmap.yaml b/deploy/charts/emqx/templates/configmap.yaml deleted file mode 100644 index 328df2000..000000000 --- a/deploy/charts/emqx/templates/configmap.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "emqx.fullname" . }}-env - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "emqx.name" . }} - helm.sh/chart: {{ include "emqx.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -data: - {{- range $index, $value := .Values.emqxConfig}} - {{$index}}: "{{ tpl (printf "%v" $value) $ }}" - {{- end}} - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "emqx.fullname" . }}-acl - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "emqx.name" . }} - helm.sh/chart: {{ include "emqx.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -data: - "acl.conf": | - {{ .Values.emqxAclConfig }} - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "emqx.fullname" . }}-loaded-plugins - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "emqx.name" . }} - helm.sh/chart: {{ include "emqx.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -data: - "loaded_plugins": | - {{ .Values.emqxLoadedPlugins }} - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "emqx.fullname" . }}-loaded-modules - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "emqx.name" . }} - helm.sh/chart: {{ include "emqx.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -data: - "loaded_modules": | - {{ .Values.emqxLoadedModules }} From cc56ad272fd0842fb3c4782abb09c8f65ab3b957 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 8 Feb 2022 14:06:39 +0800 Subject: [PATCH 178/363] fix(helm): remove the default environment variables from the template --- deploy/charts/emqx/templates/StatefulSet.yaml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 6a333bbe7..e155d71e2 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -157,18 +157,8 @@ spec: {{- if .Values.extraEnvFrom }} {{ toYaml .Values.extraEnvFrom | indent 10 }} {{- end }} - env: - - name: EMQX_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__K8S__APP_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__DISCOVERY - value: k8s - - name: EMQX_CLUSTER__K8S__SERVICE_NAME - value: {{ include "emqx.fullname" . }}-headless - - name: EMQX_CLUSTER__K8S__NAMESPACE - value: {{ .Release.Namespace }} {{- if .Values.extraEnv }} + env: {{ toYaml .Values.extraEnv | indent 10 }} {{- end }} resources: From bcb15f2a95ff358375c186bb6c5743f511fcf62d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 7 Feb 2022 15:14:24 +0800 Subject: [PATCH 179/363] fix(trace): don't return 500 when emqx_mod_trace not started. --- .../emqx_modules/src/emqx_mod_trace_api.erl | 29 +++++++++++++++---- .../emqx_modules/src/emqx_modules.appup.src | 6 ++-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl index 0b2963af6..5cc3f07a8 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -70,20 +70,36 @@ func => stream_log_file, descr => "download trace's log"}). +-define(NOT_STARTED, {error, module_not_loaded}). list_trace(Path, Params) -> - return(emqx_trace_api:list_trace(Path, Params)). + case is_started() of + true -> return(emqx_trace_api:list_trace(Path, Params)); + false -> return(?NOT_STARTED) + end. create_trace(Path, Params) -> - return(emqx_trace_api:create_trace(Path, Params)). + case is_started() of + true -> return(emqx_trace_api:create_trace(Path, Params)); + false -> return(?NOT_STARTED) + end. delete_trace(Path, Params) -> - return(emqx_trace_api:delete_trace(Path, Params)). + case is_started() of + true -> return(emqx_trace_api:delete_trace(Path, Params)); + false -> return(?NOT_STARTED) + end. clear_traces(Path, Params) -> - return(emqx_trace_api:clear_traces(Path, Params)). + case is_started() of + true -> return(emqx_trace_api:clear_traces(Path, Params)); + false -> return(?NOT_STARTED) + end. disable_trace(#{name := Name}, Params) -> - return(emqx_trace_api:update_trace(#{name => Name, operation => disable}, Params)). + case is_started() of + true -> return(emqx_trace_api:update_trace(#{name => Name, operation => disable}, Params)); + false -> return(?NOT_STARTED) + end. download_zip_log(Path, Params) -> case emqx_trace_api:download_zip_log(Path, Params) of @@ -96,3 +112,6 @@ stream_log_file(Path, Params) -> {ok, File} -> return({ok, File}); {error, Reason} -> return({error, 'NOT_FOUND', Reason}) end. + +is_started() -> + undefined =/= erlang:whereis(emqx_trace). diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index 99cb389c8..45a9194ad 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -2,12 +2,14 @@ {VSN, [{"4.4.0", [{load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_sup, brutal_purge, soft_purge, []}]}, + {load_module, emqx_mod_sup, brutal_purge, soft_purge, []}, + {load_module, emqx_mod_trace_api, brutal_purge, soft_purge, []}]}, {<<".*">>, []} ], [{"4.4.0", [{load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_sup, brutal_purge, soft_purge, []}]}, + {load_module, emqx_mod_sup, brutal_purge, soft_purge, []}, + {load_module, emqx_mod_trace_api, brutal_purge, soft_purge, []}]}, {<<".*">>, []} ] }. From 252514bfe0558c65d8ae1d16ff9b69f804a65502 Mon Sep 17 00:00:00 2001 From: lafirest Date: Tue, 8 Feb 2022 18:28:00 +0800 Subject: [PATCH 180/363] fix(emqx_slow_subs): trap the exit message --- apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index f276d7ba0..f047b7f5b 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -133,6 +133,7 @@ init_tab() -> %%-------------------------------------------------------------------- init([Conf]) -> + erlang:process_flag(trap_exit, true), expire_tick(Conf), load(Conf), {ok, #{config => Conf, @@ -201,7 +202,7 @@ load(Cfg) -> ok. unload() -> - emqx:unhook('delivery.completed', fun ?MODULE:on_delivery_completed/4 ), + emqx:unhook('delivery.completed', fun ?MODULE:on_delivery_completed/4), do_clear_history(). do_clear(Cfg, Logs) -> From d863609f4382f7eb9667372a6eed7c8db06d43b7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Feb 2022 14:04:19 +0100 Subject: [PATCH 181/363] build: use rockylinux and 'rhel' for package names --- .github/workflows/build_packages.yaml | 17 +++++++++-------- build | 6 +++--- scripts/buildx.sh | 6 ++++-- scripts/get-distro.sh | 6 +++--- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index c5b734f56..cf5f5b902 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -240,17 +240,12 @@ jobs: - ubuntu16.04 - debian10 - debian9 - # - opensuse - - centos8 + - rockylinux8 - centos7 - - centos6 - raspbian10 - # - raspbian9 exclude: - package: pkg otp: 23.3.4.9-3 - - os: centos6 - arch: arm64 - os: raspbian9 arch: amd64 - os: raspbian10 @@ -298,12 +293,18 @@ jobs: if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then export ARCH="arm" fi - + if [ "$SYSTEM" = 'centos7' ]; then + DISTRO='rhel7' + elif [ "$SYSTEM" = 'rockylinux8' ]; then + DISTRO='rhel8' + else + DISTRO=${SYSTEM} + fi mkdir -p _upgrade_base cd _upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do - package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" + package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${DISTRO}-${ARCH}" if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256 diff --git a/build b/build index 7b5e6645e..3d5d3b265 100755 --- a/build +++ b/build @@ -150,7 +150,7 @@ make_docker() { ## Name Default Example ## --------------------------------------------------------------------- ## EMQX_BASE_IMAGE current os centos:7 -## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp24.1.5-3-centos7-amd64.zip +## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp24.1.5-3-rhel7-amd64.zip ## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag ## make_docker_testing() { @@ -159,8 +159,8 @@ make_docker_testing() { ubuntu20*) EMQX_BASE_IMAGE="ubuntu:20.04" ;; - centos8) - EMQX_BASE_IMAGE="centos:8" + rhel8) + EMQX_BASE_IMAGE="rockylinux:8" ;; *) echo "Unsupported testing base image for $SYSTEM" diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 6e3ef8160..e9ebcf6e1 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -69,9 +69,11 @@ fi cd "${SRC_DIR:-.}" +set -x PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" -OTP_VSN_SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2) -PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN_SYSTEM}-${ARCH}" +OTP_VSN="$(docker run -v $(pwd):/src --rm "$BUILDER" bash /src/scripts/get-otp-vsn.sh)" +DISTRO="$(docker run -v $(pwd):/src --rm "$BUILDER" bash /src/scripts/get-distro.sh)" +PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${DISTRO}-${ARCH}" docker info docker run --rm --privileged tonistiigi/binfmt:latest --install ${ARCH} diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index 89eafc4ee..c28698af1 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -10,9 +10,9 @@ if [ "$(uname -s)" = 'Darwin' ]; then VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" elif [ "$(uname -s)" = 'Linux' ]; then - if grep -q -i 'centos' /etc/*-release; then - DIST='centos' - VERSION_ID="$(rpm --eval '%{centos_ver}')" + if grep -q -i 'rhel' /etc/*-release; then + DIST='rhel' + VERSION_ID="$(rpm --eval '%{rhel}')" else DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" From 4f6876b6e91b5a27e58b7c2a4d93b5c2ec834f8d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Feb 2022 22:16:39 +0100 Subject: [PATCH 182/363] ci: package slim-build on rockylinux --- .github/workflows/build_slim_packages.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index df02bc625..b7b948fd6 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -25,7 +25,7 @@ jobs: - 24.1.5-3 os: - ubuntu20.04 - - centos7 + - rockylinux8 container: ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.erl_otp }}-${{ matrix.os }} @@ -53,7 +53,8 @@ jobs: - name: packages test run: | PKG_VSN="$(./pkg-vsn.sh)" - PKG_NAME="${EMQX_NAME}-${PKG_VSN}-otp${{ matrix.erl_otp }}-${{ matrix.os }}-amd64" + DISTRO="$(./scripts/get-distro.sh)" + PKG_NAME="${EMQX_NAME}-${PKG_VSN}-otp${{ matrix.erl_otp }}-${DISTRO}-amd64" export CODE_PATH="$GITHUB_WORKSPACE" .ci/build_packages/tests.sh "$PKG_NAME" zip .ci/build_packages/tests.sh "$PKG_NAME" pkg From fd75756d5ce51e00978b856ae4fa5bba8e574e54 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 9 Feb 2022 22:49:08 +0100 Subject: [PATCH 183/363] ci: update to emqx-builder 4.4-5 (rockylinux) --- .ci/build_packages/Dockerfile | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/build_packages.yaml | 8 ++++---- .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 2 +- Makefile | 2 +- deploy/docker/Dockerfile | 2 +- scripts/buildx.sh | 4 ++-- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index b1edd8409..3f842dfb8 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index bd4f5f391..396f5081d 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index cf5f5b902..503f0a15b 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -16,7 +16,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles }} @@ -324,7 +324,7 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/4.4-4:${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/4.4-5:${OTP}-${SYSTEM}" - name: create sha256 working-directory: source env: @@ -393,7 +393,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -408,7 +408,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index b7b948fd6..07b18a10a 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -27,7 +27,7 @@ jobs: - ubuntu20.04 - rockylinux8 - container: ghcr.io/emqx/emqx-builder/4.4-4:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index d09270e65..98c2e8327 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 1ecbdead1..68a06f11b 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index ca4462467..1229d8414 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -226,7 +226,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -277,7 +277,7 @@ jobs: otp: - 24.1.5-3 runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index c32c13531..16f35be6b 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 9df141302..9bb3c2db3 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ $(shell $(CURDIR)/scripts/git-hooks-init.sh) REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-alpine3.14 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-alpine3.14 export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index cb8d83309..9456706cd 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-alpine3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder diff --git a/scripts/buildx.sh b/scripts/buildx.sh index e9ebcf6e1..9e803d377 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 --arch arm64 +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-debian10 --arch arm64 set -euo pipefail @@ -20,7 +20,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-debian10" } while [ "$#" -gt 0 ]; do From fd9b787e9dd7572654c12427dc20dcb7de86e55e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 7 Feb 2022 15:14:53 +0800 Subject: [PATCH 184/363] ci(build_packages): push muilt arch image for aws ecr done #6870 --- .github/workflows/build_packages.yaml | 41 ++++++++++++++------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 503f0a15b..27a7004b1 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -351,8 +351,14 @@ jobs: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + registry: + - 'docker.io' otp: - 24.1.5-3 + include: + - profile: emqx + registry: 'public.ecr.aws' + otp: 24.1.5-3 steps: - uses: actions/download-artifact@v2 @@ -366,10 +372,24 @@ jobs: with: image: tonistiigi/binfmt:latest platforms: all + - uses: aws-actions/configure-aws-credentials@v1 + if: matrix.repository == 'public.ecr.aws' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Docker login for aws ecr + if: matrix.repository == 'public.ecr.aws' + run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + - uses: docker/login-action@v1 + if: matrix.repository == 'docker.io' + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/metadata-action@v3 id: meta with: - images: ${{ github.repository_owner }}/${{ matrix.profile }} + images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} flavor: | latest=${{ !github.event.release.prerelease }} tags: | @@ -378,11 +398,6 @@ jobs: type=match,pattern=[v|e](.*),group=1 labels: org.opencontainers.image.otp.version=${{ matrix.otp }} - - uses: docker/login-action@v1 - if: github.event_name == 'release' - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/build-push-action@v2 if: matrix.profile != 'emqx-ee' with: @@ -413,20 +428,6 @@ jobs: EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise context: source - - uses: aws-actions/configure-aws-credentials@v1 - if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: Push image to aws ecr - if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx' - run: | - version=${GITHUB_REF##*/} - docker pull emqx/emqx:${version#v} - docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v} - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - docker push public.ecr.aws/emqx/emqx:${version#v} delete-artifact: runs-on: ubuntu-20.04 From 5d856041c73ea2ec8f8c3798e8b7454705a363af Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 10 Feb 2022 11:36:31 +0100 Subject: [PATCH 185/363] docs: update CHANGES-4.4 to prepare for 4.4.0 release cut removed bug fixes because 4.4.0 is the first release of 4.4 all bug fixes are forward merged from 4.3 --- CHANGES-4.4.md | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index e26663ea7..50105a4df 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,6 +1,8 @@ # EMQ X 4.4 Changes -## 4.4-beta.1 +## v4.4.0 + +**NOTE**: v4.4.0 is in sync with: v4.3.12 ### Important changes @@ -14,6 +16,10 @@ - Support dynamic modification of MQTT Keep Alive to adapt to different energy consumption strategies. +- Support 4.3 to 4.4 rolling upgrade of clustered nodes. See upgrade document for more dtails. + +- TLS for cluster backplane (RPC) connections. See clustering document for more details. + ### Minor changes - Bumpped default boot wait time from 15 seconds to 150 seconds @@ -41,21 +47,3 @@ - HTTP client performance improvement - Add openssl-1.1 to RPM dependency - -### Bug fixes - -- Fix the issue that the client process becomes unresponsive due to the blockage of RPC calls between nodes - -- Fix the issue that the lock management process `ekka_locker` crashes after killing the suspended lock owner - -- Fix the issue that the Path parameter of WebHook action in rule engine cannot use the rule engine variable - -- Fix MongoDB authentication module cannot use Replica Set mode and other issues - -- Fix the issue of out-of-sequence message forwarding between clusters. The relevant configurable item is `rpc.tcp_client_num` - -- Fix the issue of incorrect calculation of memory usage - -- Fix MQTT bridge malfunction when remote host is unreachable (hangs the connection) - -- Fix the issue that HTTP headers may be duplicated From 0bb01210ae79e344b3608f44490822b9e3ccfba2 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 11 Feb 2022 10:50:46 +0800 Subject: [PATCH 186/363] feat(ws): more client metrics --- src/emqx_channel.erl | 30 +++++++++-- src/emqx_connection.erl | 83 ++++++++++++++++--------------- src/emqx_ws_connection.erl | 61 ++++++++++++++++++----- test/emqx_ws_connection_SUITE.erl | 5 +- 4 files changed, 121 insertions(+), 58 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 0f988eed4..a1771fd14 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -118,7 +118,32 @@ quota_timer => expire_quota_limit }). --define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). +-define(INFO_KEYS, + [ conninfo + , conn_state + , clientinfo + , session + , will_msg + ]). + +-define(CHANNEL_METRICS, + [ recv_pkt + , recv_msg + , 'recv_msg.qos0' + , 'recv_msg.qos1' + , 'recv_msg.qos2' + , 'recv_msg.dropped' + , 'recv_msg.dropped.await_pubrel_timeout' + , send_pkt + , send_msg + , 'send_msg.qos0' + , 'send_msg.qos1' + , 'send_msg.qos2' + , 'send_msg.dropped' + , 'send_msg.dropped.expired' + , 'send_msg.dropped.queue_full' + , 'send_msg.dropped.too_large' + ]). -dialyzer({no_match, [shutdown/4, ensure_timer/2, interval/2]}). @@ -181,10 +206,9 @@ get_session(#channel{session = Session}) -> set_session(Session, Channel) -> Channel#channel{session = Session}. -%% TODO: Add more stats. -spec(stats(channel()) -> emqx_types:stats()). stats(#channel{session = Session})-> - emqx_session:stats(Session). + lists:append(emqx_session:stats(Session), emqx_pd:get_counters(?CHANNEL_METRICS)). -spec(caps(channel()) -> emqx_types:caps()). caps(#channel{clientinfo = #{zone := Zone}}) -> diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 7938ada74..d419a5e6a 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -109,38 +109,6 @@ -define(ACTIVE_N, 100). --define(INFO_KEYS, [ socktype - , peername - , sockname - , sockstate - , active_n - ]). - --define(CONN_STATS, [ recv_pkt - , recv_msg - , 'recv_msg.qos0' - , 'recv_msg.qos1' - , 'recv_msg.qos2' - , 'recv_msg.dropped' - , 'recv_msg.dropped.await_pubrel_timeout' - , send_pkt - , send_msg - , 'send_msg.qos0' - , 'send_msg.qos1' - , 'send_msg.qos2' - , 'send_msg.dropped' - , 'send_msg.dropped.expired' - , 'send_msg.dropped.queue_full' - , 'send_msg.dropped.too_large' - ]). - --define(SOCK_STATS, [ recv_oct - , recv_cnt - , send_oct - , send_cnt - , send_pend - ]). - -define(ENABLED(X), (X =/= undefined)). -define(ALARM_TCP_CONGEST(Channel), @@ -148,12 +116,48 @@ [emqx_channel:info(clientid, Channel), emqx_channel:info(username, Channel)]))). --define(ALARM_CONN_INFO_KEYS, [ - socktype, sockname, peername, - clientid, username, proto_name, proto_ver, connected_at -]). --define(ALARM_SOCK_STATS_KEYS, [send_pend, recv_cnt, recv_oct, send_cnt, send_oct]). --define(ALARM_SOCK_OPTS_KEYS, [high_watermark, high_msgq_watermark, sndbuf, recbuf, buffer]). +-define(INFO_KEYS, + [ socktype + , peername + , sockname + , sockstate + , active_n + ]). + +-define(SOCK_STATS, + [ recv_oct + , recv_cnt + , send_oct + , send_cnt + , send_pend + ]). + +-define(ALARM_CONN_INFO_KEYS, + [ socktype + , sockname + , peername + , clientid + , username + , proto_name + , proto_ver + , connected_at + ]). + +-define(ALARM_SOCK_STATS_KEYS, + [ send_pend + , recv_cnt + , recv_oct + , send_cnt + , send_oct + ]). + +-define(ALARM_SOCK_OPTS_KEYS, + [ high_watermark + , high_msgq_watermark + , sndbuf + , recbuf + , buffer + ]). -dialyzer({no_match, [info/2]}). -dialyzer({nowarn_function, [ init/4 @@ -214,10 +218,9 @@ stats(#state{transport = Transport, {ok, Ss} -> Ss; {error, _} -> [] end, - ConnStats = emqx_pd:get_counters(?CONN_STATS), ChanStats = emqx_channel:stats(Channel), ProcStats = emqx_misc:proc_stats(), - lists:append([SockStats, ConnStats, ChanStats, ProcStats]). + lists:append([SockStats, ChanStats, ProcStats]). %% @doc Set TCP keepalive socket options to override system defaults. %% Idle: The number of seconds a connection needs to be idle before diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index 07d437b4b..d683ebe00 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -93,9 +93,21 @@ -type(ws_cmd() :: {active, boolean()}|close). -define(ACTIVE_N, 100). --define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]). --define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]). --define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). + +-define(INFO_KEYS, + [ socktype + , peername + , sockname + , sockstate + , active_n + ]). + +-define(SOCK_STATS, + [ recv_oct + , recv_cnt + , send_oct + , send_cnt + ]). -define(ENABLED(X), (X =/= undefined)). @@ -146,10 +158,9 @@ stats(WsPid) when is_pid(WsPid) -> call(WsPid, stats); stats(#state{channel = Channel}) -> SockStats = emqx_pd:get_counters(?SOCK_STATS), - ConnStats = emqx_pd:get_counters(?CONN_STATS), ChanStats = emqx_channel:stats(Channel), ProcStats = emqx_misc:proc_stats(), - lists:append([SockStats, ConnStats, ChanStats, ProcStats]). + lists:append([SockStats, ChanStats, ProcStats]). %% kick|discard|takeover -spec(call(pid(), Req :: term()) -> Reply :: term()). @@ -615,6 +626,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> [emqx_packet:format(Packet)]), ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), + ok = inc_outgoing_stats({error, message_too_large}), <<>>; Data -> ?LOG(debug, "SEND ~s", [emqx_packet:format(Packet)]), ok = inc_outgoing_stats(Packet), @@ -641,19 +653,28 @@ inc_recv_stats(Cnt, Oct) -> inc_incoming_stats(Packet = ?PACKET(Type)) -> _ = emqx_pd:inc_counter(recv_pkt, 1), - if Type == ?PUBLISH -> - inc_counter(recv_msg, 1), - inc_counter(incoming_pubs, 1); - true -> ok + case Type of + ?PUBLISH -> + inc_counter(recv_msg, 1), + inc_qos_stats(recv_msg, Packet), + inc_counter(incoming_pubs, 1); + _ -> + ok end, emqx_metrics:inc_recv(Packet). +inc_outgoing_stats({error, message_too_large}) -> + inc_counter('send_msg.dropped', 1), + inc_counter('send_msg.dropped.too_large', 1); inc_outgoing_stats(Packet = ?PACKET(Type)) -> _ = emqx_pd:inc_counter(send_pkt, 1), - if Type == ?PUBLISH -> - inc_counter(send_msg, 1), - inc_counter(outgoing_pubs, 1); - true -> ok + case Type of + ?PUBLISH -> + inc_counter(send_msg, 1), + inc_qos_stats(send_msg, Packet), + inc_counter(outgoing_pubs, 1); + _ -> + ok end, emqx_metrics:inc_sent(Packet). @@ -666,6 +687,20 @@ inc_sent_stats(Cnt, Oct) -> inc_counter(Name, Value) -> _ = emqx_pd:inc_counter(Name, Value), ok. + +inc_qos_stats(Type, #mqtt_packet{header = #mqtt_packet_header{qos = QoS}}) when ?IS_QOS(QoS) -> + inc_counter(inc_qos_stats_key(Type, QoS), 1); +inc_qos_stats(_, _) -> + ok. + +inc_qos_stats_key(send_msg, ?QOS_0) -> 'send_msg.qos0'; +inc_qos_stats_key(send_msg, ?QOS_1) -> 'send_msg.qos1'; +inc_qos_stats_key(send_msg, ?QOS_2) -> 'send_msg.qos2'; + +inc_qos_stats_key(recv_msg, ?QOS_0) -> 'recv_msg.qos0'; +inc_qos_stats_key(recv_msg, ?QOS_1) -> 'recv_msg.qos1'; +inc_qos_stats_key(recv_msg, ?QOS_2) -> 'recv_msg.qos2'. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- diff --git a/test/emqx_ws_connection_SUITE.erl b/test/emqx_ws_connection_SUITE.erl index 3c1aad3de..f282d2eb6 100644 --- a/test/emqx_ws_connection_SUITE.erl +++ b/test/emqx_ws_connection_SUITE.erl @@ -169,8 +169,9 @@ t_stats(_) -> end end), Stats = ?ws_conn:call(WsPid, stats), - [{recv_oct, 0}, {recv_cnt, 0}, {send_oct, 0}, {send_cnt, 0}, - {recv_pkt, 0}, {recv_msg, 0}, {send_pkt, 0}, {send_msg, 0}|_] = Stats. + [lists:member(V, Stats) || + V <- [{recv_oct, 0}, {recv_cnt, 0}, {send_oct, 0}, {send_cnt, 0}, + {recv_pkt, 0}, {recv_msg, 0}, {send_pkt, 0}, {send_msg, 0}]]. t_call(_) -> Info = ?ws_conn:info(st()), From c6bafd5131dca2f3821bb3f4e912660739f6009f Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 11 Feb 2022 11:04:14 +0800 Subject: [PATCH 187/363] feat(ws): appup --- src/emqx.appup.src | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 87b70308a..ec2ca842a 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- {VSN, [{"4.4.0", - [ {load_module,emqx_metrics,brutal_purge,soft_purge,[]} + [ {load_module,emqx_channel,brutal_purge,soft_purge,[]} + , {load_module,emqx_connection,brutal_purge,soft_purge,[]} + , {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]} + , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}} , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} , {load_module,emqx_channel,brutal_purge,soft_purge,[]} @@ -17,7 +20,10 @@ {<<".*">>,[]} ], [{"4.4.0", - [ {load_module,emqx_metrics,brutal_purge,soft_purge,[]} + [ {load_module,emqx_channel,brutal_purge,soft_purge,[]} + , {load_module,emqx_connection,brutal_purge,soft_purge,[]} + , {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]} + , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} , {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}} , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} , {load_module,emqx_channel,brutal_purge,soft_purge,[]} From e9f1af809873bab38bffc24a8ea9f961f6e59275 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 11 Feb 2022 11:16:31 +0800 Subject: [PATCH 188/363] feat(ws): appup remove old --- src/emqx.appup.src | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index ec2ca842a..80ae46894 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -7,8 +7,6 @@ , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}} , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} - , {load_module,emqx_channel,brutal_purge,soft_purge,[]} - , {load_module,emqx_connection,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_os_mon,brutal_purge,soft_purge,[]} @@ -26,8 +24,6 @@ , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} , {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}} , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} - , {load_module,emqx_channel,brutal_purge,soft_purge,[]} - , {load_module,emqx_connection,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_os_mon,brutal_purge,soft_purge,[]} From e9238c6ca3359668842e07bde44f9b4f7afbe375 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sun, 13 Feb 2022 12:51:53 +0800 Subject: [PATCH 189/363] ci: use exclude instead of include for build matrix --- .github/workflows/build_packages.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 27a7004b1..4aa0b1d9b 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -351,15 +351,18 @@ jobs: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} - registry: - - 'docker.io' otp: - 24.1.5-3 - include: - - profile: emqx + registry: + - 'docker.io' + - 'public.ecr.aws' + exclude: + # we don't have an aws ecr repo for enterprise and edge yet + - profile: emqx-edge registry: 'public.ecr.aws' - otp: 24.1.5-3 - + - profile: emqx-ee + registry: 'public.ecr.aws' + steps: - uses: actions/download-artifact@v2 with: From 7a45ad8900a170f52c5568876eae7e906e6f3992 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sun, 13 Feb 2022 20:50:35 +0800 Subject: [PATCH 190/363] chore: fix a few typos --- .github/workflows/build_packages.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 4aa0b1d9b..b439c0a57 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -376,16 +376,16 @@ jobs: image: tonistiigi/binfmt:latest platforms: all - uses: aws-actions/configure-aws-credentials@v1 - if: matrix.repository == 'public.ecr.aws' + if: matrix.registry == 'public.ecr.aws' with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - name: Docker login for aws ecr - if: matrix.repository == 'public.ecr.aws' + if: matrix.registry == 'public.ecr.aws' run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - uses: docker/login-action@v1 - if: matrix.repository == 'docker.io' + if: matrix.registry == 'docker.io' with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} From 9f897a650a9b2ba9c167412d3af92c024f66477a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 14 Feb 2022 09:28:29 +0100 Subject: [PATCH 191/363] build: rename distro from 'rhel' to 'el' --- .github/workflows/build_packages.yaml | 4 ++-- build | 4 ++-- scripts/get-distro.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 503f0a15b..5c1b01789 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -294,9 +294,9 @@ jobs: export ARCH="arm" fi if [ "$SYSTEM" = 'centos7' ]; then - DISTRO='rhel7' + DISTRO='el7' elif [ "$SYSTEM" = 'rockylinux8' ]; then - DISTRO='rhel8' + DISTRO='el8' else DISTRO=${SYSTEM} fi diff --git a/build b/build index 3d5d3b265..2aa20b7a3 100755 --- a/build +++ b/build @@ -150,7 +150,7 @@ make_docker() { ## Name Default Example ## --------------------------------------------------------------------- ## EMQX_BASE_IMAGE current os centos:7 -## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp24.1.5-3-rhel7-amd64.zip +## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp24.1.5-3-el7-amd64.zip ## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag ## make_docker_testing() { @@ -159,7 +159,7 @@ make_docker_testing() { ubuntu20*) EMQX_BASE_IMAGE="ubuntu:20.04" ;; - rhel8) + el8) EMQX_BASE_IMAGE="rockylinux:8" ;; *) diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index c28698af1..f4ac5c88c 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -11,7 +11,7 @@ if [ "$(uname -s)" = 'Darwin' ]; then SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" elif [ "$(uname -s)" = 'Linux' ]; then if grep -q -i 'rhel' /etc/*-release; then - DIST='rhel' + DIST='el' VERSION_ID="$(rpm --eval '%{rhel}')" else DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" From 22ae8985c62db03554e4687b06c06ce8bcca7c5a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 14 Feb 2022 19:32:59 +0100 Subject: [PATCH 192/363] chore: compare to rel-* versions for appup --- scripts/apps-version-check.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index ae4cd22ee..e7a7d91bf 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -1,7 +1,15 @@ #!/usr/bin/env bash set -euo pipefail -latest_release=$(git describe --abbrev=0 --tags) +## compare to the latest release version tag: +## match rel-e4.4.0, v4.4.*, or e4.4.* tags +## but do not include alpha, beta and rc versions +## +## NOTE: 'rel-' tags are the merge base of enterprise release in opensource repo. +## i.e. if we are to release a new enterprise without cutting a new opensource release +## we should tag rel-e4.4.X in the opensource repo, and merge this tag to enterprise +## then cut a release from the enterprise repo. +latest_release="$(git describe --abbrev=0 --tags --match 'rel-e4.4.*' --match '[v|e]4.4*' --exclude '*beta*' --exclude '*alpha*' --exclude '*rc*')" echo "Compare base: $latest_release" bad_app_count=0 From a42927f1732259a1a8b30ac04997844624fb51b7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 16 Feb 2022 13:46:38 +0800 Subject: [PATCH 193/363] chore(emqx): fix mutiply defined module in emqx.appup.src --- src/emqx.appup.src | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index bbd216755..de0c9e00d 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -25,8 +25,7 @@ , {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_connection,brutal_purge,soft_purge,[]} , {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} - , {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}} + , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} From e7ce4ed215db3a1d3afb8ca3757690a3b7aff11f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 16 Feb 2022 18:38:55 +0800 Subject: [PATCH 194/363] chore: force check for version upgrade of emqx_dashboard same with: https://github.com/emqx/emqx/pull/5879 --- scripts/apps-version-check.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index e7a7d91bf..e9aa89349 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -55,6 +55,15 @@ check_apps() { echo "$src_file needs a vsn bump (old=$old_app_version)" echo "changed: $lines" bad_app_count=$(( bad_app_count + 1)) + elif [ "$app" = 'emqx_dashboard' ]; then + ## emqx_dashboard is ensured to be upgraded after all other plugins + ## at the end of its appup instructions, there is the final instruction + ## {apply, {emqx_plugins, load, []} + ## since we don't know which plugins are stopped during the upgrade + ## for safty, we just force a dashboard version bump for each and every release + ## even if there is nothing changed in the app + echo "$src_file needs a vsn bump to ensure plugins loaded after upgrade" + bad_app_count=$(( bad_app_count + 1)) fi fi done < <(./scripts/find-apps.sh) From 4775ea353d484a31e9503497a3032a1036045632 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 16 Feb 2022 19:40:33 +0800 Subject: [PATCH 195/363] chore: update scripts/apps-version-check.sh Co-authored-by: k32 <10274441+k32@users.noreply.github.com> --- scripts/apps-version-check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index e9aa89349..017a48c4b 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -60,7 +60,7 @@ check_apps() { ## at the end of its appup instructions, there is the final instruction ## {apply, {emqx_plugins, load, []} ## since we don't know which plugins are stopped during the upgrade - ## for safty, we just force a dashboard version bump for each and every release + ## for safety, we just force a dashboard version bump for each and every release ## even if there is nothing changed in the app echo "$src_file needs a vsn bump to ensure plugins loaded after upgrade" bad_app_count=$(( bad_app_count + 1)) From 480130c1d8e25fd99fbab4f827529f0d6cbf5ed2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 17 Feb 2022 09:23:40 +0800 Subject: [PATCH 196/363] chore(dashboard): bump version --- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 1604198dc..1354ab3c6 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "4.4.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, From 037b9e440c547b4cc5f7e66f34aa63fc30efc1c0 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 17 Feb 2022 15:20:59 +0800 Subject: [PATCH 197/363] fix(build): change header of shell script to "#!/usr/bin/env bash" --- scripts/get-otp-vsn.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-otp-vsn.sh b/scripts/get-otp-vsn.sh index a791318dc..699a16f3f 100755 --- a/scripts/get-otp-vsn.sh +++ b/scripts/get-otp-vsn.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail From ee7acdfc337c48fa74d235d9e82ccd56664fc939 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 17 Feb 2022 14:23:44 +0800 Subject: [PATCH 198/363] ci(cross build): fix build raspbian failed on armhf --- .ci/build_packages/Dockerfile | 16 -------- .ci/build_packages/tests.sh | 45 ++++++++++++---------- .github/workflows/build_slim_packages.yaml | 7 +--- scripts/buildx.sh | 8 +--- scripts/shellcheck.sh | 5 ++- 5 files changed, 32 insertions(+), 49 deletions(-) delete mode 100644 .ci/build_packages/Dockerfile diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile deleted file mode 100644 index 3f842dfb8..000000000 --- a/.ci/build_packages/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 -FROM ${BUILD_FROM} - -ARG EMQX_NAME=emqx - -COPY . /emqx - -WORKDIR /emqx - -RUN rm -rf _build/${EMQX_NAME}/lib _build/${EMQX_NAME}-pkg/lib - -RUN make ${EMQX_NAME}-zip || cat rebar3.crashdump - -RUN make ${EMQX_NAME}-pkg || cat rebar3.crashdump - -RUN /emqx/.ci/build_packages/tests.sh diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 98139dc99..c5b6ec539 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash ## This script tests built package start/stop -## Accept 2 args PACKAGE_NAME and PACKAGE_TYPE +## Accept 2 args PROFILE and PACKAGE_TYPE set -x -e -u if [ -z "${1:-}" ]; then - echo "Usage $0 zip|pkg" + echo "Usage $0 e.g. emqx, emqx-edge" exit 1 fi @@ -15,20 +15,22 @@ if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then exit 1 fi -PACKAGE_NAME="${1}" +PROFILE="${1}" PACKAGE_TYPE="${2}" export CODE_PATH=${CODE_PATH:-"/emqx"} -export EMQX_NAME=${EMQX_NAME:-"emqx"} -export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" +export PACKAGE_PATH="${CODE_PATH}/_packages/${PROFILE}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" # export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1" # export EMQX_NODE_COOKIE=$(date +%s%N) +PKG_VSN="$($CODE_PATH/pkg-vsn.sh)" +OTP_VSN="$($CODE_PATH/scripts/get-otp-vsn.sh)" +SYSTEM="$($CODE_PATH/scripts/get-distro.sh)" + if [ "$PACKAGE_TYPE" = 'zip' ]; then PKG_SUFFIX="zip" else - SYSTEM="$($CODE_PATH/scripts/get-distro.sh)" case "${SYSTEM:-}" in ubuntu*|debian*|raspbian*) PKG_SUFFIX='deb' @@ -38,15 +40,9 @@ else ;; esac fi -PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" -PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" -if ! [ -f "$PACKAGE_FILE" ]; then - echo "$PACKAGE_FILE is not a file" - exit 1 -fi - -case "$(uname -m)" in +ARCH="$(uname -m)" +case "$ARCH" in x86_64) ARCH='amd64' ;; @@ -59,6 +55,15 @@ case "$(uname -m)" in esac export ARCH +PACKAGE_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" + +PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" +if ! [ -f "$PACKAGE_FILE" ]; then + echo "$PACKAGE_FILE is not a file" + exit 1 +fi + emqx_prepare(){ mkdir -p "${PACKAGE_PATH}" if [ ! -d "/paho-mqtt-testing" ]; then @@ -111,14 +116,14 @@ emqx_test(){ running_test echo "running ${packagename} stop" - dpkg -r "${EMQX_NAME}" + dpkg -r "${PROFILE}" if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ] then echo "package remove error" exit 1 fi - dpkg -P "${EMQX_NAME}" + dpkg -P "${PROFILE}" if dpkg -l |grep -q emqx then echo "package uninstall error" @@ -127,7 +132,7 @@ emqx_test(){ ;; "rpm") yum install -y "${PACKAGE_PATH}/${packagename}" - if ! rpm -q ${EMQX_NAME} | grep -q "${EMQX_NAME}"; then + if ! rpm -q ${PROFILE} | grep -q "${PROFILE}"; then echo "package install error" exit 1 fi @@ -136,7 +141,7 @@ emqx_test(){ running_test echo "running ${packagename} stop" - rpm -e "${EMQX_NAME}" + rpm -e "${PROFILE}" if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then echo "package uninstall error" exit 1 @@ -176,7 +181,7 @@ relup_test(){ if [ -d "${RELUP_PACKAGE_PATH}" ];then cd "${RELUP_PACKAGE_PATH}" - find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" | + find . -maxdepth 1 -name "${PROFILE}-*-${ARCH}.zip" | while read -r pkg; do packagename=$(basename "${pkg}") unzip -q "$packagename" @@ -187,7 +192,7 @@ relup_test(){ fi ./emqx/bin/emqx_ctl status ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${EMQX_NAME}-${TARGET_VERSION}"-*-"${ARCH}".zip ./emqx/releases + cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*-"${ARCH}".zip ./emqx/releases ./emqx/bin/emqx install "${TARGET_VERSION}" [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 export EMQX_WAIT_FOR_STOP=300 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 07b18a10a..685dc3f94 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -52,12 +52,9 @@ jobs: path: ./rebar3.crashdump - name: packages test run: | - PKG_VSN="$(./pkg-vsn.sh)" - DISTRO="$(./scripts/get-distro.sh)" - PKG_NAME="${EMQX_NAME}-${PKG_VSN}-otp${{ matrix.erl_otp }}-${DISTRO}-amd64" export CODE_PATH="$GITHUB_WORKSPACE" - .ci/build_packages/tests.sh "$PKG_NAME" zip - .ci/build_packages/tests.sh "$PKG_NAME" pkg + .ci/build_packages/tests.sh "${EMQX_NAME}" zip + .ci/build_packages/tests.sh "${EMQX_NAME}" pkg - uses: actions/upload-artifact@v2 with: name: ${{ matrix.os }} diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 9e803d377..fd754bad6 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -70,17 +70,11 @@ fi cd "${SRC_DIR:-.}" set -x -PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" -OTP_VSN="$(docker run -v $(pwd):/src --rm "$BUILDER" bash /src/scripts/get-otp-vsn.sh)" -DISTRO="$(docker run -v $(pwd):/src --rm "$BUILDER" bash /src/scripts/get-distro.sh)" -PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${DISTRO}-${ARCH}" - docker info docker run --rm --privileged tonistiigi/binfmt:latest --install ${ARCH} docker run -i --rm \ -v "$(pwd)":/emqx \ --workdir /emqx \ --platform="linux/$ARCH" \ - -e EMQX_NAME="$PROFILE" \ "$BUILDER" \ - bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PKG_NAME $PKGTYPE" + bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" diff --git a/scripts/shellcheck.sh b/scripts/shellcheck.sh index 5f8cdfd51..158190bd6 100755 --- a/scripts/shellcheck.sh +++ b/scripts/shellcheck.sh @@ -3,7 +3,10 @@ set -euo pipefail target_files=() -while IFS='' read -r line; do target_files+=("$line"); done < <(grep -r -l --exclude-dir=.git --exclude-dir=_build "#!/bin/" .) +while IFS='' read -r line; do + target_files+=("$line"); +done < <(grep -r -l --exclude-dir=.git --exclude-dir=_build "#!/bin/" .) + return_code=0 for i in "${target_files[@]}"; do echo checking "$i" ... From fde634ac2798628a5c23f0648e7251a7f1f1ed88 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 17 Feb 2022 17:48:58 +0800 Subject: [PATCH 199/363] chore(webhook): refine appup.src --- apps/emqx_web_hook/src/emqx_web_hook.appup.src | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index 531ad308b..7b3be1b13 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -5,20 +5,24 @@ {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[3-8]">>, + {<<"4\\.3\\.[3-7]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.8", + [{load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{<<"4\\.3\\.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[3-8]">>, + {<<"4\\.3\\.[3-7]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.8", + [{load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. From a96c1284c69d8d4a2df08fe753407c89972d5f11 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 17 Feb 2022 10:59:42 +0100 Subject: [PATCH 200/363] docs(CHANGES-4.4): add changes of pckage name format --- CHANGES-4.4.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 50105a4df..f730ccdb0 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -12,6 +12,13 @@ it is still recommended that you verify and confirm again before deploying to the production environment, at least to ensure that systemd is available in your system +- Package name scheme changed comparing to 4.3. + 4.3 format: emqx-centos8-4.3.8-amd64.zip + 4.4 format: emqx-4.4.0-rc.1-otp24.1.5-3-el8-amd64.zip + * Erlang/OTP version is included in the package name, + providing the possibility to release EMQX on multiple Erlang/OTP versions + * `centos` is renamed to `el`. This is mainly due to centos8 being dead (replaced with rockylinux8) + - MongoDB authentication supports DNS SRV and TXT Records resolution, which can seamlessly connect with MongoDB Altas - Support dynamic modification of MQTT Keep Alive to adapt to different energy consumption strategies. From af8bc8ee324a22a9f3938e9bcd30a3dca7909620 Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Thu, 17 Feb 2022 18:15:19 +0800 Subject: [PATCH 201/363] fix(emqx_rule_metrics): add a function for code hot upgrade --- apps/emqx_rule_engine/src/emqx_rule_metrics.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index 0da6b1197..0dfec930e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -340,7 +340,7 @@ code_change({down, _Vsn}, State = #state{metric_ids = MIDs}, [Vsn]) -> Exception = get_actions_exception(Id), Retry = get_actions_retry(Id), ok = delete_counters(Id), - ok = create_counters(Id, 7), + ok = create_counters(Id, max_counters_size_old()), inc_rules_matched(Id, Passed), inc_actions_taken(Id, Take), inc_actions_success(Id, Success), @@ -464,6 +464,9 @@ precision(Float, N) -> %% Metrics Definitions %%------------------------------------------------------------------------------ +%% for code hot upgrade +max_counters_size_old() -> 7. + max_counters_size() -> 11. metrics_idx('rules.matched') -> 1; From cc1734490f9e90be4989eaed2340857c74134401 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 17 Feb 2022 11:20:19 +0100 Subject: [PATCH 202/363] chore: trigger a version check before pushing a tag --- scripts/git-hook-pre-push.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/git-hook-pre-push.sh b/scripts/git-hook-pre-push.sh index 5b3f2edf2..689604092 100755 --- a/scripts/git-hook-pre-push.sh +++ b/scripts/git-hook-pre-push.sh @@ -4,9 +4,13 @@ set -euo pipefail url="$2" +## ensure enterprise code is not pushed to public repo if [ -f 'EMQX_ENTERPRISE' ]; then if [[ "$url" != *emqx-enterprise* ]]; then echo "$(tput setaf 1)error: enterprise_code_to_non_enterprise_repo" exit 1 fi fi + +## this triggers a tag vs release version check before pushing a tag +./pkg-vsn.sh From 0a66d40e39957e90b3b23207621ecbea20abce49 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 17 Feb 2022 11:21:22 +0100 Subject: [PATCH 203/363] chore: prepare for 4.4.0-rc.2 release --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 534a887c4..149683661 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4-beta.1"}). +-define(EMQX_RELEASE, {opensource, "4.4.0-rc.2"}). -else. From bfa0a741a2377d9efae1de6dd3f0ba2095e83644 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 17 Feb 2022 22:41:10 +0800 Subject: [PATCH 204/363] fix(doc): add log trace changelog for 4.4.0 --- CHANGES-4.4.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index f730ccdb0..d9957620c 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -27,6 +27,8 @@ - TLS for cluster backplane (RPC) connections. See clustering document for more details. +- Support real-time trace filtering logs for the ClientID or Topic or IP via dashboard. + ### Minor changes - Bumpped default boot wait time from 15 seconds to 150 seconds From b044eda31f32c956ea9e9622244271d3b5406fce Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 17 Feb 2022 23:10:13 +0800 Subject: [PATCH 205/363] fix(doc): Update CHANGES-4.4.md Co-authored-by: Zaiming (Stone) Shi --- CHANGES-4.4.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index d9957620c..56c77d458 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -27,7 +27,7 @@ - TLS for cluster backplane (RPC) connections. See clustering document for more details. -- Support real-time trace filtering logs for the ClientID or Topic or IP via dashboard. +- Support real-time tracing in the dashboard, with Client ID, Client IP address, and topic name based filtering. ### Minor changes From c13f32b2a5b1b02139a9463fb392fabfe310b91a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 18 Feb 2022 09:50:33 +0800 Subject: [PATCH 206/363] chore: update dashboard vsn --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9bb3c2db3..30e8b4c0a 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X -export EMQX_CE_DASHBOARD_VERSION ?= v4.4.0-beta.1 +export EMQX_CE_DASHBOARD_VERSION ?= v4.4.0 export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) From 24bd7371b70ee8809ad23c91b9543a5f7e8d0448 Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 18 Feb 2022 09:51:48 +0800 Subject: [PATCH 207/363] fix(docs): add slow subs change log --- CHANGES-4.4.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 56c77d458..17b6c607b 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -29,6 +29,8 @@ - Support real-time tracing in the dashboard, with Client ID, Client IP address, and topic name based filtering. +- Refactor the Slow subscribers statistics module, now, the feature will rank The per clientid-topic latency measurements. + ### Minor changes - Bumpped default boot wait time from 15 seconds to 150 seconds From 0ba6262d06b9214ec3d520a2ec75add452b70e94 Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 18 Feb 2022 11:10:57 +0800 Subject: [PATCH 208/363] fix(docs): update slow subs change log --- CHANGES-4.4.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 17b6c607b..6dbe1ff97 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -29,7 +29,7 @@ - Support real-time tracing in the dashboard, with Client ID, Client IP address, and topic name based filtering. -- Refactor the Slow subscribers statistics module, now, the feature will rank The per clientid-topic latency measurements. +- Add the Slow Subscriptions module to count the time spent during the message transmission. This feature will list the Clients and Topics with higher time consumption in Dashboard ### Minor changes From 8a75b5305234b9197c2c3f4073438e6de0f8cd68 Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 18 Feb 2022 11:03:32 +0800 Subject: [PATCH 209/363] fix(emqx_slow_subs): add upgrade/downgrade action --- .../src/emqx_plugin_libs.appup.src | 12 +++---- .../src/emqx_slow_subs/emqx_slow_subs.erl | 31 ++++++++++++++++++- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index 64e2b7d24..10095eed0 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -1,13 +1,13 @@ %% -*- mode: erlang -*- {VSN, [{"4.4.0", - [ {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} + [ {update, emqx_slow_subs, {advanced, ["4.4.0"]}} , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} ]}, {<<".*">>,[]}], - [{"4.4.0", - [ {load_module,emqx_slow_subs,brutal_purge,soft_purge,[]} - , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} - ]}, - {<<".*">>,[]}] + [{"4.4.0", + [ {update, emqx_slow_subs, {advanced, ["4.4.0"]}} + , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} + ]}, + {<<".*">>,[]}] }. diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index f047b7f5b..947a0e4b9 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -180,7 +180,36 @@ terminate(_Reason, _) -> unload(), ok. -code_change(_OldVsn, State, _Extra) -> +code_change({down, _Vsn}, #{config := Cfg} = State, ["4.4.0"]) -> + unload(), + + MaxSize = get_value(top_k_num, Cfg), + _ = emqx:hook('message.slow_subs_stats', + {?MODULE, on_stats_update, [#{max_size => MaxSize}]}), + + erlang:send_after(?EXPIRE_CHECK_INTERVAL, self(), ?FUNCTION_NAME), + + {ok, State}; + +code_change(_OldVsn, #{config := Conf} = State, ["4.4.0"]) -> + HookPoint = 'message.slow_subs_stats', + case emqx_hooks:lookup(HookPoint) of + [Action | _] -> + emqx_hooks:del(HookPoint, Action); + _ -> + ok + end, + try + ets:delete_all_objects(?TOPK_TAB) + catch _:_ -> + ok + end, + init_tab(), + expire_tick(Conf), + load(Conf), + {ok, State}; + +code_change(_OldVsn, State, _Extras) -> {ok, State}. %%-------------------------------------------------------------------- From 1a1415a9961a566a825724e95f7b2579848ff3fb Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 18 Feb 2022 16:27:22 +0800 Subject: [PATCH 210/363] fix(emqx_slow_subs): fix upgrade action error --- .../src/emqx_slow_subs/emqx_slow_subs.erl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl index 947a0e4b9..07cab1bd8 100644 --- a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -192,19 +192,23 @@ code_change({down, _Vsn}, #{config := Cfg} = State, ["4.4.0"]) -> {ok, State}; code_change(_OldVsn, #{config := Conf} = State, ["4.4.0"]) -> + %% clear old data HookPoint = 'message.slow_subs_stats', - case emqx_hooks:lookup(HookPoint) of - [Action | _] -> - emqx_hooks:del(HookPoint, Action); - _ -> - ok - end, + Callbacks = emqx_hooks:lookup(HookPoint), + _ = [emqx_hooks:del(HookPoint, Action) || + {callback, Action, _Filter, _Priority} <- Callbacks], try ets:delete_all_objects(?TOPK_TAB) catch _:_ -> ok end, + + %% add new table init_tab(), + [_Sup, SupPid] = erlang:get('$ancestors'), + ets:give_away(?INDEX_TAB, SupPid, undefined), + + %% enable expire_tick(Conf), load(Conf), {ok, State}; From 82285980a7221aa2fbe172c7592be0b05fa93c8f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 18 Feb 2022 19:14:30 +0800 Subject: [PATCH 211/363] chore: bump version --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 149683661..24f6feb57 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.0-rc.2"}). +-define(EMQX_RELEASE, {opensource, "4.4.0"}). -else. From 3a5a2047e91ad6adaa81119b0225e7cb323f07e4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 20 Feb 2022 09:19:56 +0100 Subject: [PATCH 212/363] build: fix windows build --- .ci/build_packages/tests.sh | 76 +++++++++------------ .gitattributes | 1 + .github/workflows/build_packages.yaml | 25 ++----- Makefile | 7 +- bin/install_upgrade.escript | 2 +- build | 96 ++++++++++++++++++--------- deploy/packages/deb/Makefile | 5 +- deploy/packages/rpm/Makefile | 4 +- scripts/ensure-rebar3.sh | 29 ++++---- scripts/find-apps.sh | 11 ++- scripts/get-distro.sh | 36 ++++++---- scripts/pkg-full-vsn.sh | 45 +++++++++++++ 12 files changed, 204 insertions(+), 133 deletions(-) create mode 100755 scripts/pkg-full-vsn.sh diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index c5b6ec539..628a798e9 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -24,8 +24,6 @@ export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" # export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1" # export EMQX_NODE_COOKIE=$(date +%s%N) -PKG_VSN="$($CODE_PATH/pkg-vsn.sh)" -OTP_VSN="$($CODE_PATH/scripts/get-otp-vsn.sh)" SYSTEM="$($CODE_PATH/scripts/get-distro.sh)" if [ "$PACKAGE_TYPE" = 'zip' ]; then @@ -41,21 +39,8 @@ else esac fi -ARCH="$(uname -m)" -case "$ARCH" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm - ;; -esac -export ARCH - -PACKAGE_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +PACKAGE_NAME="${PROFILE}-$($CODE_PATH/scripts/pkg-full-vsn.sh)" +OLD_PACKAGE_PATTERN="${PROFILE}-$($CODE_PATH/scripts/pkg-full-vsn.sh 'vsn_matcher')" PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" @@ -79,7 +64,7 @@ emqx_test(){ "zip") unzip -q "${PACKAGE_PATH}/${packagename}" export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \ - EMQX_MQTT__MAX_TOPIC_ALIAS=10 + EMQX_MQTT__MAX_TOPIC_ALIAS=10 sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins echo "running ${packagename} start" @@ -178,34 +163,33 @@ running_test(){ relup_test(){ TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh)" - if [ -d "${RELUP_PACKAGE_PATH}" ];then - cd "${RELUP_PACKAGE_PATH}" - - find . -maxdepth 1 -name "${PROFILE}-*-${ARCH}.zip" | - while read -r pkg; do - packagename=$(basename "${pkg}") - unzip -q "$packagename" - if ! ./emqx/bin/emqx start; then - cat emqx/log/erlang.log.1 || true - cat emqx/log/emqx.log.1 || true - exit 1 - fi - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*-"${ARCH}".zip ./emqx/releases - ./emqx/bin/emqx install "${TARGET_VERSION}" - [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 - export EMQX_WAIT_FOR_STOP=300 - ./emqx/bin/emqx_ctl status - if ! ./emqx/bin/emqx stop; then - cat emqx/log/erlang.log.1 || true - cat emqx/log/emqx.log.1 || true - echo "failed to stop emqx" - exit 1 - fi - rm -rf emqx - done - fi + if [ ! -d "${RELUP_PACKAGE_PATH}" ];then + return 0 + fi + cd "${RELUP_PACKAGE_PATH}" + while read -r pkg; do + packagename=$(basename "${pkg}") + unzip -q "$packagename" + if ! ./emqx/bin/emqx start; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + exit 1 + fi + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx versions + cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*.zip ./emqx/releases/ + ./emqx/bin/emqx install "${TARGET_VERSION}" + [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 + export EMQX_WAIT_FOR_STOP=300 + ./emqx/bin/emqx_ctl status + if ! ./emqx/bin/emqx stop; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + echo "failed to stop emqx" + exit 1 + fi + rm -rf emqx + done < <(find . -maxdepth 1 -name "${OLD_PACKAGE_PATTERN}.zip") } emqx_prepare diff --git a/.gitattributes b/.gitattributes index 4ed73da9a..8ecb2ae1a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ +build text eol=lf * text=auto *.* text eol=lf *.jpg -text diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 39b387e1c..94bfccb22 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -91,19 +91,11 @@ jobs: env: PYTHON: python DIAGNOSTIC: 1 + PROFILE: emqx working-directory: source run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" erl -eval "erlang:display(crypto:info_lib())" -s init stop - - $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) - if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { - $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" - $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip" - } - else { - $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" - } ## We do not build/release bcrypt for windows package Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ if (Test-Path rebar.lock) { @@ -115,11 +107,7 @@ jobs: head -2 rebar3 which rebar3 rebar3 --help - make ${{ matrix.profile }} - mkdir -p _packages/${{ matrix.profile }} - Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name - mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }} - sha256sum "_packages/${{ matrix.profile }}/$pkg_name" | head -c 64 > "_packages/${{ matrix.profile }}/${pkg_name}.sha256" + make ${{ matrix.profile }}-zip - name: run emqx timeout-minutes: 1 working-directory: source @@ -132,7 +120,7 @@ jobs: - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-windows path: source/_packages/${{ matrix.profile }}/. mac: @@ -214,8 +202,6 @@ jobs: exit 1 fi rm -rf emqx - #sha256sum $pkg_name | head -c64 > $pkg_name.sha256 - openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256 - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: @@ -368,7 +354,7 @@ jobs: registry: 'public.ecr.aws' - profile: emqx-ee registry: 'public.ecr.aws' - + steps: - uses: actions/download-artifact@v2 with: @@ -459,6 +445,9 @@ jobs: otp: - 23.3.4.9-3 - 24.1.5-3 + include: + - profile: emqx + otp: windows # otp version on windows is rather fixed steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 30e8b4c0a..703a2d551 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none + FIND=/usr/bin/find +else + FIND=find endif PROFILE ?= emqx @@ -91,8 +94,8 @@ $(PROFILES:%=clean-%): @if [ -d _build/$(@:clean-%=%) ]; then \ rm rebar.lock \ rm -rf _build/$(@:clean-%=%)/rel; \ - find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ - find _build/$(@:clean-%=%) -type l -delete; \ + $(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ + $(FIND) _build/$(@:clean-%=%) -type l -delete; \ fi .PHONY: clean-all diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 97548cba8..f88af106d 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -365,7 +365,7 @@ start_distribution(TargetNode, NameTypeArg, Cookie) -> make_script_node(Node) -> [Name, Host] = string:tokens(atom_to_list(Node), "@"), - list_to_atom(lists:concat([Name, "_upgrader_", os:getpid(), "@", Host])). + list_to_atom(lists:concat(["remsh_", Name, "_upgrader_", os:getpid(), "@", Host])). %% get name type from arg get_name_type(NameTypeArg) -> diff --git a/build b/build index 2aa20b7a3..f51d2c491 100755 --- a/build +++ b/build @@ -17,20 +17,6 @@ export PKG_VSN SYSTEM="$(./scripts/get-distro.sh)" -ARCH="$(uname -m)" -case "$ARCH" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm - ;; -esac -export ARCH - ## ## Support RPM and Debian based linux systems ## @@ -45,6 +31,28 @@ if [ "$(uname -s)" = 'Linux' ]; then esac fi +if [ "${SYSTEM}" = 'windows' ]; then + # windows does not like the find + FIND="/usr/bin/find" +else + FIND='find' +fi + +UNAME="$(uname -m)" +case "$UNAME" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac +# used in -pkg Makefile:s +export ARCH + log() { local msg="$1" # rebar3 prints ===>, so we print ===< @@ -52,15 +60,16 @@ log() { } make_rel() { - # shellcheck disable=SC1010 - ./rebar3 as "$PROFILE" do release,tar + ./rebar3 as "$PROFILE" tar } ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup make_relup() { local lib_dir="_build/$PROFILE/rel/emqx/lib" local releases_dir="_build/$PROFILE/rel/emqx/releases" - mkdir -p "$lib_dir" "$releases_dir" + local name_pattern + name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')" + mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base' local releases=() if [ -d "$releases_dir" ]; then while read -r zip; do @@ -76,7 +85,7 @@ make_relup() { rm -rf "$tmp_dir" fi releases+=( "$base_vsn" ) - done < <(find _upgrade_base -maxdepth 1 -name "${PROFILE}-*-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" -type f) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.zip" -type f) fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -87,6 +96,8 @@ make_relup() { ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" } +## try to be portable for zip packages. +## for DEB and RPM packages the dependencies are resoved by yum and apt cp_dyn_libs() { local rel_dir="$1" local target_dir="${rel_dir}/dynlibs" @@ -96,36 +107,55 @@ cp_dyn_libs() { mkdir -p "$target_dir" while read -r so_file; do cp -L "$so_file" "$target_dir/" - done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ + done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ | xargs -0 ldd \ | grep -E '(libcrypto)|(libtinfo)' \ | awk '{print $3}' \ | sort -u) } + ## make_zip turns .tar.gz into a .zip with a slightly different name. ## It assumes the .tar.gz has been built -- relies on Makefile dependency make_zip() { # build the tarball again to ensure relup is included make_rel - - tard="/tmp/emqx_untar_${PKG_VSN}" - rm -rf "${tard}" + # use relative path because abs path is tricky in windows + tard="tmp/zip-wd-${PKG_VSN}" + rm -rf "${tard}/emqx" mkdir -p "${tard}/emqx" local relpath="_build/${PROFILE}/rel/emqx" local pkgpath="_packages/${PROFILE}" + local pkgname + pkgname="${PROFILE}-$(./scripts/pkg-full-vsn.sh).zip" mkdir -p "${pkgpath}" - local tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" - if [ ! -f "$tarball" ]; then - log "ERROR: $tarball is not found" - fi - local zipball - zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" + local tarname="emqx-${PKG_VSN}.tar.gz" + local tarball="${relpath}/${tarname}" + local target_zip="${pkgpath}/${pkgname}" tar zxf "${tarball}" -C "${tard}/emqx" - ## try to be portable for zip packages. - ## for DEB and RPM packages the dependencies are resoved by yum and apt cp_dyn_libs "${tard}/emqx" - (cd "${tard}" && zip -qr - emqx) > "${zipball}" + pushd "${tard}" >/dev/null + case "$SYSTEM" in + windows*) + 7z a "${pkgname}" emqx + ;; + *) + zip -qr "${pkgname}" emqx + ;; + esac + popd >/dev/null + mv "${tard}/${pkgname}" "${target_zip}" + case "$SYSTEM" in + macos*) + # sha256sum may not be available on macos + openssl dgst -sha256 "${target_zip}" | cut -d ' ' -f 2 > "${target_zip}.sha256" + ;; + *) + sha256sum "${target_zip}" | head -c 64 > "${target_zip}.sha256" + ;; + esac + log "Zip package successfully created: ${target_zip}" + log "Zip package sha256sum: $(cat "${target_zip}.sha256")" } ## This function builds the default docker image based on alpine:3.14 (by default) @@ -170,7 +200,7 @@ make_docker_testing() { fi EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" local defaultzip - defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" + defaultzip="_packages/${PROFILE}/${PROFILE}-$(./scripts/pkg-full-vsn.sh).zip" local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}" if [ ! -f "$zip" ]; then log "ERROR: $zip not built?" @@ -201,7 +231,7 @@ case "$ARTIFACT" in log "Skipped making deb/rpm package for $SYSTEM" exit 0 fi - make -C "deploy/packages/${PKGERDIR}" clean + EMQX_REL="$(pwd)" make -C "deploy/packages/${PKGERDIR}" clean EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}" ;; docker) diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 2cb3679ee..15665fc07 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -1,4 +1,3 @@ -ARCH ?= amd64 TOPDIR := /tmp/emqx # Keep this short to avoid bloating beam files with long file path info SRCDIR := $(TOPDIR)/$(PKG_VSN) @@ -8,7 +7,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh) .PHONY: all all: | $(BUILT) @@ -22,6 +21,8 @@ all: | $(BUILT) cd $(SRCDIR) && dpkg-buildpackage -us -uc mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb + sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb | head -c 64 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb.sha256 $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index acee8b51c..31c1919ff 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -17,8 +17,8 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -48,6 +48,8 @@ all: | $(BUILT) emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm + sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm | head -c 64 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm.sha256 $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index 89058a298..e52a82821 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -2,7 +2,21 @@ set -euo pipefail -VERSION="3.14.3-emqx-8" +## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5. +## we have to use an otp24-compiled rebar3 because the defination of record #application{} +## in systools.hrl is changed in otp24. +case ${OTP_VSN} in + 23*) + VERSION="3.14.3-emqx-8" + ;; + 24*) + VERSION="3.18.0-emqx-1" + ;; + *) + echo "Unsupporetd Erlang/OTP version $OTP_VSN" + exit 1 + ;; +esac # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." @@ -14,10 +28,6 @@ download() { curl -f -L "${DOWNLOAD_URL}/${VERSION}/rebar3" -o ./rebar3 } -version_gte() { - test "$(printf '%s\n' "$1" "$2" | sort -V | head -n 1)" = "$2" -} - # get the version number from the second line of the escript # because command `rebar3 -v` tries to load rebar.config # which is slow and may print some logs @@ -25,16 +35,7 @@ version() { head -n 2 ./rebar3 | tail -n 1 | tr ' ' '\n' | grep -E '^.+-emqx-.+' } -echo "OTP_VSN: ${OTP_VSN}" -if version_gte "${OTP_VSN}" "24.0"; then - ## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5. - ## we have to use an otp24-compiled rebar3 because the defination of record #application{} - ## in systools.hrl is changed in otp24. - VERSION="3.18.0-emqx-1" -fi - if [ -f 'rebar3' ] && [ "$(version)" = "$VERSION" ]; then - echo "rebar3 ${VERSION} already exists" exit 0 fi diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index fabec239e..c4a0eec62 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -5,9 +5,16 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")/.." +if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then + # Otherwise windows may resolve to find.exe + FIND="/usr/bin/find" +else + FIND='find' +fi + find_app() { local appdir="$1" - find "${appdir}" -mindepth 1 -maxdepth 1 -type d + "$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d } # append emqx application first @@ -23,4 +30,4 @@ fi ## find directories in lib-extra find_app 'lib-extra' ## find symlinks in lib-extra -find 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print +"$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index f4ac5c88c..d017a7d9d 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -5,18 +5,26 @@ set -euo pipefail -if [ "$(uname -s)" = 'Darwin' ]; then - DIST='macos' - VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') - SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" -elif [ "$(uname -s)" = 'Linux' ]; then - if grep -q -i 'rhel' /etc/*-release; then - DIST='el' - VERSION_ID="$(rpm --eval '%{rhel}')" - else - DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" - VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" - fi - SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" -fi +UNAME="$(uname -s)" + +case "$UNAME" in + Darwin) + DIST='macos' + VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') + SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" + ;; + Linux) + if grep -q -i 'rhel' /etc/*-release; then + DIST='el' + VERSION_ID="$(rpm --eval '%{rhel}')" + else + DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" + VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" + fi + SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" + ;; + CYGWIN*|MSYS*|MINGW*) + SYSTEM="windows" + ;; +esac echo "$SYSTEM" diff --git a/scripts/pkg-full-vsn.sh b/scripts/pkg-full-vsn.sh new file mode 100755 index 000000000..de32e11ef --- /dev/null +++ b/scripts/pkg-full-vsn.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +## This script print the package full vsn based on current build environment + +## Arg 1 is either 'vsn_exact' (default) or 'vsn_matcher' +## when 'vsn_exact' is given, the version number is the output of pkg-vsn.sh +## otherwise '*' is used for 'find' command to find old versions (as upgrade base) + +set -euo pipefail + +VSN_MATCH="${1:-vsn_exact}" + +case "${VSN_MATCH}" in + vsn_exact) + PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" + ;; + vsn_matcher) + PKG_VSN='*' + ;; + *) + echo "$0 ERROR: second arg must " + exit 1 + ;; +esac + +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" +SYSTEM="$(./scripts/get-distro.sh)" + +UNAME="$(uname -m)" +case "$UNAME" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac + +echo "${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" From 62ced62aac8b3f5f8e22e19d31022313f8340b2f Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 21 Feb 2022 14:33:36 +0800 Subject: [PATCH 213/363] fix(emqx_mgmt_http): add slow subs api into emqx_mgmt_http api list --- apps/emqx_management/src/emqx_mgmt_http.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index 8e92b7371..010a4fce2 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -89,7 +89,8 @@ listener_name(Proto) -> http_handlers() -> Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), - [{"/api/v4", minirest:handler(#{apps => Plugins ++ [emqx_modules] -- ?EXCEPT_PLUGIN, + [{"/api/v4", minirest:handler(#{apps => Plugins ++ + [emqx_plugin_libs, emqx_modules] -- ?EXCEPT_PLUGIN, except => ?EXCEPT, filter => fun ?MODULE:filter/1}), [{authorization, fun ?MODULE:authorize_appid/1}]}]. @@ -128,6 +129,7 @@ filter(_) -> true. -else. filter(#{app := emqx_modules}) -> true; +filter(#{app := emqx_plugin_libs}) -> true; filter(#{app := App}) -> case emqx_plugins:find_plugin(App) of false -> false; From 9abbe4eafc28c7e37f287d7b23bcef418ccea52e Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 21 Feb 2022 14:47:20 +0800 Subject: [PATCH 214/363] chore(emqx_management): bump app version --- apps/emqx_management/src/emqx_management.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 1efd30dcc..c83969abc 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.4.1"}, % strict semver, bump manually! + {vsn, "4.4.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, From 14575fed63fea0994f9dbf193b3e9c7c61e89393 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 21 Feb 2022 00:21:48 +0100 Subject: [PATCH 215/363] chore: prepare for release v4.4.1-rc.1 --- include/emqx_release.hrl | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 24f6feb57..5e5a7a4b7 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.0"}). +-define(EMQX_RELEASE, {opensource, "4.4.1-rc.1"}). -else. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 1354ab3c6..7adcdc7db 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.4.1"}, % strict semver, bump manually! + {vsn, "4.4.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, From e13f2101f2592ad9e3d1773cf8be1c9014b37bfb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 21 Feb 2022 11:34:23 +0100 Subject: [PATCH 216/363] chore: prepare for v4.4.1 release --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 5e5a7a4b7..f3e6ab562 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.1-rc.1"}). +-define(EMQX_RELEASE, {opensource, "4.4.1"}). -else. From 8226ed2b494491afe5b2de2ddd2a1e43fb3ee525 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 21 Feb 2022 11:36:49 +0100 Subject: [PATCH 217/363] chore: update CHANGES-4.4 --- CHANGES-4.4.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 6dbe1ff97..29bdaf75f 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,5 +1,9 @@ # EMQ X 4.4 Changes +## v4.4.1 + +This patch release is only to fix windows build which failed on v4.4.0. + ## v4.4.0 **NOTE**: v4.4.0 is in sync with: v4.3.12 From 89e2cdb5a48a4ac293b199701196371d8fe3c2cb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 21 Feb 2022 11:55:19 +0100 Subject: [PATCH 218/363] docs: update windows.md to use otp 24 --- Windows.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Windows.md b/Windows.md index f02b40920..b30a65b81 100644 --- a/Windows.md +++ b/Windows.md @@ -29,7 +29,7 @@ The second path is for CMD to setup environment variables. ### Erlang/OTP -Install Erlang/OTP 23.2 from https://www.erlang.org/downloads +Install Erlang/OTP 24.2.1 from https://www.erlang.org/downloads You may need to edit the `Path` environment variable to allow running Erlang commands such as `erl` from CMD. @@ -45,7 +45,7 @@ e.g. ``` PS C:\Users\zmsto> erl -Eshell V11.1.4 (abort with ^G) +Eshell V12.2.1 (abort with ^G) 1> halt(). ``` From 3c22878d5499032c9cc56ef8e701b8c1f9e5cb79 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 21 Feb 2022 12:55:34 +0100 Subject: [PATCH 219/363] ci: update windows ci to use erlef action and add windows to slim build --- .github/workflows/build_packages.yaml | 25 ++++----------- .github/workflows/build_slim_packages.yaml | 37 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 94bfccb22..190077a97 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -61,19 +61,16 @@ jobs: windows: runs-on: windows-2019 - needs: prepare if: endsWith(github.repository, 'emqx') - strategy: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.2 + - 24.2.1 exclude: - profile: emqx-edge - steps: - uses: actions/download-artifact@v2 with: @@ -82,9 +79,7 @@ jobs: - name: unzip source code run: Expand-Archive -Path source.zip -DestinationPath ./ - uses: ilammy/msvc-dev-cmd@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang - ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows + - uses: erlef/setup-beam@v1 with: otp-version: ${{ matrix.otp }} - name: build @@ -94,19 +89,7 @@ jobs: PROFILE: emqx working-directory: source run: | - $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" erl -eval "erlang:display(crypto:info_lib())" -s init stop - ## We do not build/release bcrypt for windows package - Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ - if (Test-Path rebar.lock) { - Remove-Item -Force -Path rebar.lock - } - make ensure-rebar3 - copy rebar3 "${{ steps.install_erlang.outputs.erlpath }}\bin" - ls "${{ steps.install_erlang.outputs.erlpath }}\bin" - head -2 rebar3 - which rebar3 - rebar3 --help make ${{ matrix.profile }}-zip - name: run emqx timeout-minutes: 1 @@ -114,9 +97,13 @@ jobs: run: | ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start Start-Sleep -s 5 + echo "EMQX started" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop + echo "EMQX stopped" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install + echo "EQMX installed" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall + echo "EQMX uninstaled" - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 685dc3f94..d8482faf2 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -60,8 +60,43 @@ jobs: name: ${{ matrix.os }} path: _packages/**/*.zip - mac: + windows: + runs-on: windows-2019 + if: endsWith(github.repository, 'emqx') + strategy: + fail-fast: false + matrix: + profile: + - emqx + otp: + - 24.2.1 + steps: + - uses: actions/checkout@v2 + - uses: ilammy/msvc-dev-cmd@v1 + - uses: erlef/setup-beam@v1 + with: + otp-version: ${{ matrix.otp }} + - name: build + env: + PYTHON: python + DIAGNOSTIC: 1 + run: | + erl -eval "erlang:display(crypto:info_lib())" -s init stop + make ${{ matrix.profile }}-zip + - name: run emqx + timeout-minutes: 1 + run: | + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start + Start-Sleep -s 5 + echo "EMQX started" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop + echo "EMQX stopped" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install + echo "EQMX installed" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall + echo "EQMX uninstaled" + mac: strategy: fail-fast: false matrix: From 99ef6fbb2aa48d8f05ce1a312fc377dde68fe9d0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 21 Feb 2022 15:08:23 +0100 Subject: [PATCH 220/363] docs: update CHANGES-4.4 --- CHANGES-4.4.md | 6 ++++++ lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 29bdaf75f..c776c8e15 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,5 +1,11 @@ # EMQ X 4.4 Changes +## v4.4.2 + +### Minor changes + +* Windows package is built on Erlang/OTP 24 + ## v4.4.1 This patch release is only to fix windows build which failed on v4.4.0. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 7adcdc7db..0156ea865 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.4.2"}, % strict semver, bump manually! + {vsn, "4.4.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, From 6b720286c6fdbec3abf67ad9c1d84c1884c3cef1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 22 Feb 2022 11:55:20 +0800 Subject: [PATCH 221/363] chore: fix bad sytanx for emqx.appup.src --- src/emqx.appup.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index e339ad952..c95a1beaf 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -8,7 +8,7 @@ ]}, {"4.4.0", [ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]} - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + , {load_module,emqx_banned,brutal_purge,soft_purge,[]} , {load_module,emqx_ctl,brutal_purge,soft_purge,[]} , {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_connection,brutal_purge,soft_purge,[]} From 18feec9030595bbd0bfceaa40706803cf8fe001d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 22 Feb 2022 13:43:46 +0800 Subject: [PATCH 222/363] chore(ci): fix the download filename --- scripts/relup-base-packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index bedc050c7..90e79edbb 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -54,7 +54,7 @@ mkdir -p _upgrade_base pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do - filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" + filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip" url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename" echo "downloading base package from ${url} ..." if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then From fb7944391d25ee7d52b6dd1ebad0b98f11af99b1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 09:40:32 +0100 Subject: [PATCH 223/363] build: fix shellcheck --- .ci/build_packages/tests.sh | 8 ++++---- scripts/buildx.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 628a798e9..04d8018ac 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -24,7 +24,7 @@ export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" # export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1" # export EMQX_NODE_COOKIE=$(date +%s%N) -SYSTEM="$($CODE_PATH/scripts/get-distro.sh)" +SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)" if [ "$PACKAGE_TYPE" = 'zip' ]; then PKG_SUFFIX="zip" @@ -39,8 +39,8 @@ else esac fi -PACKAGE_NAME="${PROFILE}-$($CODE_PATH/scripts/pkg-full-vsn.sh)" -OLD_PACKAGE_PATTERN="${PROFILE}-$($CODE_PATH/scripts/pkg-full-vsn.sh 'vsn_matcher')" +PACKAGE_NAME="${PROFILE}-$("$CODE_PATH"/scripts/pkg-full-vsn.sh)" +OLD_PACKAGE_PATTERN="${PROFILE}-$("$CODE_PATH"/scripts/pkg-full-vsn.sh 'vsn_matcher')" PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" @@ -117,7 +117,7 @@ emqx_test(){ ;; "rpm") yum install -y "${PACKAGE_PATH}/${packagename}" - if ! rpm -q ${PROFILE} | grep -q "${PROFILE}"; then + if ! rpm -q "${PROFILE}" | grep -q "${PROFILE}"; then echo "package install error" exit 1 fi diff --git a/scripts/buildx.sh b/scripts/buildx.sh index fd754bad6..94da5fb9b 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -71,7 +71,7 @@ cd "${SRC_DIR:-.}" set -x docker info -docker run --rm --privileged tonistiigi/binfmt:latest --install ${ARCH} +docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" docker run -i --rm \ -v "$(pwd)":/emqx \ --workdir /emqx \ From 4473a832bc903bb419a05aa6c872e8ea831fc94c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 09:43:06 +0100 Subject: [PATCH 224/363] build: fix macos -> macos* so matches macos10 and macos11 --- scripts/relup-base-packages.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 90e79edbb..f86185e0a 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -42,10 +42,15 @@ case "$ARCH" in ;; esac -SHASUM="sha256sum" -if [ "$SYSTEM" = "macos" ]; then - SHASUM="shasum -a 256" -fi + +case "$SYSTEM" in + macos*) + SHASUM="shasum -a 256" + ;; + *) + SHASUM="sha256sum" + ;; +esac # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." From 9d50d5e99daba597028b1ab7d20223cd806c0682 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 23 Feb 2022 17:15:35 +0800 Subject: [PATCH 225/363] ci(cross build): splitting cross builds and upload assets --- .github/workflows/build_packages.yaml | 102 ++++----------------- .github/workflows/build_slim_packages.yaml | 4 - .github/workflows/release.yaml | 96 +++++++++++++++++++ 3 files changed, 113 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 94bfccb22..8315ae834 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -7,9 +7,10 @@ concurrency: on: schedule: - cron: '0 */6 * * *' - release: - types: - - published + push: + tags: + - v* + - e* workflow_dispatch: jobs: @@ -424,14 +425,6 @@ jobs: file: source/deploy/docker/Dockerfile.enterprise context: source - delete-artifact: - runs-on: ubuntu-20.04 - needs: [prepare, mac, linux, docker] - steps: - - uses: geekyeggo/delete-artifact@v1 - with: - name: source - upload: runs-on: ubuntu-20.04 @@ -450,12 +443,6 @@ jobs: otp: windows # otp version on windows is rather fixed steps: - - uses: actions/checkout@v2 - - name: get_version - run: | - echo 'version<> $GITHUB_ENV - echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - uses: actions/download-artifact@v2 with: name: ${{ matrix.profile }}-${{ matrix.otp }} @@ -471,72 +458,17 @@ jobs: echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 done cd - + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - name: upload aws s3 - run: | - set -e -u - if [ "${{ matrix.profile }}" == "emqx" ];then - broker="emqx-ce" - else - broker=${{ matrix.profile }} - fi - aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }} - - aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }} - aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*" - - uses: Rory-Z/upload-release-asset@v1 - if: github.event_name == 'release' && matrix.profile != 'emqx-ee' - with: - repo: emqx - path: "_packages/${{ matrix.profile }}/emqx-*" - token: ${{ github.token }} - - uses: Rory-Z/upload-release-asset@v1 - if: github.event_name == 'release' && matrix.profile == 'emqx-ee' - with: - repo: emqx-enterprise - path: "_packages/${{ matrix.profile }}/emqx-*" - token: ${{ github.token }} - - name: update to emqx.io - if: github.event_name == 'release' - run: | - set -e -x -u - curl -w %{http_code} \ - --insecure \ - -H "Content-Type: application/json" \ - -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \ - -X POST \ - -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ - ${{ secrets.EMQX_IO_RELEASE_API }} - - name: update repo.emqx.io - if: github.event_name == 'release' && matrix.profile == 'emqx-ee' - run: | - curl --silent --show-error \ - -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - -X POST \ - -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ - "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - - name: update repo.emqx.io - if: github.event_name == 'release' && matrix.profile == 'emqx' - run: | - curl --silent --show-error \ - -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - -X POST \ - -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ - "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - - name: update homebrew packages - if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' - run: | - if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then - curl --silent --show-error \ - -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - -X POST \ - -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ - "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" - fi - - uses: geekyeggo/delete-artifact@v1 - with: - name: ${{ matrix.profile }} + run: | + if [ "${{ matrix.profile }}" == "emqx" ];then + s3dir="emqx-ce" + else + s3dir=${{ matrix.profile }} + fi + aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} + aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*" \ No newline at end of file diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 685dc3f94..fb6f130b7 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -6,10 +6,6 @@ concurrency: on: - push: - tags: - - v* - - e* pull_request: workflow_dispatch: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..1d58fea86 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,96 @@ +name: Upload release assets +on: + release: + types: + - published + +jobs: + prepare: + runs-on: ubuntu-20.04 + container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + + outputs: + profiles: ${{ steps.set_profile.outputs.profiles}} + + steps: + - uses: actions/checkout@v2 + with: + path: source + fetch-depth: 0 + - name: set profile + id: set_profile + shell: bash + run: | + cd source + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "::set-output name=profiles::[\"emqx-ee\"]" + else + echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]" + fi + + upload: + runs-on: ubuntu-20.04 + needs: prepare + strategy: + fail-fast: false + matrix: + profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + + steps: + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: Get packages + run: | + if [ "${{ matrix.profile }}" == "emqx" ];then + s3dir="emqx-ce" + else + s3dir=${{ matrix.profile }} + fi + aws s3 cp --recursive s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} packages + - uses: alexellis/upload-assets@0.2.2 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + asset_paths: '["packages/*"]' + - name: update to emqx.io + run: | + set -e -x -u + curl -w %{http_code} \ + --insecure \ + -H "Content-Type: application/json" \ + -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \ + -X POST \ + -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \ + ${{ secrets.EMQX_IO_RELEASE_API }} + - name: update repo.emqx.io + if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' + run: | + curl --silent --show-error \ + -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + -X POST \ + -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\", \"emqx_ee\": \"true\"}}" \ + "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" + - name: update repo.emqx.io + if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' + run: | + curl --silent --show-error \ + -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + -X POST \ + -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\", \"emqx_ce\": \"true\"}}" \ + "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" + - name: update homebrew packages + if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' + run: | + if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then + curl --silent --show-error \ + -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + -X POST \ + -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \ + "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" + fi \ No newline at end of file From 7d52bc3d3a23c3d63dc24897d819bfebcd3a72f2 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 23 Feb 2022 18:29:49 +0800 Subject: [PATCH 226/363] ci(cross build): fix syntax error --- .github/workflows/build_packages.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 8315ae834..a2826708b 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -464,11 +464,11 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - name: upload aws s3 - run: | - if [ "${{ matrix.profile }}" == "emqx" ];then - s3dir="emqx-ce" - else - s3dir=${{ matrix.profile }} - fi - aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} - aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*" \ No newline at end of file + run: | + if [ "${{ matrix.profile }}" == "emqx" ];then + s3dir="emqx-ce" + else + s3dir=${{ matrix.profile }} + fi + aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} + aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*" \ No newline at end of file From 2796113c20f3dc8b9b900414c9bfdb7579c33e30 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 23 Feb 2022 20:03:33 +0800 Subject: [PATCH 227/363] ci(cross build): fix syntax error --- .github/workflows/build_packages.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index a2826708b..a873a56e7 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -470,5 +470,5 @@ jobs: else s3dir=${{ matrix.profile }} fi - aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} - aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*" \ No newline at end of file + aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} + aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*" \ No newline at end of file From 91c46de4aa3d3a910033f91803148a614749964a Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 26 Feb 2022 20:33:42 +0800 Subject: [PATCH 228/363] fix(relup): inject relup instructions to the end of relup file --- rebar.config.erl | 3 +- scripts/inject-relup.escript | 58 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100755 scripts/inject-relup.escript diff --git a/rebar.config.erl b/rebar.config.erl index ce8933570..11db1abb1 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -129,7 +129,8 @@ prod_overrides() -> [{add, [ {erl_opts, [deterministic]}]}]. relup_deps(Profile) -> - {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}. + {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", release, + "scripts/inject-relup.escript " ++ filename:join(["_build", Profile, "rel", "emqx"])}]}. profiles() -> Vsn = get_vsn(), diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript new file mode 100755 index 000000000..a7ba23689 --- /dev/null +++ b/scripts/inject-relup.escript @@ -0,0 +1,58 @@ +#!/usr/bin/env escript + +%% This script injects implicit relup instructions for emqx applications. + +-mode(compile). + +-define(ERROR(FORMAT, ARGS), io:format(standard_error, "[inject-relup] " ++ FORMAT ++"~n", ARGS)). +-define(INFO(FORMAT, ARGS), io:format(user, "[inject-relup] " ++ FORMAT ++"~n", ARGS)). + +usage() -> + "Usage: " ++ escript:script_name() ++ " ". + +main([Dir]) -> + case filelib:is_dir(Dir) of + true -> + ok = inject(Dir); + false -> + ?ERROR("not a valid dir: ~p", [Dir]), + erlang:halt(1) + end; +main(_Args) -> + ?ERROR("~s", [usage()]), + erlang:halt(1). + +inject(Dir) -> + RelupFiles = filelib:wildcard(filename:join([Dir, "releases", "*", "relup"])), + lists:foreach(fun(File) -> + case file:consult(File) of + {ok, [{CurrRelVsn, UpVsnRUs, DnVsnRUs}]} -> + ?INFO("injecting instructions to: ~p", [File]), + UpdatedContent = [{CurrRelVsn, inject_relup_instrs(CurrRelVsn, UpVsnRUs), + inject_relup_instrs(CurrRelVsn, DnVsnRUs)}], + file:write_file("/tmp/" ++ filename:basename(File), term_to_text(UpdatedContent)); + {ok, _BadFormat} -> + ?ERROR("bad formatted relup file: ~p", [File]); + {error, Reason} -> + ?ERROR("read relup file ~p failed: ~p", [File, Reason]) + end + end, RelupFiles). + +inject_relup_instrs(CurrRelVsn, RUs) -> + [{Vsn, Desc, append_emqx_relup_instrs(CurrRelVsn, Vsn, Instrs)} || {Vsn, Desc, Instrs} <- RUs]. + +%% The `{apply, emqx_relup, post_release_upgrade, []}` will be appended to the end of +%% the instruction lists. +append_emqx_relup_instrs(CurrRelVsn, Vsn, Instrs) -> + case lists:reverse(Instrs) of + [{apply, {emqx_relup, post_release_upgrade, _}} | _] -> + Instrs; + RInstrs -> + lists:reverse([instr_post_release_upgrade(CurrRelVsn, Vsn) | RInstrs]) + end. + +instr_post_release_upgrade(CurrRelVsn, Vsn) -> + {apply, {emqx_relup, post_release_upgrade, [CurrRelVsn, Vsn]}}. + +term_to_text(Term) -> + io_lib:format("~p.", [Term]). \ No newline at end of file From 692d1e2a3371f633099d6bee2c06558e7f12ef22 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 26 Feb 2022 21:31:09 +0800 Subject: [PATCH 229/363] fix(relup): OTP_VSN: unbound variable --- scripts/relup-base-packages.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index f86185e0a..2a47b5135 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -28,6 +28,7 @@ case $PROFILE in esac SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" ARCH="${ARCH:-$(uname -m)}" case "$ARCH" in From 0931a426cd53da182bff391c7149049fcff0d473 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 26 Feb 2022 16:44:11 +0100 Subject: [PATCH 230/363] chore(update_appup.escript): make it work with dirty src dirs --- scripts/update_appup.escript | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 965de1efe..50a01b708 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -32,8 +32,8 @@ Options: --make-command A command used to assemble the release --release-dir Release directory --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' - --binary-rel-url Binary release URL pattern. %TAG% variable is substituted with the release tag. - E.g. \"https://github.com/emqx/emqx/releases/download/v%TAG%/emqx-centos7-%TAG%-amd64.zip\" + --binary-rel-url Binary release URL pattern. + E.g. https://www.emqx.com/en/downloads/broker/v4.4.1/emqx-4.4.1-otp24.1.5-3-el7-amd64.zip ". -record(app, @@ -208,8 +208,8 @@ find_appup_actions(App, {OldUpgrade0, OldDowngrade0} = find_old_appup_actions(App, PrevVersion), OldUpgrade = ensure_all_patch_versions(App, CurrVersion, OldUpgrade0), OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0), - Upgrade = merge_update_actions(App, diff_app(App, CurrAppIdx, PrevAppIdx), OldUpgrade), - Downgrade = merge_update_actions(App, diff_app(App, PrevAppIdx, CurrAppIdx), OldDowngrade), + Upgrade = merge_update_actions(App, diff_app(up, App, CurrAppIdx, PrevAppIdx), OldUpgrade), + Downgrade = merge_update_actions(App, diff_app(down, App, PrevAppIdx, CurrAppIdx), OldDowngrade), if OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade -> %% The appup file has been already updated: []; @@ -521,7 +521,7 @@ index_app(AppFile) -> , modules = Modules }}. -diff_app(App, +diff_app(UpOrDown, App, #app{version = NewVersion, modules = NewModules}, #app{version = OldVersion, modules = OldModules}) -> {New, Changed} = @@ -540,13 +540,15 @@ diff_app(App, ), Deleted = maps:keys(maps:without(maps:keys(NewModules), OldModules)), NChanges = length(New) + length(Changed) + length(Deleted), - if NewVersion =:= OldVersion andalso NChanges > 0 -> + case NewVersion =:= OldVersion of + true when NChanges =:= 0 -> + %% no change + ok; + true -> set_invalid(), log("ERROR: Application '~p' contains changes, but its version is not updated~n", [App]); - NewVersion > OldVersion -> - log("INFO: Application '~p' has been updated: ~p -> ~p~n", [App, OldVersion, NewVersion]), - ok; - true -> + false -> + log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [App, OldVersion, UpOrDown, NewVersion]), ok end, {New, Changed, Deleted}. @@ -607,13 +609,18 @@ locate(ebin_current, App, Suffix) -> locate(src, App, Suffix) -> AppStr = atom_to_list(App), SrcDirs = getopt(src_dirs), - case filelib:wildcard(SrcDirs ++ AppStr ++ Suffix) of + case find_app(SrcDirs ++ AppStr ++ Suffix) of [File] -> {ok, File}; [] -> undefined end. +find_app(Pattern) -> + %% exclude _build dir inside apps + lists:filter(fun(S) -> string:find(S, "/_build/") =:= nomatch end, + filelib:wildcard(Pattern)). + bash(Script) -> bash(Script, []). From cc8168ba239462d4fe79e1b348aad3b5692e1704 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 26 Feb 2022 16:45:55 +0100 Subject: [PATCH 231/363] chore(appup): commit script updated emqx.app.src --- src/emqx.appup.src | 103 ++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 48 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index c95a1beaf..b9d4f5a16 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,54 +1,61 @@ %% -*- mode: erlang -*- {VSN, [{"4.4.1", - [ {load_module,emqx_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_banned,brutal_purge,soft_purge,[]} - , {load_module,emqx_ctl,brutal_purge,soft_purge,[]} - , {load_module,emqx_channel,brutal_purge,soft_purge,[]} - ]}, + [{load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]} - , {load_module,emqx_banned,brutal_purge,soft_purge,[]} - , {load_module,emqx_ctl,brutal_purge,soft_purge,[]} - , {load_module,emqx_channel,brutal_purge,soft_purge,[]} - , {load_module,emqx_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} - , {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}} - , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} - , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} - , {load_module,emqx_session,brutal_purge,soft_purge,[]} - , {load_module,emqx_os_mon,brutal_purge,soft_purge,[]} - , {load_module,emqx,brutal_purge,soft_purge,[]} - , {load_module,emqx_app,brutal_purge,soft_purge,[]} - , {load_module,emqx_message,brutal_purge,soft_purge,[]} - , {load_module,emqx_limiter,brutal_purge,soft_purge,[]} - ]}, - {<<".*">>,[]} - ], + [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], [{"4.4.1", - [ {load_module,emqx_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_banned,brutal_purge,soft_purge,[]} - , {load_module,emqx_ctl,brutal_purge,soft_purge,[]} - , {load_module,emqx_channel,brutal_purge,soft_purge,[]} - ]}, + [{load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]} - , {load_module,emqx_banned,brutal_purge,soft_purge,[]} - , {load_module,emqx_ctl,brutal_purge,soft_purge,[]} - , {load_module,emqx_channel,brutal_purge,soft_purge,[]} - , {load_module,emqx_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]} - , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} - , {load_module,emqx_access_control,brutal_purge,soft_purge,[]} - , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} - , {load_module,emqx_session,brutal_purge,soft_purge,[]} - , {load_module,emqx_os_mon,brutal_purge,soft_purge,[]} - , {load_module,emqx,brutal_purge,soft_purge,[]} - , {load_module,emqx_app,brutal_purge,soft_purge,[]} - , {load_module,emqx_message,brutal_purge,soft_purge,[]} - , {load_module,emqx_limiter,brutal_purge,soft_purge,[]} - ]}, - {<<".*">>,[]} - ] -}. + [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. From 8654600ea24c01984981e70523cde4f946e54f84 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 26 Feb 2022 17:00:05 +0100 Subject: [PATCH 232/363] fix(update_appup): use curl command instead of wget --- scripts/update_appup.escript | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 50a01b708..d3d5fd11c 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -171,9 +171,10 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) - BaseDir = "/tmp/emqx-baseline-bin/", Dir = filename:basename(Repo, ".git") ++ [$-|Tag], Filename = filename:join(BaseDir, Dir), - Script = "mkdir -p ${OUTFILE} && - wget -c -O ${OUTFILE}.zip ${URL} && - unzip -n -d ${OUTFILE} ${OUTFILE}.zip", + Script = "echo \"Download: ${OUTFILE}\" && + mkdir -p ${OUTFILE} && + curl -f -L -o ${OUTFILE}.zip ${URL} && + unzip -q -n -d ${OUTFILE} ${OUTFILE}.zip", Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}], bash(Script, Env), {ok, Filename}. From 2365d1e983d0d9c56c6de31bb94f04ef3ce31475 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 27 Feb 2022 09:28:35 +0800 Subject: [PATCH 233/363] fix(relup): inject relup only for the current rel vsn --- rebar.config.erl | 17 +++++---- scripts/inject-relup.escript | 68 ++++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 37 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index 11db1abb1..447e58f97 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -128,31 +128,34 @@ prod_compile_opts() -> prod_overrides() -> [{add, [ {erl_opts, [deterministic]}]}]. -relup_deps(Profile) -> - {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", release, - "scripts/inject-relup.escript " ++ filename:join(["_build", Profile, "rel", "emqx"])}]}. +relup_deps(Profile, Vsn) -> + InjectCmd = "scripts/inject-relup.escript " ++ filename:join(["_build", Profile, "rel", "emqx", "releases", Vsn]), + {post_hooks, + [ {"(linux|darwin|solaris|freebsd|netbsd|openbsd)", relup, InjectCmd} + ] + }. profiles() -> Vsn = get_vsn(), [ {'emqx', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin)} , {overrides, prod_overrides()} - , relup_deps('emqx') + , relup_deps('emqx', Vsn) ]} , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg)} , {overrides, prod_overrides()} - , relup_deps('emqx-pkg') + , relup_deps('emqx-pkg', Vsn) ]} , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin)} , {overrides, prod_overrides()} - , relup_deps('emqx-edge') + , relup_deps('emqx-edge', Vsn) ]} , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg)} , {overrides, prod_overrides()} - , relup_deps('emqx-edge-pkg') + , relup_deps('emqx-edge-pkg', Vsn) ]} , {check, [ {erl_opts, common_compile_opts()} ]} diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript index a7ba23689..86c541ada 100755 --- a/scripts/inject-relup.escript +++ b/scripts/inject-relup.escript @@ -8,51 +8,59 @@ -define(INFO(FORMAT, ARGS), io:format(user, "[inject-relup] " ++ FORMAT ++"~n", ARGS)). usage() -> - "Usage: " ++ escript:script_name() ++ " ". + "Usage: " ++ escript:script_name() ++ " ". -main([Dir]) -> - case filelib:is_dir(Dir) of - true -> - ok = inject(Dir); +main([DirOrFile]) -> + case filelib:is_dir(DirOrFile) of + true -> ok = inject_dir(DirOrFile); false -> - ?ERROR("not a valid dir: ~p", [Dir]), - erlang:halt(1) + case filelib:is_regular(DirOrFile) of + true -> inject_file(DirOrFile); + false -> + ?ERROR("not a valid file: ~p", [DirOrFile]), + erlang:halt(1) + end end; main(_Args) -> ?ERROR("~s", [usage()]), erlang:halt(1). -inject(Dir) -> - RelupFiles = filelib:wildcard(filename:join([Dir, "releases", "*", "relup"])), - lists:foreach(fun(File) -> - case file:consult(File) of - {ok, [{CurrRelVsn, UpVsnRUs, DnVsnRUs}]} -> - ?INFO("injecting instructions to: ~p", [File]), - UpdatedContent = [{CurrRelVsn, inject_relup_instrs(CurrRelVsn, UpVsnRUs), - inject_relup_instrs(CurrRelVsn, DnVsnRUs)}], - file:write_file("/tmp/" ++ filename:basename(File), term_to_text(UpdatedContent)); - {ok, _BadFormat} -> - ?ERROR("bad formatted relup file: ~p", [File]); - {error, Reason} -> - ?ERROR("read relup file ~p failed: ~p", [File, Reason]) - end - end, RelupFiles). +inject_dir(Dir) -> + RelupFiles = filelib:wildcard(filename:join([Dir, "**", "relup"])), + lists:foreach(fun inject_file/1, RelupFiles). -inject_relup_instrs(CurrRelVsn, RUs) -> - [{Vsn, Desc, append_emqx_relup_instrs(CurrRelVsn, Vsn, Instrs)} || {Vsn, Desc, Instrs} <- RUs]. +inject_file(File) -> + case file:script(File) of + {ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} -> + ?INFO("injecting instructions to: ~p", [File]), + UpdatedContent = {CurrRelVsn, inject_relup_instrs(up, CurrRelVsn, UpVsnRUs), + inject_relup_instrs(down, CurrRelVsn, DnVsnRUs)}, + ok = file:write_file(File, term_to_text(UpdatedContent)); + {ok, _BadFormat} -> + ?ERROR("bad formatted relup file: ~p", [File]); + {error, Reason} -> + ?ERROR("read relup file ~p failed: ~p", [File, Reason]) + end. + +inject_relup_instrs(Type, CurrRelVsn, RUs) -> + [{Vsn, Desc, append_emqx_relup_instrs(Type, CurrRelVsn, Vsn, Instrs)} || {Vsn, Desc, Instrs} <- RUs]. %% The `{apply, emqx_relup, post_release_upgrade, []}` will be appended to the end of %% the instruction lists. -append_emqx_relup_instrs(CurrRelVsn, Vsn, Instrs) -> +append_emqx_relup_instrs(Type, CurrRelVsn, Vsn, Instrs) -> + CallbackFun = relup_callback_func(Type), case lists:reverse(Instrs) of - [{apply, {emqx_relup, post_release_upgrade, _}} | _] -> + [{apply, {emqx_relup, CallbackFun, _}} | _] -> Instrs; RInstrs -> - lists:reverse([instr_post_release_upgrade(CurrRelVsn, Vsn) | RInstrs]) + lists:reverse([ {load_object_code, {emqx, CurrRelVsn, [emqx_relup]}} + , {load, {emqx_relup, brutal_purge, soft_purge}} + , {apply, {emqx_relup, CallbackFun, [CurrRelVsn, Vsn]}} + | RInstrs]) end. -instr_post_release_upgrade(CurrRelVsn, Vsn) -> - {apply, {emqx_relup, post_release_upgrade, [CurrRelVsn, Vsn]}}. +relup_callback_func(up) -> post_release_upgrade; +relup_callback_func(down) -> post_release_downgrade. term_to_text(Term) -> - io_lib:format("~p.", [Term]). \ No newline at end of file + io_lib:format("~p.", [Term]). From 17440b2b53da924343e8d58c64d29fa3ad70630d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 27 Feb 2022 10:23:14 +0800 Subject: [PATCH 234/363] fix(relup): add module emqx_relup --- scripts/inject-relup.escript | 3 ++- src/emqx_relup.erl | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 src/emqx_relup.erl diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript index 86c541ada..e414b9c31 100755 --- a/scripts/inject-relup.escript +++ b/scripts/inject-relup.escript @@ -49,13 +49,14 @@ inject_relup_instrs(Type, CurrRelVsn, RUs) -> %% the instruction lists. append_emqx_relup_instrs(Type, CurrRelVsn, Vsn, Instrs) -> CallbackFun = relup_callback_func(Type), + Extra = #{}, %% we may need some extended args case lists:reverse(Instrs) of [{apply, {emqx_relup, CallbackFun, _}} | _] -> Instrs; RInstrs -> lists:reverse([ {load_object_code, {emqx, CurrRelVsn, [emqx_relup]}} , {load, {emqx_relup, brutal_purge, soft_purge}} - , {apply, {emqx_relup, CallbackFun, [CurrRelVsn, Vsn]}} + , {apply, {emqx_relup, CallbackFun, [CurrRelVsn, Vsn, Extra]}} | RInstrs]) end. diff --git a/src/emqx_relup.erl b/src/emqx_relup.erl new file mode 100644 index 000000000..e844ea0ba --- /dev/null +++ b/src/emqx_relup.erl @@ -0,0 +1,17 @@ +-module(emqx_relup). + +-export([ post_release_upgrade/3 + , post_release_downgrade/3 + ]). + +post_release_upgrade(_CurrRelVsn, _FromVsn, _) -> + reload_components(). + +post_release_downgrade(_CurrRelVsn, _ToVsn, _) -> + reload_components(). + +reload_components() -> + io:format("reloading resource providers ..."), + emqx_rule_engine:load_providers(), + io:format("loading plugins ..."), + emqx_plugins:load(). From 28bd2fcfa463dc04a01bdb07d11521e51f0294d1 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 27 Feb 2022 20:04:21 +0800 Subject: [PATCH 235/363] fix(relup): release upgrade failed with {bad_lib_vsn,emqx,"4.4.2"} --- scripts/inject-relup.escript | 46 +++++++++++++++++++++++++++--------- src/emqx_relup.erl | 14 +++++++++-- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript index e414b9c31..1a2dae78a 100755 --- a/scripts/inject-relup.escript +++ b/scripts/inject-relup.escript @@ -30,38 +30,62 @@ inject_dir(Dir) -> lists:foreach(fun inject_file/1, RelupFiles). inject_file(File) -> + EmqxVsn = emqx_vsn_from_rel_file(File), case file:script(File) of {ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} -> ?INFO("injecting instructions to: ~p", [File]), - UpdatedContent = {CurrRelVsn, inject_relup_instrs(up, CurrRelVsn, UpVsnRUs), - inject_relup_instrs(down, CurrRelVsn, DnVsnRUs)}, + UpdatedContent = {CurrRelVsn, inject_relup_instrs(up, EmqxVsn, CurrRelVsn, UpVsnRUs), + inject_relup_instrs(down, EmqxVsn, CurrRelVsn, DnVsnRUs)}, ok = file:write_file(File, term_to_text(UpdatedContent)); {ok, _BadFormat} -> - ?ERROR("bad formatted relup file: ~p", [File]); + ?ERROR("bad formatted relup file: ~p", [File]), + error({bad_relup_format, File}); {error, Reason} -> - ?ERROR("read relup file ~p failed: ~p", [File, Reason]) + ?ERROR("read relup file ~p failed: ~p", [File, Reason]), + error({read_relup_error, Reason}) end. -inject_relup_instrs(Type, CurrRelVsn, RUs) -> - [{Vsn, Desc, append_emqx_relup_instrs(Type, CurrRelVsn, Vsn, Instrs)} || {Vsn, Desc, Instrs} <- RUs]. +inject_relup_instrs(Type, EmqxVsn, CurrRelVsn, RUs) -> + [{Vsn, Desc, append_emqx_relup_instrs(Type, EmqxVsn, CurrRelVsn, Vsn, Instrs)} + || {Vsn, Desc, Instrs} <- RUs]. %% The `{apply, emqx_relup, post_release_upgrade, []}` will be appended to the end of %% the instruction lists. -append_emqx_relup_instrs(Type, CurrRelVsn, Vsn, Instrs) -> +append_emqx_relup_instrs(Type, EmqxVsn, CurrRelVsn, Vsn, Instrs) -> CallbackFun = relup_callback_func(Type), Extra = #{}, %% we may need some extended args case lists:reverse(Instrs) of [{apply, {emqx_relup, CallbackFun, _}} | _] -> Instrs; RInstrs -> - lists:reverse([ {load_object_code, {emqx, CurrRelVsn, [emqx_relup]}} - , {load, {emqx_relup, brutal_purge, soft_purge}} - , {apply, {emqx_relup, CallbackFun, [CurrRelVsn, Vsn, Extra]}} - | RInstrs]) + Instrs2 = lists:reverse( + [ {apply, {emqx_relup, CallbackFun, [CurrRelVsn, Vsn, Extra]}} + , {load, {emqx_relup, brutal_purge, soft_purge}} + | RInstrs]), + %% we have to put 'load_object_code' before 'point_of_no_return' + %% so here we simply put it to the beginning + [{load_object_code, {emqx, EmqxVsn, [emqx_relup]}} | Instrs2] end. relup_callback_func(up) -> post_release_upgrade; relup_callback_func(down) -> post_release_downgrade. +emqx_vsn_from_rel_file(RelupFile) -> + RelDir = filename:dirname(RelupFile), + RelFile = filename:join([RelDir, "emqx.rel"]), + case file:script(RelFile) of + {ok, {release, {_RelName, _RelVsn}, _Erts, Apps}} -> + case lists:keysearch(emqx, 1, Apps) of + {value, {emqx, EmqxVsn}} -> + EmqxVsn; + false -> + error({emqx_vsn_cannot_found, RelFile}) + end; + {ok, _BadFormat} -> + ?ERROR("bad formatted .rel file: ~p", [RelFile]); + {error, Reason} -> + ?ERROR("read .rel file ~p failed: ~p", [RelFile, Reason]) + end. + term_to_text(Term) -> io_lib:format("~p.", [Term]). diff --git a/src/emqx_relup.erl b/src/emqx_relup.erl index e844ea0ba..b435f8ae8 100644 --- a/src/emqx_relup.erl +++ b/src/emqx_relup.erl @@ -10,8 +10,18 @@ post_release_upgrade(_CurrRelVsn, _FromVsn, _) -> post_release_downgrade(_CurrRelVsn, _ToVsn, _) -> reload_components(). +-ifdef(EMQX_ENTERPRISE). reload_components() -> - io:format("reloading resource providers ..."), + io:format("reloading resource providers ...~n"), emqx_rule_engine:load_providers(), - io:format("loading plugins ..."), + io:format("reloading module providers ...~n"), + emqx_modules:load_providers(), + io:format("loading plugins ...~n"), emqx_plugins:load(). +-else. +reload_components() -> + io:format("reloading resource providers ...~n"), + emqx_rule_engine:load_providers(), + io:format("loading plugins ...~n"), + emqx_plugins:load(). +-endif. From bbbb0edca11f422217e7a01bf68295befca2aaea Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 27 Feb 2022 14:54:29 +0100 Subject: [PATCH 236/363] chore: skip downloading upgrade base images for windows because we do not support relup for windows for now --- scripts/relup-base-packages.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index f86185e0a..2b2f7e92f 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -44,6 +44,10 @@ esac case "$SYSTEM" in + windows*) + echo "WARNING: skipped downloading relup base for windows because we do not support relup for windows yet." + exit 0 + ;; macos*) SHASUM="shasum -a 256" ;; From b7d07d7a96f30b29e9611cf795b69b6e54456563 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 28 Feb 2022 10:05:41 +0800 Subject: [PATCH 237/363] fix(appup): remove the post relup scripts from emqx_dashboard.appup.src The instructions have been move into module `emqx_relup`. --- lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src index 902585ffb..270c65b5e 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src @@ -1,18 +1,11 @@ %% -*- mode: erlang -*- {VSN, [ {<<".*">>, - %% load all plugins - %% NOTE: this depends on the fact that emqx_dashboard is always - %% the last application gets upgraded - [ {apply, {emqx_rule_engine, load_providers, []}} - , {restart_application, emqx_dashboard} - , {apply, {emqx_plugins, load, []}} + [ {restart_application, emqx_dashboard} ]} ], [ {<<".*">>, - [ {apply, {emqx_rule_engine, load_providers, []}} - , {restart_application, emqx_dashboard} - , {apply, {emqx_plugins, load, []}} + [ {restart_application, emqx_dashboard} ]} ] -}. +}. \ No newline at end of file From f25b8801b4a83e7a15e2a6a574f3d85460bc7336 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 28 Feb 2022 19:07:38 +0800 Subject: [PATCH 238/363] fix(build): also make SYSTEM configurable by env variable --- build | 2 +- scripts/pkg-full-vsn.sh | 2 +- scripts/relup-base-packages.sh | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build b/build index a339f803b..b4c9d0548 100755 --- a/build +++ b/build @@ -18,7 +18,7 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN -SYSTEM="$(./scripts/get-distro.sh)" +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" ## ## Support RPM and Debian based linux systems diff --git a/scripts/pkg-full-vsn.sh b/scripts/pkg-full-vsn.sh index de32e11ef..e118643c9 100755 --- a/scripts/pkg-full-vsn.sh +++ b/scripts/pkg-full-vsn.sh @@ -27,7 +27,7 @@ esac cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" -SYSTEM="$(./scripts/get-distro.sh)" +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" UNAME="$(uname -m)" case "$UNAME" in diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 2a47b5135..070f3926a 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -62,10 +62,11 @@ pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip" url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename" - echo "downloading base package from ${url} ..." if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then + echo "downloading base package from ${url} ..." curl -L -o "${filename}" "${url}" if [ "$SYSTEM" != "centos6" ]; then + echo "downloading sha256 sum from ${url}.sha256 ..." curl -L -o "${filename}.sha256" "${url}.sha256" SUMSTR=$(cat "${filename}.sha256") echo "got sha265sum: ${SUMSTR}" From a4523995d24365d7dd1a5f66aee18fe20ed630b5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Feb 2022 18:52:16 +0100 Subject: [PATCH 239/363] build: git clone silent --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 08a7a615f..23ecd0cfb 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ REL_PROFILES := emqx emqx-edge PKG_PROFILES := emqx-pkg emqx-edge-pkg PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default -export REBAR_GIT_CLONE_OPTIONS += --depth=1 +export REBAR_GIT_CLONE_OPTIONS += --depth=1 --quiet .PHONY: default default: $(REBAR) $(PROFILE) From fe597cd3f9a9a0c1c834fd31a76c36ffa4d02ccb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Feb 2022 18:52:43 +0100 Subject: [PATCH 240/363] ci: delete ci step which creates sha256sum the latest build commands create sha256 sum right after the build e.g. command `make emqx-zip` creates two files: - emqx-.zip - emqx-.zip.sha256 --- .github/workflows/build_packages.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 69da9aa59..c9969a5da 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -266,18 +266,6 @@ jobs: --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ --builder "ghcr.io/emqx/emqx-builder/4.4-5:${OTP}-${SYSTEM}" - - name: create sha256 - working-directory: source - env: - PROFILE: ${{ matrix.profile}} - run: | - if [ -d _packages/$PROFILE ]; then - cd _packages/$PROFILE - for var in $(ls emqx-* ); do - sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" - done - cd - - fi - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: From a7791b6c586f238a5b2f172aff01ec480bccb477 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 1 Mar 2022 01:58:55 +0800 Subject: [PATCH 241/363] fix(relup): download to 4.4.0 failed with error bad_lib_vsn --- rebar.config.erl | 2 +- scripts/inject-relup.escript | 175 +++++++++++++++++++++-------------- src/emqx_relup.erl | 10 +- 3 files changed, 112 insertions(+), 75 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index 447e58f97..32e3444d1 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -129,7 +129,7 @@ prod_overrides() -> [{add, [ {erl_opts, [deterministic]}]}]. relup_deps(Profile, Vsn) -> - InjectCmd = "scripts/inject-relup.escript " ++ filename:join(["_build", Profile, "rel", "emqx", "releases", Vsn]), + InjectCmd = "scripts/inject-relup.escript " ++ filename:join(["_build", Profile, "rel", "emqx"]) ++ " " ++ Vsn, {post_hooks, [ {"(linux|darwin|solaris|freebsd|netbsd|openbsd)", relup, InjectCmd} ] diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript index 1a2dae78a..b860ab0d9 100755 --- a/scripts/inject-relup.escript +++ b/scripts/inject-relup.escript @@ -4,88 +4,123 @@ -mode(compile). --define(ERROR(FORMAT, ARGS), io:format(standard_error, "[inject-relup] " ++ FORMAT ++"~n", ARGS)). --define(INFO(FORMAT, ARGS), io:format(user, "[inject-relup] " ++ FORMAT ++"~n", ARGS)). +-define(ERROR(FORMAT, ARGS), io:format(standard_error, "[inject-relup] " ++ FORMAT ++ "~n", ARGS)). +-define(INFO(FORMAT, ARGS), io:format(user, "[inject-relup] " ++ FORMAT ++ "~n", ARGS)). usage() -> - "Usage: " ++ escript:script_name() ++ " ". + "Usage: " ++ escript:script_name() ++ " ". -main([DirOrFile]) -> - case filelib:is_dir(DirOrFile) of - true -> ok = inject_dir(DirOrFile); - false -> - case filelib:is_regular(DirOrFile) of - true -> inject_file(DirOrFile); +main([RelRootDir, CurrRelVsn]) -> + case filelib:is_dir(filename:join([RelRootDir, "releases"])) andalso + filelib:is_dir(filename:join([RelRootDir, "lib"])) of + true -> + EmqxAppVsns = get_emqx_app_vsns(RelRootDir), + ok = inject_relup_file(RelRootDir, CurrRelVsn, EmqxAppVsns); false -> - ?ERROR("not a valid file: ~p", [DirOrFile]), - erlang:halt(1) - end - end; + ?ERROR("not a valid root dir of release: ~p, for example: _build/emqx/rel/emqx", + [RelRootDir]), + erlang:halt(1) + end; main(_Args) -> - ?ERROR("~s", [usage()]), - erlang:halt(1). + ?ERROR("~s", [usage()]), + erlang:halt(1). -inject_dir(Dir) -> - RelupFiles = filelib:wildcard(filename:join([Dir, "**", "relup"])), - lists:foreach(fun inject_file/1, RelupFiles). +inject_relup_file(RelRootDir, CurrRelVsn, EmqxAppVsns) -> + RelupFile = filename:join([RelRootDir, "releases", CurrRelVsn, "relup"]), + inject_file(RelupFile, EmqxAppVsns). -inject_file(File) -> - EmqxVsn = emqx_vsn_from_rel_file(File), - case file:script(File) of - {ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} -> - ?INFO("injecting instructions to: ~p", [File]), - UpdatedContent = {CurrRelVsn, inject_relup_instrs(up, EmqxVsn, CurrRelVsn, UpVsnRUs), - inject_relup_instrs(down, EmqxVsn, CurrRelVsn, DnVsnRUs)}, - ok = file:write_file(File, term_to_text(UpdatedContent)); - {ok, _BadFormat} -> - ?ERROR("bad formatted relup file: ~p", [File]), - error({bad_relup_format, File}); - {error, Reason} -> - ?ERROR("read relup file ~p failed: ~p", [File, Reason]), - error({read_relup_error, Reason}) - end. +inject_file(File, EmqxAppVsns) -> + case file:script(File) of + {ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} -> + ?INFO("injecting instructions to: ~p", [File]), + UpdatedContent = {CurrRelVsn, + inject_relup_instrs(up, EmqxAppVsns, CurrRelVsn, UpVsnRUs), + inject_relup_instrs(down, EmqxAppVsns, CurrRelVsn, DnVsnRUs)}, + file:write_file(File, term_to_text(UpdatedContent)); + {ok, _BadFormat} -> + ?ERROR("bad formatted relup file: ~p", [File]), + error({bad_relup_format, File}); + {error, enoent} -> + ?INFO("relup file not found: ~p", [File]), + ok; + {error, Reason} -> + ?ERROR("read relup file ~p failed: ~p", [File, Reason]), + error({read_relup_error, Reason}) + end. -inject_relup_instrs(Type, EmqxVsn, CurrRelVsn, RUs) -> - [{Vsn, Desc, append_emqx_relup_instrs(Type, EmqxVsn, CurrRelVsn, Vsn, Instrs)} - || {Vsn, Desc, Instrs} <- RUs]. +inject_relup_instrs(Type, EmqxAppVsns, CurrRelVsn, RUs) -> + lists:map(fun + ({Vsn, "(relup-injected) " ++ _ = Desc, Instrs}) -> %% already injected + {Vsn, Desc, Instrs}; + ({Vsn, Desc, Instrs}) -> + {Vsn, "(relup-injected) " ++ Desc, + append_emqx_relup_instrs(Type, EmqxAppVsns, CurrRelVsn, Vsn, Instrs)} + end, RUs). %% The `{apply, emqx_relup, post_release_upgrade, []}` will be appended to the end of %% the instruction lists. -append_emqx_relup_instrs(Type, EmqxVsn, CurrRelVsn, Vsn, Instrs) -> - CallbackFun = relup_callback_func(Type), - Extra = #{}, %% we may need some extended args - case lists:reverse(Instrs) of - [{apply, {emqx_relup, CallbackFun, _}} | _] -> - Instrs; - RInstrs -> - Instrs2 = lists:reverse( - [ {apply, {emqx_relup, CallbackFun, [CurrRelVsn, Vsn, Extra]}} - , {load, {emqx_relup, brutal_purge, soft_purge}} - | RInstrs]), - %% we have to put 'load_object_code' before 'point_of_no_return' - %% so here we simply put it to the beginning - [{load_object_code, {emqx, EmqxVsn, [emqx_relup]}} | Instrs2] - end. +append_emqx_relup_instrs(up, EmqxAppVsns, CurrRelVsn, FromRelVsn, Instrs) -> + {EmqxVsn, true} = maps:get(CurrRelVsn, EmqxAppVsns), + Extra = #{}, %% we may need some extended args + %% we have to put 'load_object_code' before 'point_of_no_return' + %% so here we simply put it to the beginning + Instrs0 = [ {load_object_code, {emqx, EmqxVsn, [emqx_relup]}} + | Instrs], + Instrs0 ++ + [ {load, {emqx_relup, brutal_purge, soft_purge}} + , {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, Extra]}} + ]; -relup_callback_func(up) -> post_release_upgrade; -relup_callback_func(down) -> post_release_downgrade. +append_emqx_relup_instrs(down, EmqxAppVsns, _CurrRelVsn, ToRelVsn, Instrs) -> + Extra = #{}, %% we may need some extended args + case maps:get(ToRelVsn, EmqxAppVsns) of + {EmqxVsn, true} -> + Instrs0 = [ {load_object_code, {emqx, EmqxVsn, [emqx_relup]}} + | Instrs], + Instrs0 ++ + [ {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, Extra]}} + , {load, {emqx_relup, brutal_purge, soft_purge}} + ]; + {_EmqxVsn, false} -> + Instrs ++ + [ {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, Extra]}} + , {remove, {emqx_relup, brutal_purge, soft_purge}} + ] + end. -emqx_vsn_from_rel_file(RelupFile) -> - RelDir = filename:dirname(RelupFile), - RelFile = filename:join([RelDir, "emqx.rel"]), - case file:script(RelFile) of - {ok, {release, {_RelName, _RelVsn}, _Erts, Apps}} -> - case lists:keysearch(emqx, 1, Apps) of - {value, {emqx, EmqxVsn}} -> - EmqxVsn; - false -> - error({emqx_vsn_cannot_found, RelFile}) - end; - {ok, _BadFormat} -> - ?ERROR("bad formatted .rel file: ~p", [RelFile]); - {error, Reason} -> - ?ERROR("read .rel file ~p failed: ~p", [RelFile, Reason]) - end. +get_emqx_app_vsns(RelRootDir) -> + RelFiles = filelib:wildcard(filename:join([RelRootDir, "releases", "*", "emqx.rel"])), + lists:foldl(fun(RelFile, AppVsns) -> + {ok, RelVsn, EmqxVsn} = read_emqx_vsn_from_rel_file(RelFile), + AppVsns#{RelVsn => {EmqxVsn, has_relup_module(RelRootDir, EmqxVsn)}} + end, #{}, RelFiles). + +read_emqx_vsn_from_rel_file(RelFile) -> + case file:script(RelFile) of + {ok, {release, {_RelName, RelVsn}, _Erts, Apps}} -> + case lists:keysearch(emqx, 1, Apps) of + {value, {emqx, EmqxVsn}} -> + {ok, RelVsn, EmqxVsn}; + false -> + error({emqx_vsn_cannot_found, RelFile}) + end; + {ok, _BadFormat} -> + ?ERROR("bad formatted .rel file: ~p", [RelFile]); + {error, Reason} -> + ?ERROR("read .rel file ~p failed: ~p", [RelFile, Reason]) + end. + +has_relup_module(RelRootDir, EmqxVsn) -> + AppFile = filename:join([RelRootDir, "lib", "emqx-" ++ EmqxVsn, "ebin", "emqx.app"]), + case file:script(AppFile) of + {ok, {application, emqx, AppInfo}} -> + {value, {_, EmqxVsn}} = lists:keysearch(vsn, 1, AppInfo), %% assert + {value, {_, Modules}} = lists:keysearch(modules, 1, AppInfo), + lists:member(emqx_relup, Modules); + {error, Reason} -> + ?ERROR("read .app file ~p failed: ~p", [AppFile, Reason]), + error({read_app_file_error, AppFile, Reason}) + end. term_to_text(Term) -> - io_lib:format("~p.", [Term]). + io_lib:format("~p.", [Term]). diff --git a/src/emqx_relup.erl b/src/emqx_relup.erl index b435f8ae8..66a5281da 100644 --- a/src/emqx_relup.erl +++ b/src/emqx_relup.erl @@ -1,13 +1,15 @@ -module(emqx_relup). --export([ post_release_upgrade/3 - , post_release_downgrade/3 +-export([ post_release_upgrade/2 + , post_release_downgrade/2 ]). -post_release_upgrade(_CurrRelVsn, _FromVsn, _) -> +%% what to do after upgraded from a old release vsn. +post_release_upgrade(_FromRelVsn, _) -> reload_components(). -post_release_downgrade(_CurrRelVsn, _ToVsn, _) -> +%% what to do after downgraded to a old release vsn. +post_release_downgrade(_ToRelVsn, _) -> reload_components(). -ifdef(EMQX_ENTERPRISE). From 769e79e2cd99f3e5a89f4a59574bfa6c49e6acd7 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 1 Mar 2022 15:09:04 +0800 Subject: [PATCH 242/363] chore(relup): also inject emqx_app automatically --- scripts/inject-relup.escript | 56 +++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript index b860ab0d9..3a779d239 100755 --- a/scripts/inject-relup.escript +++ b/scripts/inject-relup.escript @@ -59,32 +59,54 @@ inject_relup_instrs(Type, EmqxAppVsns, CurrRelVsn, RUs) -> %% The `{apply, emqx_relup, post_release_upgrade, []}` will be appended to the end of %% the instruction lists. -append_emqx_relup_instrs(up, EmqxAppVsns, CurrRelVsn, FromRelVsn, Instrs) -> +append_emqx_relup_instrs(up, EmqxAppVsns, CurrRelVsn, FromRelVsn, Instrs0) -> {EmqxVsn, true} = maps:get(CurrRelVsn, EmqxAppVsns), Extra = #{}, %% we may need some extended args + LoadObjEmqxMods = {load_object_code, {emqx, EmqxVsn, [emqx_relup, emqx_app]}}, + LoadCodeEmqxRelup = {load, {emqx_relup, brutal_purge, soft_purge}}, + LoadCodeEmqxApp = {load, {emqx_app, brutal_purge, soft_purge}}, + ApplyEmqxRelup = {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, Extra]}}, + Instrs1 = Instrs0 -- [LoadCodeEmqxRelup, LoadCodeEmqxApp], %% we have to put 'load_object_code' before 'point_of_no_return' - %% so here we simply put it to the beginning - Instrs0 = [ {load_object_code, {emqx, EmqxVsn, [emqx_relup]}} - | Instrs], - Instrs0 ++ - [ {load, {emqx_relup, brutal_purge, soft_purge}} - , {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, Extra]}} + %% so here we simply put them to the beginning of the instruction list + Instrs2 = [ LoadObjEmqxMods + | Instrs1], + %% the `load` must be put after the 'point_of_no_return' + Instrs2 ++ + [ LoadCodeEmqxRelup + , LoadCodeEmqxApp + , ApplyEmqxRelup ]; -append_emqx_relup_instrs(down, EmqxAppVsns, _CurrRelVsn, ToRelVsn, Instrs) -> +append_emqx_relup_instrs(down, EmqxAppVsns, _CurrRelVsn, ToRelVsn, Instrs0) -> Extra = #{}, %% we may need some extended args + ApplyEmqxRelup = {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, Extra]}}, case maps:get(ToRelVsn, EmqxAppVsns) of {EmqxVsn, true} -> - Instrs0 = [ {load_object_code, {emqx, EmqxVsn, [emqx_relup]}} - | Instrs], - Instrs0 ++ - [ {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, Extra]}} - , {load, {emqx_relup, brutal_purge, soft_purge}} + LoadObjEmqxMods = {load_object_code, {emqx, EmqxVsn, [emqx_relup, emqx_app]}}, + LoadCodeEmqxRelup = {load, {emqx_relup, brutal_purge, soft_purge}}, + LoadCodeEmqxApp = {load, {emqx_app, brutal_purge, soft_purge}}, + Instrs1 = Instrs0 -- [LoadCodeEmqxRelup, LoadCodeEmqxApp, ApplyEmqxRelup], + Instrs2 = [ LoadObjEmqxMods + | Instrs1], + %% NOTE: We apply emqx_relup:post_release_downgrade/2 first, and then reload + %% the old vsn code of emqx_relup. + Instrs2 ++ + [ LoadCodeEmqxApp + , ApplyEmqxRelup + , LoadCodeEmqxRelup ]; - {_EmqxVsn, false} -> - Instrs ++ - [ {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, Extra]}} - , {remove, {emqx_relup, brutal_purge, soft_purge}} + {EmqxVsn, false} -> + LoadObjEmqxApp = {load_object_code, {emqx, EmqxVsn, [emqx_app]}}, + LoadCodeEmqxApp = {load, {emqx_app, brutal_purge, soft_purge}}, + RemoveCodeEmqxRelup = {remove, {emqx_relup, brutal_purge, soft_purge}}, + Instrs1 = Instrs0 -- [LoadCodeEmqxApp, RemoveCodeEmqxRelup, ApplyEmqxRelup], + Instrs2 = [ LoadObjEmqxApp + | Instrs1], + Instrs2 ++ + [ LoadCodeEmqxApp + , ApplyEmqxRelup + , RemoveCodeEmqxRelup ] end. From 7698ad7c4b32701e8e3b3a25435d8d7d9bb135a1 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 1 Mar 2022 17:15:19 +0800 Subject: [PATCH 243/363] chore(relup): print more info after release upgrade complete --- src/emqx_relup.erl | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/emqx_relup.erl b/src/emqx_relup.erl index 66a5281da..cb71c6305 100644 --- a/src/emqx_relup.erl +++ b/src/emqx_relup.erl @@ -4,26 +4,31 @@ , post_release_downgrade/2 ]). +-define(INFO(FORMAT), io:format("[emqx_relup] " ++ FORMAT ++ "~n")). +-define(INFO(FORMAT, ARGS), io:format("[emqx_relup] " ++ FORMAT ++ "~n", ARGS)). + %% what to do after upgraded from a old release vsn. post_release_upgrade(_FromRelVsn, _) -> + ?INFO("emqx has been upgraded to ~s", [emqx_app:get_release()]), reload_components(). %% what to do after downgraded to a old release vsn. post_release_downgrade(_ToRelVsn, _) -> + ?INFO("emqx has been downgrade to ~s", [emqx_app:get_release()]), reload_components(). -ifdef(EMQX_ENTERPRISE). reload_components() -> - io:format("reloading resource providers ...~n"), + ?INFO("reloading resource providers ..."), emqx_rule_engine:load_providers(), - io:format("reloading module providers ...~n"), + ?INFO("reloading module providers ..."), emqx_modules:load_providers(), - io:format("loading plugins ...~n"), + ?INFO("loading plugins ..."), emqx_plugins:load(). -else. reload_components() -> - io:format("reloading resource providers ...~n"), + ?INFO("reloading resource providers ..."), emqx_rule_engine:load_providers(), - io:format("loading plugins ...~n"), + ?INFO("loading plugins ..."), emqx_plugins:load(). -endif. From 41afbd2b13ddcbd77c911b9193c4cfa7a7e2f1d4 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 1 Mar 2022 22:01:00 +0800 Subject: [PATCH 244/363] refactor(relup): don't collect relvsn -> appvsn mappings --- build | 3 + .../src/emqx_dashboard.appup.src | 2 +- rebar.config.erl | 11 -- scripts/inject-relup.escript | 165 +++++++----------- src/emqx.appup.src | 14 +- src/emqx_relup.erl | 30 +++- 6 files changed, 107 insertions(+), 118 deletions(-) diff --git a/build b/build index b4c9d0548..8bf435357 100755 --- a/build +++ b/build @@ -136,6 +136,9 @@ make_zip() { local tarball="${relpath}/${tarname}" local target_zip="${pkgpath}/${pkgname}" tar zxf "${tarball}" -C "${tard}/emqx" + if ! [[ $SYSTEM == windows* ]]; then + ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup" + fi cp_dyn_libs "${tard}/emqx" pushd "${tard}" >/dev/null case "$SYSTEM" in diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src index 270c65b5e..513757418 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src @@ -8,4 +8,4 @@ [ {restart_application, emqx_dashboard} ]} ] -}. \ No newline at end of file +}. diff --git a/rebar.config.erl b/rebar.config.erl index 32e3444d1..492707867 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -128,34 +128,23 @@ prod_compile_opts() -> prod_overrides() -> [{add, [ {erl_opts, [deterministic]}]}]. -relup_deps(Profile, Vsn) -> - InjectCmd = "scripts/inject-relup.escript " ++ filename:join(["_build", Profile, "rel", "emqx"]) ++ " " ++ Vsn, - {post_hooks, - [ {"(linux|darwin|solaris|freebsd|netbsd|openbsd)", relup, InjectCmd} - ] - }. - profiles() -> Vsn = get_vsn(), [ {'emqx', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin)} , {overrides, prod_overrides()} - , relup_deps('emqx', Vsn) ]} , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg)} , {overrides, prod_overrides()} - , relup_deps('emqx-pkg', Vsn) ]} , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin)} , {overrides, prod_overrides()} - , relup_deps('emqx-edge', Vsn) ]} , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg)} , {overrides, prod_overrides()} - , relup_deps('emqx-edge-pkg', Vsn) ]} , {check, [ {erl_opts, common_compile_opts()} ]} diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript index 3a779d239..93fe34a90 100755 --- a/scripts/inject-relup.escript +++ b/scripts/inject-relup.escript @@ -8,34 +8,27 @@ -define(INFO(FORMAT, ARGS), io:format(user, "[inject-relup] " ++ FORMAT ++ "~n", ARGS)). usage() -> - "Usage: " ++ escript:script_name() ++ " ". + "Usage: " ++ escript:script_name() ++ " ". -main([RelRootDir, CurrRelVsn]) -> - case filelib:is_dir(filename:join([RelRootDir, "releases"])) andalso - filelib:is_dir(filename:join([RelRootDir, "lib"])) of +main([RelupFile]) -> + case filelib:is_regular(RelupFile) of true -> - EmqxAppVsns = get_emqx_app_vsns(RelRootDir), - ok = inject_relup_file(RelRootDir, CurrRelVsn, EmqxAppVsns); + ok = inject_relup_file(RelupFile); false -> - ?ERROR("not a valid root dir of release: ~p, for example: _build/emqx/rel/emqx", - [RelRootDir]), + ?ERROR("not a valid file: ~p", [RelupFile]), erlang:halt(1) end; main(_Args) -> ?ERROR("~s", [usage()]), erlang:halt(1). -inject_relup_file(RelRootDir, CurrRelVsn, EmqxAppVsns) -> - RelupFile = filename:join([RelRootDir, "releases", CurrRelVsn, "relup"]), - inject_file(RelupFile, EmqxAppVsns). - -inject_file(File, EmqxAppVsns) -> +inject_relup_file(File) -> case file:script(File) of {ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} -> ?INFO("injecting instructions to: ~p", [File]), UpdatedContent = {CurrRelVsn, - inject_relup_instrs(up, EmqxAppVsns, CurrRelVsn, UpVsnRUs), - inject_relup_instrs(down, EmqxAppVsns, CurrRelVsn, DnVsnRUs)}, + inject_relup_instrs(up, UpVsnRUs), + inject_relup_instrs(down, DnVsnRUs)}, file:write_file(File, term_to_text(UpdatedContent)); {ok, _BadFormat} -> ?ERROR("bad formatted relup file: ~p", [File]), @@ -48,101 +41,77 @@ inject_file(File, EmqxAppVsns) -> error({read_relup_error, Reason}) end. -inject_relup_instrs(Type, EmqxAppVsns, CurrRelVsn, RUs) -> - lists:map(fun - ({Vsn, "(relup-injected) " ++ _ = Desc, Instrs}) -> %% already injected - {Vsn, Desc, Instrs}; - ({Vsn, Desc, Instrs}) -> - {Vsn, "(relup-injected) " ++ Desc, - append_emqx_relup_instrs(Type, EmqxAppVsns, CurrRelVsn, Vsn, Instrs)} +inject_relup_instrs(Type, RUs) -> + lists:map(fun({Vsn, Desc, Instrs}) -> + {Vsn, Desc, append_emqx_relup_instrs(Type, Vsn, Instrs)} end, RUs). %% The `{apply, emqx_relup, post_release_upgrade, []}` will be appended to the end of %% the instruction lists. -append_emqx_relup_instrs(up, EmqxAppVsns, CurrRelVsn, FromRelVsn, Instrs0) -> - {EmqxVsn, true} = maps:get(CurrRelVsn, EmqxAppVsns), +append_emqx_relup_instrs(up, FromRelVsn, Instrs0) -> Extra = #{}, %% we may need some extended args - LoadObjEmqxMods = {load_object_code, {emqx, EmqxVsn, [emqx_relup, emqx_app]}}, - LoadCodeEmqxRelup = {load, {emqx_relup, brutal_purge, soft_purge}}, - LoadCodeEmqxApp = {load, {emqx_app, brutal_purge, soft_purge}}, - ApplyEmqxRelup = {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, Extra]}}, - Instrs1 = Instrs0 -- [LoadCodeEmqxRelup, LoadCodeEmqxApp], - %% we have to put 'load_object_code' before 'point_of_no_return' - %% so here we simply put them to the beginning of the instruction list - Instrs2 = [ LoadObjEmqxMods - | Instrs1], - %% the `load` must be put after the 'point_of_no_return' - Instrs2 ++ - [ LoadCodeEmqxRelup - , LoadCodeEmqxApp - , ApplyEmqxRelup + filter_and_check_instrs(up, Instrs0) ++ + [ {load, {emqx_app, brutal_purge, soft_purge}} + , {load, {emqx_relup, brutal_purge, soft_purge}} + , {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, Extra]}} ]; -append_emqx_relup_instrs(down, EmqxAppVsns, _CurrRelVsn, ToRelVsn, Instrs0) -> +append_emqx_relup_instrs(down, ToRelVsn, Instrs0) -> Extra = #{}, %% we may need some extended args - ApplyEmqxRelup = {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, Extra]}}, - case maps:get(ToRelVsn, EmqxAppVsns) of - {EmqxVsn, true} -> - LoadObjEmqxMods = {load_object_code, {emqx, EmqxVsn, [emqx_relup, emqx_app]}}, - LoadCodeEmqxRelup = {load, {emqx_relup, brutal_purge, soft_purge}}, - LoadCodeEmqxApp = {load, {emqx_app, brutal_purge, soft_purge}}, - Instrs1 = Instrs0 -- [LoadCodeEmqxRelup, LoadCodeEmqxApp, ApplyEmqxRelup], - Instrs2 = [ LoadObjEmqxMods - | Instrs1], - %% NOTE: We apply emqx_relup:post_release_downgrade/2 first, and then reload - %% the old vsn code of emqx_relup. - Instrs2 ++ - [ LoadCodeEmqxApp - , ApplyEmqxRelup - , LoadCodeEmqxRelup - ]; - {EmqxVsn, false} -> - LoadObjEmqxApp = {load_object_code, {emqx, EmqxVsn, [emqx_app]}}, - LoadCodeEmqxApp = {load, {emqx_app, brutal_purge, soft_purge}}, - RemoveCodeEmqxRelup = {remove, {emqx_relup, brutal_purge, soft_purge}}, - Instrs1 = Instrs0 -- [LoadCodeEmqxApp, RemoveCodeEmqxRelup, ApplyEmqxRelup], - Instrs2 = [ LoadObjEmqxApp - | Instrs1], - Instrs2 ++ - [ LoadCodeEmqxApp - , ApplyEmqxRelup - , RemoveCodeEmqxRelup - ] + %% NOTE: When downgrading, we apply emqx_relup:post_release_downgrade/2 before reloading + %% or removing the emqx_relup module. + Instrs1 = filter_and_check_instrs(down, Instrs0) ++ + [ {load, {emqx_app, brutal_purge, soft_purge}} + , {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, Extra]}} + ], + %% emqx_relup does not exist before release "4.4.2" + LoadInsts = + case ToRelVsn of + ToRelVsn when ToRelVsn =:= "4.4.1"; ToRelVsn =:= "4.4.0" -> + [{remove, {emqx_relup, brutal_purge, brutal_purge}}]; + _ -> + [{load, {emqx_relup, brutal_purge, soft_purge}}] + end, + Instrs1 ++ LoadInsts. + +filter_and_check_instrs(Type, Instrs) -> + case take_emqx_vsn_and_modules(Instrs) of + {EmqxAppVsn, EmqxMods, RemainInstrs} when EmqxAppVsn =/= not_found, EmqxMods =/= [] -> + assert_mandatory_modules(Type, EmqxMods), + [{load_object_code, {emqx, EmqxAppVsn, EmqxMods}} | RemainInstrs]; + {_, _, _} -> + ?ERROR("cannot found 'load_module' instructions for app emqx", []), + error({instruction_not_found, load_object_code}) end. -get_emqx_app_vsns(RelRootDir) -> - RelFiles = filelib:wildcard(filename:join([RelRootDir, "releases", "*", "emqx.rel"])), - lists:foldl(fun(RelFile, AppVsns) -> - {ok, RelVsn, EmqxVsn} = read_emqx_vsn_from_rel_file(RelFile), - AppVsns#{RelVsn => {EmqxVsn, has_relup_module(RelRootDir, EmqxVsn)}} - end, #{}, RelFiles). +take_emqx_vsn_and_modules(Instrs) -> + lists:foldl(fun + ({load_object_code, {emqx, AppVsn, Mods}}, {_EmqxAppVsn, EmqxMods, RemainInstrs}) -> + {AppVsn, EmqxMods ++ Mods, RemainInstrs}; + ({load, {Mod, _, _}}, {EmqxAppVsn, EmqxMods, RemainInstrs}) + when Mod =:= emqx_relup; Mod =:= emqx_app -> + {EmqxAppVsn, EmqxMods, RemainInstrs}; + ({remove, {emqx_relup, _, _}}, {EmqxAppVsn, EmqxMods, RemainInstrs}) -> + {EmqxAppVsn, EmqxMods, RemainInstrs}; + ({apply, {emqx_relup, _, _}}, {EmqxAppVsn, EmqxMods, RemainInstrs}) -> + {EmqxAppVsn, EmqxMods, RemainInstrs}; + (Instr, {EmqxAppVsn, EmqxMods, RemainInstrs}) -> + {EmqxAppVsn, EmqxMods, RemainInstrs ++ [Instr]} + end, {not_found, [], []}, Instrs). -read_emqx_vsn_from_rel_file(RelFile) -> - case file:script(RelFile) of - {ok, {release, {_RelName, RelVsn}, _Erts, Apps}} -> - case lists:keysearch(emqx, 1, Apps) of - {value, {emqx, EmqxVsn}} -> - {ok, RelVsn, EmqxVsn}; - false -> - error({emqx_vsn_cannot_found, RelFile}) - end; - {ok, _BadFormat} -> - ?ERROR("bad formatted .rel file: ~p", [RelFile]); - {error, Reason} -> - ?ERROR("read .rel file ~p failed: ~p", [RelFile, Reason]) - end. +assert_mandatory_modules(up, Mods) -> + assert(lists:member(emqx_relup, Mods) andalso lists:member(emqx_app, Mods), + "cannot found 'load_module' instructions for emqx_app and emqx_rel: ~p", [Mods]); -has_relup_module(RelRootDir, EmqxVsn) -> - AppFile = filename:join([RelRootDir, "lib", "emqx-" ++ EmqxVsn, "ebin", "emqx.app"]), - case file:script(AppFile) of - {ok, {application, emqx, AppInfo}} -> - {value, {_, EmqxVsn}} = lists:keysearch(vsn, 1, AppInfo), %% assert - {value, {_, Modules}} = lists:keysearch(modules, 1, AppInfo), - lists:member(emqx_relup, Modules); - {error, Reason} -> - ?ERROR("read .app file ~p failed: ~p", [AppFile, Reason]), - error({read_app_file_error, AppFile, Reason}) - end. +assert_mandatory_modules(down, Mods) -> + assert(lists:member(emqx_app, Mods), + "cannot found 'load_module' instructions for emqx_app", []). + +assert(true, _, _) -> + ok; +assert(false, Msg, Args) -> + ?ERROR(Msg, Args), + error(assert_failed). term_to_text(Term) -> io_lib:format("~p.", [Term]). diff --git a/src/emqx.appup.src b/src/emqx.appup.src index b9d4f5a16..53b4a9ff7 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- {VSN, [{"4.4.1", - [{load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {add_module,emqx_relup}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -10,7 +12,8 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + [{add_module,emqx_relup}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, @@ -31,7 +34,9 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.1", - [{load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {delete_module,emqx_relup}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -40,7 +45,8 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_relup}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_relup.erl b/src/emqx_relup.erl index cb71c6305..dc443947a 100644 --- a/src/emqx_relup.erl +++ b/src/emqx_relup.erl @@ -1,5 +1,25 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + -module(emqx_relup). +%% NOTE: DO NOT remove this `-include`. +%% We use this to forece this module to upgraded every release. +-include("emqx_release.hrl"). + -export([ post_release_upgrade/2 , post_release_downgrade/2 ]). @@ -8,13 +28,15 @@ -define(INFO(FORMAT, ARGS), io:format("[emqx_relup] " ++ FORMAT ++ "~n", ARGS)). %% what to do after upgraded from a old release vsn. -post_release_upgrade(_FromRelVsn, _) -> - ?INFO("emqx has been upgraded to ~s", [emqx_app:get_release()]), +post_release_upgrade(FromRelVsn, _) -> + {_, CurrRelVsn} = ?EMQX_RELEASE, + ?INFO("emqx has been upgraded to from ~s to ~s!", [FromRelVsn, CurrRelVsn]), reload_components(). %% what to do after downgraded to a old release vsn. -post_release_downgrade(_ToRelVsn, _) -> - ?INFO("emqx has been downgrade to ~s", [emqx_app:get_release()]), +post_release_downgrade(ToRelVsn, _) -> + {_, CurrRelVsn} = ?EMQX_RELEASE, + ?INFO("emqx has been downgraded to from ~s to ~s!", [CurrRelVsn, ToRelVsn]), reload_components(). -ifdef(EMQX_ENTERPRISE). From 77e3c1d3ac35888097b958201f1e8a2bee066b55 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 2 Mar 2022 10:48:10 +0800 Subject: [PATCH 245/363] feat(relup): support providing 'Extra' parameter from appup.src --- scripts/inject-relup.escript | 68 +++++++++++++++++++++--------------- src/emqx_relup.erl | 6 ++-- 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript index 93fe34a90..32c944246 100755 --- a/scripts/inject-relup.escript +++ b/scripts/inject-relup.escript @@ -49,20 +49,20 @@ inject_relup_instrs(Type, RUs) -> %% The `{apply, emqx_relup, post_release_upgrade, []}` will be appended to the end of %% the instruction lists. append_emqx_relup_instrs(up, FromRelVsn, Instrs0) -> - Extra = #{}, %% we may need some extended args - filter_and_check_instrs(up, Instrs0) ++ + {{UpExtra, _}, Instrs1} = filter_and_check_instrs(up, Instrs0), + Instrs1 ++ [ {load, {emqx_app, brutal_purge, soft_purge}} , {load, {emqx_relup, brutal_purge, soft_purge}} - , {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, Extra]}} + , {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, UpExtra]}} ]; append_emqx_relup_instrs(down, ToRelVsn, Instrs0) -> - Extra = #{}, %% we may need some extended args + {{_, DnExtra}, Instrs1} = filter_and_check_instrs(down, Instrs0) %% NOTE: When downgrading, we apply emqx_relup:post_release_downgrade/2 before reloading %% or removing the emqx_relup module. - Instrs1 = filter_and_check_instrs(down, Instrs0) ++ + Instrs1 ++ [ {load, {emqx_app, brutal_purge, soft_purge}} - , {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, Extra]}} + , {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}} ], %% emqx_relup does not exist before release "4.4.2" LoadInsts = @@ -75,37 +75,49 @@ append_emqx_relup_instrs(down, ToRelVsn, Instrs0) -> Instrs1 ++ LoadInsts. filter_and_check_instrs(Type, Instrs) -> - case take_emqx_vsn_and_modules(Instrs) of - {EmqxAppVsn, EmqxMods, RemainInstrs} when EmqxAppVsn =/= not_found, EmqxMods =/= [] -> + case filter_fetch_emqx_mods_and_extra(Instrs) of + {_, _, [], _} -> + ?ERROR("cannot find any 'load_object_code' instructions for app emqx", []), + error({instruction_not_found, load_object_code}); + {UpExtra, DnExtra, EmqxMods, RemainInstrs} -> assert_mandatory_modules(Type, EmqxMods), - [{load_object_code, {emqx, EmqxAppVsn, EmqxMods}} | RemainInstrs]; - {_, _, _} -> - ?ERROR("cannot found 'load_module' instructions for app emqx", []), - error({instruction_not_found, load_object_code}) + {{UpExtra, DnExtra}, RemainInstrs} end. -take_emqx_vsn_and_modules(Instrs) -> - lists:foldl(fun - ({load_object_code, {emqx, AppVsn, Mods}}, {_EmqxAppVsn, EmqxMods, RemainInstrs}) -> - {AppVsn, EmqxMods ++ Mods, RemainInstrs}; - ({load, {Mod, _, _}}, {EmqxAppVsn, EmqxMods, RemainInstrs}) - when Mod =:= emqx_relup; Mod =:= emqx_app -> - {EmqxAppVsn, EmqxMods, RemainInstrs}; - ({remove, {emqx_relup, _, _}}, {EmqxAppVsn, EmqxMods, RemainInstrs}) -> - {EmqxAppVsn, EmqxMods, RemainInstrs}; - ({apply, {emqx_relup, _, _}}, {EmqxAppVsn, EmqxMods, RemainInstrs}) -> - {EmqxAppVsn, EmqxMods, RemainInstrs}; - (Instr, {EmqxAppVsn, EmqxMods, RemainInstrs}) -> - {EmqxAppVsn, EmqxMods, RemainInstrs ++ [Instr]} - end, {not_found, [], []}, Instrs). +filter_fetch_emqx_mods_and_extra(Instrs) -> + lists:foldl(fun do_filter_and_get/2, {UpExtra, DnExtra, [], []}, Instrs). + +%% collect modules for emqx app +do_filter_and_get({load_object_code, {emqx, _AppVsn, Mods}} = Instr, + {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods ++ Mods, RemainInstrs ++ [Instr]}; +%% remove 'load' instrs +do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) + when Mod =:= emqx_relup; Mod =:= emqx_app -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs}; +%% remove 'remove' instrs +do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs}; +%% remove 'apply' instrs for upgrade, and collect the 'Extra' parameter +do_filter_and_get({apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}}, + {_, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra0, DnExtra, EmqxMods, RemainInstrs}; +%% remove 'apply' instrs for downgrade, and collect the 'Extra' parameter +do_filter_and_get({apply, {emqx_relup, post_release_downgrade, [_, DnExtra0]}}, + {UpExtra, _, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra0, EmqxMods, RemainInstrs}; +%% keep all other instrs unchanged +do_filter_and_get(Instr, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs ++ [Instr]}. + assert_mandatory_modules(up, Mods) -> assert(lists:member(emqx_relup, Mods) andalso lists:member(emqx_app, Mods), - "cannot found 'load_module' instructions for emqx_app and emqx_rel: ~p", [Mods]); + "cannot find any 'load_object_code' instructions for emqx_app and emqx_rel: ~p", [Mods]); assert_mandatory_modules(down, Mods) -> assert(lists:member(emqx_app, Mods), - "cannot found 'load_module' instructions for emqx_app", []). + "cannot find any 'load_object_code' instructions for emqx_app", []). assert(true, _, _) -> ok; diff --git a/src/emqx_relup.erl b/src/emqx_relup.erl index dc443947a..fcc90d088 100644 --- a/src/emqx_relup.erl +++ b/src/emqx_relup.erl @@ -17,7 +17,7 @@ -module(emqx_relup). %% NOTE: DO NOT remove this `-include`. -%% We use this to forece this module to upgraded every release. +%% We use this to force this module to be upgraded every release. -include("emqx_release.hrl"). -export([ post_release_upgrade/2 @@ -27,13 +27,13 @@ -define(INFO(FORMAT), io:format("[emqx_relup] " ++ FORMAT ++ "~n")). -define(INFO(FORMAT, ARGS), io:format("[emqx_relup] " ++ FORMAT ++ "~n", ARGS)). -%% what to do after upgraded from a old release vsn. +%% What to do after upgraded from an old release vsn. post_release_upgrade(FromRelVsn, _) -> {_, CurrRelVsn} = ?EMQX_RELEASE, ?INFO("emqx has been upgraded to from ~s to ~s!", [FromRelVsn, CurrRelVsn]), reload_components(). -%% what to do after downgraded to a old release vsn. +%% What to do after downgraded to an old release vsn. post_release_downgrade(ToRelVsn, _) -> {_, CurrRelVsn} = ?EMQX_RELEASE, ?INFO("emqx has been downgraded to from ~s to ~s!", [CurrRelVsn, ToRelVsn]), From 8be8f538a101ab138d1e0a21822b52f0e653ab7b Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 2 Mar 2022 11:36:42 +0800 Subject: [PATCH 246/363] fix(relup): verify emqx_relup call in upgrade and downgrade clauses --- scripts/inject-relup.escript | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript index 32c944246..e2d3de795 100755 --- a/scripts/inject-relup.escript +++ b/scripts/inject-relup.escript @@ -46,8 +46,6 @@ inject_relup_instrs(Type, RUs) -> {Vsn, Desc, append_emqx_relup_instrs(Type, Vsn, Instrs)} end, RUs). -%% The `{apply, emqx_relup, post_release_upgrade, []}` will be appended to the end of -%% the instruction lists. append_emqx_relup_instrs(up, FromRelVsn, Instrs0) -> {{UpExtra, _}, Instrs1} = filter_and_check_instrs(up, Instrs0), Instrs1 ++ @@ -57,10 +55,10 @@ append_emqx_relup_instrs(up, FromRelVsn, Instrs0) -> ]; append_emqx_relup_instrs(down, ToRelVsn, Instrs0) -> - {{_, DnExtra}, Instrs1} = filter_and_check_instrs(down, Instrs0) + {{_, DnExtra}, Instrs1} = filter_and_check_instrs(down, Instrs0), %% NOTE: When downgrading, we apply emqx_relup:post_release_downgrade/2 before reloading %% or removing the emqx_relup module. - Instrs1 ++ + Instrs2 = Instrs1 ++ [ {load, {emqx_app, brutal_purge, soft_purge}} , {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}} ], @@ -68,14 +66,24 @@ append_emqx_relup_instrs(down, ToRelVsn, Instrs0) -> LoadInsts = case ToRelVsn of ToRelVsn when ToRelVsn =:= "4.4.1"; ToRelVsn =:= "4.4.0" -> - [{remove, {emqx_relup, brutal_purge, brutal_purge}}]; + [ {remove, {emqx_relup, brutal_purge, brutal_purge}} + , {purge, [emqx_relup]} + ]; _ -> [{load, {emqx_relup, brutal_purge, soft_purge}}] end, - Instrs1 ++ LoadInsts. + Instrs2 ++ LoadInsts. filter_and_check_instrs(Type, Instrs) -> case filter_fetch_emqx_mods_and_extra(Instrs) of + {_, DnExtra, _, _} when Type =:= up, DnExtra =/= undefined -> + ?ERROR("got '{apply,{emqx_relup,post_release_downgrade,[_,Extra]}}'" + " from the upgrade instruction list, should be 'post_release_upgrade'", []), + error({instruction_not_found, load_object_code}); + {UpExtra, _, _, _} when Type =:= down, UpExtra =/= undefined -> + ?ERROR("got '{apply,{emqx_relup,post_release_upgrade,[_,Extra]}}'" + " from the downgrade instruction list, should be 'post_release_downgrade'", []), + error({instruction_not_found, load_object_code}); {_, _, [], _} -> ?ERROR("cannot find any 'load_object_code' instructions for app emqx", []), error({instruction_not_found, load_object_code}); @@ -85,19 +93,21 @@ filter_and_check_instrs(Type, Instrs) -> end. filter_fetch_emqx_mods_and_extra(Instrs) -> - lists:foldl(fun do_filter_and_get/2, {UpExtra, DnExtra, [], []}, Instrs). + lists:foldl(fun do_filter_and_get/2, {undefined, undefined, [], []}, Instrs). %% collect modules for emqx app do_filter_and_get({load_object_code, {emqx, _AppVsn, Mods}} = Instr, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> {UpExtra, DnExtra, EmqxMods ++ Mods, RemainInstrs ++ [Instr]}; -%% remove 'load' instrs +%% remove 'load' instrs for emqx_relup and emqx_app do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) when Mod =:= emqx_relup; Mod =:= emqx_app -> {UpExtra, DnExtra, EmqxMods, RemainInstrs}; -%% remove 'remove' instrs +%% remove 'remove' and 'purge' instrs for emqx_relup do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> {UpExtra, DnExtra, EmqxMods, RemainInstrs}; +do_filter_and_get({purge, [emqx_relup]}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs}; %% remove 'apply' instrs for upgrade, and collect the 'Extra' parameter do_filter_and_get({apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}}, {_, DnExtra, EmqxMods, RemainInstrs}) -> From a94dc87c346c8c41a9e16cced65472650707652b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 3 Mar 2022 11:20:50 +0800 Subject: [PATCH 247/363] chore: add missed appup instructions --- src/emqx.appup.src | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 53b4a9ff7..dad0ca480 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -7,6 +7,7 @@ {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, @@ -16,6 +17,7 @@ {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, @@ -40,6 +42,7 @@ {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, @@ -49,6 +52,7 @@ {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, From c5a6b4739058a86ca077335bf395114c4365c19d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 3 Mar 2022 11:50:59 +0800 Subject: [PATCH 248/363] build(ci): fix package name --- .github/workflows/apps_version_check.yaml | 4 ++-- scripts/update-appup.sh | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 67355155f..a1bf859ba 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -9,11 +9,11 @@ jobs: strategy: matrix: erl_otp: - - erl23.2.7.2-emqx-3 + - 24.1.5-3 os: - ubuntu20.04 - container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index db0759695..5fd26355e 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -40,6 +40,8 @@ PREV_VERSION="${PREV_VERSION#[e|v]}" shift 1 ESCRIPT_ARGS="$*" +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" + SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" if [ -z "${ARCH:-}" ]; then UNAME="$(uname -m)" @@ -56,7 +58,7 @@ if [ -z "${ARCH:-}" ]; then esac fi -PACKAGE_NAME="${PROFILE}-${SYSTEM}-${PREV_VERSION}-${ARCH}.zip" +PACKAGE_NAME="${PROFILE}-${PREV_VERSION}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" DOWNLOAD_URL="https://www.emqx.com/downloads/${DIR}/v${PREV_VERSION}/${PACKAGE_NAME}" # shellcheck disable=SC2086 From dbacfb2bb4a56cb27eb2c9d56c33ed0a91063b89 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 3 Mar 2022 22:02:27 +0800 Subject: [PATCH 249/363] test(mgmt): ensure application loaded --- apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl index 1878dec45..94d354e25 100644 --- a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl @@ -47,6 +47,7 @@ groups() -> ]}]. init_per_suite(Config) -> + application:load(emqx_plugin_libs), emqx_ct_helpers:start_apps( [emqx_modules, emqx_management, emqx_auth_mnesia] , fun set_special_configs/1 ), From a27ff13cebe3f87ac7928f24a6dae8900969229e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 4 Mar 2022 21:08:29 +0800 Subject: [PATCH 250/363] ci(release): update container image --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0d7446de3..0d90195ca 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} From 521f47ff8080e650affc2ffda6b2828755ebba84 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 5 Mar 2022 10:42:38 +0100 Subject: [PATCH 251/363] build: add 4.4 dashboard download --- scripts/get-dashboard.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index 39dd4c7d5..a83294568 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -11,6 +11,11 @@ case "${PKG_VSN}" in EMQX_CE_DASHBOARD_VERSION='v4.3.5' EMQX_EE_DASHBOARD_VERSION='v4.3.15' ;; + 4.4*) + # keep the above 4.3 untouched, otherwise conflicts! + EMQX_CE_DASHBOARD_VERSION='v4.4.0' + EMQX_EE_DASHBOARD_VERSION='see-enterprise-repo' + ;; *) echo "Unsupported version $PKG_VSN" >&2 exit 1 From 03374cdf147534f1ad2dd35bbc978669b20ce49b Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 7 Mar 2022 17:54:13 +0800 Subject: [PATCH 252/363] test(slow_subs): fix the flaky case, relax the range check --- lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl index 6cfbdea23..6434fedd0 100644 --- a/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl @@ -73,8 +73,8 @@ t_log_and_pub(_) -> timer:sleep(2000), Size = ets:info(?TOPK_TAB, size), - %% some time record maybe delete due to it expired - ?assert(Size =< 6 andalso Size >= 4, + %% some time record maybe delete due to it expired or the ets size exceeds 5 due to race conditions + ?assert(Size =< 8 andalso Size >= 3, unicode:characters_to_binary(io_lib:format("size is :~p~n", [Size]))), timer:sleep(3000), From 27617539f16defaece2522da2e20e27cf72481ac Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 8 Mar 2022 19:31:21 +0100 Subject: [PATCH 253/363] fix(appup): adjust update-appup.sh for 4.4. --- scripts/update-appup.sh | 20 +------------------- src/emqx.appup.src | 12 ++++++++---- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index 8fb5310e6..df4f56e74 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -72,25 +72,7 @@ if [ "${SKIP_BUILD:-}" != 'yes' ]; then make "${PROFILE}" fi -OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" - -SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" -if [ -z "${ARCH:-}" ]; then - UNAME="$(uname -m)" - case "$UNAME" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH='arm' - ;; - esac -fi - -PACKAGE_NAME="${PROFILE}-${PREV_VERSION}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" +PACKAGE_NAME="${PROFILE}-$(env PKG_VSN="$PREV_VERSION" ./scripts/pkg-full-vsn.sh).zip" DOWNLOAD_URL="https://www.emqx.com/downloads/${DIR}/v${PREV_VERSION}/${PACKAGE_NAME}" PREV_DIR_BASE="/tmp/emqx-appup-base" diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 906c6507e..68ca21a3b 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.1", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -14,7 +15,8 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{add_module,emqx_relup}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {add_module,emqx_relup}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -37,7 +39,8 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.1", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -49,7 +52,8 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{delete_module,emqx_relup}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {delete_module,emqx_relup}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, From 2da3333879fbb4929fa55ad1c3f70b6d3f5675a0 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 11 Mar 2022 16:24:30 +0800 Subject: [PATCH 254/363] fix: check backup file type & legal json --- .../src/emqx_management.app.src | 2 +- .../src/emqx_mgmt_api_data.erl | 69 +++----- .../src/emqx_mgmt_data_backup.erl | 166 ++++++++++++++++-- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 19 ++ .../test/emqx_mgmt_api_SUITE.erl | 8 +- .../src/emqx_trace/emqx_trace.erl | 8 +- rebar.config.erl | 1 + 7 files changed, 207 insertions(+), 66 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index c83969abc..f7e82e266 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -3,7 +3,7 @@ {vsn, "4.4.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, - {applications, [kernel,stdlib,minirest]}, + {applications, [kernel,stdlib,emqx_plugin_libs,minirest]}, {mod, {emqx_mgmt_app,[]}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index e389a1313..e9a00907b 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -73,39 +73,17 @@ export(_Bindings, _Params) -> case emqx_mgmt_data_backup:export() of {ok, File = #{filename := Filename}} -> - minirest:return({ok, File#{filename => filename:basename(Filename)}}); + minirest:return({ok, File#{filename => list_to_binary(filename:basename(Filename))}}); Return -> minirest:return(Return) end. list_exported(_Bindings, _Params) -> - List = [ rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes() ], + List = [rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes()], NList = lists:map(fun({_, FileInfo}) -> FileInfo end, lists:keysort(1, lists:append(List))), minirest:return({ok, NList}). get_list_exported() -> - Dir = emqx:get_env(data_dir), - {ok, Files} = file:list_dir_all(Dir), - lists:foldl( - fun(File, Acc) -> - case filename:extension(File) =:= ".json" of - true -> - FullFile = filename:join([Dir, File]), - case file:read_file_info(FullFile) of - {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} -> - CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), - Seconds = calendar:datetime_to_gregorian_seconds(CTime), - [{Seconds, [{filename, list_to_binary(File)}, - {size, Size}, - {created_at, list_to_binary(CreatedAt)}, - {node, node()} - ]} | Acc]; - {error, Reason} -> - logger:error("Read file info of ~s failed with: ~p", [File, Reason]), - Acc - end; - false -> Acc - end - end, [], Files). + emqx_mgmt_data_backup:list_backup_file(). import(_Bindings, Params) -> case proplists:get_value(<<"filename">>, Params) of @@ -121,22 +99,26 @@ import(_Bindings, Params) -> case lists:member(Node, [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ] ) of - true -> minirest:return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename])); + true -> + N = erlang:binary_to_atom(Node, utf8), + case rpc:call(N, ?MODULE, do_import, [Filename]) of + {badrpc, Reason} -> + minirest:return({error, Reason}); + Res -> + minirest:return(Res) + end; false -> minirest:return({error, no_existent_node}) end end end. do_import(Filename) -> - FullFilename = fullname(Filename), - emqx_mgmt_data_backup:import(FullFilename, "{}"). + emqx_mgmt_data_backup:import(Filename, "{}"). download(#{filename := Filename}, _Params) -> - FullFilename = fullname(Filename), - case file:read_file(FullFilename) of - {ok, Bin} -> - {ok, #{filename => list_to_binary(Filename), - file => Bin}}; + case emqx_mgmt_data_backup:read_backup_file(Filename) of + {ok, Res} -> + {ok, Res}; {error, Reason} -> minirest:return({error, Reason}) end. @@ -146,8 +128,7 @@ upload(Bindings, Params) -> do_upload(_Bindings, #{<<"filename">> := Filename, <<"file">> := Bin}) -> - FullFilename = fullname(Filename), - case file:write_file(FullFilename, Bin) of + case emqx_mgmt_data_backup:upload_backup_file(Filename, Bin) of ok -> minirest:return({ok, [{node, node()}]}); {error, Reason} -> @@ -159,8 +140,7 @@ do_upload(_Bindings, _Params) -> minirest:return({error, missing_required_params}). delete(#{filename := Filename}, _Params) -> - FullFilename = fullname(Filename), - case file:delete(FullFilename) of + case emqx_mgmt_data_backup:delete_backup_file(Filename) of ok -> minirest:return(); {error, Reason} -> @@ -168,17 +148,14 @@ delete(#{filename := Filename}, _Params) -> end. import_content(Content) -> - File = dump_to_tmp_file(Content), - do_import(File). - -dump_to_tmp_file(Content) -> Bin = emqx_json:encode(Content), Filename = tmp_filename(), - ok = file:write_file(fullname(Filename), Bin), - Filename. - -fullname(Name) -> - filename:join(emqx:get_env(data_dir), Name). + case emqx_mgmt_data_backup:upload_backup_file(Filename, Bin) of + ok -> + do_import(Filename); + {error, Reason} -> + {error, Reason} + end. tmp_filename() -> Seconds = erlang:system_time(second), diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 07bd3d27c..9ecc4d57e 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -31,6 +31,8 @@ ]). -endif. +-define(BACKUP_DIR, backup). + -export([ export_rules/0 , export_resources/0 , export_blacklist/0 @@ -53,8 +55,18 @@ -export([ export/0 , import/2 + , upload_backup_file/2 + , list_backup_file/0 + , read_backup_file/1 + , delete_backup_file/1 ]). +-ifdef(TEST). +-export([ backup_dir/0 + , delete_all_backup_file/0 + ]). +-endif. + %%-------------------------------------------------------------------- %% Data Export and Import %%-------------------------------------------------------------------- @@ -600,19 +612,101 @@ to_version(Version) when is_binary(Version) -> to_version(Version) when is_list(Version) -> Version. +upload_backup_file(Filename0, Bin) -> + case ensure_file_name(Filename0) of + {ok, Filename} -> + case check_json(Bin) of + {ok, _} -> + logger:info("write backup file ~p", [Filename]), + file:write_file(Filename, Bin); + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. + +list_backup_file() -> + {ok, Files} = file:list_dir_all(backup_dir()), + lists:foldl( + fun(File, Acc) -> + case filename:extension(File) =:= ".json" of + true -> + {ok, FileName} = ensure_file_name(File), + case file:read_file_info(FileName) of + {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} -> + CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), + Seconds = calendar:datetime_to_gregorian_seconds(CTime), + [{Seconds, [{filename, list_to_binary(File)}, + {size, Size}, + {created_at, list_to_binary(CreatedAt)}, + {node, node()} + ]} | Acc]; + {error, Reason} -> + logger:error("Read file info of ~s failed with: ~p", [File, Reason]), + Acc + end; + false -> Acc + end + end, [], Files). + +read_backup_file(Filename0) -> + case ensure_file_name(Filename0) of + {ok, Filename} -> + case file:read_file(Filename) of + {ok, Bin} -> + {ok, #{filename => to_binary(Filename0), + file => Bin}}; + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. + +delete_backup_file(Filename0) -> + case ensure_file_name(Filename0) of + {ok, Filename} -> + case file:read_file_info(Filename) of + {ok, #file_info{}} -> + case file:delete(Filename) of + ok -> + logger:info("delete backup file ~p", [Filename]), + ok; + {error, Reason} -> + logger:error( + "delete backup file ~p error:~p", [Filename, Reason]), + {error, Reason} + end; + _ -> + {error, not_found} + end; + {error, _Reason} -> + {error, not_found} + end. + +-ifdef(TEST). +%% clean all for test +delete_all_backup_file() -> + [begin + Filename = proplists:get_value(filename, Info), + _ = delete_backup_file(Filename) + end || {_, Info} <- list_backup_file()], + ok. +-endif. + export() -> Seconds = erlang:system_time(second), Data = do_export_data() ++ [{date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}], {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), - Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), - NFilename = filename:join([emqx:get_env(data_dir), Filename]), - ok = filelib:ensure_dir(NFilename), - case file:write_file(NFilename, emqx_json:encode(Data)) of + BaseFilename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), + {ok, Filename} = ensure_file_name(BaseFilename), + case file:write_file(Filename, emqx_json:encode(Data)) of ok -> - case file:read_file_info(NFilename) of + case file:read_file_info(Filename) of {ok, #file_info{size = Size, ctime = {{Y1, M1, D1}, {H1, MM1, S1}}}} -> CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y1, M1, D1, H1, MM1, S1]), - {ok, #{filename => list_to_binary(NFilename), + {ok, #{filename => Filename, size => Size, created_at => list_to_binary(CreatedAt), node => node() @@ -648,9 +742,8 @@ do_export_extra_data() -> []. -ifdef(EMQX_ENTERPRISE). import(Filename, OverridesJson) -> - case file:read_file(Filename) of - {ok, Json} -> - Imported = emqx_json:decode(Json, [return_maps]), + case check_import_json(Filename) of + {ok, Imported} -> Overrides = emqx_json:decode(OverridesJson, [return_maps]), Data = maps:merge(Imported, Overrides), Version = to_version(maps:get(<<"version">>, Data)), @@ -663,13 +756,13 @@ import(Filename, OverridesJson) -> logger:error("The emqx data import failed: ~0p", [{Class, Reason, Stack}]), {error, import_failed} end; - Error -> Error + {error, Reason} -> + {error, Reason} end. -else. import(Filename, OverridesJson) -> - case file:read_file(Filename) of - {ok, Json} -> - Imported = emqx_json:decode(Json, [return_maps]), + case check_import_json(Filename) of + {ok, Imported} -> Overrides = emqx_json:decode(OverridesJson, [return_maps]), Data = maps:merge(Imported, Overrides), Version = to_version(maps:get(<<"version">>, Data)), @@ -688,10 +781,52 @@ import(Filename, OverridesJson) -> logger:error("Unsupported version: ~p", [Version]), {error, unsupported_version, Version} end; - Error -> Error + {error, Reason} -> + {error, Reason} end. -endif. +-spec(check_import_json(binary() | string()) -> {ok, map()} | {error, term()}). +check_import_json(Filename) -> + ReadFile = fun(F) -> file:read_file(F) end, + FunList = [fun ensure_file_name/1, ReadFile, fun check_json/1], + check_import_json(Filename, FunList). + +check_import_json(Res, []) -> + {ok, Res}; +check_import_json(Acc, [Fun | FunList]) -> + case Fun(Acc) of + {ok, Next} -> + check_import_json(Next, FunList); + {error, Reason} -> + {error, Reason} + end. + +ensure_file_name(Filename) -> + case legal_filename(Filename) of + true -> + {ok, filename:join(backup_dir(), Filename)}; + false -> + {error, bad_filename} + end. + +backup_dir() -> + Dir = filename:join(emqx:get_env(data_dir), ?BACKUP_DIR), + ok = filelib:ensure_dir(filename:join([Dir, dummy])), + Dir. + +legal_filename(Filename) -> + MaybeJson = filename:extension(Filename), + MaybeJson == ".json" orelse MaybeJson == <<".json">>. + +check_json(MaybeJson) -> + case emqx_json:safe_decode(MaybeJson, [return_maps]) of + {ok, Json} -> + {ok, Json}; + {error, _} -> + {error, bad_json} + end. + do_import_data(Data, Version) -> do_import_extra_data(Data, Version), import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), @@ -800,3 +935,6 @@ get_old_type() -> set_old_type(Type) -> application:set_env(emqx_auth_mnesia, as, Type). + +to_binary(Bin) when is_binary(Bin) -> Bin; +to_binary(Str) when is_list(Str) -> list_to_binary(Str). diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index d2fc8cb49..07f97f89e 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -368,6 +368,25 @@ t_cli(_) -> [?assertMatch({match, _}, re:run(Value, "mgmt")) || Value <- emqx_mgmt_cli:mgmt([""])]. +t_backup_file(_)-> + Filename = <<"test.json">>, + BadFilename = <<"bad.notjson">>, + Bin = emqx_json:encode(#{a => b}), + BadBin = <<"[bad json]">>, + + {error, bad_filename} = emqx_mgmt_data_backup:upload_backup_file(BadFilename, Bin), + {error, bad_json} = emqx_mgmt_data_backup:upload_backup_file(Filename, BadBin), + + ok = emqx_mgmt_data_backup:upload_backup_file(Filename, Bin), + {ok, #{file := <<"{\"a\":\"b\"}">>, filename := <<"test.json">>}} = + emqx_mgmt_data_backup:read_backup_file(Filename), + [{_, FileInfoList}] = emqx_mgmt_data_backup:list_backup_file(), + Filename = proplists:get_value(filename, FileInfoList), + ok = emqx_mgmt_data_backup:delete_backup_file(Filename), + + {error, not_found} = emqx_mgmt_data_backup:delete_backup_file(BadFilename), + ok. + mock_print() -> catch meck:unload(emqx_ctl), meck:new(emqx_ctl, [non_strict, passthrough]), diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index a2bcbf44e..5631e5b78 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -689,6 +689,7 @@ t_data(_) -> ok = emqx_dashboard_admin:mnesia(boot), application:ensure_all_started(emqx_rule_engine), application:ensure_all_started(emqx_dashboard), + emqx_mgmt_data_backup:delete_all_backup_file(), {ok, Data} = request_api(post, api_path(["data","export"]), [], auth_header_(), [#{}]), #{<<"filename">> := Filename, <<"node">> := Node} = emqx_ct_http:get_http_data(Data), {ok, DataList} = request_api(get, api_path(["data","export"]), auth_header_()), @@ -701,6 +702,8 @@ t_data(_) -> ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename})), + _ = emqx_mgmt_data_backup:delete_backup_file(Filename), + emqx_mgmt_data_backup:delete_all_backup_file(), application:stop(emqx_rule_engine), application:stop(emqx_dashboard), ok. @@ -710,13 +713,16 @@ t_data_import_content(_) -> ok = emqx_dashboard_admin:mnesia(boot), application:ensure_all_started(emqx_rule_engine), application:ensure_all_started(emqx_dashboard), + emqx_mgmt_data_backup:delete_all_backup_file(), {ok, Data} = request_api(post, api_path(["data","export"]), [], auth_header_(), [#{}]), #{<<"filename">> := Filename} = emqx_ct_http:get_http_data(Data), - Dir = emqx:get_env(data_dir), + Dir = emqx_mgmt_data_backup:backup_dir(), {ok, Bin} = file:read_file(filename:join(Dir, Filename)), Content = emqx_json:decode(Bin), ?assertMatch({ok, "{\"code\":0}"}, request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), + + emqx_mgmt_data_backup:delete_all_backup_file(), application:stop(emqx_rule_engine), application:stop(emqx_dashboard). diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index f074055f9..fd0110a36 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -189,8 +189,8 @@ init([]) -> ok = create_table(), erlang:process_flag(trap_exit, true), OriginLogLevel = emqx_logger:get_primary_log_level(), - ok = filelib:ensure_dir(trace_dir()), - ok = filelib:ensure_dir(zip_dir()), + ok = filelib:ensure_dir(filename:join([trace_dir(), dummy])), + ok = filelib:ensure_dir(filename:join([zip_dir(), dummy])), {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), Traces = get_enable_trace(), ok = update_log_primary_level(Traces, OriginLogLevel), @@ -462,10 +462,10 @@ to_system_second(At) -> end. zip_dir() -> - trace_dir() ++ "zip/". + filename:join(trace_dir(), "zip"). trace_dir() -> - filename:join(emqx:get_env(data_dir), "trace") ++ "/". + filename:join(emqx:get_env(data_dir), "trace"). log_file(Name, Start) -> filename:join(trace_dir(), filename(Name, Start)). diff --git a/rebar.config.erl b/rebar.config.erl index 492707867..788e99200 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -333,6 +333,7 @@ relx_overlay(ReleaseType) -> , {mkdir, "data/configs"} , {mkdir, "data/patches"} , {mkdir, "data/scripts"} + , {mkdir, "data/backup"} , {template, "data/loaded_plugins.tmpl", "data/loaded_plugins"} , {template, "data/loaded_modules.tmpl", "data/loaded_modules"} , {template, "data/emqx_vars", "releases/emqx_vars"} From 1ec0377a697db864dfbdaf709739adb098d3bddc Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 14 Mar 2022 15:08:34 +0800 Subject: [PATCH 255/363] fix: api file path with filename:join(List) & app up --- .../src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_plugin_libs.appup.src | 20 +++++++++++++++---- .../src/emqx_trace/emqx_trace_api.erl | 8 +++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 67337af21..fe4fff0b5 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.4.1"}, + {vsn, "4.4.2"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index 10095eed0..46e76ed82 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -1,12 +1,24 @@ %% -*- mode: erlang -*- {VSN, - [{"4.4.0", - [ {update, emqx_slow_subs, {advanced, ["4.4.0"]}} + [{"4.4.1", + [ {load_module,emqx_trace,brutal_purge,soft_purge,[]} + , {load_module,emqx_trace_api,brutal_purge,soft_purge,[]} + ]}, + {"4.4.0", + [ {load_module,emqx_trace,brutal_purge,soft_purge,[]} + , {load_module,emqx_trace_api,brutal_purge,soft_purge,[]} + , {update, emqx_slow_subs, {advanced, ["4.4.0"]}} , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} ]}, {<<".*">>,[]}], - [{"4.4.0", - [ {update, emqx_slow_subs, {advanced, ["4.4.0"]}} + [{"4.4.1", + [ {load_module,emqx_trace,brutal_purge,soft_purge,[]} + , {load_module,emqx_trace_api,brutal_purge,soft_purge,[]} + ]}, + {"4.4.0", + [ {load_module,emqx_trace,brutal_purge,soft_purge,[]} + , {load_module,emqx_trace_api,brutal_purge,soft_purge,[]} + , {update, emqx_slow_subs, {advanced, ["4.4.0"]}} , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} ]}, {<<".*">>,[]}] diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 0e298698e..548b58ae9 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -95,7 +95,8 @@ download_zip_log(#{name := Name}, _Param) -> TraceFiles = collect_trace_file(TraceLog), ZipDir = emqx_trace:zip_dir(), Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), - ZipFileName = ZipDir ++ binary_to_list(Name) ++ ".zip", + ZipFileName0 = binary_to_list(Name) ++ ".zip", + ZipFileName = filename:join([Zips, ZipFileName0]), {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]), emqx_trace:delete_files_after_send(ZipFileName, Zips), {ok, ZipFile}; @@ -107,9 +108,10 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> lists:foldl(fun(Res, Acc) -> case Res of {ok, Node, Bin} -> - ZipName = ZipDir ++ Node ++ "-" ++ TraceLog, + FileName = Node ++ "-" ++ TraceLog, + ZipName = filename:join([ZipDir, FileName]), case file:write_file(ZipName, Bin) of - ok -> [Node ++ "-" ++ TraceLog | Acc]; + ok -> [FileName | Acc]; _ -> Acc end; {error, Node, Reason} -> From 4adc9143724b93eb535b2e6997af67376f386e9a Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 15 Mar 2022 10:40:01 +0800 Subject: [PATCH 256/363] feat: Support set keepalive via queryString & Body HTTP API. --- CHANGES-4.3.md | 4 ++-- apps/emqx_management/src/emqx_mgmt_api_clients.erl | 5 ++++- src/emqx_channel.erl | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 7f6d2b5a3..881618918 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -22,8 +22,8 @@ File format: * CLI `emqx_ctl pem_cache clean` to force purge x509 certificate cache, to force an immediate reload of all certificates after the files are updated on disk. * Refactor the ExProto so that anonymous clients can also be displayed on the dashboard [#6983] -* Force shutdown of processe that cannot answer takeover event [#7026] - +* Force shutdown of processes that cannot answer takeover event [#7026] +* Support set keepalive via queryString & Body HTTP API. * `topic` parameter in bridge configuration can have `${node}` substitution (just like in `clientid` parameter) ### Bug fixes diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 985752f73..4df1fa856 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -257,7 +257,7 @@ set_keepalive(#{clientid := ClientId}, Params) -> undefined -> minirest:return({error, ?ERROR7, params_not_found}); Interval0 -> - Interval = binary_to_integer(Interval0), + Interval = to_integer(Interval0), case emqx_mgmt:set_keepalive(emqx_mgmt_util:urldecode(ClientId), Interval) of ok -> minirest:return(); {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); @@ -266,6 +266,9 @@ set_keepalive(#{clientid := ClientId}, Params) -> end end. +to_integer(Int)when is_integer(Int) -> Int; +to_integer(Bin) when is_binary(Bin) -> binary_to_integer(Bin). + %% @private %% S = 100,1s %% | 100KB, 1m diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index a9f5a3ab0..4ac7ece05 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -977,7 +977,9 @@ handle_call({quota, Policy}, Channel) -> reply(ok, Channel#channel{quota = Quota}); handle_call({keepalive, Interval}, Channel = #channel{keepalive = KeepAlive, - conninfo = ConnInfo}) -> + conninfo = ConnInfo, timers = Timers}) -> + AliveTimer = maps:get(alive_timer, Timers, undefined), + emqx_misc:cancel_timer(AliveTimer), ClientId = info(clientid, Channel), NKeepalive = emqx_keepalive:set(interval, Interval * 1000, KeepAlive), NConnInfo = maps:put(keepalive, Interval, ConnInfo), From 02e7f3bfe0b52b239d2e065fcd30be1a372f1618 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 15 Mar 2022 15:05:48 +0800 Subject: [PATCH 257/363] feat: backup data support old versions dir --- .../src/emqx_mgmt_data_backup.erl | 86 ++++++++++++------- 1 file changed, 57 insertions(+), 29 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 9ecc4d57e..d5c6c7b97 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -627,45 +627,67 @@ upload_backup_file(Filename0, Bin) -> end. list_backup_file() -> - {ok, Files} = file:list_dir_all(backup_dir()), - lists:foldl( - fun(File, Acc) -> - case filename:extension(File) =:= ".json" of - true -> - {ok, FileName} = ensure_file_name(File), - case file:read_file_info(FileName) of - {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} -> - CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), - Seconds = calendar:datetime_to_gregorian_seconds(CTime), - [{Seconds, [{filename, list_to_binary(File)}, - {size, Size}, - {created_at, list_to_binary(CreatedAt)}, - {node, node()} - ]} | Acc]; - {error, Reason} -> - logger:error("Read file info of ~s failed with: ~p", [File, Reason]), - Acc - end; - false -> Acc + Filter = + fun(File) -> + case file:read_file_info(File) of + {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} -> + Seconds = calendar:datetime_to_gregorian_seconds(CTime), + BaseFilename = to_binary(filename:basename(File)), + CreatedAt = to_binary(io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S])), + Info = { + Seconds, + [{filename, BaseFilename}, + {size, Size}, + {created_at, CreatedAt}, + {node, node()} + ] + }, + {true, Info}; + _ -> + false end - end, [], Files). + end, + lists:filtermap(Filter, backup_files()). + +backup_files() -> + backup_files(backup_dir()) ++ backup_files(backup_dir_odl_version()). + +backup_files(Dir) -> + {ok, FilesAll} = file:list_dir_all(Dir), + Files = lists:filtermap(fun legal_filename/1, FilesAll), + [filename:join([Dir, File]) || File <- Files]. + +look_up_file(Filename) when is_binary(Filename) -> + look_up_file(binary_to_list(Filename)); +look_up_file(Filename) -> + Filter = + fun(MaybeFile) -> + filename:basename(MaybeFile) == Filename + end, + case lists:filter(Filter, backup_files()) of + [] -> + {error, not_found}; + List -> + {ok, hd(List)} + end. read_backup_file(Filename0) -> - case ensure_file_name(Filename0) of + case look_up_file(Filename0) of {ok, Filename} -> case file:read_file(Filename) of {ok, Bin} -> {ok, #{filename => to_binary(Filename0), file => Bin}}; {error, Reason} -> - {error, Reason} + logger:error("read file ~p failed ~p", [Filename, Reason]), + {error, bad_file} end; - {error, Reason} -> - {error, Reason} + {error, not_found} -> + {error, not_found} end. delete_backup_file(Filename0) -> - case ensure_file_name(Filename0) of + case look_up_file(Filename0) of {ok, Filename} -> case file:read_file_info(Filename) of {ok, #file_info{}} -> @@ -681,7 +703,7 @@ delete_backup_file(Filename0) -> _ -> {error, not_found} end; - {error, _Reason} -> + {error, not_found} -> {error, not_found} end. @@ -788,8 +810,11 @@ import(Filename, OverridesJson) -> -spec(check_import_json(binary() | string()) -> {ok, map()} | {error, term()}). check_import_json(Filename) -> - ReadFile = fun(F) -> file:read_file(F) end, - FunList = [fun ensure_file_name/1, ReadFile, fun check_json/1], + FunList = [ + fun look_up_file/1, + fun(F) -> file:read_file(F) end, + fun check_json/1 + ], check_import_json(Filename, FunList). check_import_json(Res, []) -> @@ -815,6 +840,9 @@ backup_dir() -> ok = filelib:ensure_dir(filename:join([Dir, dummy])), Dir. +backup_dir_odl_version() -> + emqx:get_env(data_dir). + legal_filename(Filename) -> MaybeJson = filename:extension(Filename), MaybeJson == ".json" orelse MaybeJson == <<".json">>. From 95cc5a19c856544e107eb92996818a8b01b5be98 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 15 Mar 2022 21:57:51 +0800 Subject: [PATCH 258/363] fix: bad list & bad SUITE --- apps/emqx_management/src/emqx_mgmt_api_data.erl | 2 +- .../test/emqx_auth_mnesia_migration_SUITE.erl | 5 ++++- .../test/emqx_bridge_mqtt_data_export_import_SUITE.erl | 10 +++++++--- apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 3 ++- .../test/emqx_webhook_data_export_import_SUITE.erl | 7 +++++-- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index e9a00907b..93d5498f1 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -160,4 +160,4 @@ import_content(Content) -> tmp_filename() -> Seconds = erlang:system_time(second), {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), - io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]). + list_to_binary(io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S])). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl index 19ec8d61e..313f7767b 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl @@ -183,7 +183,10 @@ do_import(File, Config, Overrides) -> mnesia:clear_table(?ACL_TABLE2), mnesia:clear_table(emqx_user), emqx_acl_mnesia_migrator:migrate_records(), - Filename = filename:join(proplists:get_value(data_dir, Config), File), + Filename = filename:basename(File), + FilePath = filename:join([proplists:get_value(data_dir, Config), File]), + {ok, Bin} = file:read_file(FilePath), + ok = emqx_mgmt_data_backup:upload_backup_file(Filename, Bin), emqx_mgmt_data_backup:import(Filename, Overrides). test_import(username, {Username, Password}) -> diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index bae633045..11ee96d4f 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -34,14 +34,18 @@ init_per_suite(Cfg) -> Cfg. end_per_suite(Cfg) -> + emqx_mgmt_data_backup:delete_all_backup_file(), emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]), Cfg. get_data_path() -> emqx_ct_helpers:deps_path(emqx_management, "test/emqx_bridge_mqtt_data_export_import_SUITE_data/"). -import(FilePath, Version) -> - ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), +import(FilePath0, Version) -> + Filename = filename:basename(FilePath0), + FilePath = filename:join([get_data_path(), FilePath0]), + {ok, Bin} = file:read_file(FilePath), + ok = emqx_mgmt_data_backup:upload_backup_file(Filename, Bin), timer:sleep(500), lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> case Id of @@ -181,4 +185,4 @@ remove_resources() -> lists:foreach(fun(#resource{id = Id}) -> emqx_rule_engine:delete_resource(Id) end, emqx_rule_registry:get_resources()), - timer:sleep(500). \ No newline at end of file + timer:sleep(500). diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index 5631e5b78..27d3a2cbf 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -719,9 +719,10 @@ t_data_import_content(_) -> Dir = emqx_mgmt_data_backup:backup_dir(), {ok, Bin} = file:read_file(filename:join(Dir, Filename)), Content = emqx_json:decode(Bin), + ct:pal("Content:::: ~p~n", [Content]), ?assertMatch({ok, "{\"code\":0}"}, request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), - + emqx_mgmt_data_backup:delete_all_backup_file(), application:stop(emqx_rule_engine), application:stop(emqx_dashboard). diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl index 2965b7ad0..e22222e4c 100644 --- a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl @@ -46,8 +46,11 @@ remove_resource(Id) -> emqx_rule_registry:remove_resource(Id), emqx_rule_registry:remove_resource_params(Id). -import(FilePath, Version) -> - ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), +import(FilePath0, Version) -> + Filename = filename:basename(FilePath0), + FilePath = filename:join([get_data_path(), FilePath0]), + {ok, Bin} = file:read_file(FilePath), + ok = emqx_mgmt_data_backup:upload_backup_file(Filename, Bin), lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> case Id of <<"webhook">> -> From 21ed258d5806825bf3311da310534c130aa3ae9b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 18 Mar 2022 12:23:13 +0100 Subject: [PATCH 259/363] fix(session): compatible to 4.3 takeover --- src/emqx.appup.src | 2 ++ src/emqx_cm.erl | 5 +++-- src/emqx_session.erl | 12 ++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 68ca21a3b..4678aca89 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -3,6 +3,7 @@ {VSN, [{"4.4.1", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, @@ -40,6 +41,7 @@ {<<".*">>,[]}], [{"4.4.1", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 4aa953caa..4b06f35a8 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -242,7 +242,8 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> {ok, #{session => Session, present => false}} end, case takeover_session(ClientId) of - {ok, ConnMod, ChanPid, Session} -> + {ok, ConnMod, ChanPid, Session0} -> + Session = emqx_session:upgrade(ClientInfo, Session0), ok = emqx_session:resume(ClientInfo, Session), case request_stepdown({takeover, 'end'}, ConnMod, ChanPid) of {ok, Pendings} -> @@ -289,7 +290,7 @@ takeover_session(ClientId, ChanPid) when node(ChanPid) == node() -> ConnMod when is_atom(ConnMod) -> case request_stepdown({takeover, 'begin'}, ConnMod, ChanPid) of {ok, Session} -> - {ok, ConnMod, ChanPid, Session}; + {ok, ConnMod, ChanPid, emqx_session:downgrade(Session)}; {error, Reason} -> {error, Reason} end diff --git a/src/emqx_session.erl b/src/emqx_session.erl index bb5e84ac4..8612d9f16 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -83,6 +83,8 @@ -export([ takeover/1 , resume/2 , replay/2 + , upgrade/2 + , downgrade/1 ]). -export([expire/2]). @@ -196,6 +198,16 @@ init_mqueue(Zone) -> default_priority => get_env(Zone, mqueue_default_priority, lowest) }). +%% @doc uprade from 4.3 +upgrade(CInfo, S) -> + [session | Fields] = tuple_to_list(S), + #session{} = list_to_tuple([session, ?GET_CLIENT_ID(CInfo) | Fields] ++ [#{}]). + +%% @doc Downgrade to 4.3 +downgrade(#session{} = S) -> + [session, _ClientID | Fields] = tuple_to_list(S), + list_to_tuple([session | lists:reverse(tl(lists:reverse(Fields)))]). + %%-------------------------------------------------------------------- %% Info, Stats %%-------------------------------------------------------------------- From d56650e78b253f313994688fb3ff27fd330042a7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 18 Mar 2022 17:57:17 +0100 Subject: [PATCH 260/363] test: fix session downgrade/upgrade in tests --- src/emqx_session.erl | 7 +++++++ test/emqx_cm_SUITE.erl | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 8612d9f16..0364bde61 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -165,6 +165,7 @@ -ifdef(TEST). -define(GET_CLIENT_ID(C), maps:get(clientid, C, <<>>)). +-export([dummy/0]). -else. -define(GET_CLIENT_ID(C), maps:get(clientid, C)). -endif. @@ -200,14 +201,20 @@ init_mqueue(Zone) -> %% @doc uprade from 4.3 upgrade(CInfo, S) -> + ?LOG(warning, "upgrading from 4.3", []), [session | Fields] = tuple_to_list(S), #session{} = list_to_tuple([session, ?GET_CLIENT_ID(CInfo) | Fields] ++ [#{}]). %% @doc Downgrade to 4.3 downgrade(#session{} = S) -> + ?LOG(warning, "downgrading to 4.3", []), [session, _ClientID | Fields] = tuple_to_list(S), list_to_tuple([session | lists:reverse(tl(lists:reverse(Fields)))]). +-ifdef(TEST). +dummy() -> + #session{}. +-endif. %%-------------------------------------------------------------------- %% Info, Stats %%-------------------------------------------------------------------- diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl index 4fcaaf473..84b362c0a 100644 --- a/test/emqx_cm_SUITE.erl +++ b/test/emqx_cm_SUITE.erl @@ -332,15 +332,17 @@ t_discard_session_race(_) -> t_takeover_session(_) -> #{conninfo := ConnInfo} = ?ChanInfo, {error, not_found} = emqx_cm:takeover_session(<<"clientid">>), + Dummy = emqx_session:dummy(), + Downgraded = emqx_session:downgrade(Dummy), erlang:spawn_link(fun() -> ok = emqx_cm:register_channel(<<"clientid">>, self(), ConnInfo), receive {'$gen_call', From, {takeover, 'begin'}} -> - gen_server:reply(From, test), ok + gen_server:reply(From, Dummy), ok end end), timer:sleep(100), - {ok, emqx_connection, _, test} = emqx_cm:takeover_session(<<"clientid">>), + {ok, emqx_connection, _, Downgraded} = emqx_cm:takeover_session(<<"clientid">>), emqx_cm:unregister_channel(<<"clientid">>). t_all_channels(_) -> From ed9266b6c034f5ba485b53955397dcef88e4d131 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Mar 2022 12:05:55 +0100 Subject: [PATCH 261/363] ci: update to latest builder image --- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/apps_version_check.yaml | 2 +- .github/workflows/build_packages.yaml | 12 ++++++------ .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 2 +- CHANGES-4.4.md | 5 +++++ Makefile | 4 ++-- build | 3 ++- deploy/docker/Dockerfile | 4 ++-- scripts/buildx.sh | 4 ++-- 14 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 396f5081d..ffd35548c 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index a1bf859ba..c67c4daff 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -13,7 +13,7 @@ jobs: os: - ubuntu20.04 - container: ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-7:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f13e5b8b2..816fd3d53 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -264,7 +264,7 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/4.4-5:${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/4.4-7:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: @@ -339,8 +339,8 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.otp }}-alpine3.14 - RUN_FROM=alpine:3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-7:${{ matrix.otp }}-alpine3.15.1 + RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile context: source @@ -354,8 +354,8 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.otp }}-alpine3.14 - RUN_FROM=alpine:3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-7:${{ matrix.otp }}-alpine3.15.1 + RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise context: source diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index f91e1dcd0..298380f0c 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -23,7 +23,7 @@ jobs: - ubuntu20.04 - rockylinux8 - container: ghcr.io/emqx/emqx-builder/4.4-5:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-7:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 98c2e8327..6e7f05e3f 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0d90195ca..c8f041816 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 68a06f11b..a291c3927 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index ced28a683..fdec2765a 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -224,7 +224,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -275,7 +275,7 @@ jobs: otp: - 24.1.5-3 runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 16f35be6b..94c9f83b0 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index c776c8e15..8e0b26378 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -2,9 +2,14 @@ ## v4.4.2 +### Important changes + +* Docker image is based on alpine-3.15.1 (OpenSSL-1.1.1n) + ### Minor changes * Windows package is built on Erlang/OTP 24 +* Added debian11 build ## v4.4.1 diff --git a/Makefile b/Makefile index 00c6936b4..55bd72667 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-alpine3.14 -export EMQX_DEFAULT_RUNNER = alpine:3.14 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-alpine3.15.1 +export EMQX_DEFAULT_RUNNER = alpine:3.15.1 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export DOCKERFILE := deploy/docker/Dockerfile diff --git a/build b/build index 8bf435357..83190bcbd 100755 --- a/build +++ b/build @@ -164,7 +164,8 @@ make_zip() { log "Zip package sha256sum: $(cat "${target_zip}.sha256")" } -## This function builds the default docker image based on alpine:3.14 (by default) +## This function builds the default docker image +## based images is by default $EMQX_DEFAULT_BUILDER (see Makefile) make_docker() { EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}" EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}" diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 6f48d3432..eaa9b0227 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,5 +1,5 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-alpine3.14 -ARG RUN_FROM=alpine:3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-alpine3.15.1 +ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder RUN apk add --no-cache \ diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 94da5fb9b..2cb5a718d 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-debian10 --arch arm64 +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-debian10 --arch arm64 set -euo pipefail @@ -20,7 +20,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-debian10" } while [ "$#" -gt 0 ]; do From 692e2c79a27536fc48037f6964319f1f736bb6d0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Mar 2022 20:58:34 +0100 Subject: [PATCH 262/363] chore: update appups after merge --- .../src/emqx_rule_engine.appup.src | 44 ++++++++++--------- .../emqx_modules/src/emqx_modules.appup.src | 25 +++++------ src/emqx.appup.src | 19 +++++--- 3 files changed, 50 insertions(+), 38 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index b4abcd68c..8c42ac493 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,27 +1,31 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.1", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - ]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {update, emqx_rule_metrics, {advanced, ["4.4.0"]}} - , {load_module,emqx_rule_events,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]} - ]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {update,emqx_rule_metrics,{advanced,["4.4.0"]}}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.1", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - ]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]} - , {update, emqx_rule_metrics, {advanced, ["4.4.0"]}} - , {load_module,emqx_rule_events,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]} - , {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]} - ]}, - {<<".*">>,[]}] -}. + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {update,emqx_rule_metrics,{advanced,["4.4.0"]}}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index f3f49f24f..dd4895a7e 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,16 +1,15 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.0", - [{load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_sup, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_trace_api, brutal_purge, soft_purge, []}]}, - {<<".*">>, []} - ], - [{"4.4.0", - [{load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_sup, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_trace_api, brutal_purge, soft_purge, []}]}, - {<<".*">>, []} - ] -}. + [{"4.4.0", + [{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.4.0", + [{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 7b4d5617a..383db43f1 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,9 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.1", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}, @@ -16,7 +18,9 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -40,7 +44,9 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.1", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, @@ -51,9 +57,12 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {delete_module,emqx_relup}]}, {"4.4.0", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, From 42d94b478fcb7d11061f1300a15bda7fd52b3e79 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Mar 2022 21:08:20 +0100 Subject: [PATCH 263/363] fix: update app vsn and update appup --- apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src | 2 +- apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src | 10 ++++++---- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- lib-ce/emqx_modules/src/emqx_modules.appup.src | 6 ++++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src index ecd948944..d899f35c4 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mongo, [{description, "EMQ X Authentication/ACL with MongoDB"}, - {vsn, "4.4.1"}, % strict semver, bump manually! + {vsn, "4.4.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mongo_sup]}, {applications, [kernel,stdlib,mongodb,ecpool]}, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src index de7107e99..f05fa9de1 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src @@ -1,13 +1,15 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.0", + [{"4.4.1",[{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, + {"4.4.0", [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.0", + [{"4.4.1",[{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, + {"4.4.0", [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}] -}. + {<<".*">>,[]}]}. diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index a54c10418..fbcc6fc68 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.4.1"}, + {vsn, "4.4.2"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index dd4895a7e..9176467c3 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,13 +1,15 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.0", + [{"4.4.1",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, + {"4.4.0", [{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.0", + [{"4.4.1",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, + {"4.4.0", [{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, From a5d1dbaf66a5f91c7c5384257857601c49a75ae4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Mar 2022 21:26:38 +0100 Subject: [PATCH 264/363] docs: sync 4.3.13 changes to 4.4.2 --- CHANGES-4.4.md | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 8e0b26378..d3889309c 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -2,14 +2,52 @@ ## v4.4.2 +**NOTE**: v4.4.2 is in sync with: v4.3.13 + ### Important changes * Docker image is based on alpine-3.15.1 (OpenSSL-1.1.1n) +* For docker image, /opt/emqx/etc has been removed from the VOLUME list, + this made it easier for the users to rebuild image on top with changed configs. -### Minor changes +### Enhancements * Windows package is built on Erlang/OTP 24 -* Added debian11 build + +### Enhancements (synced from v4.3.13) + +* CLI `emqx_ctl pem_cache clean` to force purge x509 certificate cache, + to force an immediate reload of all certificates after the files are updated on disk. +* Refactor the ExProto so that anonymous clients can also be displayed on the dashboard [#6983] +* Force shutdown of processes that cannot answer takeover event [#7026] +* Support set keepalive via queryString & Body HTTP API. +* `topic` parameter in bridge configuration can have `${node}` substitution (just like in `clientid` parameter) +* Add UTF-8 string validity check in `strict_mode` for MQTT packet. + When set to true, invalid UTF-8 strings will cause the client to be disconnected. i.e. client ID, topic name. [#7261] +* Changed systemd service restart delay from 10 seconds to 60 seconds. +* MQTT-SN gateway supports initiative to synchronize registered topics after session resumed. [#7300] +* Add load control app for future development. +* Change the precision of float to 17 digits after the decimal point when formatting a + float using payload templates of rule actions. The old precision is 10 digits before + this change. + +### Bug fixes (synced from v4.3.13) + +* Fix the `{error,eexist}` error when do release upgrade again if last run failed. [#7121] +* Fix case where publishing to a non-existent topic alias would crash the connection [#6979] +* Fix HTTP-API 500 error on querying the lwm2m client list on the another node [#7009] +* Fix the ExProto connection registry is not released after the client process abnormally exits [#6983] +* Fix Server-KeepAlive wrongly applied on MQTT v3.0/v3.1 [#7085] +* Fix Stomp client can not trigger `$event/client_connection` message [#7096] +* Fix system memory false alarm at boot +* Fix the MQTT-SN message replay when the topic is not registered to the client [#6970] +* Fix rpc get node info maybe crash when other nodes is not ready. +* Fix false alert level log “cannot_find_plugins” caused by duplicate plugin names in `loaded_plugins` files. +* Prompt user how to change the dashboard's initial default password when emqx start. +* Fix errno=13 'Permission denied' Cannot create FIFO boot error in Amazon Linux 2022 (el8 package) +* Fix user or appid created, name only allow `^[A-Za-z]+[A-Za-z0-9-_]*$` +* Fix subscribe http api crash by bad_qos `/mqtt/subscribe`,`/mqtt/subscribe_batch`. +* Send DISCONNECT packet with reason code 0x98 if connection has been kicked [#7309] ## v4.4.1 From 5a042645e2056b261354c09efd78eb49a1317a48 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 21 Mar 2022 10:30:12 +0800 Subject: [PATCH 265/363] fix: make all traces stopped when disable emqx_trace_module --- CHANGES-4.4.md | 1 + apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl | 6 ++++++ .../src/emqx_trace/emqx_trace_api.erl | 13 ++++++++----- src/emqx.appup.src | 5 +++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index d3889309c..59c62d9c5 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -48,6 +48,7 @@ * Fix user or appid created, name only allow `^[A-Za-z]+[A-Za-z0-9-_]*$` * Fix subscribe http api crash by bad_qos `/mqtt/subscribe`,`/mqtt/subscribe_batch`. * Send DISCONNECT packet with reason code 0x98 if connection has been kicked [#7309] +* Fix make all traces stopped when emqx_trace_module is disabled. ## v4.4.1 diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index fd0110a36..84673b4a7 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -43,8 +43,10 @@ , trace_dir/0 , trace_file/1 , delete_files_after_send/2 + , is_enable/0 ]). + -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -define(TRACE, ?MODULE). @@ -102,6 +104,10 @@ start_link() -> list() -> ets:match_object(?TRACE, #?TRACE{_ = '_'}). +-spec is_enable() -> boolean(). +is_enable() -> + undefined =/= erlang:whereis(?MODULE). + -spec list(boolean()) -> [tuple()]. list(Enable) -> ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 548b58ae9..b8aa5c059 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -50,11 +50,12 @@ list_trace(_, _Params) -> FileName = emqx_trace:filename(Name, Start), LogSize = collect_file_size(Nodes, FileName, AllFileSize), Trace0 = maps:without([enable, filter], Trace), + ModEnable = emqx_trace:is_enable(), Trace0#{ log_size => LogSize , Type => iolist_to_binary(Filter) , start_at => list_to_binary(calendar:system_time_to_rfc3339(Start)) , end_at => list_to_binary(calendar:system_time_to_rfc3339(End)) - , status => status(Enable, Start, End, Now) + , status => status(ModEnable, Enable, Start, End, Now) } end, List), {ok, Traces} @@ -208,7 +209,9 @@ collect_file_size(Nodes, FileName, AllFiles) -> Acc#{Node => Size} end, #{}, Nodes). -status(false, _Start, _End, _Now) -> <<"stopped">>; -status(true, Start, _End, Now) when Now < Start -> <<"waiting">>; -status(true, _Start, End, Now) when Now >= End -> <<"stopped">>; -status(true, _Start, _End, _Now) -> <<"running">>. +%% if the module is not running, it will return stopped, user can download the trace file. +status(false, _Enable, _Start, _End, _Now) -> <<"stopped">>; +status(true, false, _Start, _End, _Now) -> <<"stopped">>; +status(true, true, Start, _End, Now) when Now < Start -> <<"waiting">>; +status(true, true, _Start, End, Now) when Now >= End -> <<"stopped">>; +status(true, true, _Start, _End, _Now) -> <<"running">>. diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 383db43f1..94ad634c7 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -7,6 +7,7 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -23,6 +24,7 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -49,6 +51,8 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {delete_module,emqx_relup}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -65,6 +69,7 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, From d2abc2fcd0bcad05d6a8e093d28d9f0c39e49702 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 21 Mar 2022 15:20:35 +0800 Subject: [PATCH 266/363] fix: revert emqx_trace in emqx.appup.src --- src/emqx.appup.src | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 94ad634c7..f1c6a2a78 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -7,7 +7,6 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -24,7 +23,6 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -51,7 +49,6 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -69,7 +66,6 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, From ea915339125e6ca8e99a18cc7bde3fb94aa0acf0 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 23 Mar 2022 11:18:46 +0800 Subject: [PATCH 267/363] fix: delete multiply defined module(emqx_relup) in emqx.appup.src --- src/emqx.appup.src | 1 - 1 file changed, 1 deletion(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index f1c6a2a78..383db43f1 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -49,7 +49,6 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {delete_module,emqx_relup}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, From 4c4604d39715223babf4a5b7d2277817a8fa2cb8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Mar 2022 21:28:58 +0100 Subject: [PATCH 268/363] build: add debian11 --- .github/workflows/build_packages.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 816fd3d53..ff236f966 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -214,6 +214,7 @@ jobs: - ubuntu20.04 - ubuntu18.04 - ubuntu16.04 + - debian11 - debian10 - debian9 - rockylinux8 From 929ca787f476efa6c7b97750107454d83d4a4678 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 23 Mar 2022 13:12:57 +0100 Subject: [PATCH 269/363] test: fix data import test failure --- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 19 ++++++++++++------- ...emqx_mongo_auth_module_migration_SUITE.erl | 5 ++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 07f97f89e..5e9d05b5a 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -292,8 +292,7 @@ t_subscriptions_cmd(_) -> [?assertMatch({match, _} , re:run(Result, "b/b/c")) || Result <- emqx_mgmt_cli:subscriptions(["show", <<"client">>])], ?assertEqual(emqx_mgmt_cli:subscriptions(["add", "client", "b/b/c", "0"]), "ok\n"), - ?assertEqual(emqx_mgmt_cli:subscriptions(["del", "client", "b/b/c"]), "ok\n"), - unmock_print(). + ?assertEqual(emqx_mgmt_cli:subscriptions(["del", "client", "b/b/c"]), "ok\n"). t_listeners_cmd_old(_) -> ok = emqx_listeners:ensure_all_started(), @@ -322,8 +321,7 @@ t_listeners_cmd_new(_) -> emqx_mgmt_cli:listeners(["restart", "bad:listener:identifier"]), "Failed to restart bad:listener:identifier listener:" " {no_such_listener,\"bad:listener:identifier\"}\n" - ), - unmock_print(). + ). t_plugins_cmd(_) -> ?assertEqual(emqx_mgmt_cli:plugins(["list"]), ok), @@ -338,8 +336,7 @@ t_plugins_cmd(_) -> ?assertEqual( emqx_mgmt_cli:plugins(["unload", "emqx_management"]), "Plugin emqx_management can not be unloaded.\n" - ), - unmock_print(). + ). t_cli(_) -> ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([""]), "status")), @@ -388,7 +385,7 @@ t_backup_file(_)-> ok. mock_print() -> - catch meck:unload(emqx_ctl), + ok = safe_unmeck(emqx_ctl), meck:new(emqx_ctl, [non_strict, passthrough]), meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg, []) end), meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end), @@ -397,3 +394,11 @@ mock_print() -> unmock_print() -> meck:unload(emqx_ctl). + +safe_unmeck(Module) -> + %% isolate exits + {Pid, Ref} = erlang:spawn_monitor(fun() -> meck:unload(Module) end), + receive + {'DOWN', Ref, process, Pid, _} -> + ok + end. diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl index 3a697a17d..748424a8f 100644 --- a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl @@ -60,7 +60,10 @@ t_import_4_3(Config) -> import(File, Config) -> Filename = filename:join(proplists:get_value(data_dir, Config), File), - emqx_mgmt_data_backup:import(Filename, "{}"). + {ok, Content} = file:read_file(Filename), + BackupFile = filename:join(emqx:get_env(data_dir), File), + ok = file:write_file(BackupFile, Content), + emqx_mgmt_data_backup:import(File, "{}"). delete_modules() -> [emqx_modules_registry:remove_module(Mod) || Mod <- emqx_modules_registry:get_modules()]. From 1c448d6016183569055c6303c7a28a3a423a6590 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 23 Mar 2022 13:51:13 +0100 Subject: [PATCH 270/363] test: use try catch instaed of sopan a process --- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 5e9d05b5a..de695e468 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -396,9 +396,10 @@ unmock_print() -> meck:unload(emqx_ctl). safe_unmeck(Module) -> - %% isolate exits - {Pid, Ref} = erlang:spawn_monitor(fun() -> meck:unload(Module) end), - receive - {'DOWN', Ref, process, Pid, _} -> + try + meck:unload(Module), + ok + catch + _ : _ -> ok end. From 56683328eca68315516a9268d217345f2a19268c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 23 Mar 2022 15:49:58 +0100 Subject: [PATCH 271/363] docs: delete '4.3' from 4.4 config file comment --- etc/emqx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 8b8c0aca4..3b4cca025 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1,4 +1,4 @@ -## EMQX Configuration 4.3 +## EMQX Configuration ## NOTE: Do not change format of CONFIG_SECTION_{BGN,END} comments! From b99627b86670b1bcbd6f42221960412825e803d4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 23 Mar 2022 15:50:36 +0100 Subject: [PATCH 272/363] chore(emqx_auth_mongo_sup): port changes from ee back to ce --- apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl index 55263494a..982b20727 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl @@ -34,7 +34,8 @@ init([]) -> {ok, {{one_for_all, 10, 100}, [PoolSpec]}}. may_parse_srv_and_txt_records(Opts) when is_list(Opts) -> - maps:to_list(may_parse_srv_and_txt_records(maps:from_list(Opts))); + Default = #{srv_record => false}, + maps:to_list(may_parse_srv_and_txt_records(maps:merge(Default, maps:from_list(Opts)))); may_parse_srv_and_txt_records(#{type := Type, srv_record := false, From 7e01a5245710ad291ef2fcc4f3d45a858299afbe Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 23 Mar 2022 15:56:51 +0100 Subject: [PATCH 273/363] fix: typo in function name --- apps/emqx_management/src/emqx_mgmt_data_backup.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index d5c6c7b97..aa2cbc40f 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -650,7 +650,7 @@ list_backup_file() -> lists:filtermap(Filter, backup_files()). backup_files() -> - backup_files(backup_dir()) ++ backup_files(backup_dir_odl_version()). + backup_files(backup_dir()) ++ backup_files(backup_dir_old_version()). backup_files(Dir) -> {ok, FilesAll} = file:list_dir_all(Dir), @@ -840,7 +840,7 @@ backup_dir() -> ok = filelib:ensure_dir(filename:join([Dir, dummy])), Dir. -backup_dir_odl_version() -> +backup_dir_old_version() -> emqx:get_env(data_dir). legal_filename(Filename) -> From ada29de3d5235f028f8214b1bfc78d608daf39b0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 23 Mar 2022 15:58:07 +0100 Subject: [PATCH 274/363] fix(appup): add changed modules --- apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src index f05fa9de1..641ce3e93 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src @@ -1,15 +1,21 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.1",[{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, + [{"4.4.1", + [{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.1",[{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, + [{"4.4.1", + [{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. From 6e08ede0b026fd209b26732ade0735122269295e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 24 Mar 2022 07:38:42 +0800 Subject: [PATCH 275/363] chore: update appup.src files by update-appup.sh --- .../src/emqx_rule_engine.appup.src | 8 ++++++-- lib-ce/emqx_modules/src/emqx_modules.appup.src | 14 ++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 8c42ac493..0028cc988 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,7 +2,9 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.1", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", @@ -16,7 +18,9 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.1", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index 9176467c3..7e79063f8 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,16 +1,22 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.1",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, + [{"4.4.1", + [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, + [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.1",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, + [{"4.4.1", + [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, + [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, From 65e11bcb01f868017f6ac9728224d0adca37d7dc Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 24 Mar 2022 14:26:52 +0800 Subject: [PATCH 276/363] chore: bump vsn for 4.4.2-rc.1 --- include/emqx_release.hrl | 2 +- scripts/get-dashboard.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index f3e6ab562..a000946e3 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.1"}). +-define(EMQX_RELEASE, {opensource, "4.4.2-rc.1"}). -else. diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index a83294568..9dd638a40 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -13,8 +13,8 @@ case "${PKG_VSN}" in ;; 4.4*) # keep the above 4.3 untouched, otherwise conflicts! - EMQX_CE_DASHBOARD_VERSION='v4.4.0' - EMQX_EE_DASHBOARD_VERSION='see-enterprise-repo' + EMQX_CE_DASHBOARD_VERSION='v4.4.1' + EMQX_EE_DASHBOARD_VERSION='v4.4.7' ;; *) echo "Unsupported version $PKG_VSN" >&2 From 3410e20fbe09081e4010cc5b68bd5da9c100f7b4 Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Thu, 24 Mar 2022 14:49:48 +0800 Subject: [PATCH 277/363] feat(emqx_rule_engine_events): add client_connack event --- .../emqx_rule_engine/src/emqx_rule_events.erl | 66 +++++++++++++++++++ .../test/emqx_rule_engine_SUITE.erl | 26 ++++++++ 2 files changed, 92 insertions(+) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 540b1cbbd..0ef00ea85 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -31,6 +31,7 @@ -export([ on_client_connected/3 , on_client_disconnected/4 + , on_client_connack/4 , on_session_subscribed/4 , on_session_unsubscribed/4 , on_message_publish/2 @@ -48,6 +49,7 @@ -define(SUPPORTED_HOOK, [ 'client.connected' , 'client.disconnected' + , 'client.connack' , 'session.subscribed' , 'session.unsubscribed' , 'message.publish' @@ -106,6 +108,10 @@ on_client_disconnected(ClientInfo, Reason, ConnInfo, Env) -> may_publish_and_apply('client.disconnected', fun() -> eventmsg_disconnected(ClientInfo, ConnInfo, Reason) end, Env). +on_client_connack(ConnInfo, Reason, _, Env) -> + may_publish_and_apply('client.connack', + fun() -> eventmsg_connack(ConnInfo, Reason) end, Env). + on_session_subscribed(ClientInfo, Topic, SubOpts, Env) -> may_publish_and_apply('session.subscribed', fun() -> eventmsg_sub_or_unsub('session.subscribed', ClientInfo, Topic, SubOpts) end, Env). @@ -220,6 +226,34 @@ eventmsg_disconnected(_ClientInfo = #{ disconnected_at => DisconnectedAt }). +eventmsg_connack(_ConnInfo = #{ + clientid := ClientId, + clean_start := CleanStart, + username := Username, + peername := PeerName, + sockname := SockName, + proto_name := ProtoName, + proto_ver := ProtoVer, + keepalive := Keepalive, + connected_at := ConnectedAt, + conn_props := ConnProps, + expiry_interval := ExpiryInterval + }, Reason) -> + with_basic_columns('client.connack', + #{reason_code => reason(Reason), + clientid => ClientId, + clean_start => CleanStart, + username => Username, + peername => ntoa(PeerName), + sockname => ntoa(SockName), + proto_name => ProtoName, + proto_ver => ProtoVer, + keepalive => Keepalive, + expiry_interval => ExpiryInterval, + connected_at => ConnectedAt, + conn_props => printable_maps(ConnProps) + }). + eventmsg_sub_or_unsub(Event, _ClientInfo = #{ clientid := ClientId, username := Username, @@ -372,6 +406,7 @@ event_info() -> , event_info_delivery_dropped() , event_info_client_connected() , event_info_client_disconnected() + , event_info_client_connack() , event_info_session_subscribed() , event_info_session_unsubscribed() ]. @@ -427,6 +462,13 @@ event_info_client_disconnected() -> {<<"client disconnected">>, <<"连接断开"/utf8>>}, <<"SELECT * FROM \"$events/client_disconnected\" WHERE topic =~ 't/#'">> ). +event_info_client_connack() -> + event_info_common( + 'client.connack', + {<<"client connack">>, <<"连接确认"/utf8>>}, + {<<"client connack">>, <<"连接确认"/utf8>>}, + <<"SELECT * FROM \"$events/client_connack\"">> + ). event_info_session_subscribed() -> event_info_common( 'session.subscribed', @@ -485,6 +527,11 @@ test_columns('client.disconnected') -> , {<<"username">>, <<"u_emqx">>} , {<<"reason">>, <<"normal">>} ]; +test_columns('client.connack') -> + [ {<<"clientid">>, <<"c_emqx">>} + , {<<"username">>, <<"u_emqx">>} + , {<<"reason_code">>, <<"sucess">>} + ]; test_columns('session.unsubscribed') -> test_columns('session.subscribed'); test_columns('session.subscribed') -> @@ -607,6 +654,23 @@ columns_with_exam('client.disconnected') -> , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; +columns_with_exam('client.connack') -> + [ {<<"event">>, 'client.connected'} + , {<<"reason_code">>, success} + , {<<"clientid">>, <<"c_emqx">>} + , {<<"username">>, <<"u_emqx">>} + , {<<"peername">>, <<"192.168.0.10:56431">>} + , {<<"sockname">>, <<"0.0.0.0:1883">>} + , {<<"proto_name">>, <<"MQTT">>} + , {<<"proto_ver">>, 5} + , {<<"keepalive">>, 60} + , {<<"clean_start">>, true} + , {<<"expiry_interval">>, 3600} + , {<<"connected_at">>, erlang:system_time(millisecond)} + , columns_example_props(conn_props) + , {<<"timestamp">>, erlang:system_time(millisecond)} + , {<<"node">>, node()} + ]; columns_with_exam('session.subscribed') -> [ {<<"event">>, 'session.subscribed'} , {<<"clientid">>, <<"c_emqx">>} @@ -694,6 +758,7 @@ ntoa(IpAddr) -> event_name(<<"$events/client_connected", _/binary>>) -> 'client.connected'; event_name(<<"$events/client_disconnected", _/binary>>) -> 'client.disconnected'; +event_name(<<"$events/client_connack", _/binary>>) -> 'client.connack'; event_name(<<"$events/session_subscribed", _/binary>>) -> 'session.subscribed'; event_name(<<"$events/session_unsubscribed", _/binary>>) -> 'session.unsubscribed'; @@ -705,6 +770,7 @@ event_name(_) -> 'message.publish'. event_topic('client.connected') -> <<"$events/client_connected">>; event_topic('client.disconnected') -> <<"$events/client_disconnected">>; +event_topic('client.connack') -> <<"$events/client_connack">>; event_topic('session.subscribed') -> <<"$events/session_subscribed">>; event_topic('session.unsubscribed') -> <<"$events/session_unsubscribed">>; event_topic('message.delivered') -> <<"$events/message_delivered">>; diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 02e0f607c..ca4b8db0c 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -101,6 +101,7 @@ groups() -> t_sqlselect_2_2, t_sqlselect_2_3, t_sqlselect_3, + t_sqlselect_3_1, t_sqlparse_event_1, t_sqlparse_event_2, t_sqlparse_event_3, @@ -1442,6 +1443,31 @@ t_sqlselect_3(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). +t_sqlselect_3_1(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% republish the client.connected msg + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/client_connack\" " + "WHERE username = 'emqx1'", + <<"clientid=${clientid}">>), + {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + ct:sleep(200), + {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), + {ok, _} = emqtt:connect(Client1), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"clientid=c_emqx1">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + t_metrics(_Config) -> ok = emqx_rule_engine:load_providers(), TopicRule = create_simple_repub_rule( From 326b01968bfdf61c712322870b1a10b79426d1d4 Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Thu, 24 Mar 2022 16:52:01 +0800 Subject: [PATCH 278/363] fix(appup): load_module emqx_rule_events --- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 0028cc988..d5b2c6319 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.1", - [{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, @@ -18,7 +19,8 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.1", - [{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, From 2d97b1bb8931fa3f0fec3ebec842b32624904636 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 24 Mar 2022 12:26:09 +0100 Subject: [PATCH 279/363] chore: delete warning message for session upgrade/downgrade --- src/emqx_session.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 0364bde61..a5b723b59 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -201,13 +201,11 @@ init_mqueue(Zone) -> %% @doc uprade from 4.3 upgrade(CInfo, S) -> - ?LOG(warning, "upgrading from 4.3", []), [session | Fields] = tuple_to_list(S), #session{} = list_to_tuple([session, ?GET_CLIENT_ID(CInfo) | Fields] ++ [#{}]). %% @doc Downgrade to 4.3 downgrade(#session{} = S) -> - ?LOG(warning, "downgrading to 4.3", []), [session, _ClientID | Fields] = tuple_to_list(S), list_to_tuple([session | lists:reverse(tl(lists:reverse(Fields)))]). From 489dbec76bb60401e3bea36a743cf39a61dd58d4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 24 Mar 2022 20:39:08 +0100 Subject: [PATCH 280/363] ci: exclude 4.4.0 and 4.4.1 for debian11 --- scripts/relup-base-vsns.sh | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 8f391b01b..91c26823e 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -58,6 +58,9 @@ case "${EDITION}" in ;; esac +# must not be empty for MacOS (bash 3.x) +TAGS=( 'dummy' ) +TAGS_EXCLUDE=( 'dummy' ) while read -r git_tag; do # shellcheck disable=SC2207 semver=($(parse_semver "$git_tag")) @@ -68,7 +71,25 @@ while read -r git_tag; do # because current version is not an upgrade base true else - echo "$git_tag" + TAGS+=( "$git_tag" ) fi fi done < <(git tag -l "${GIT_TAG_PREFIX}${CUR_SEMVER[0]}.${CUR_SEMVER[1]}.*") + +# debian11 is introduced since v4.4.2 and e4.4.2 +# exclude tags before them +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" +if [ "$SYSTEM" = 'debian11' ]; then + TAGS_EXCLUDE+=( 'v4.4.0' 'v4.4.1' ) + TAGS_EXCLUDE+=( 'e4.4.0' 'e4.4.1' ) +fi + +for tag_to_del in "${TAGS_EXCLUDE[@]}"; do + TAGS=( "${TAGS[@]/$tag_to_del}" ) +done + +for tag in "${TAGS[@]}"; do + if [ "$tag" != '' ]; then + echo "$tag" + fi +done From e0d142c625497e0e782cca86253cb1461c513fff Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Fri, 25 Mar 2022 11:08:30 +0800 Subject: [PATCH 281/363] feat: add client.check_acl_complete event --- .../emqx_rule_engine/src/emqx_rule_events.erl | 52 +++++++++++- .../test/emqx_rule_engine_SUITE.erl | 79 ++++++++++++------- src/emqx_access_control.erl | 5 +- 3 files changed, 106 insertions(+), 30 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 0ef00ea85..1a03d01fc 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -39,6 +39,7 @@ , on_message_delivered/3 , on_message_acked/3 , on_delivery_dropped/4 + , on_client_check_acl_complete/5 ]). -export([ event_info/0 @@ -57,6 +58,7 @@ , 'message.acked' , 'message.dropped' , 'delivery.dropped' + , 'client.check_acl_complete' ]). -ifdef(TEST). @@ -112,6 +114,13 @@ on_client_connack(ConnInfo, Reason, _, Env) -> may_publish_and_apply('client.connack', fun() -> eventmsg_connack(ConnInfo, Reason) end, Env). +on_client_check_acl_complete(ClientInfo, PubSub, Topic, Result, Env) -> + may_publish_and_apply('client.check_acl_complete', + fun() -> eventmsg_check_acl_complete(ClientInfo, + PubSub, + Topic, + Result) end, Env). + on_session_subscribed(ClientInfo, Topic, SubOpts, Env) -> may_publish_and_apply('session.subscribed', fun() -> eventmsg_sub_or_unsub('session.subscribed', ClientInfo, Topic, SubOpts) end, Env). @@ -253,6 +262,19 @@ eventmsg_connack(_ConnInfo = #{ connected_at => ConnectedAt, conn_props => printable_maps(ConnProps) }). +eventmsg_check_acl_complete(_ClientInfo = #{ + clientid := ClientId, + username := Username, + peerhost := PeerHost + }, PubSub, Topic, Result) -> + with_basic_columns('client.check_acl_complete', + #{clientid => ClientId, + username => Username, + peerhost => ntoa(PeerHost), + topic => Topic, + action => PubSub, + result => Result + }). eventmsg_sub_or_unsub(Event, _ClientInfo = #{ clientid := ClientId, @@ -409,6 +431,7 @@ event_info() -> , event_info_client_connack() , event_info_session_subscribed() , event_info_session_unsubscribed() + , event_info_client_check_acl_complete() ]. event_info_message_publish() -> @@ -483,6 +506,13 @@ event_info_session_unsubscribed() -> {<<"session unsubscribed">>, <<"会话取消订阅完成"/utf8>>}, <<"SELECT * FROM \"$events/session_unsubscribed\" WHERE topic =~ 't/#'">> ). +event_info_client_check_acl_complete() -> + event_info_common( + 'client.check_acl_complete', + {<<"client check acl complete">>, <<"鉴权结果"/utf8>>}, + {<<"client check acl complete">>, <<"鉴权结果"/utf8>>}, + <<"SELECT * FROM \"$events/client_check_acl_complete\"">> + ). event_info_common(Event, {TitleEN, TitleZH}, {DescrEN, DescrZH}, SqlExam) -> #{event => event_topic(Event), @@ -539,6 +569,13 @@ test_columns('session.subscribed') -> , {<<"username">>, <<"u_emqx">>} , {<<"topic">>, <<"t/a">>} , {<<"qos">>, 1} + ]; +test_columns('client.check_acl_complete') -> + [ {<<"clientid">>, <<"c_emqx">>} + , {<<"username">>, <<"u_emqx">>} + , {<<"topic">>, <<"t/1">>} + , {<<"action">>, <<"publish">>} + , {<<"result">>, <<"allow">>} ]. columns_with_exam('message.publish') -> @@ -692,6 +729,17 @@ columns_with_exam('session.unsubscribed') -> , columns_example_props(unsub_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} + ]; +columns_with_exam('client.check_acl_complete') -> + [ {<<"event">>, 'client.check_acl_complete'} + , {<<"clientid">>, <<"c_emqx">>} + , {<<"username">>, <<"u_emqx">>} + , {<<"peerhost">>, <<"192.168.0.10">>} + , {<<"topic">>, <<"t/a">>} + , {<<"action">>, <<"publish">>} + , {<<"result">>, <<"allow">>} + , {<<"timestamp">>, erlang:system_time(millisecond)} + , {<<"node">>, node()} ]. columns_example_props(PropType) -> @@ -766,6 +814,7 @@ event_name(<<"$events/message_delivered", _/binary>>) -> 'message.delivered'; event_name(<<"$events/message_acked", _/binary>>) -> 'message.acked'; event_name(<<"$events/message_dropped", _/binary>>) -> 'message.dropped'; event_name(<<"$events/delivery_dropped", _/binary>>) -> 'delivery.dropped'; +event_name(<<"$events/client_check_acl_complete", _/binary>>) -> 'client.check_acl_complete'; event_name(_) -> 'message.publish'. event_topic('client.connected') -> <<"$events/client_connected">>; @@ -777,7 +826,8 @@ event_topic('message.delivered') -> <<"$events/message_delivered">>; event_topic('message.acked') -> <<"$events/message_acked">>; event_topic('message.dropped') -> <<"$events/message_dropped">>; event_topic('delivery.dropped') -> <<"$events/delivery_dropped">>; -event_topic('message.publish') -> <<"$events/message_publish">>. +event_topic('message.publish') -> <<"$events/message_publish">>; +event_topic('client.check_acl_complete') -> <<"$events/client_check_acl_complete">>. printable_maps(undefined) -> #{}; printable_maps(Headers) -> diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index ca4b8db0c..136db6695 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -101,7 +101,6 @@ groups() -> t_sqlselect_2_2, t_sqlselect_2_3, t_sqlselect_3, - t_sqlselect_3_1, t_sqlparse_event_1, t_sqlparse_event_2, t_sqlparse_event_3, @@ -198,6 +197,8 @@ init_per_testcase(t_events, Config) -> description = #{en => <<"Hook metrics action">>}}), SQL = "SELECT * FROM \"$events/client_connected\", " "\"$events/client_disconnected\", " + "\"$events/client_connack\", " + "\"$events/client_check_acl_complete\", " "\"$events/session_subscribed\", " "\"$events/session_unsubscribed\", " "\"$events/message_acked\", " @@ -1014,7 +1015,7 @@ t_events(_Config) -> , {proto_ver, v5} , {properties, #{'Session-Expiry-Interval' => 60}} ]), - ct:pal("====== verify $events/client_connected"), + ct:pal("====== verify $events/client_connected, $events/client_connack"), client_connected(Client, Client2), ct:pal("====== verify $events/session_subscribed"), session_subscribed(Client2), @@ -1040,8 +1041,10 @@ message_publish(Client) -> client_connected(Client, Client2) -> {ok, _} = emqtt:connect(Client), {ok, _} = emqtt:connect(Client2), + verify_event('client.connack'), verify_event('client.connected'), ok. + client_disconnected(Client, Client2) -> ok = emqtt:disconnect(Client, 0, #{'User-Property' => {<<"reason">>, <<"normal">>}}), ok = emqtt:disconnect(Client2, 0, #{'User-Property' => {<<"reason">>, <<"normal">>}}), @@ -1054,6 +1057,7 @@ session_subscribed(Client2) -> , 1 ), verify_event('session.subscribed'), + verify_event('client.check_acl_complete'), ok. session_unsubscribed(Client2) -> {ok, _, _} = emqtt:unsubscribe( Client2 @@ -1443,31 +1447,6 @@ t_sqlselect_3(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_3_1(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% republish the client.connected msg - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/client_connack\" " - "WHERE username = 'emqx1'", - <<"clientid=${clientid}">>), - {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ct:sleep(200), - {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), - {ok, _} = emqtt:connect(Client1), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - t_metrics(_Config) -> ok = emqx_rule_engine:load_providers(), TopicRule = create_simple_repub_rule( @@ -2670,6 +2649,37 @@ verify_event_fields('client.disconnected', Fields) -> ?assert(0 =< RcvdAtElapse andalso RcvdAtElapse =< 60*1000), ?assert(EventAt =< Timestamp); +verify_event_fields('client.connack', Fields) -> + #{clientid := ClientId, + clean_start := CleanStart, + username := Username, + peername := PeerName, + sockname := SockName, + proto_name := ProtoName, + proto_ver := ProtoVer, + keepalive := Keepalive, + expiry_interval := ExpiryInterval, + conn_props := Properties, + timestamp := Timestamp, + connected_at := EventAt + } = Fields, + Now = erlang:system_time(millisecond), + TimestampElapse = Now - Timestamp, + RcvdAtElapse = Now - EventAt, + ?assert(lists:member(ClientId, [<<"c_event">>, <<"c_event2">>])), + ?assert(lists:member(Username, [<<"u_event">>, <<"u_event2">>])), + verify_peername(PeerName), + verify_peername(SockName), + ?assertEqual(<<"MQTT">>, ProtoName), + ?assertEqual(5, ProtoVer), + ?assert(is_integer(Keepalive)), + ?assert(is_boolean(CleanStart)), + ?assertEqual(60, ExpiryInterval), + ?assertMatch(#{'Session-Expiry-Interval' := 60}, Properties), + ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60*1000), + ?assert(0 =< RcvdAtElapse andalso RcvdAtElapse =< 60*1000), + ?assert(EventAt =< Timestamp); + verify_event_fields(SubUnsub, Fields) when SubUnsub == 'session.subscribed' ; SubUnsub == 'session.unsubscribed' -> #{clientid := ClientId, @@ -2793,7 +2803,20 @@ verify_event_fields('message.acked', Fields) -> ?assert(is_map(PubAckProps)), ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60*1000), ?assert(0 =< RcvdAtElapse andalso RcvdAtElapse =< 60*1000), - ?assert(EventAt =< Timestamp). + ?assert(EventAt =< Timestamp); + +verify_event_fields('client.check_acl_complete', Fields) -> + #{clientid := ClientId, + action := Action, + result := Result, + topic := Topic, + username := Username + } = Fields, + ?assertEqual(<<"t1">>, Topic), + ?assert(lists:member(Action, [subscribe, publish])), + ?assert(lists:member(Result, [allow, deny])), + ?assert(lists:member(ClientId, [<<"c_event">>, <<"c_event2">>])), + ?assert(lists:member(Username, [<<"u_event">>, <<"u_event2">>])). verify_peername(PeerName) -> case string:split(PeerName, ":") of diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index fb0741c0c..8de296a6d 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -49,7 +49,10 @@ check_acl(ClientInfo, PubSub, Topic) -> true -> check_acl_cache(ClientInfo, PubSub, Topic); false -> do_check_acl(ClientInfo, PubSub, Topic) end, - inc_acl_metrics(Result), Result. + inc_acl_metrics(Result), + emqx:run_hook('client.check_acl_complete', [ClientInfo, PubSub, Topic, Result]), + %% io:format(standard_error, "~p, ~p, ~p, ~p, ~n", [ClientInfo, PubSub, Topic, Result]), + Result. check_acl_cache(ClientInfo, PubSub, Topic) -> case emqx_acl_cache:get_acl_cache(PubSub, Topic) of From 0ffb66ee7fa57f721f36a990febad32ac7eaec26 Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Fri, 25 Mar 2022 11:12:55 +0800 Subject: [PATCH 282/363] fix(CHANGES): update CHANGES-4.4.md --- CHANGES-4.4.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 59c62d9c5..6ac86a0f1 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,5 +1,8 @@ # EMQ X 4.4 Changes +### Enhancements +* Add rule events: client.connack, client.check_acl_complete + ## v4.4.2 **NOTE**: v4.4.2 is in sync with: v4.3.13 From 3eab6b436bdf44ee267e07d9b7384864620f4e54 Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Fri, 25 Mar 2022 11:15:23 +0800 Subject: [PATCH 283/363] fix(appup): load_module emqx_access_control --- apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl | 3 +-- src/emqx.appup.src | 6 ++++-- src/emqx_access_control.erl | 1 - 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 136db6695..bc4f968aa 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -1017,7 +1017,7 @@ t_events(_Config) -> ]), ct:pal("====== verify $events/client_connected, $events/client_connack"), client_connected(Client, Client2), - ct:pal("====== verify $events/session_subscribed"), + ct:pal("====== verify $events/session_subscribed, $events/client_check_acl_complete"), session_subscribed(Client2), ct:pal("====== verify t1"), message_publish(Client), @@ -1044,7 +1044,6 @@ client_connected(Client, Client2) -> verify_event('client.connack'), verify_event('client.connected'), ok. - client_disconnected(Client, Client2) -> ok = emqtt:disconnect(Client, 0, #{'User-Property' => {<<"reason">>, <<"normal">>}}), ok = emqtt:disconnect(Client2, 0, #{'User-Property' => {<<"reason">>, <<"normal">>}}), diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 383db43f1..372e26df7 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.1", - [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, + [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -44,7 +45,8 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.1", - [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, + [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index 8de296a6d..4598bea38 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -51,7 +51,6 @@ check_acl(ClientInfo, PubSub, Topic) -> end, inc_acl_metrics(Result), emqx:run_hook('client.check_acl_complete', [ClientInfo, PubSub, Topic, Result]), - %% io:format(standard_error, "~p, ~p, ~p, ~p, ~n", [ClientInfo, PubSub, Topic, Result]), Result. check_acl_cache(ClientInfo, PubSub, Topic) -> From 44f4dfa49854d51d9abd342b60dcd1f683e1b648 Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Fri, 25 Mar 2022 13:40:29 +0800 Subject: [PATCH 284/363] fix(CHANGES): update CHANGES-4.4.md --- CHANGES-4.4.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 6ac86a0f1..3b1c4c911 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -2,6 +2,8 @@ ### Enhancements * Add rule events: client.connack, client.check_acl_complete +- client.connack The rule event is triggered when the server sends a CONNACK packet to the client. reason_code contains the error reason code. +- client.check_acl_complete The rule event is triggered when the client check acl complete. ## v4.4.2 From 059fc6e3c7dd5bda9a518bbc802153cabc13d58b Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Fri, 25 Mar 2022 18:05:25 +0800 Subject: [PATCH 285/363] feat(emqx_rule_events): add field 'is_cache' --- apps/emqx_rule_engine/src/emqx_rule_events.erl | 11 +++++++---- .../emqx_rule_engine/test/emqx_rule_engine_SUITE.erl | 2 ++ src/emqx_access_control.erl | 12 +++++++----- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 1a03d01fc..1cef282c8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -39,7 +39,7 @@ , on_message_delivered/3 , on_message_acked/3 , on_delivery_dropped/4 - , on_client_check_acl_complete/5 + , on_client_check_acl_complete/6 ]). -export([ event_info/0 @@ -114,12 +114,13 @@ on_client_connack(ConnInfo, Reason, _, Env) -> may_publish_and_apply('client.connack', fun() -> eventmsg_connack(ConnInfo, Reason) end, Env). -on_client_check_acl_complete(ClientInfo, PubSub, Topic, Result, Env) -> +on_client_check_acl_complete(ClientInfo, PubSub, Topic, Result, IsCache, Env) -> may_publish_and_apply('client.check_acl_complete', fun() -> eventmsg_check_acl_complete(ClientInfo, PubSub, Topic, - Result) end, Env). + Result, + IsCache) end, Env). on_session_subscribed(ClientInfo, Topic, SubOpts, Env) -> may_publish_and_apply('session.subscribed', @@ -266,13 +267,14 @@ eventmsg_check_acl_complete(_ClientInfo = #{ clientid := ClientId, username := Username, peerhost := PeerHost - }, PubSub, Topic, Result) -> + }, PubSub, Topic, Result, IsCache) -> with_basic_columns('client.check_acl_complete', #{clientid => ClientId, username => Username, peerhost => ntoa(PeerHost), topic => Topic, action => PubSub, + is_cache => IsCache, result => Result }). @@ -737,6 +739,7 @@ columns_with_exam('client.check_acl_complete') -> , {<<"peerhost">>, <<"192.168.0.10">>} , {<<"topic">>, <<"t/a">>} , {<<"action">>, <<"publish">>} + , {<<"is_cache">>, <<"false">>} , {<<"result">>, <<"allow">>} , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index bc4f968aa..2a0498d2f 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -2809,11 +2809,13 @@ verify_event_fields('client.check_acl_complete', Fields) -> action := Action, result := Result, topic := Topic, + is_cache := IsCache, username := Username } = Fields, ?assertEqual(<<"t1">>, Topic), ?assert(lists:member(Action, [subscribe, publish])), ?assert(lists:member(Result, [allow, deny])), + ?assert(lists:member(IsCache, [true, false])), ?assert(lists:member(ClientId, [<<"c_event">>, <<"c_event2">>])), ?assert(lists:member(Username, [<<"u_event">>, <<"u_event2">>])). diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index 4598bea38..11eb5efb2 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -50,7 +50,6 @@ check_acl(ClientInfo, PubSub, Topic) -> false -> do_check_acl(ClientInfo, PubSub, Topic) end, inc_acl_metrics(Result), - emqx:run_hook('client.check_acl_complete', [ClientInfo, PubSub, Topic, Result]), Result. check_acl_cache(ClientInfo, PubSub, Topic) -> @@ -61,15 +60,18 @@ check_acl_cache(ClientInfo, PubSub, Topic) -> AclResult; AclResult -> inc_acl_metrics(cache_hit), + emqx:run_hook('client.check_acl_complete', [ClientInfo, PubSub, Topic, AclResult, true]), AclResult end. do_check_acl(ClientInfo = #{zone := Zone}, PubSub, Topic) -> Default = emqx_zone:get_env(Zone, acl_nomatch, deny), - case run_hooks('client.check_acl', [ClientInfo, PubSub, Topic], Default) of - allow -> allow; - _Other -> deny - end. + Result = case run_hooks('client.check_acl', [ClientInfo, PubSub, Topic], Default) of + allow -> allow; + _Other -> deny + end, + emqx:run_hook('client.check_acl_complete', [ClientInfo, PubSub, Topic, Result, false]), + Result. default_auth_result(Zone) -> case emqx_zone:get_env(Zone, allow_anonymous, false) of From 0ac979361e3e549972d8d4be8d72efa7c9998409 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 31 Mar 2022 09:31:53 +0200 Subject: [PATCH 286/363] fix(appup): re-generate emqx.appup.src --- src/emqx.appup.src | 102 ++++++++++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 44 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 372e26df7..c793df943 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,33 +2,40 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.1", - [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {add_module,emqx_relup}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, - {"4.4.0", - [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {add_module,emqx_relup}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {add_module,emqx_relup}]}, + {"4.4.0", + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {add_module,emqx_relup}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -45,33 +52,39 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.1", - [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}]}, {"4.4.0", - [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {delete_module,emqx_relup}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -84,5 +97,6 @@ {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, + {delete_module,emqx_relup}]}, {<<".*">>,[]}]}. From 88c30e3a4f46bf250178b34dc31f9853da99c06c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 31 Mar 2022 10:14:01 +0200 Subject: [PATCH 287/363] ci: bump to latest emqx-builder image 4.4-8 --- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/apps_version_check.yaml | 2 +- .github/workflows/build_packages.yaml | 8 ++++---- .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 2 +- Makefile | 2 +- deploy/docker/Dockerfile | 2 +- scripts/buildx.sh | 4 ++-- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index ffd35548c..819522dbd 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 0946b7c54..16dc5c218 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -13,7 +13,7 @@ jobs: os: - ubuntu20.04 - container: ghcr.io/emqx/emqx-builder/4.4-7:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index ff236f966..d84371b94 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -265,7 +265,7 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/4.4-7:${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/4.4-8:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: @@ -340,7 +340,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-7:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -355,7 +355,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-7:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 298380f0c..dab02d1ec 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -23,7 +23,7 @@ jobs: - ubuntu20.04 - rockylinux8 - container: ghcr.io/emqx/emqx-builder/4.4-7:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 6e7f05e3f..2b49d4d77 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c8f041816..ffcb69247 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index a291c3927..3b11a9368 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index fdec2765a..dbc79a8f5 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -224,7 +224,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -275,7 +275,7 @@ jobs: otp: - 24.1.5-3 runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 94c9f83b0..efca499ea 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 55bd72667..5cfccf879 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-alpine3.15.1 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-alpine3.15.1 export EMQX_DEFAULT_RUNNER = alpine:3.15.1 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index eaa9b0227..d3a58ce74 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-alpine3.15.1 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-alpine3.15.1 ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 2cb5a718d..6efc6bcc0 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-debian10 --arch arm64 +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-debian10 --arch arm64 set -euo pipefail @@ -20,7 +20,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-7:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-debian10" } while [ "$#" -gt 0 ]; do From 4438b9e59d0fa0517537392a9a02febc0040e9d3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 31 Mar 2022 10:17:43 +0200 Subject: [PATCH 288/363] docs: update 4.4 change log to cover changes made for centos 7 --- CHANGES-4.4.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 3b1c4c911..601585858 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,4 +1,4 @@ -# EMQ X 4.4 Changes +# EMQX 4.4 Changes ### Enhancements * Add rule events: client.connack, client.check_acl_complete @@ -14,6 +14,9 @@ * Docker image is based on alpine-3.15.1 (OpenSSL-1.1.1n) * For docker image, /opt/emqx/etc has been removed from the VOLUME list, this made it easier for the users to rebuild image on top with changed configs. +* CentOS 7 Erlang runtime is rebuilt on OpenSSL-1.1.1n (previously on 1.0), + Prior to v4.4.1, EMQX may pick certain cipher suites proposed by the clients, + but then fail to handshake resulting in a `malformed_handshake_data` exception. ### Enhancements From 5fa30b2d516f4405470ff0fc8e56cf39355a64d4 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 31 Mar 2022 21:55:57 +0800 Subject: [PATCH 289/363] chore: bump vsn to 4.4.2-rc.2 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index a000946e3..128237310 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.2-rc.1"}). +-define(EMQX_RELEASE, {opensource, "4.4.2-rc.2"}). -else. From 363242fd610f27a8bd0bea835d73dcec44755693 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 31 Mar 2022 22:32:47 +0200 Subject: [PATCH 290/363] build: no upgrade base for debian11 at version 4.4.2 --- build | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/build b/build index 83190bcbd..fa6a41615 100755 --- a/build +++ b/build @@ -136,7 +136,22 @@ make_zip() { local tarball="${relpath}/${tarname}" local target_zip="${pkgpath}/${pkgname}" tar zxf "${tarball}" -C "${tard}/emqx" - if ! [[ $SYSTEM == windows* ]]; then + has_relup='yes' + case "$SYSTEM" in + windows*) + # no relup support for windows + has_relup='no' + ;; + debian11) + case "$PKG_VSN" in + 4.4.2*) + # this is the first version for debian11, no relup + has_relup='no' + ;; + esac + ;; + esac + if [ "$has_relup" = 'yes' ]; then ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup" fi cp_dyn_libs "${tard}/emqx" From 3bbc7bf31fb0f48974f629508beb6de0c0536fcc Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 1 Apr 2022 18:05:39 +0800 Subject: [PATCH 291/363] chore: release 4.4.2 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index f4a219706..7c29d0ff2 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.2-rc.3"}). +-define(EMQX_RELEASE, {opensource, "4.4.2"}). -else. From 905e831edc9008586bf4c0817dd728c9166d5f2d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 6 Apr 2022 14:22:54 +0800 Subject: [PATCH 292/363] chore: bump emqx to 4.4.3 --- src/emqx.app.src | 2 +- src/emqx.appup.src | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/emqx.app.src b/src/emqx.app.src index 54b9dfc01..986221c5a 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.4.2"}, % strict semver, bump manually! + {vsn, "4.4.3"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index c793df943..fa24eedd2 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.1", + [{"4.4.2", + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_relup}]}, + {"4.4.1", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -51,7 +54,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.1", + [{"4.4.2", + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_relup}]}, + {"4.4.1", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, From bad227c45d30db48137acc56af5e619c0d099760 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Apr 2022 13:33:33 +0200 Subject: [PATCH 293/363] chore: re-generate appup files --- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.appup.src | 16 +++++-- .../src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_plugin_libs.appup.src | 46 ++++++++++--------- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 38 ++++++++++++--- .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- src/emqx.appup.src | 24 +++++++--- 9 files changed, 90 insertions(+), 44 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 7d784e3b2..e88f6daf1 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.4.0"}, % strict semver, bump manually! + {vsn, "4.4.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 715060df4..500f0ef05 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,6 +1,6 @@ {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.4.0"}, + {vsn, "4.4.1"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index 9cb9fc2a9..a5e90bbba 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,7 +1,13 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<".*">>, []} - ], - [{<<".*">>, []} - ] -}. + [{"4.4.0", + [{load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.4.0", + [{load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index fe4fff0b5..5fbd21fab 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.4.2"}, + {vsn, "4.4.3"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index 46e76ed82..b183c5e0a 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -1,25 +1,27 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.1", - [ {load_module,emqx_trace,brutal_purge,soft_purge,[]} - , {load_module,emqx_trace_api,brutal_purge,soft_purge,[]} - ]}, - {"4.4.0", - [ {load_module,emqx_trace,brutal_purge,soft_purge,[]} - , {load_module,emqx_trace_api,brutal_purge,soft_purge,[]} - , {update, emqx_slow_subs, {advanced, ["4.4.0"]}} - , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} - ]}, + [{"4.4.2",[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.0", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}, + {update,emqx_slow_subs,{advanced,["4.4.0"]}}, + {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.1", - [ {load_module,emqx_trace,brutal_purge,soft_purge,[]} - , {load_module,emqx_trace_api,brutal_purge,soft_purge,[]} - ]}, - {"4.4.0", - [ {load_module,emqx_trace,brutal_purge,soft_purge,[]} - , {load_module,emqx_trace_api,brutal_purge,soft_purge,[]} - , {update, emqx_slow_subs, {advanced, ["4.4.0"]}} - , {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]} - ]}, - {<<".*">>,[]}] -}. + [{"4.4.2",[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.0", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}, + {update,emqx_slow_subs,{advanced,["4.4.0"]}}, + {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index bcf2ee469..d7b77f41d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.4.2"}, % strict semver, bump manually! + {vsn, "4.4.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index d5b2c6319..2c01ba70d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,15 +1,28 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.1", - [{load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + [{"4.4.2", + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {update,emqx_rule_metrics,{advanced,["4.4.0"]}}, @@ -18,15 +31,28 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.1", - [{load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + [{"4.4.2", + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {update,emqx_rule_metrics,{advanced,["4.4.0"]}}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 0156ea865..b02625117 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.4.3"}, % strict semver, bump manually! + {vsn, "4.4.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index fa24eedd2..88003d6df 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,10 +2,15 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.2", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, {"4.4.1", - [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + [{load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, @@ -26,7 +31,8 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}]}, {"4.4.0", - [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -55,10 +61,15 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.2", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, {"4.4.1", - [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + [{load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, @@ -79,7 +90,8 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}]}, {"4.4.0", - [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, From 243a30dacb92b026e6a6718f2d9424aeeb3e0652 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 6 Apr 2022 20:22:39 +0800 Subject: [PATCH 294/363] chore: update appup.src for v4.4 --- apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 4 ++-- apps/emqx_exhook/src/emqx_exhook.appup.src | 4 ++-- apps/emqx_exhook/src/emqx_exhook_mngr.erl | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index 684b4fa93..dca3ef2c1 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,13 +1,13 @@ %% -*-: erlang -*- {VSN, [ - {<<"4\\.3\\.[0-1]">>, [ + {<<"4\\.4\\.0">>, [ {load_module, emqx_auth_jwt_svr, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ - {<<"4\\.3\\.[0-1]">>, [ + {<<"4\\.4\\.0">>, [ {load_module, emqx_auth_jwt_svr, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index a5e90bbba..05590198e 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -4,10 +4,10 @@ [{"4.4.0", [{load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}]}, + {update, emqx_exhook_mngr, {advanced, ["4.4.0"]}}]}, {<<".*">>,[]}], [{"4.4.0", [{load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, - {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}]}, + {update, emqx_exhook_mngr, {advanced, ["4.4.0"]}}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl index c0748f20b..b3d7ea089 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mngr.erl @@ -218,7 +218,7 @@ terminate(_Reason, State = #state{running = Running}) -> %% in the emqx_exhook:v4.3.5, we have added one new field in the state last: %% - hooks_options :: map() code_change({down, _Vsn}, State, [ToVsn]) -> - case re:run(ToVsn, "4\\.3\\.[0-4]") of + case re:run(ToVsn, "4\\.4\\.0") of {match, _} -> NState = list_to_tuple( lists:droplast( @@ -228,7 +228,7 @@ code_change({down, _Vsn}, State, [ToVsn]) -> {ok, State} end; code_change(_Vsn, State, [FromVsn]) -> - case re:run(FromVsn, "4\\.3\\.[0-4]") of + case re:run(FromVsn, "4\\.4\\.0") of {match, _} -> NState = list_to_tuple( tuple_to_list(State) ++ [?DEFAULT_HOOK_OPTS]), From d1e773d83cfadd3ce6a58f3f96cbfc561ff1b3cc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Apr 2022 21:09:10 +0200 Subject: [PATCH 295/363] test: fix float point number compare --- .../test/emqx_rule_engine_SUITE.erl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index c25c43126..562254285 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -377,11 +377,15 @@ t_reset_metrics(_Config) -> end || _ <- lists:seq(1,10)], emqx_rule_metrics:reset_metrics(Id), - ?assertEqual(#{exception => 0,failed => 0, - matched => 0,no_result => 0,passed => 0, - speed => 0.0,speed_last5m => 0.0,speed_max => 0}, - emqx_rule_metrics:get_rule_metrics(Id)), - ?assertEqual(#{failed => 0,success => 0,taken => 0}, + Expected = #{exception => 0, failed => 0, matched => 0, no_result => 0, + passed => 0, speed => 0.0, speed_last5m => 0.0, speed_max => 0}, + Got = emqx_rule_metrics:get_rule_metrics(Id), + %% use == instead of =:=, so that 0 and 0.0 are compared equal + case Expected == Got of + true -> ok; + false -> ?assertEqual(Expected, Got) + end, + ?assertEqual(#{failed => 0, success => 0, taken => 0}, emqx_rule_metrics:get_action_metrics(ResId)), emqtt:stop(Client), emqx_rule_registry:remove_rule(Id), From af0c0c8ee1f188196fbed139794f28e8b06595a6 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 7 Apr 2022 15:32:14 +0800 Subject: [PATCH 296/363] chore: update renamed module in helm deploy file --- deploy/charts/emqx/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index ff412b637..bf66a2251 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -128,7 +128,7 @@ emqxLoadedModules: > {emqx_mod_acl_internal, true}. {emqx_mod_presence, true}. {emqx_mod_trace, false}. - {emqx_mod_st_statistics, false}. + {emqx_mod_slow_subs, false}. {emqx_mod_delayed, false}. {emqx_mod_rewrite, false}. {emqx_mod_subscription, false}. From bee3973a828139633b4cb55e84fc407be3a3e4fb Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 7 Apr 2022 17:02:51 +0800 Subject: [PATCH 297/363] test: fix flaky test --- lib-ce/emqx_modules/test/emqx_modules_SUITE.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl index a149d9991..02f80455c 100644 --- a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl @@ -46,7 +46,7 @@ end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]). init_per_testcase(t_ensure_default_loaded_modules_file, Config) -> - LoadedModulesFilepath = application:get_env(emqx, modules_loaded_file), + {ok, LoadedModulesFilepath} = application:get_env(emqx, modules_loaded_file), ok = application:stop(emqx_modules), TmpFilepath = filename:join(["/", "tmp", "loaded_modules_tmp"]), case file:delete(TmpFilepath) of @@ -86,8 +86,10 @@ t_ensure_default_loaded_modules_file(_Config) -> , {emqx_mod_delayed,false} , {emqx_mod_presence,true} , {emqx_mod_rewrite,false} + , {emqx_mod_slow_subs,false} , {emqx_mod_subscription,false} , {emqx_mod_topic_metrics,false} + , {emqx_mod_trace,false} ], lists:sort(emqx_modules:list())), ok. From 3c67947c82c408e0b66a4559b4cec2debe9ca8b9 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 8 Apr 2022 14:26:35 +0800 Subject: [PATCH 298/363] chore: update dashboard for 4.4 --- scripts/get-dashboard.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index a7eef35ef..26e129cfd 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -13,8 +13,8 @@ case "${PKG_VSN}" in ;; 4.4*) # keep the above 4.3 untouched, otherwise conflicts! - EMQX_CE_DASHBOARD_VERSION='v4.4.1' - EMQX_EE_DASHBOARD_VERSION='v4.4.7' + EMQX_CE_DASHBOARD_VERSION='v4.4.2' + EMQX_EE_DASHBOARD_VERSION='v4.4.9' ;; *) echo "Unsupported version $PKG_VSN" >&2 From e0c3071a49e8756acd51cee70b0c9e116b58c746 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 8 Apr 2022 19:59:24 +0800 Subject: [PATCH 299/363] chore: fix appup.src for exhook --- apps/emqx_exhook/src/emqx_exhook.appup.src | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index ad45d7426..5a81af051 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -6,6 +6,7 @@ {load_module,emqx_exhook,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, {update, emqx_exhook_mngr, {advanced, ["4.4.0"]}}]}, {<<".*">>,[]}], [{"4.4.0", @@ -13,5 +14,6 @@ {load_module,emqx_exhook,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, {update, emqx_exhook_mngr, {advanced, ["4.4.0"]}}]}, {<<".*">>,[]}]}. From 9feb4dba3032e43b29ff5daf52ea756a00a119fb Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 8 Apr 2022 20:54:46 +0800 Subject: [PATCH 300/363] chore: fix duplicated load_module for emqx.appup.src --- src/emqx.appup.src | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index fb6cf7db7..a3baf9294 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -25,7 +25,6 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -44,7 +43,6 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -89,7 +87,6 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -108,7 +105,6 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, From 6b636d0786d5ac406a638e842fb29cd8c5b2c4b2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 9 Apr 2022 01:17:38 +0800 Subject: [PATCH 301/363] chore: bump vsn to 4.4.3-rc.1 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 7c29d0ff2..cf21aba2c 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.2"}). +-define(EMQX_RELEASE, {opensource, "4.4.3-rc.1"}). -else. From 3f9f53a1e0e40ef3ffac06ca0d51c20a651e89b4 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 12 Apr 2022 13:48:43 +0800 Subject: [PATCH 302/363] chore: bump vsn --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index cf21aba2c..b3b06c23f 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.3-rc.1"}). +-define(EMQX_RELEASE, {opensource, "4.4.3-rc.2"}). -else. From 12874aa688061b58b0c3df6951a21a8805ad09f5 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 15 Apr 2022 16:12:40 +0800 Subject: [PATCH 303/363] fix: backup data filename, uri decode --- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt_api_data.erl | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index f7e82e266..250c3e991 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.4.2"}, % strict semver, bump manually! + {vsn, "4.4.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,emqx_plugin_libs,minirest]}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index 93d5498f1..719f691f1 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -115,7 +115,8 @@ import(_Bindings, Params) -> do_import(Filename) -> emqx_mgmt_data_backup:import(Filename, "{}"). -download(#{filename := Filename}, _Params) -> +download(#{filename := Filename0}, _Params) -> + Filename = filename_decode(Filename0), case emqx_mgmt_data_backup:read_backup_file(Filename) of {ok, Res} -> {ok, Res}; @@ -139,7 +140,8 @@ do_upload(Bindings, Params = #{<<"file">> := _}) -> do_upload(_Bindings, _Params) -> minirest:return({error, missing_required_params}). -delete(#{filename := Filename}, _Params) -> +delete(#{filename := Filename0}, _Params) -> + Filename = filename_decode(Filename0), case emqx_mgmt_data_backup:delete_backup_file(Filename) of ok -> minirest:return(); @@ -161,3 +163,6 @@ tmp_filename() -> Seconds = erlang:system_time(second), {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), list_to_binary(io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S])). + +filename_decode(Filename) -> + uri_string:percent_decode(Filename). From 09a100fc2d082d40f50433ae86d1191d6deea74a Mon Sep 17 00:00:00 2001 From: JimMoen Date: Sat, 16 Apr 2022 11:45:54 +0800 Subject: [PATCH 304/363] chore: apps version check ignore comments in erl/hrl file --- scripts/apps-version-check.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 017a48c4b..d5c9645aa 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -14,6 +14,9 @@ echo "Compare base: $latest_release" bad_app_count=0 +no_comment_re='(^[^\s?%])' +## TODO: c source code comments re (in $app_path/c_src dirs) + get_vsn() { commit="$1" app_src_file="$2" @@ -47,11 +50,12 @@ check_apps() { if [ -z "${old_app_version:-}" ]; then echo "skiped checking new app ${app}" elif [ "$old_app_version" = "$now_app_version" ]; then - lines="$(git diff --name-only "$latest_release"...HEAD \ - -- "$app_path/src" \ - -- "$app_path/priv" \ - -- "$app_path/c_src")" - if [ "$lines" != '' ]; then + lines="$(git diff "$latest_release"...HEAD --ignore-blank-lines -G "$no_comment_re" \ + -- "$app_path/src" \ + -- ":(exclude)'$app_path/src/*.appup.src'" \ + -- "$app_path/priv" \ + -- "$app_path/c_src" | wc -l ) " + if [ "$lines" -gt 0 ]; then echo "$src_file needs a vsn bump (old=$old_app_version)" echo "changed: $lines" bad_app_count=$(( bad_app_count + 1)) From 0bf55b99e392ecc2b9a0b83a40d47f2089c2aca9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 23 Apr 2022 10:08:44 +0200 Subject: [PATCH 305/363] chore: fix appup after merge from 4.3 --- .../src/emqx_rule_engine.appup.src | 28 +++++++++---- .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- src/emqx.app.src | 2 +- src/emqx.appup.src | 40 ++++++++++++++----- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 360f85c34..7f6b40da8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,8 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.2", - [{load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + [{"4.4.3", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}]}, + {"4.4.2", + [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -12,7 +16,8 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.1", - [{load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, @@ -23,7 +28,8 @@ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, @@ -34,8 +40,12 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.2", - [{load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + [{"4.4.3", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}]}, + {"4.4.2", + [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -45,7 +55,8 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.1", - [{load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, @@ -56,7 +67,8 @@ {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 40aa06839..95cca4e1c 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQX Web Dashboard"}, - {vsn, "4.4.4"}, % strict semver, bump manually! + {vsn, "4.4.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/src/emqx.app.src b/src/emqx.app.src index 986221c5a..156dc57c7 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.4.3"}, % strict semver, bump manually! + {vsn, "4.4.4"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index a3baf9294..a1b764592 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,8 +1,13 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.2", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{"4.4.3", + [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}]}, + {"4.4.2", + [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {add_module,emqx_tracer}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -11,7 +16,9 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, {"4.4.1", - [{load_module,emqx,brutal_purge,soft_purge,[]}, + [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {add_module,emqx_tracer}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -34,7 +41,9 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}]}, {"4.4.0", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {add_module,emqx_tracer}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, @@ -63,17 +72,23 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.2", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{"4.4.3", + [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}]}, + {"4.4.2", + [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_relup}]}, + {load_module,emqx_relup}, + {delete_module,emqx_tracer}]}, {"4.4.1", - [{load_module,emqx,brutal_purge,soft_purge,[]}, + [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -94,9 +109,11 @@ {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {delete_module,emqx_relup}]}, + {delete_module,emqx_relup}, + {delete_module,emqx_tracer}]}, {"4.4.0", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, @@ -122,5 +139,6 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, - {delete_module,emqx_relup}]}, + {delete_module,emqx_relup}, + {delete_module,emqx_tracer}]}, {<<".*">>,[]}]}. From 42aa0e5d0ad0c2064c5d5f1ec06ab3bb8de4e95f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 23 Apr 2022 10:43:58 +0200 Subject: [PATCH 306/363] chore: add debug output to update-appup.sh --- scripts/update-appup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index 55bcc0122..4cfe4a06f 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -6,6 +6,7 @@ ## Arg1: EMQX PROFILE set -euo pipefail +set -x usage() { echo "$0 PROFILE" From 4330edd674e61ff3d18b666415191101a946198d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 23 Apr 2022 10:53:39 +0200 Subject: [PATCH 307/363] fix(appup): bump to 4.4.4-beta.1 also fix emqx appup instructions --- include/emqx_release.hrl | 2 +- scripts/inject-relup.escript | 2 +- src/emqx.appup.src | 22 ++++++++++------------ 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 5a460f91b..f95494ed8 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.3"}). +-define(EMQX_RELEASE, {opensource, "4.4.4-beta.1"}). -else. diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript index e2d3de795..0ca54361b 100755 --- a/scripts/inject-relup.escript +++ b/scripts/inject-relup.escript @@ -123,7 +123,7 @@ do_filter_and_get(Instr, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> assert_mandatory_modules(up, Mods) -> assert(lists:member(emqx_relup, Mods) andalso lists:member(emqx_app, Mods), - "cannot find any 'load_object_code' instructions for emqx_app and emqx_rel: ~p", [Mods]); + "cannot find any 'load_object_code' instructions for emqx_app and emqx_relup: ~p", [Mods]); assert_mandatory_modules(down, Mods) -> assert(lists:member(emqx_app, Mods), diff --git a/src/emqx.appup.src b/src/emqx.appup.src index a1b764592..7a04551b2 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -3,10 +3,11 @@ {VSN, [{"4.4.3", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_relup}]}, {"4.4.2", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {add_module,emqx_tracer}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -17,7 +18,6 @@ {load_module,emqx_relup}]}, {"4.4.1", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {add_module,emqx_tracer}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -42,7 +42,6 @@ {add_module,emqx_relup}]}, {"4.4.0", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {add_module,emqx_tracer}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -73,8 +72,10 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.3", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.2", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -84,8 +85,7 @@ {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_relup}, - {delete_module,emqx_tracer}]}, + {load_module,emqx_relup}]}, {"4.4.1", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, @@ -109,8 +109,7 @@ {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {delete_module,emqx_relup}, - {delete_module,emqx_tracer}]}, + {delete_module,emqx_relup}]}, {"4.4.0", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -139,6 +138,5 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, - {delete_module,emqx_relup}, - {delete_module,emqx_tracer}]}, + {delete_module,emqx_relup}]}, {<<".*">>,[]}]}. From a9acfc0ee1ef2ed2e5b4ca3019c836400d8b4de2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 23 Apr 2022 13:22:05 +0200 Subject: [PATCH 308/363] fix(bin/emqx): fix sed replace with '/n' work for macos 10 --- bin/emqx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index f409f19ee..8caede90b 100755 --- a/bin/emqx +++ b/bin/emqx @@ -361,7 +361,10 @@ generate_config() { ## transform a single line args list like '-config ... -args_file ... -vm_args ...' to lines and get path for each file respectively ## NOTE: the -args_file and -vm_args are the same file passed twice because args_file is used by beam, but not possible to get at runtime ## by calling init:get_arguments/0 - lines="$(echo "$CUTTLEFISH_OUTPUT" | tail -1 | sed 's/-config/\nconfig=/g' | sed 's/-args_file/\nargs_file=/g' | sed 's/-vm_args/\nvm_args=/g')" + lines="$(echo "$CUTTLEFISH_OUTPUT" | tail -1 \ + | sed -e $'s/-config/\\\nconfig=/g' \ + | sed -e $'s/-args_file/\\\nargs_file=/g' \ + | sed -e $'s/-vm_args/\\\nvm_args=/g')" CONFIG_FILE="$(trim "$(echo -e "$lines" | grep 'config=' | sed 's/config=//g')")" CUTTLE_GEN_ARG_FILE="$(trim "$(echo -e "$lines" | grep 'vm_args=' | sed 's/vm_args=//g')")" From ffd51e1fe6c36ce4f5cef6cf58e73440400e5db3 Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Tue, 26 Apr 2022 19:07:00 +0800 Subject: [PATCH 309/363] fix: remove error field --- .../emqx_rule_engine/src/emqx_rule_events.erl | 12 +++--- .../test/emqx_rule_engine_SUITE.erl | 38 +++++++++++++++---- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index c1f019692..6c2ff97e4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -236,19 +236,18 @@ eventmsg_disconnected(_ClientInfo = #{ disconnected_at => DisconnectedAt }). -eventmsg_connack(_ConnInfo = #{ +eventmsg_connack(ConnInfo = #{ clientid := ClientId, clean_start := CleanStart, username := Username, peername := PeerName, sockname := SockName, proto_name := ProtoName, - proto_ver := ProtoVer, - keepalive := Keepalive, - connected_at := ConnectedAt, - conn_props := ConnProps, - expiry_interval := ExpiryInterval + proto_ver := ProtoVer }, Reason) -> + Keepalive = maps:get(keepalive, ConnInfo, 0), + ConnProps = maps:get(conn_props, ConnInfo, #{}), + ExpiryInterval = maps:get(expiry_interval, ConnInfo, 0), with_basic_columns('client.connack', #{reason_code => reason(Reason), clientid => ClientId, @@ -260,7 +259,6 @@ eventmsg_connack(_ConnInfo = #{ proto_ver => ProtoVer, keepalive => Keepalive, expiry_interval => ExpiryInterval, - connected_at => ConnectedAt, conn_props => printable_maps(ConnProps) }). eventmsg_check_acl_complete(_ClientInfo = #{ diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index df9ebbb78..11f470e3b 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -1058,6 +1058,7 @@ t_events(_Config) -> , {proto_ver, v5} , {properties, #{'Session-Expiry-Interval' => 60}} ]), + ct:pal("====== verify $events/client_connected, $events/client_connack"), client_connected(Client, Client2), ct:pal("====== verify $events/session_subscribed, $events/client_check_acl_complete"), @@ -1074,6 +1075,8 @@ t_events(_Config) -> message_dropped(Client), ct:pal("====== verify $events/client_disconnected"), client_disconnected(Client, Client2), + ct:pal("====== verify $events/client_connack"), + client_connack_failed(), ok. message_publish(Client) -> @@ -1081,12 +1084,33 @@ message_publish(Client) -> <<"{\"id\": 1, \"name\": \"ha\"}">>, [{qos, 1}]), verify_event('message.publish'), ok. + client_connected(Client, Client2) -> {ok, _} = emqtt:connect(Client), {ok, _} = emqtt:connect(Client2), verify_event('client.connack'), verify_event('client.connected'), ok. + +client_connack_failed() -> + {ok, Client} = emqtt:start_link( + [ {username, <<"u_event3">>} + , {clientid, <<"c_event3">>} + , {proto_ver, v5} + , {properties, #{'Session-Expiry-Interval' => 60}} + ]), + try + meck:new(emqx_access_control, [non_strict, passthrough]), + meck:expect(emqx_access_control, authenticate, + fun(_) -> {error, bad_username_or_password} end), + process_flag(trap_exit, true), + ?assertMatch({error, _}, emqtt:connect(Client)), + timer:sleep(300), + verify_event('client.connack') + after + meck:unload(emqx_access_control) + end, + ok. client_disconnected(Client, Client2) -> ok = emqtt:disconnect(Client, 0, #{'User-Property' => {<<"reason">>, <<"normal">>}}), ok = emqtt:disconnect(Client2, 0, #{'User-Property' => {<<"reason">>, <<"normal">>}}), @@ -2726,14 +2750,14 @@ verify_event_fields('client.connack', Fields) -> keepalive := Keepalive, expiry_interval := ExpiryInterval, conn_props := Properties, - timestamp := Timestamp, - connected_at := EventAt + reason_code := Reason, + timestamp := Timestamp } = Fields, Now = erlang:system_time(millisecond), TimestampElapse = Now - Timestamp, - RcvdAtElapse = Now - EventAt, - ?assert(lists:member(ClientId, [<<"c_event">>, <<"c_event2">>])), - ?assert(lists:member(Username, [<<"u_event">>, <<"u_event2">>])), + ?assert(lists:member(Reason, [success, bad_username_or_password])), + ?assert(lists:member(ClientId, [<<"c_event">>, <<"c_event2">>, <<"c_event3">>])), + ?assert(lists:member(Username, [<<"u_event">>, <<"u_event2">>, <<"u_event3">>])), verify_peername(PeerName), verify_peername(SockName), ?assertEqual(<<"MQTT">>, ProtoName), @@ -2742,9 +2766,7 @@ verify_event_fields('client.connack', Fields) -> ?assert(is_boolean(CleanStart)), ?assertEqual(60, ExpiryInterval), ?assertMatch(#{'Session-Expiry-Interval' := 60}, Properties), - ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60*1000), - ?assert(0 =< RcvdAtElapse andalso RcvdAtElapse =< 60*1000), - ?assert(EventAt =< Timestamp); + ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60*1000); verify_event_fields(SubUnsub, Fields) when SubUnsub == 'session.subscribed' ; SubUnsub == 'session.unsubscribed' -> From e67b710facb88c723cc54e1c5eef34049fbbe267 Mon Sep 17 00:00:00 2001 From: EMQ-YangM Date: Wed, 27 Apr 2022 09:35:20 +0800 Subject: [PATCH 310/363] fix: update emqx_rule_engine.appup.src --- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 7f6b40da8..93e38c44d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.3", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}]}, {"4.4.2", [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, @@ -41,7 +42,8 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.3", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}]}, {"4.4.2", [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, From 315bfa03854574ea00af679b8397d0267ef4ba2f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 28 Apr 2022 14:58:35 -0300 Subject: [PATCH 311/363] fix(helm): use tag if defined The previous code was ignoring `.Values.image.tag` and always using `.Char.AppVersion`. --- deploy/charts/emqx/templates/StatefulSet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index c0f4e85ad..fc798bbe5 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -5,7 +5,7 @@ {{ $configData := printf "%s\n%s\n%s\n%s" $cfgEnv $cfgAcl $cfgPlugins $cfgModules}} ## Compatible with previous misspellings {{ $licenseSecretName := coalesce .Values.emqxLicenseSecretName .Values.emqxLicneseSecretName }} -{{ $image := printf "%s:%s" .Values.image.repository (default .Values.image.tag .Chart.AppVersion) }} +{{ $image := printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }} apiVersion: apps/v1 kind: StatefulSet From 787530459f57aa2060487d1164e1b822c3f02cbb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 29 Apr 2022 20:22:21 +0200 Subject: [PATCH 312/363] chore: bump emqx_management app version to 4.4.4 --- apps/emqx_management/src/emqx_management.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 250c3e991..fd941f8b8 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.4.3"}, % strict semver, bump manually! + {vsn, "4.4.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,emqx_plugin_libs,minirest]}, From 4b0fe016f35be63cea57cf60387ee3962cfda222 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 6 May 2022 14:49:50 +0200 Subject: [PATCH 313/363] build(macos): do not depend on gsed --- scripts/get-distro.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index 48d52122c..4bf81afb3 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -10,8 +10,8 @@ UNAME="$(uname -s)" case "$UNAME" in Darwin) DIST='macos' - VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') - SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" + VERSION_ID="$(sw_vers | grep 'ProductVersion' | cut -d ':' -f2 | cut -d'.' -f1 | tr -d ' \t')" + SYSTEM="${DIST}${VERSION_ID}" ;; Linux) if grep -q -i 'rhel' /etc/*-release; then From d655bea16a7c91bc54ace220b3dc451493c42dd8 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 17 May 2022 14:39:09 +0800 Subject: [PATCH 314/363] fix(mgmt): pubsub api use bad params caused sub client crash --- .../src/emqx_mgmt_api_pubsub.erl | 77 +++++++++++++------ 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index b41c801d4..3fe859811 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -71,19 +71,10 @@ subscribe(_Bindings, Params) -> publish(_Bindings, Params) -> logger:debug("API publish Params:~p", [Params]), - {ClientId, Topic, Qos, Retain, Payload, UserProps} = parse_publish_params(Params), - case do_publish(ClientId, Topic, Qos, Retain, Payload, UserProps) of - {ok, MsgIds} -> - case proplists:get_value(<<"return">>, Params, undefined) of - undefined -> minirest:return(ok); - _Val -> - case proplists:get_value(<<"topics">>, Params, undefined) of - undefined -> minirest:return({ok, #{msgid => lists:last(MsgIds)}}); - _ -> minirest:return({ok, #{msgids => MsgIds}}) - end - end; - Result -> - minirest:return(Result) + try parse_publish_params(Params) of + Result -> do_publish(Params, Result) + catch + _E : _R -> minirest:return({ok, ?ERROR8, bad_params}) end. unsubscribe(_Bindings, Params) -> @@ -124,14 +115,19 @@ loop_publish(Params) -> loop_publish([], Result) -> lists:reverse(Result); loop_publish([Params | ParamsN], Acc) -> - {ClientId, Topic, Qos, Retain, Payload, UserProps} = parse_publish_params(Params), - Code = case do_publish(ClientId, Topic, Qos, Retain, Payload, UserProps) of - {ok, _} -> 0; - {_, Code0, _} -> Code0 - end, - Result = #{topic => resp_topic(proplists:get_value(<<"topic">>, Params), - proplists:get_value(<<"topics">>, Params, <<"">>)), - code => Code}, + Result = + try parse_publish_params(Params) of + Res -> + Code = case do_publish(Params, Res) of + {ok, _} -> 0; + {_, Code0, _} -> Code0 + end, + #{topic => resp_topic(proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), + code => Code} + catch + _E : _R -> #{code => ?ERROR8, message => <<"bad_params">>} + end, loop_publish(ParamsN, [Result | Acc]). loop_unsubscribe(Params) -> @@ -163,6 +159,21 @@ do_subscribe(ClientId, Topics, QoS) -> _ -> ok end. +do_publish(Params, {ClientId, Topic, Qos, Retain, Payload, UserProps}) -> + case do_publish(ClientId, Topic, Qos, Retain, Payload, UserProps) of + {ok, MsgIds} -> + case proplists:get_value(<<"return">>, Params, undefined) of + undefined -> minirest:return(ok); + _Val -> + case proplists:get_value(<<"topics">>, Params, undefined) of + undefined -> minirest:return({ok, #{msgid => lists:last(MsgIds)}}); + _ -> minirest:return({ok, #{msgids => MsgIds}}) + end + end; + Result -> + minirest:return(Result) + end. + do_publish(ClientId, _Topics, _Qos, _Retain, _Payload, _UserProps) when not (is_binary(ClientId) or (ClientId =:= undefined)) -> {ok, ?ERROR8, <<"bad clientid: must be string">>}; @@ -207,7 +218,7 @@ parse_publish_params(Params) -> Qos = proplists:get_value(<<"qos">>, Params, 0), Retain = proplists:get_value(<<"retain">>, Params, false), Payload1 = maybe_maps_to_binary(Payload), - UserProps = check_user_props(proplists:get_value(<<"user_properties">>, Params, [])), + UserProps = generate_user_props(proplists:get_value(<<"user_properties">>, Params, [])), {ClientId, Topics, Qos, Retain, Payload1, UserProps}. parse_unsubscribe_params(Params) -> @@ -263,7 +274,23 @@ maybe_maps_to_binary(Payload) -> error({encode_payload_fail, S}) end. -check_user_props(UserProps) when is_list(UserProps) -> - UserProps; -check_user_props(UserProps) -> +generate_user_props(UserProps) when is_list(UserProps)-> + generate_user_props_(UserProps, []); +generate_user_props(UserProps) -> error({user_properties_type_error, UserProps}). + +generate_user_props_([{Name, Value} | Rest], Acc) -> + generate_user_props_(Rest, [{bin(Name), bin(Value)} | Acc]); +generate_user_props_([], Acc) -> + lists:reverse(Acc). + +bin(Bin) when is_binary(Bin) -> Bin; +bin(Num) when is_number(Num) -> number_to_binary(Num); +bin(Boolean) when is_boolean(Boolean) -> atom_to_binary(Boolean); +bin(Other) -> error({user_properties_type_error, Other}). + +-define(FLOAT_PRECISION, 17). +number_to_binary(Int) when is_integer(Int) -> + integer_to_binary(Int); +number_to_binary(Float) when is_float(Float) -> + float_to_binary(Float, [{decimals, ?FLOAT_PRECISION}, compact]). From 77fb8bfc192f197ed95e3a081e8203c4eef0e22f Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 17 May 2022 14:39:43 +0800 Subject: [PATCH 315/363] test(mgmt): use pub api with bad params --- apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index aa2047ef6..0029b866f 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -447,6 +447,20 @@ t_pubsub(_) -> <<"payload">> => <<"hello">>}), ?assertEqual(?ERROR8, get(<<"code">>, BadClient2)), + {ok, BadParams} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), + #{<<"clientid">> => 1, + <<"topics">> => <<"mytopic">>, + <<"qos">> => 1, + <<"payload">> => <<"hello">>, + <<"user_properties">> => + #{<<"id">> => 10010, + <<"name">> => <<"emqx">>, + <<"foo">> => ["bad_properties1", "bad_properties2"], + <<"boolean">> => false + } + }), + ?assertEqual(?ERROR8, get(<<"code">>, BadParams)), + {ok, BadClient3} = request_api(post, api_path(["mqtt/unsubscribe"]), [], auth_header_(), #{<<"clientid">> => 1, <<"topic">> => <<"mytopic">>}), From 67e3e2de9610e4e0fd8d7a43d8b9c7034a758ec8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 19 May 2022 11:24:57 -0300 Subject: [PATCH 316/363] fix(backup): accept files outside `data/dir` when importing Fixes https://github.com/emqx/emqx/issues/7990 Currently, when importing a data backup using `emqx_ctl data import /some/data.json`, it'll only search in the `data/backup` directory and fail if the file is not inside that dir. --- .../src/emqx_mgmt_data_backup.erl | 5 ++++- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index f5eaaa7ab..5a83528fd 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -666,7 +666,10 @@ look_up_file(Filename) -> end, case lists:filter(Filter, backup_files()) of [] -> - {error, not_found}; + case filelib:is_file(Filename) of + true -> {ok, Filename}; + false -> {error, not_found} + end; List -> {ok, hd(List)} end. diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 45afbd860..48a771d7a 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -72,6 +72,13 @@ init_per_testcase(t_plugins_cmd, Config) -> meck:expect(emqx_plugins, reload, fun(_) -> ok end), mock_print(), Config; +init_per_testcase(t_import_outside_backup_dir, Config) -> + RandomName = emqx_guid:to_hexstr(emqx_guid:gen()), + Filepath = "/tmp/" ++ binary_to_list(RandomName) ++ ".json", + FakeData = #{version => "4.4"}, + ok = file:write_file(Filepath, emqx_json:encode(FakeData)), + [ {tmp_file, Filepath} + | Config]; init_per_testcase(_Case, Config) -> mock_print(), Config. @@ -79,6 +86,10 @@ init_per_testcase(_Case, Config) -> end_per_testcase(t_plugins_cmd, _Config) -> meck:unload(emqx_plugins), unmock_print(); +end_per_testcase(t_import_outside_backup_dir, Config) -> + Filepath = ?config(tmp_file, Config), + file:delete(Filepath), + ok; end_per_testcase(_Case, _Config) -> unmock_print(). @@ -384,6 +395,12 @@ t_backup_file(_)-> {error, not_found} = emqx_mgmt_data_backup:delete_backup_file(BadFilename), ok. +t_import_outside_backup_dir(Config) -> + Filepath = ?config(tmp_file, Config), + Env = "{}", + ?assertEqual(ok, emqx_mgmt_data_backup:import(Filepath, Env)), + ok. + mock_print() -> ok = safe_unmeck(emqx_ctl), meck:new(emqx_ctl, [non_strict, passthrough]), From 78c5cb4aacc87ae5ad8d120a24c21afd8e3a48c5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 20 May 2022 09:37:09 -0300 Subject: [PATCH 317/363] refactor: only import checks for external backup file --- .../src/emqx_mgmt_data_backup.erl | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 5a83528fd..47b590530 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -660,16 +660,19 @@ backup_files(Dir) -> look_up_file(Filename) when is_binary(Filename) -> look_up_file(binary_to_list(Filename)); look_up_file(Filename) -> + DefOnNotFound = fun(_Filename) -> {error, not_found} end, + do_look_up_file(Filename, DefOnNotFound). + +do_look_up_file(Filename, OnNotFound) when is_binary(Filename) -> + do_look_up_file(binary_to_list(Filename), OnNotFound); +do_look_up_file(Filename, OnNotFound) -> Filter = fun(MaybeFile) -> filename:basename(MaybeFile) == Filename end, case lists:filter(Filter, backup_files()) of [] -> - case filelib:is_file(Filename) of - true -> {ok, Filename}; - false -> {error, not_found} - end; + OnNotFound(Filename); List -> {ok, hd(List)} end. @@ -813,19 +816,26 @@ import(Filename, OverridesJson) -> -spec(check_import_json(binary() | string()) -> {ok, map()} | {error, term()}). check_import_json(Filename) -> + OnNotFound = + fun(F) -> + case filelib:is_file(F) of + true -> {ok, F}; + false -> {error, not_found} + end + end, FunList = [ - fun look_up_file/1, + fun(F) -> do_look_up_file(F, OnNotFound) end, fun(F) -> file:read_file(F) end, fun check_json/1 ], - check_import_json(Filename, FunList). + do_check_import_json(Filename, FunList). -check_import_json(Res, []) -> +do_check_import_json(Res, []) -> {ok, Res}; -check_import_json(Acc, [Fun | FunList]) -> +do_check_import_json(Acc, [Fun | FunList]) -> case Fun(Acc) of {ok, Next} -> - check_import_json(Next, FunList); + do_check_import_json(Next, FunList); {error, Reason} -> {error, Reason} end. From ca842aa869cf0390535f5986e34a45c43f6315f9 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 23 May 2022 11:23:27 +0800 Subject: [PATCH 318/363] fix: merge main-v4.3 to main-v4.4 --- .ci/build_packages/Dockerfile | 16 - .ci/build_packages/tests.sh | 250 +++-- .ci/docker-compose-file/docker-compose.yaml | 2 +- .ci/fvt_tests/local_relup_test_run.sh | 4 + .ci/fvt_tests/relup.lux | 8 +- .gitattributes | 1 + .github/workflows/apps_version_check.yaml | 4 +- .github/workflows/build_packages.yaml | 170 ++- .github/workflows/build_slim_packages.yaml | 80 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .../workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_automate_tests.yaml | 41 +- .github/workflows/run_fvt_tests.yaml | 103 +- .github/workflows/run_test_cases.yaml | 2 +- .tool-versions | 2 +- CHANGES-4.3.md | 1 + CHANGES-4.4.md | 122 +++ Makefile | 24 +- Windows.md | 4 +- apps/emqx_auth_jwt/rebar.config | 2 +- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 4 +- .../emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 19 +- .../src/emqx_auth_mnesia_cli.erl | 2 +- .../test/emqx_acl_mnesia_SUITE.erl | 1 + apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 7 +- .../priv/emqx_auth_mongo.schema | 37 +- .../src/emqx_auth_mongo.app.src | 2 +- .../src/emqx_auth_mongo.appup.src | 20 +- .../src/emqx_auth_mongo_sup.erl | 94 +- apps/emqx_auth_pgsql/rebar.config | 2 +- .../src/emqx_auth_pgsql.app.src | 2 +- .../src/emqx_auth_pgsql.appup.src | 24 +- apps/emqx_exhook/etc/emqx_exhook.conf | 6 + apps/emqx_exhook/priv/emqx_exhook.schema | 4 + apps/emqx_exhook/priv/protos/exhook.proto | 25 + apps/emqx_exhook/rebar.config | 2 +- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.appup.src | 46 +- apps/emqx_exhook/src/emqx_exhook_handler.erl | 67 +- apps/emqx_exhook/src/emqx_exhook_mngr.erl | 22 +- apps/emqx_exhook/src/emqx_exhook_server.erl | 17 +- apps/emqx_exhook/src/emqx_exhook_sup.erl | 5 +- .../emqx_exhook/test/emqx_exhook_demo_svr.erl | 18 +- .../test/props/prop_exhook_hooks.erl | 19 +- apps/emqx_exproto/rebar.config | 2 +- apps/emqx_lwm2m/rebar.config | 4 +- apps/emqx_management/include/emqx_mgmt.hrl | 2 +- .../src/emqx_management.app.src | 4 +- .../src/emqx_management.appup.src | 22 +- apps/emqx_management/src/emqx_mgmt.erl | 47 +- apps/emqx_management/src/emqx_mgmt_api.erl | 33 +- .../src/emqx_mgmt_api_clients.erl | 71 +- .../src/emqx_mgmt_api_data.erl | 81 +- .../src/emqx_mgmt_api_pubsub.erl | 111 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 248 +++-- .../src/emqx_mgmt_data_backup.erl | 250 ++++- apps/emqx_management/src/emqx_mgmt_http.erl | 4 +- .../test/emqx_auth_mnesia_migration_SUITE.erl | 5 +- ...x_bridge_mqtt_data_export_import_SUITE.erl | 10 +- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 140 ++- .../test/emqx_mgmt_api_SUITE.erl | 197 +++- ...emqx_mongo_auth_module_migration_SUITE.erl | 71 ++ .../e4.2.8.json | 1 + .../e4.3.5.json | 1 + .../emqx_webhook_data_export_import_SUITE.erl | 7 +- .../include/emqx_slow_subs.hrl | 38 + .../src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_plugin_libs.appup.src | 41 +- .../src/emqx_slow_subs/emqx_slow_subs.erl | 337 ++++++ .../src/emqx_slow_subs/emqx_slow_subs_api.erl | 116 +++ .../src/emqx_trace/emqx_trace.erl | 496 +++++++++ .../src/emqx_trace/emqx_trace_api.erl | 217 ++++ .../test/emqx_trace_SUITE.erl | 348 +++++++ apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- .../emqx_retainer/src/emqx_retainer.appup.src | 15 +- apps/emqx_retainer/src/emqx_retainer.erl | 45 +- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 246 +---- .../emqx_rule_engine/src/emqx_rule_events.erl | 164 ++- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 26 +- .../src/emqx_rule_metrics.erl | 29 +- .../src/emqx_rule_sqltester.erl | 17 +- .../test/emqx_rule_engine_SUITE.erl | 78 +- apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 378 +++++-- bin/emqx | 47 +- build | 157 ++- deploy/charts/emqx/templates/StatefulSet.yaml | 16 +- .../charts/emqx/templates/configmap.env.yaml | 2 +- deploy/charts/emqx/templates/rbac.yaml | 6 +- deploy/charts/emqx/values.yaml | 15 + deploy/docker/Dockerfile | 15 +- deploy/docker/Dockerfile.testing | 43 + deploy/docker/docker-entrypoint.sh | 20 +- deploy/packages/deb/Makefile | 5 +- deploy/packages/deb/debian/control | 2 +- deploy/packages/rpm/Makefile | 16 +- deploy/packages/rpm/emqx.spec | 2 +- etc/emqx.conf | 32 +- include/emqx_mqtt.hrl | 18 + include/emqx_release.hrl | 2 +- .../emqx_dashboard/src/emqx_dashboard.app.src | 4 +- .../src/emqx_dashboard.appup.src | 11 +- lib-ce/emqx_dashboard/src/emqx_dashboard.erl | 19 +- .../test/emqx_dashboard_SUITE.erl | 1 + .../emqx_modules/src/emqx_mod_slow_subs.erl | 49 + lib-ce/emqx_modules/src/emqx_mod_sup.erl | 24 +- lib-ce/emqx_modules/src/emqx_mod_trace.erl | 39 + .../emqx_modules/src/emqx_mod_trace_api.erl | 117 +++ lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- .../emqx_modules/src/emqx_modules.appup.src | 40 +- .../emqx_modules/test/emqx_mod_sup_SUITE.erl | 3 +- .../test/emqx_mod_trace_api_SUITE.erl | 187 ++++ .../emqx_modules/test/emqx_modules_SUITE.erl | 4 +- .../test/emqx_slow_subs_SUITE.erl | 117 +++ .../test/emqx_slow_subs_api_SUITE.erl | 163 +++ priv/emqx.schema | 94 +- rebar.config | 6 +- rebar.config.erl | 12 +- scripts/apps-version-check.sh | 104 +- scripts/buildx.sh | 80 ++ scripts/ensure-rebar3.sh | 17 +- scripts/get-dashboard.sh | 5 + scripts/get-distro.sh | 6 +- scripts/get-otp-vsn.sh | 5 + scripts/git-hook-pre-push.sh | 4 + scripts/inject-relup.escript | 139 +++ scripts/pkg-full-vsn.sh | 45 + scripts/relup-base-packages.sh | 23 +- scripts/relup-base-vsns.sh | 23 +- scripts/update-appup.sh | 1 + src/emqx.app.src | 2 +- src/emqx.appup.src | 970 ++---------------- src/emqx_access_control.erl | 20 +- src/emqx_broker.erl | 53 +- src/emqx_channel.erl | 60 +- src/emqx_cm.erl | 41 +- src/emqx_connection.erl | 91 +- src/emqx_guid.erl | 4 +- src/emqx_keepalive.erl | 20 +- src/emqx_logger.erl | 49 +- src/emqx_misc.erl | 51 +- src/emqx_packet.erl | 59 +- src/emqx_relup.erl | 56 + src/emqx_session.erl | 148 ++- src/emqx_slow_subs/emqx_moving_average.erl | 90 ++ src/emqx_stats.erl | 13 +- src/emqx_trace_handler.erl | 259 +++++ src/emqx_tracer.erl | 168 --- src/emqx_types.erl | 3 +- src/emqx_ws_connection.erl | 61 +- test/emqx_broker_SUITE.erl | 366 ++++++- test/emqx_channel_SUITE.erl | 2 + test/emqx_cm_SUITE.erl | 6 +- test/emqx_connection_SUITE.erl | 18 +- test/emqx_listeners_SUITE.erl | 7 +- test/emqx_session_SUITE.erl | 38 +- test/emqx_trace_handler_SUITE.erl | 233 +++++ test/emqx_trace_handler_tests.erl | 44 + test/emqx_tracer_SUITE.erl | 120 --- test/emqx_ws_connection_SUITE.erl | 16 +- 161 files changed, 7240 insertions(+), 2767 deletions(-) delete mode 100644 .ci/build_packages/Dockerfile create mode 100644 CHANGES-4.4.md create mode 100644 apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl create mode 100644 apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json create mode 100644 apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json create mode 100644 apps/emqx_plugin_libs/include/emqx_slow_subs.hrl create mode 100644 apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl create mode 100644 apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl create mode 100644 apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl create mode 100644 apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl create mode 100644 apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl create mode 100644 deploy/docker/Dockerfile.testing create mode 100644 lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl create mode 100644 lib-ce/emqx_modules/src/emqx_mod_trace.erl create mode 100644 lib-ce/emqx_modules/src/emqx_mod_trace_api.erl create mode 100644 lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl create mode 100644 lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl create mode 100644 lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl create mode 100755 scripts/buildx.sh create mode 100755 scripts/get-otp-vsn.sh create mode 100755 scripts/inject-relup.escript create mode 100755 scripts/pkg-full-vsn.sh create mode 100644 src/emqx_relup.erl create mode 100644 src/emqx_slow_subs/emqx_moving_average.erl create mode 100644 src/emqx_trace_handler.erl delete mode 100644 src/emqx_tracer.erl create mode 100644 test/emqx_trace_handler_SUITE.erl create mode 100644 test/emqx_trace_handler_tests.erl delete mode 100644 test/emqx_tracer_SUITE.erl diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile deleted file mode 100644 index 3c7e401ae..000000000 --- a/.ci/build_packages/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 -FROM ${BUILD_FROM} - -ARG EMQX_NAME=emqx - -COPY . /emqx - -WORKDIR /emqx - -RUN rm -rf _build/${EMQX_NAME}/lib _build/${EMQX_NAME}-pkg/lib - -RUN make ${EMQX_NAME}-zip || cat rebar3.crashdump - -RUN make ${EMQX_NAME}-pkg || cat rebar3.crashdump - -RUN /emqx/.ci/build_packages/tests.sh diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index b74d61ff2..04d8018ac 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -1,28 +1,56 @@ -#!/bin/bash +#!/usr/bin/env bash + +## This script tests built package start/stop +## Accept 2 args PROFILE and PACKAGE_TYPE + set -x -e -u + +if [ -z "${1:-}" ]; then + echo "Usage $0 e.g. emqx, emqx-edge" + exit 1 +fi + +if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then + echo "Usage $0 zip|pkg" + exit 1 +fi + +PROFILE="${1}" +PACKAGE_TYPE="${2}" + export CODE_PATH=${CODE_PATH:-"/emqx"} -export EMQX_NAME=${EMQX_NAME:-"emqx"} -export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" +export PACKAGE_PATH="${CODE_PATH}/_packages/${PROFILE}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" # export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1" # export EMQX_NODE_COOKIE=$(date +%s%N) -case "$(uname -m)" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm - ;; -esac -export ARCH +SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)" + +if [ "$PACKAGE_TYPE" = 'zip' ]; then + PKG_SUFFIX="zip" +else + case "${SYSTEM:-}" in + ubuntu*|debian*|raspbian*) + PKG_SUFFIX='deb' + ;; + *) + PKG_SUFFIX='rpm' + ;; + esac +fi + +PACKAGE_NAME="${PROFILE}-$("$CODE_PATH"/scripts/pkg-full-vsn.sh)" +OLD_PACKAGE_PATTERN="${PROFILE}-$("$CODE_PATH"/scripts/pkg-full-vsn.sh 'vsn_matcher')" +PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" + +PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" +if ! [ -f "$PACKAGE_FILE" ]; then + echo "$PACKAGE_FILE is not a file" + exit 1 +fi emqx_prepare(){ mkdir -p "${PACKAGE_PATH}" - if [ ! -d "/paho-mqtt-testing" ]; then git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho-mqtt-testing fi @@ -31,83 +59,80 @@ emqx_prepare(){ emqx_test(){ cd "${PACKAGE_PATH}" + local packagename="${PACKAGE_FILE_NAME}" + case "$PKG_SUFFIX" in + "zip") + unzip -q "${PACKAGE_PATH}/${packagename}" + export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \ + EMQX_MQTT__MAX_TOPIC_ALIAS=10 + sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins - for var in "$PACKAGE_PATH"/"${EMQX_NAME}"-*;do - case ${var##*.} in - "zip") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip) - unzip -q "${PACKAGE_PATH}/${packagename}" - export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \ - EMQX_MQTT__MAX_TOPIC_ALIAS=10 - sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins - - echo "running ${packagename} start" - "${PACKAGE_PATH}"/emqx/bin/emqx start || ( tail "${PACKAGE_PATH}"/emqx/log/emqx.log.1 && exit 1 ) - IDLE_TIME=0 - while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' - do - if [ $IDLE_TIME -gt 10 ] - then - echo "emqx running error" - exit 1 - fi - sleep 10 - IDLE_TIME=$((IDLE_TIME+1)) - done - pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic - "${PACKAGE_PATH}"/emqx/bin/emqx stop - echo "running ${packagename} stop" - rm -rf "${PACKAGE_PATH}"/emqx - ;; - "deb") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.deb) - dpkg -i "${PACKAGE_PATH}/${packagename}" - if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ] + echo "running ${packagename} start" + if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then + cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true + cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true + exit 1 + fi + IDLE_TIME=0 + while ! "${PACKAGE_PATH}"/emqx/bin/emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' + do + if [ $IDLE_TIME -gt 10 ] then - echo "package install error" + echo "emqx running error" exit 1 fi + sleep 10 + IDLE_TIME=$((IDLE_TIME+1)) + done + pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic + "${PACKAGE_PATH}"/emqx/bin/emqx stop + echo "running ${packagename} stop" + rm -rf "${PACKAGE_PATH}"/emqx + ;; + "deb") + dpkg -i "${PACKAGE_PATH}/${packagename}" + if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ] + then + echo "package install error" + exit 1 + fi - echo "running ${packagename} start" - running_test - echo "running ${packagename} stop" + echo "running ${packagename} start" + running_test + echo "running ${packagename} stop" - dpkg -r "${EMQX_NAME}" - if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ] - then - echo "package remove error" - exit 1 - fi + dpkg -r "${PROFILE}" + if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ] + then + echo "package remove error" + exit 1 + fi - dpkg -P "${EMQX_NAME}" - if dpkg -l |grep -q emqx - then - echo "package uninstall error" - exit 1 - fi - ;; - "rpm") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm) + dpkg -P "${PROFILE}" + if dpkg -l |grep -q emqx + then + echo "package uninstall error" + exit 1 + fi + ;; + "rpm") + yum install -y "${PACKAGE_PATH}/${packagename}" + if ! rpm -q "${PROFILE}" | grep -q "${PROFILE}"; then + echo "package install error" + exit 1 + fi - rpm -ivh "${PACKAGE_PATH}/${packagename}" - if ! rpm -q emqx | grep -q emqx; then - echo "package install error" - exit 1 - fi + echo "running ${packagename} start" + running_test + echo "running ${packagename} stop" - echo "running ${packagename} start" - running_test - echo "running ${packagename} stop" - - rpm -e "${EMQX_NAME}" - if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then - echo "package uninstall error" - exit 1 - fi - ;; - - esac - done + rpm -e "${PROFILE}" + if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then + echo "package uninstall error" + exit 1 + fi + ;; + esac } running_test(){ @@ -115,7 +140,11 @@ running_test(){ EMQX_MQTT__MAX_TOPIC_ALIAS=10 sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins - emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 ) + if ! emqx start; then + cat /var/log/emqx/erlang.log.1 || true + cat /var/log/emqx/emqx.log.1 || true + exit 1 + fi IDLE_TIME=0 while ! emqx_ctl status | grep -qE 'Node\s.*@.*\sis\sstarted' do @@ -134,30 +163,33 @@ running_test(){ relup_test(){ TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh)" - if [ -d "${RELUP_PACKAGE_PATH}" ];then - cd "${RELUP_PACKAGE_PATH}" - - find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" | - while read -r pkg; do - packagename=$(basename "${pkg}") - unzip -q "$packagename" - ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 ) - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases - ./emqx/bin/emqx install "${TARGET_VERSION}" - [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 - export EMQX_WAIT_FOR_STOP=300 - ./emqx/bin/emqx_ctl status - if ! ./emqx/bin/emqx stop; then - cat emqx/log/erlang.log.1 || true - cat emqx/log/emqx.log.1 || true - echo "failed to stop emqx" - exit 1 - fi - rm -rf emqx - done - fi + if [ ! -d "${RELUP_PACKAGE_PATH}" ];then + return 0 + fi + cd "${RELUP_PACKAGE_PATH}" + while read -r pkg; do + packagename=$(basename "${pkg}") + unzip -q "$packagename" + if ! ./emqx/bin/emqx start; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + exit 1 + fi + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx versions + cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*.zip ./emqx/releases/ + ./emqx/bin/emqx install "${TARGET_VERSION}" + [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 + export EMQX_WAIT_FOR_STOP=300 + ./emqx/bin/emqx_ctl status + if ! ./emqx/bin/emqx stop; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + echo "failed to stop emqx" + exit 1 + fi + rm -rf emqx + done < <(find . -maxdepth 1 -name "${OLD_PACKAGE_PATTERN}.zip") } emqx_prepare diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 6b5c1b602..819522dbd 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.ci/fvt_tests/local_relup_test_run.sh b/.ci/fvt_tests/local_relup_test_run.sh index 0be98b117..649af9587 100755 --- a/.ci/fvt_tests/local_relup_test_run.sh +++ b/.ci/fvt_tests/local_relup_test_run.sh @@ -15,6 +15,8 @@ PROFILE="$1" VSN="$2" OLD_VSN="$3" PACKAGE_PATH="$4" +FROM_OTP_VSN="${5:-24.1.5-3}" +TO_OTP_VSN="${6:-24.1.5-3}" TEMPDIR=$(mktemp -d) trap '{ rm -rf -- "$TEMPDIR"; }' EXIT @@ -37,4 +39,6 @@ exec docker run \ --var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ + --var FROM_OTP_VSN="$FROM_OTP_VSN" \ + --var TO_OTP_VSN="$TO_OTP_VSN" \ relup.lux diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index ce3b014fa..e3d28a2e8 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -3,6 +3,8 @@ [config var=ONE_MORE_EMQX_PATH] [config var=VSN] [config var=OLD_VSN] +[config var=FROM_OTP_VSN] +[config var=TO_OTP_VSN] [config shell_cmd=/bin/bash] [config timeout=600000] @@ -20,7 +22,7 @@ [shell emqx] !OLD_VSN=$(echo $OLD_VSN | sed -r 's/[v|e]//g') !cd $PACKAGE_PATH - !unzip -q -o $PROFILE-ubuntu20.04-$${OLD_VSN}-amd64.zip + !unzip -q -o $PROFILE-$(echo $OLD_VSN | sed -r 's/[v|e]//g')-otp${FROM_OTP_VSN}-ubuntu20.04-amd64.zip ?SH-PROMPT !cd emqx @@ -82,7 +84,7 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/ ## upgrade to the new version !./bin/emqx install $VSN @@ -128,7 +130,7 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-otp${TO_OTP_VSN}-ubuntu20.04-amd64.zip releases/ ## upgrade to the new version !./bin/emqx install $VSN diff --git a/.gitattributes b/.gitattributes index 4ed73da9a..8ecb2ae1a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ +build text eol=lf * text=auto *.* text eol=lf *.jpg -text diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 2ef18934f..b787a2afe 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -9,11 +9,11 @@ jobs: strategy: matrix: erl_otp: - - erl23.2.7.2-emqx-3 + - 24.1.5-3 os: - ubuntu20.04 - container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index fc2a23b99..09e9aaa6a 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -18,7 +18,8 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + # prepare source with any OTP version, no need for a matrix + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -31,8 +32,8 @@ jobs: - name: set profile id: set_profile shell: bash + working-directory: source run: | - cd source if make emqx-ee --dry-run > /dev/null 2>&1; then echo "::set-output name=profiles::[\"emqx-ee\"]" else @@ -43,7 +44,7 @@ jobs: run: | make -C source deps-all zip -ryq source.zip source/* source/.[^.]* - - name: get_all_deps + - name: get_all_deps_ee if: endsWith(github.repository, 'enterprise') run: | echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials @@ -60,13 +61,13 @@ jobs: needs: prepare if: endsWith(github.repository, 'emqx') strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.13 + - 24.2.1 exclude: - profile: emqx-edge - steps: - uses: actions/download-artifact@v2 with: @@ -82,57 +83,44 @@ jobs: env: PYTHON: python DIAGNOSTIC: 1 + PROFILE: emqx + working-directory: source run: | erl -eval "erlang:display(crypto:info_lib())" -s init stop - $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) - if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { - $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" - $pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip" - } - else { - $pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip" - } - cd source - ## We do not build/release bcrypt for windows package - Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ - if (Test-Path rebar.lock) { - Remove-Item -Force -Path rebar.lock - } - make ensure-rebar3 - make ${{ matrix.profile }} - mkdir -p _packages/${{ matrix.profile }} - Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name - mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }} - sha256sum "_packages/${{ matrix.profile }}/$pkg_name" | head -c 64 > "_packages/${{ matrix.profile }}/${pkg_name}.sha256" + make ${{ matrix.profile }}-zip - name: run emqx timeout-minutes: 1 + working-directory: source run: | - cd source ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start Start-Sleep -s 5 + echo "EMQX started" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop + echo "EMQX stopped" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install + echo "EQMX installed" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall + echo "EQMX uninstaled" - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-windows path: source/_packages/${{ matrix.profile }}/. mac: needs: prepare - strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} - erl_otp: - - 23.2.7.2-emqx-3 + otp: + - 24.1.5-3 + macos: + - macos-11 + - macos-10.15 exclude: - profile: emqx-edge - macos: - - macos-10.15 - runs-on: ${{ matrix.macos }} - + runs-on: ${{ matrix.macos }} steps: - uses: actions/download-artifact@v2 with: @@ -149,8 +137,8 @@ jobs: - uses: actions/cache@v2 id: cache with: - path: ~/.kerl/${{ matrix.erl_otp }} - key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }} + path: ~/.kerl/${{ matrix.otp }} + key: otp-install-${{ matrix.otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 @@ -159,21 +147,22 @@ jobs: OTP_GITHUB_URL: https://github.com/emqx/otp run: | kerl update releases - kerl build ${{ matrix.erl_otp }} - kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} + kerl build ${{ matrix.otp }} + kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} - name: build + working-directory: source run: | - . $HOME/.kerl/${{ matrix.erl_otp }}/activate - cd source + . $HOME/.kerl/${{ matrix.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 rm -rf _build/${{ matrix.profile }}/lib make ${{ matrix.profile }}-zip - name: test + working-directory: source run: | - cd source - pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip) - unzip -q _packages/${{ matrix.profile }}/$pkg_name + set -x + pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) + unzip -q $pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' @@ -197,12 +186,10 @@ jobs: exit 1 fi rm -rf emqx - #sha256sum ./_packages/${{ matrix.profile }}/$pkg_name | head -c64 > ./_packages/${{ matrix.profile }}/$pkg_name.sha256 - openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}' > ./_packages/${{ matrix.profile }}/$pkg_name.sha256 - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-${{ matrix.otp }} path: source/_packages/${{ matrix.profile }}/. linux: @@ -214,6 +201,12 @@ jobs: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + package: + - zip + - pkg + otp: + - 23.3.4.9-3 + - 24.1.5-3 arch: - amd64 - arm64 @@ -221,17 +214,15 @@ jobs: - ubuntu20.04 - ubuntu18.04 - ubuntu16.04 + - debian11 - debian10 - debian9 - # - opensuse - - centos8 + - rockylinux8 - centos7 - - centos6 - raspbian10 - # - raspbian9 exclude: - - os: centos6 - arch: arm64 + - package: pkg + otp: 23.3.4.9-3 - os: raspbian9 arch: amd64 - os: raspbian10 @@ -250,15 +241,11 @@ jobs: shell: bash steps: - - name: prepare docker - run: | - mkdir -p $HOME/.docker - echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json - echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker info - docker buildx create --use --name mybuild - docker run --rm --privileged tonistiigi/binfmt --install all + - uses: docker/setup-buildx-action@v1 + - uses: docker/setup-qemu-action@v1 + with: + image: tonistiigi/binfmt:latest + platforms: all - uses: actions/download-artifact@v2 with: name: source @@ -267,54 +254,34 @@ jobs: run: unzip -q source.zip - name: build emqx packages env: - ERL_OTP: erl23.2.7.2-emqx-3 + OTP: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} + PACKAGE: ${{ matrix.package}} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} + working-directory: source run: | - set -e -u - cd source - docker buildx build --no-cache \ - --platform=linux/$ARCH \ - -t cross_build_emqx_for_$SYSTEM \ - -f .ci/build_packages/Dockerfile \ - --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \ - --build-arg EMQX_NAME=$PROFILE \ - --output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar . - - mkdir -p /tmp/packages/$PROFILE - tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/* - mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/ - rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar - - docker rm -f $(docker ps -a -q) - docker volume prune -f - - name: create sha256 - env: - PROFILE: ${{ matrix.profile}} - run: | - if [ -d /tmp/packages/$PROFILE ]; then - cd /tmp/packages/$PROFILE - for var in $(ls emqx-* ); do - bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" - done - cd - - fi + ./scripts/buildx.sh \ + --profile "${PROFILE}" \ + --pkgtype "${PACKAGE}" \ + --arch "${ARCH}" \ + --builder "ghcr.io/emqx/emqx-builder/4.4-8:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.profile }} - path: /tmp/packages/${{ matrix.profile }}/. + name: ${{ matrix.profile }}-${{ matrix.otp }} + path: source/_packages/${{ matrix.profile }}/. docker: runs-on: ubuntu-20.04 - needs: prepare strategy: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + otp: + - 24.1.5-3 registry: - 'docker.io' - 'public.ecr.aws' @@ -375,8 +342,8 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine - RUN_FROM=alpine:3.12 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.otp }}-alpine3.15.1 + RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile context: source @@ -391,8 +358,8 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine - RUN_FROM=alpine:3.12 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.otp }}-alpine3.15.1 + RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise context: source @@ -408,10 +375,17 @@ jobs: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + otp: + - 23.3.4.9-3 + - 24.1.5-3 + include: + - profile: emqx + otp: windows # otp version on windows is rather fixed + steps: - uses: actions/download-artifact@v2 with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-${{ matrix.otp }} path: packages/${{ matrix.profile }} - name: install dos2unix run: sudo apt-get update && sudo apt install -y dos2unix diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index a10e72258..dab02d1ec 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -1,5 +1,10 @@ name: Build slim packages +concurrency: + group: slim-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + + on: pull_request: workflow_dispatch: @@ -9,14 +14,16 @@ jobs: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: erl_otp: - - erl23.2.7.2-emqx-3 + - 23.3.4.9-3 + - 24.1.5-3 os: - ubuntu20.04 - - centos7 + - rockylinux8 - container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 @@ -38,23 +45,64 @@ jobs: with: name: rebar3.crashdump path: ./rebar3.crashdump - - name: pakcages test + - name: packages test run: | - export CODE_PATH=$GITHUB_WORKSPACE - .ci/build_packages/tests.sh + export CODE_PATH="$GITHUB_WORKSPACE" + .ci/build_packages/tests.sh "${EMQX_NAME}" zip + .ci/build_packages/tests.sh "${EMQX_NAME}" pkg - uses: actions/upload-artifact@v2 with: name: ${{ matrix.os }} path: _packages/**/*.zip + windows: + runs-on: windows-2019 + if: endsWith(github.repository, 'emqx') + strategy: + fail-fast: false + matrix: + profile: + - emqx + otp: + - 24.2.1 + steps: + - uses: actions/checkout@v2 + - uses: ilammy/msvc-dev-cmd@v1 + - uses: erlef/setup-beam@v1 + with: + otp-version: ${{ matrix.otp }} + - name: build + env: + PYTHON: python + DIAGNOSTIC: 1 + run: | + erl -eval "erlang:display(crypto:info_lib())" -s init stop + make ${{ matrix.profile }}-zip + - name: run emqx + timeout-minutes: 1 + run: | + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start + Start-Sleep -s 5 + echo "EMQX started" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop + echo "EMQX stopped" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install + echo "EQMX installed" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall + echo "EQMX uninstaled" + mac: strategy: + fail-fast: false matrix: - erl_otp: - - 23.2.7.2-emqx-3 + otp: + - 24.1.5-3 macos: - macos-11 - runs-on: ${{ matrix.macos }} + - macos-10.15 + + runs-on: ${{ matrix.macos }} + steps: - uses: actions/checkout@v1 - name: prepare @@ -75,8 +123,8 @@ jobs: - uses: actions/cache@v2 id: cache with: - path: ~/.kerl/${{ matrix.erl_otp }} - key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }} + path: ~/.kerl/${{ matrix.otp }} + key: otp-install-${{ matrix.otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 @@ -85,11 +133,11 @@ jobs: OTP_GITHUB_URL: https://github.com/emqx/otp run: | kerl update releases - kerl build ${{ matrix.erl_otp }} - kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} + kerl build ${{ matrix.otp }} + kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} - name: build run: | - . $HOME/.kerl/${{ matrix.erl_otp }}/activate + . $HOME/.kerl/${{ matrix.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 make ${EMQX_NAME}-zip @@ -100,8 +148,8 @@ jobs: path: ./rebar3.crashdump - name: test run: | - pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) - unzip -q _packages/${EMQX_NAME}/$pkg_name + pkg_name=$(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip) + unzip -q $pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index e58afcc1a..2b49d4d77 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0d7446de3..ffcb69247 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 0f001ac04..3b11a9368 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index a9c4b1edf..aa8bfd46f 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -12,8 +12,8 @@ jobs: build: runs-on: ubuntu-latest outputs: - imgname: ${{ steps.build_docker.outputs.imgname}} - version: ${{ steps.build_docker.outputs.version}} + imgname: ${{ steps.prepare.outputs.imgname}} + version: ${{ steps.prepare.outputs.version}} steps: - name: download jmeter id: dload_jmeter @@ -27,28 +27,31 @@ jobs: name: apache-jmeter.tgz path: /tmp/apache-jmeter.tgz - uses: actions/checkout@v2 - - name: build docker - id: build_docker + - name: prepare + id: prepare run: | if [ -f EMQX_ENTERPRISE ]; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store make deps-emqx-ee make clean - make emqx-ee-docker echo "::set-output name=imgname::emqx-ee" echo "::set-output name=version::$(./pkg-vsn.sh)" - docker save emqx/emqx-ee:$(./pkg-vsn.sh) -o emqx.tar else make emqx-docker echo "::set-output name=imgname::emqx" echo "::set-output name=version::$(./pkg-vsn.sh)" - docker save emqx/emqx:$(./pkg-vsn.sh) -o emqx.tar fi + - name: build docker image + env: + OTP_VSN: 24.1.5-3 + run: | + make ${{ steps.prepare.outputs.imgname }}-docker + docker save emqx/${{ steps.prepare.outputs.imgname }}:${{ steps.prepare.outputs.version }} -o image.tar.gz - uses: actions/upload-artifact@v2 with: - name: emqx-docker-image - path: emqx.tar + name: image + path: image.tar.gz webhook: runs-on: ubuntu-latest @@ -64,10 +67,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image + name: image path: /tmp - name: load docker image - run: docker load < /tmp/emqx.tar + run: | + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: @@ -162,10 +166,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image + name: image path: /tmp - name: load docker image - run: docker load < /tmp/emqx.tar + run: | + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: @@ -267,10 +272,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image + name: image path: /tmp - name: load docker image - run: docker load < /tmp/emqx.tar + run: | + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: @@ -361,10 +367,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 with: - name: emqx-docker-image + name: image path: /tmp - name: load docker image - run: docker load < /tmp/emqx.tar + run: | + docker load < /tmp/image.tar.gz - name: docker compose up timeout-minutes: 5 env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 1eb189720..dbc79a8f5 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -13,24 +13,27 @@ jobs: steps: - uses: actions/checkout@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang - with: - otp-version: 23.2 - - name: make docker + - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store make deps-emqx-ee + make clean echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV + echo "PROFILE=emqx-ee" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV make emqx-ee-docker else echo "TARGET=emqx/emqx" >> $GITHUB_ENV + echo "PROFILE=emqx" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV make emqx-docker fi + - name: make emqx image + env: + OTP_VSN: 24.1.5-3 + run: make ${PROFILE}-docker - name: run emqx timeout-minutes: 5 run: | @@ -63,25 +66,34 @@ jobs: helm_test: runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + discovery: + - k8s + - dns steps: - uses: actions/checkout@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang - with: - otp-version: 23.2 - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store make deps-emqx-ee + make clean echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV - make emqx-ee-docker + echo "PROFILE=emqx-ee" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV else echo "TARGET=emqx/emqx" >> $GITHUB_ENV - make emqx-docker + echo "PROFILE=emqx" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi + - name: make emqx image + env: + OTP_VSN: 24.1.5-3 + run: make ${PROFILE}-docker - name: install k3s env: KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" @@ -98,18 +110,18 @@ jobs: sudo chmod 700 get_helm.sh sudo ./get_helm.sh helm version - - name: run emqx on chart - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - timeout-minutes: 5 + - name: setup emqx chart run: | - version=$(./pkg-vsn.sh) - sudo docker save ${TARGET}:$version -o emqx.tar.gz + sudo docker save ${TARGET}:${EMQX_TAG} -o emqx.tar.gz sudo k3s ctr image import emqx.tar.gz - sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml + sed -i -r "s/^appVersion: .*$/appVersion: \"${EMQX_TAG}\"/g" deploy/charts/emqx/Chart.yaml sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml - + - name: run emqx on chart + if: matrix.discovery == 'k8s' + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | helm install emqx \ --set image.repository=${TARGET} \ --set image.pullPolicy=Never \ @@ -119,7 +131,29 @@ jobs: --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ deploy/charts/emqx \ --debug - + - name: run emqx on chart + if: matrix.discovery == 'dns' + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | + helm install emqx \ + --set emqxConfig.EMQX_CLUSTER__DISCOVERY="dns" \ + --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \ + --set emqxConfig.EMQX_CLUSTER__DNS__APP="emqx" \ + --set emqxConfig.EMQX_CLUSTER__DNS__TYPE="srv" \ + --set image.repository=${TARGET} \ + --set image.pullPolicy=Never \ + --set emqxAclConfig="" \ + --set image.pullPolicy=Never \ + --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ + deploy/charts/emqx \ + --debug + - name: waiting emqx started + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + timeout-minutes: 5 + run: | while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \ != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do echo "=============================="; @@ -128,6 +162,18 @@ jobs: echo "waiting emqx started"; sleep 10; done + - name: Check ${{ matrix.kind[0]}} cluster + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + timeout-minutes: 10 + run: | + while + nodes="$(kubectl exec -i emqx-0 -- curl --silent --basic -u admin:public -X GET http://localhost:8081/api/v4/brokers | jq '.data|length')"; + [ "$nodes" != "3" ]; + do + echo "waiting emqx cluster scale" + sleep 1 + done - name: get emqx-0 pods log if: failure() env: @@ -178,7 +224,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -223,8 +269,13 @@ jobs: relup_test_build: needs: relup_test_plan + strategy: + fail-fast: false + matrix: + otp: + - 24.1.5-3 runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 defaults: run: shell: bash @@ -259,11 +310,13 @@ jobs: - relup_test_plan - relup_test_build runs-on: ubuntu-20.04 - container: emqx/relup-test-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: emqx/relup-test-env:erl23.2.7.2-emqx-2-ubuntu20.04 strategy: fail-fast: false matrix: old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }} + otp: + - 24.1.5-3 env: OLD_VSN: "${{ matrix.old_vsn }}" PROFILE: "${{ needs.relup_test_plan.outputs.profile }}" @@ -290,7 +343,7 @@ jobs: mkdir -p packages cp emqx_built/_packages/*/*.zip packages cd packages - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-ubuntu20.04-${OLD_VSN#[e|v]}-amd64.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-${OLD_VSN#[e|v]}-otp${{ matrix.otp }}-ubuntu20.04-amd64.zip - name: Run relup test scenario timeout-minutes: 5 run: | @@ -302,6 +355,8 @@ jobs: --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ --var VSN="$VSN" \ --var OLD_VSN="$OLD_VSN" \ + --var FROM_OTP_VSN="24.1.5-3" \ + --var TO_OTP_VSN="24.1.5-3" \ emqx_built/.ci/fvt_tests/relup.lux - uses: actions/upload-artifact@v2 name: Save debug data diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 9a40964bc..efca499ea 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.tool-versions b/.tool-versions index 0b6392b95..a6568713b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 23.2.7.2-emqx-3 +erlang 24.1.5-3 diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index b74eae09e..0245e23ab 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -92,6 +92,7 @@ File format: to force an immediate reload of all certificates after the files are updated on disk. * Refactor the ExProto so that anonymous clients can also be displayed on the dashboard [#6983] * Force shutdown of processes that cannot answer takeover event [#7026] +* Support set keepalive via queryString & Body HTTP API. * `topic` parameter in bridge configuration can have `${node}` substitution (just like in `clientid` parameter) * Add UTF-8 string validity check in `strict_mode` for MQTT packet. When set to true, invalid UTF-8 strings will cause the client to be disconnected. i.e. client ID, topic name. [#7261] diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md new file mode 100644 index 000000000..601585858 --- /dev/null +++ b/CHANGES-4.4.md @@ -0,0 +1,122 @@ +# EMQX 4.4 Changes + +### Enhancements +* Add rule events: client.connack, client.check_acl_complete +- client.connack The rule event is triggered when the server sends a CONNACK packet to the client. reason_code contains the error reason code. +- client.check_acl_complete The rule event is triggered when the client check acl complete. + +## v4.4.2 + +**NOTE**: v4.4.2 is in sync with: v4.3.13 + +### Important changes + +* Docker image is based on alpine-3.15.1 (OpenSSL-1.1.1n) +* For docker image, /opt/emqx/etc has been removed from the VOLUME list, + this made it easier for the users to rebuild image on top with changed configs. +* CentOS 7 Erlang runtime is rebuilt on OpenSSL-1.1.1n (previously on 1.0), + Prior to v4.4.1, EMQX may pick certain cipher suites proposed by the clients, + but then fail to handshake resulting in a `malformed_handshake_data` exception. + +### Enhancements + +* Windows package is built on Erlang/OTP 24 + +### Enhancements (synced from v4.3.13) + +* CLI `emqx_ctl pem_cache clean` to force purge x509 certificate cache, + to force an immediate reload of all certificates after the files are updated on disk. +* Refactor the ExProto so that anonymous clients can also be displayed on the dashboard [#6983] +* Force shutdown of processes that cannot answer takeover event [#7026] +* Support set keepalive via queryString & Body HTTP API. +* `topic` parameter in bridge configuration can have `${node}` substitution (just like in `clientid` parameter) +* Add UTF-8 string validity check in `strict_mode` for MQTT packet. + When set to true, invalid UTF-8 strings will cause the client to be disconnected. i.e. client ID, topic name. [#7261] +* Changed systemd service restart delay from 10 seconds to 60 seconds. +* MQTT-SN gateway supports initiative to synchronize registered topics after session resumed. [#7300] +* Add load control app for future development. +* Change the precision of float to 17 digits after the decimal point when formatting a + float using payload templates of rule actions. The old precision is 10 digits before + this change. + +### Bug fixes (synced from v4.3.13) + +* Fix the `{error,eexist}` error when do release upgrade again if last run failed. [#7121] +* Fix case where publishing to a non-existent topic alias would crash the connection [#6979] +* Fix HTTP-API 500 error on querying the lwm2m client list on the another node [#7009] +* Fix the ExProto connection registry is not released after the client process abnormally exits [#6983] +* Fix Server-KeepAlive wrongly applied on MQTT v3.0/v3.1 [#7085] +* Fix Stomp client can not trigger `$event/client_connection` message [#7096] +* Fix system memory false alarm at boot +* Fix the MQTT-SN message replay when the topic is not registered to the client [#6970] +* Fix rpc get node info maybe crash when other nodes is not ready. +* Fix false alert level log “cannot_find_plugins” caused by duplicate plugin names in `loaded_plugins` files. +* Prompt user how to change the dashboard's initial default password when emqx start. +* Fix errno=13 'Permission denied' Cannot create FIFO boot error in Amazon Linux 2022 (el8 package) +* Fix user or appid created, name only allow `^[A-Za-z]+[A-Za-z0-9-_]*$` +* Fix subscribe http api crash by bad_qos `/mqtt/subscribe`,`/mqtt/subscribe_batch`. +* Send DISCONNECT packet with reason code 0x98 if connection has been kicked [#7309] +* Fix make all traces stopped when emqx_trace_module is disabled. + +## v4.4.1 + +This patch release is only to fix windows build which failed on v4.4.0. + +## v4.4.0 + +**NOTE**: v4.4.0 is in sync with: v4.3.12 + +### Important changes + +- **For Debian/Ubuntu users**, Debian/Ubuntu package (deb) installed EMQ X is now started from systemd. + This is to use systemd's supervision functionality to ensure that EMQ X service restarts after a crash. + The package installation service upgrade from init.d to systemd has been verified, + it is still recommended that you verify and confirm again before deploying to the production environment, + at least to ensure that systemd is available in your system + +- Package name scheme changed comparing to 4.3. + 4.3 format: emqx-centos8-4.3.8-amd64.zip + 4.4 format: emqx-4.4.0-rc.1-otp24.1.5-3-el8-amd64.zip + * Erlang/OTP version is included in the package name, + providing the possibility to release EMQX on multiple Erlang/OTP versions + * `centos` is renamed to `el`. This is mainly due to centos8 being dead (replaced with rockylinux8) + +- MongoDB authentication supports DNS SRV and TXT Records resolution, which can seamlessly connect with MongoDB Altas + +- Support dynamic modification of MQTT Keep Alive to adapt to different energy consumption strategies. + +- Support 4.3 to 4.4 rolling upgrade of clustered nodes. See upgrade document for more dtails. + +- TLS for cluster backplane (RPC) connections. See clustering document for more details. + +- Support real-time tracing in the dashboard, with Client ID, Client IP address, and topic name based filtering. + +- Add the Slow Subscriptions module to count the time spent during the message transmission. This feature will list the Clients and Topics with higher time consumption in Dashboard + +### Minor changes + +- Bumpped default boot wait time from 15 seconds to 150 seconds + because in some simulated environments it may take up to 70 seconds to boot in build CI + +- Dashboard supports relative paths and custom access paths + +- Supports configuring whether to forward retained messages with empty payload to suit users + who are still using MQTT v3.1. The relevant configurable item is `retainer.stop_publish_clear_msg` + +- Multi-language hook extension (ExHook) supports dynamic cancellation of subsequent forwarding of client messages + +- Rule engine SQL supports the use of single quotes in `FROM` clauses, for example: `SELECT * FROM 't/#'` + +- Change the default value of the `max_topic_levels` configurable item to 128. + Previously, it had no limit (configured to 0), which may be a potential DoS threat + +- Improve the error log content when the Proxy Protocol message is received without `proxy_protocol` configured. + +- Add additional message attributes to the message reported by the gateway. + Messages from gateways such as CoAP, LwM2M, Stomp, ExProto, etc., when converted to EMQ X messages, + add fields such as protocol name, protocol version, user name, client IP, etc., + which can be used for multi-language hook extension (ExHook) + +- HTTP client performance improvement + +- Add openssl-1.1 to RPM dependency diff --git a/Makefile b/Makefile index 0fb5c63e5..0ba09b68c 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,14 @@ $(shell $(CURDIR)/scripts/git-hooks-init.sh) -REBAR_VERSION = 3.14.3-emqx-8 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine -export EMQX_DEFAULT_RUNNER = alpine:3.12 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-alpine3.15.1 +export EMQX_DEFAULT_RUNNER = alpine:3.15.1 +export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export DOCKERFILE := deploy/docker/Dockerfile +export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none FIND=/usr/bin/find @@ -20,7 +21,7 @@ REL_PROFILES := emqx emqx-edge PKG_PROFILES := emqx-pkg emqx-edge-pkg PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default -export REBAR_GIT_CLONE_OPTIONS += --depth=1 +export REBAR_GIT_CLONE_OPTIONS += --depth=1 --quiet .PHONY: default default: $(REBAR) $(PROFILE) @@ -31,7 +32,7 @@ all: $(REBAR) $(PROFILES) .PHONY: ensure-rebar3 ensure-rebar3: @$(SCRIPTS)/fail-on-old-otp-version.escript - @$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION) + @$(SCRIPTS)/ensure-rebar3.sh $(REBAR): ensure-rebar3 @@ -98,6 +99,7 @@ $(PROFILES:%=clean-%): .PHONY: clean-all clean-all: + @rm -f rebar.lock @rm -rf _build @rm -f rebar.lock @@ -172,6 +174,18 @@ endef ALL_ZIPS = $(REL_PROFILES) $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) +## emqx-docker-testing +## emqx-ee-docker-testing +## is to directly copy a unzipped zip-package to a +## base image such as ubuntu20.04. Mostly for testing +.PHONY: $(REL_PROFILES:%=%-docker-testing) +define gen-docker-target-testing +$1-docker-testing: $(COMMON_DEPS) + @$(BUILD) $1 docker-testing +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt)))) + .PHONY: run run: $(PROFILE) quickrun diff --git a/Windows.md b/Windows.md index 6b6eff60a..6ffc3d8a6 100644 --- a/Windows.md +++ b/Windows.md @@ -29,7 +29,7 @@ The second path is for CMD to setup environment variables. ### Erlang/OTP -Install Erlang/OTP 23.2 from https://www.erlang.org/downloads +Install Erlang/OTP 24.2.1 from https://www.erlang.org/downloads You may need to edit the `Path` environment variable to allow running Erlang commands such as `erl` from CMD. @@ -45,7 +45,7 @@ e.g. ``` PS C:\Users\zmsto> erl -Eshell V11.1.4 (abort with ^G) +Eshell V12.2.1 (abort with ^G) 1> halt(). ``` diff --git a/apps/emqx_auth_jwt/rebar.config b/apps/emqx_auth_jwt/rebar.config index 5e7575881..b0a07eb8c 100644 --- a/apps/emqx_auth_jwt/rebar.config +++ b/apps/emqx_auth_jwt/rebar.config @@ -1,6 +1,6 @@ {deps, [ - {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} + {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 3d84c5b54..cc0029e0f 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,13 +1,13 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.4.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, {mod, {emqx_auth_jwt_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, + {maintainers, ["EMQX Team "]}, {links, [{"Homepage", "https://emqx.io/"}, {"Github", "https://github.com/emqx/emqx-auth-jwt"} ]} diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index b94159225..34e964e65 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,9 +1,16 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[0-2]">>, - [{restart_application,emqx_auth_jwt}]}, - {<<".*">>,[]}], - [{<<"4\\.3\\.[0-2]">>, - [{restart_application,emqx_auth_jwt}]}, - {<<".*">>,[]}]}. + [ + {<<"4\\.4\\.[0-1]">>, [ + {restart_application,emqx_auth_jwt} + ]}, + {<<".*">>, []} + ], + [ + {<<"4\\.4\\.[0-1]">>, [ + {restart_application,emqx_auth_jwt} + ]}, + {<<".*">>, []} + ] +}. diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl index bf919f5bf..86572c0a6 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl @@ -58,7 +58,7 @@ insert_user(User = #emqx_user{login = Login}) -> [_|_] -> mnesia:abort(existed) end. --spec(add_default_user(clientid | username, tuple(), binary()) -> ok | {error, any()}). +-spec(add_default_user(clientid | username, binary(), binary()) -> ok | {error, any()}). add_default_user(Type, Key, Password) -> Login = {Type, Key}, case add_user(Login, Password) of diff --git a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl index 941ebedb9..96e66d3c0 100644 --- a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl @@ -48,6 +48,7 @@ groups() -> ]}]. init_per_suite(Config) -> + application:load(emqx_plugin_libs), emqx_ct_helpers:start_apps( [emqx_modules, emqx_management, emqx_auth_mnesia] , fun set_special_configs/1 ), diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index f72e9859b..46a938d05 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -7,6 +7,12 @@ ## Value: single | unknown | sharded | rs auth.mongo.type = single +## Whether to use SRV and TXT records. +## +## Value: true | false +## Default: false +auth.mongo.srv_record = false + ## The set name if type is rs. ## ## Value: String @@ -37,7 +43,6 @@ auth.mongo.pool = 8 ## MongoDB AuthSource ## ## Value: String -## Default: mqtt ## auth.mongo.auth_source = admin ## MongoDB database diff --git a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema index 0c491e005..59d9a28fe 100644 --- a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema +++ b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema @@ -6,8 +6,12 @@ {datatype, {enum, [single, unknown, sharded, rs]}} ]}. +{mapping, "auth.mongo.srv_record", "emqx_auth_mongo.server", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + {mapping, "auth.mongo.rs_set_name", "emqx_auth_mongo.server", [ - {default, "mqtt"}, {datatype, string} ]}. @@ -41,7 +45,6 @@ ]}. {mapping, "auth.mongo.auth_source", "emqx_auth_mongo.server", [ - {default, "mqtt"}, {datatype, string} ]}. @@ -101,9 +104,9 @@ ]}. {translation, "emqx_auth_mongo.server", fun(Conf) -> - H = cuttlefish:conf_get("auth.mongo.server", Conf), - Hosts = string:tokens(H, ","), - Type0 = cuttlefish:conf_get("auth.mongo.type", Conf), + SrvRecord = cuttlefish:conf_get("auth.mongo.srv_record", Conf, false), + Server = cuttlefish:conf_get("auth.mongo.server", Conf), + Type = cuttlefish:conf_get("auth.mongo.type", Conf), Pool = cuttlefish:conf_get("auth.mongo.pool", Conf), %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 Login = cuttlefish:conf_get("auth.mongo.username", Conf, @@ -111,7 +114,10 @@ ), Passwd = cuttlefish:conf_get("auth.mongo.password", Conf), DB = cuttlefish:conf_get("auth.mongo.database", Conf), - AuthSrc = cuttlefish:conf_get("auth.mongo.auth_source", Conf), + AuthSource = case cuttlefish:conf_get("auth.mongo.auth_source", Conf, undefined) of + undefined -> []; + AuthSource0 -> [{auth_source, list_to_binary(AuthSource0)}] + end, R = cuttlefish:conf_get("auth.mongo.w_mode", Conf), W = cuttlefish:conf_get("auth.mongo.r_mode", Conf), Login0 = case Login =:= [] of @@ -157,8 +163,8 @@ false -> [] end, - WorkerOptions = [{database, list_to_binary(DB)}, {auth_source, list_to_binary(AuthSrc)}] - ++ Login0 ++ Passwd0 ++ W0 ++ R0 ++ Ssl, + WorkerOptions = [{database, list_to_binary(DB)}] + ++ Login0 ++ Passwd0 ++ W0 ++ R0 ++ Ssl ++ AuthSource, Vars = cuttlefish_variable:fuzzy_matches(["auth", "mongo", "topology", "$name"], Conf), Options = lists:map(fun({_, Name}) -> @@ -175,16 +181,17 @@ {list_to_atom(Name2), cuttlefish:conf_get("auth.mongo.topology."++Name, Conf)} end, Vars), - Type = case Type0 =:= rs of - true -> {Type0, list_to_binary(cuttlefish:conf_get("auth.mongo.rs_set_name", Conf))}; - false -> Type0 - end, - [{type, Type}, - {hosts, Hosts}, + ReplicaSet = case cuttlefish:conf_get("auth.mongo.rs_set_name", Conf, undefined) of + undefined -> []; + ReplicaSet0 -> [{rs_set_name, list_to_binary(ReplicaSet0)}] + end, + [{srv_record, SrvRecord}, + {type, Type}, + {server, Server}, {options, Options}, {worker_options, WorkerOptions}, {auto_reconnect, 1}, - {pool_size, Pool}] + {pool_size, Pool}] ++ ReplicaSet end}. %% The mongodb operation timeout is specified by the value of `cursor_timeout` from application config, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src index 2b6fedbb5..13e83387c 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mongo, [{description, "EMQ X Authentication/ACL with MongoDB"}, - {vsn, "4.3.4"}, % strict semver, bump manually! + {vsn, "4.4.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mongo_sup]}, {applications, [kernel,stdlib,mongodb,ecpool]}, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src index 1907b7fa7..67fc5c1db 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src @@ -1,19 +1,15 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[1-3]">>, - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + [{<<"4.4.[0-3]">>, + [{load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[1-3]">>, - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + [{<<"4.4.[0-3]">>, + [{load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl index 416c6e731..c0384d27b 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl @@ -28,7 +28,97 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - {ok, PoolEnv} = application:get_env(?APP, server), - PoolSpec = ecpool:pool_spec(?APP, ?APP, ?APP, PoolEnv), + {ok, Opts} = application:get_env(?APP, server), + NOpts = may_parse_srv_and_txt_records(Opts), + PoolSpec = ecpool:pool_spec(?APP, ?APP, ?APP, NOpts), {ok, {{one_for_all, 10, 100}, [PoolSpec]}}. +may_parse_srv_and_txt_records(Opts) when is_list(Opts) -> + Default = #{srv_record => false}, + maps:to_list(may_parse_srv_and_txt_records(maps:merge(Default, maps:from_list(Opts)))); + +may_parse_srv_and_txt_records(#{type := Type, + srv_record := false, + server := Server} = Opts) -> + Hosts = to_hosts(Server), + case Type =:= rs of + true -> + case maps:get(rs_set_name, Opts, undefined) of + undefined -> + error({missing_parameter, rs_set_name}); + ReplicaSet -> + Opts#{type => {rs, ReplicaSet}, + hosts => Hosts} + end; + false -> + Opts#{hosts => Hosts} + end; + +may_parse_srv_and_txt_records(#{type := Type, + srv_record := true, + server := Server, + worker_options := WorkerOptions} = Opts) -> + Hosts = parse_srv_records(Server), + Opts0 = parse_txt_records(Type, Server), + NWorkerOptions = maps:to_list(maps:merge(maps:from_list(WorkerOptions), maps:with([auth_source], Opts0))), + NOpts = Opts#{hosts => Hosts, worker_options => NWorkerOptions}, + case Type =:= rs of + true -> + case maps:get(rs_set_name, Opts0, maps:get(rs_set_name, NOpts, undefined)) of + undefined -> + error({missing_parameter, rs_set_name}); + ReplicaSet -> + NOpts#{type => {Type, ReplicaSet}} + end; + false -> + NOpts + end. + +to_hosts(Server) -> + [string:trim(H) || H <- string:tokens(Server, ",")]. + +parse_srv_records(Server) -> + case inet_res:lookup("_mongodb._tcp." ++ Server, in, srv) of + [] -> + error(service_not_found); + Services -> + [Host ++ ":" ++ integer_to_list(Port) || {_, _, Port, Host} <- Services] + end. + +parse_txt_records(Type, Server) -> + case inet_res:lookup(Server, in, txt) of + [] -> + #{}; + [[QueryString]] -> + case uri_string:dissect_query(QueryString) of + {error, _, _} -> + error({invalid_txt_record, invalid_query_string}); + Options -> + Fields = case Type of + rs -> ["authSource", "replicaSet"]; + _ -> ["authSource"] + end, + take_and_convert(Fields, Options) + end; + _ -> + error({invalid_txt_record, multiple_records}) + end. + +take_and_convert(Fields, Options) -> + take_and_convert(Fields, Options, #{}). + +take_and_convert([], [_ | _], _Acc) -> + error({invalid_txt_record, invalid_option}); +take_and_convert([], [], Acc) -> + Acc; +take_and_convert([Field | More], Options, Acc) -> + case lists:keytake(Field, 1, Options) of + {value, {"authSource", V}, NOptions} -> + take_and_convert(More, NOptions, Acc#{auth_source => list_to_binary(V)}); + {value, {"replicaSet", V}, NOptions} -> + take_and_convert(More, NOptions, Acc#{rs_set_name => list_to_binary(V)}); + {value, _, _} -> + error({invalid_txt_record, invalid_option}); + false -> + take_and_convert(More, Options, Acc) + end. diff --git a/apps/emqx_auth_pgsql/rebar.config b/apps/emqx_auth_pgsql/rebar.config index be395993b..7a6aaf411 100644 --- a/apps/emqx_auth_pgsql/rebar.config +++ b/apps/emqx_auth_pgsql/rebar.config @@ -1,5 +1,5 @@ {deps, - [{epgsql, {git, "https://github.com/epgsql/epgsql.git", {tag, "4.6.0"}}} + [{epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} ]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src index 35cae1622..7313747ee 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_pgsql, [{description, "EMQ X Authentication/ACL with PostgreSQL"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.4.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_pgsql_sup]}, {applications, [kernel,stdlib,epgsql,ecpool]}, diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src index d1c10d6fa..35f180341 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src @@ -1,25 +1,15 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[1-2]">>, + [{<<"4\\.4\\.[0-2]">>, %% epgsql 4.4.0 -> 4.6.0. - %% epgsql has no appup ,so we can only restart it. + %% epgsql has no appup, so we can only restart it. [{restart_application,epgsql}, - {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{restart_application,epgsql}, - {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, + {restart_application,emqx_auth_pgsql}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[1-2]">>, + [{<<"4\\.4\\.[0-2]">>, + %% epgsql 4.4.0 -> 4.6.0. + %% epgsql has no appup, so we can only restart it. [{restart_application,epgsql}, - {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}, - {restart_application,epgsql}, - {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, + {restart_application,emqx_auth_pgsql}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_exhook/etc/emqx_exhook.conf b/apps/emqx_exhook/etc/emqx_exhook.conf index 6a4725e02..70dba4e06 100644 --- a/apps/emqx_exhook/etc/emqx_exhook.conf +++ b/apps/emqx_exhook/etc/emqx_exhook.conf @@ -24,6 +24,12 @@ ## Value: false | Duration #exhook.auto_reconnect = 60s +## The process pool size for gRPC client +## +## Default: Equals cpu cores +## Value: Integer +#exhook.pool_size = 16 + ## The exhook execution priority on the Chain of the emqx hooks. ## ## Modify the field to fix the exhook execute order before/after other plugins/modules. diff --git a/apps/emqx_exhook/priv/emqx_exhook.schema b/apps/emqx_exhook/priv/emqx_exhook.schema index 4f6419c6a..277e05f0c 100644 --- a/apps/emqx_exhook/priv/emqx_exhook.schema +++ b/apps/emqx_exhook/priv/emqx_exhook.schema @@ -31,6 +31,10 @@ end end}. +{mapping, "exhook.pool_size", "emqx_exhook.pool_size", [ + {datatype, integer} +]}. + {mapping, "exhook.server.$name.url", "emqx_exhook.servers", [ {datatype, string} ]}. diff --git a/apps/emqx_exhook/priv/protos/exhook.proto b/apps/emqx_exhook/priv/protos/exhook.proto index c4f444759..740aa060e 100644 --- a/apps/emqx_exhook/priv/protos/exhook.proto +++ b/apps/emqx_exhook/priv/protos/exhook.proto @@ -403,6 +403,31 @@ message Message { bytes payload = 6; uint64 timestamp = 7; + + // The key of header can be: + // - username: + // * Readonly + // * The username of sender client + // * Value type: utf8 string + // - protocol: + // * Readonly + // * The protocol name of sender client + // * Value type: string enum with "mqtt", "mqtt-sn", ... + // - peerhost: + // * Readonly + // * The peerhost of sender client + // * Value type: ip address string + // - allow_publish: + // * Writable + // * Whether to allow the message to be published by emqx + // * Value type: string enum with "true", "false", default is "true" + // + // Notes: All header may be missing, which means that the message does not + // carry these headers. We can guarantee that clients coming from MQTT, + // MQTT-SN, CoAP, LwM2M and other natively supported protocol clients will + // carry these headers, but there is no guarantee that messages published + // by other means will do, e.g. messages published by HTTP-API + map headers = 8; } message Property { diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index 3529b6314..d1cc4d778 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -5,7 +5,7 @@ ]}. {deps, - [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.3"}}} + [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} ]}. {grpc, diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 6075d50db..2004230be 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.3.5"}, + {vsn, "4.4.1"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index 8245576ec..5a81af051 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,31 +1,19 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {"4.3.4", [ - {load_module, emqx_exhook_sup, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_handler, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook, brutal_purge, soft_purge, []}, - {update, emqx_exhook_mngr, {advanced, ["4.3.4"]}} - ]}, - {<<"4\\.3\\.[0-3]">>, [ - {restart_application, emqx_exhook} - ]}, - {<<".*">>, []} - ], - [ - {"4.3.4", [ - {load_module, emqx_exhook_sup, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_server, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_handler, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, - {load_module, emqx_exhook, brutal_purge, soft_purge, []}, - {update, emqx_exhook_mngr, {advanced, ["4.3.4"]}} - ]}, - {<<"4\\.3\\.[0-3]">>, [ - {restart_application, emqx_exhook} - ]}, - {<<".*">>, []} - ] -}. + [{"4.4.0", + [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, + {update, emqx_exhook_mngr, {advanced, ["4.4.0"]}}]}, + {<<".*">>,[]}], + [{"4.4.0", + [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, + {update, emqx_exhook_mngr, {advanced, ["4.4.0"]}}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index 1809d81ee..7e78e29bb 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -50,6 +50,7 @@ %% Utils -export([ message/1 + , headers/1 , stringfy/1 , merge_responsed_bool/2 , merge_responsed_message/2 @@ -62,6 +63,8 @@ , call_fold/3 ]). +-elvis([{elvis_style, god_modules, disable}]). + %%-------------------------------------------------------------------- %% Clients %%-------------------------------------------------------------------- @@ -258,17 +261,58 @@ clientinfo(ClientInfo = cn => maybe(maps:get(cn, ClientInfo, undefined)), dn => maybe(maps:get(dn, ClientInfo, undefined))}. -message(#message{id = Id, qos = Qos, from = From, topic = Topic, payload = Payload, timestamp = Ts}) -> +message(#message{id = Id, qos = Qos, from = From, topic = Topic, + payload = Payload, timestamp = Ts, headers = Headers}) -> #{node => stringfy(node()), id => emqx_guid:to_hexstr(Id), qos => Qos, from => stringfy(From), topic => Topic, payload => Payload, - timestamp => Ts}. + timestamp => Ts, + headers => headers(Headers) + }. -assign_to_message(#{qos := Qos, topic := Topic, payload := Payload}, Message) -> - Message#message{qos = Qos, topic = Topic, payload = Payload}. +headers(Headers) -> + Ls = [username, protocol, peerhost, allow_publish], + maps:fold( + fun + (_, undefined, Acc) -> + Acc; %% Ignore undefined value + (K, V, Acc) -> + case lists:member(K, Ls) of + true -> + Acc#{atom_to_binary(K) => bin(K, V)}; + _ -> + Acc + end + end, #{}, Headers). + +bin(K, V) when K == username; + K == protocol; + K == allow_publish -> + bin(V); +bin(peerhost, V) -> + bin(inet:ntoa(V)). + +bin(V) when is_binary(V) -> V; +bin(V) when is_atom(V) -> atom_to_binary(V); +bin(V) when is_list(V) -> iolist_to_binary(V). + +assign_to_message(InMessage = #{qos := Qos, topic := Topic, + payload := Payload}, Message) -> + NMsg = Message#message{qos = Qos, topic = Topic, payload = Payload}, + enrich_header(maps:get(headers, InMessage, #{}), NMsg). + +enrich_header(Headers, Message) -> + case maps:get(<<"allow_publish">>, Headers, undefined) of + <<"false">> -> + emqx_message:set_header(allow_publish, false, Message); + <<"true">> -> + emqx_message:set_header(allow_publish, true, Message); + _ -> + Message + end. topicfilters(Tfs) when is_list(Tfs) -> [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs]. @@ -301,11 +345,7 @@ merge_responsed_bool(_Req, #{type := 'IGNORE'}) -> ignore; merge_responsed_bool(Req, #{type := Type, value := {bool_result, NewBool}}) when is_boolean(NewBool) -> - NReq = Req#{result => NewBool}, - case Type of - 'CONTINUE' -> {ok, NReq}; - 'STOP_AND_RETURN' -> {stop, NReq} - end; + {ret(Type), Req#{result => NewBool}}; merge_responsed_bool(_Req, Resp) -> ?LOG(warning, "Unknown responsed value ~0p to merge to callback chain", [Resp]), ignore. @@ -313,11 +353,10 @@ merge_responsed_bool(_Req, Resp) -> merge_responsed_message(_Req, #{type := 'IGNORE'}) -> ignore; merge_responsed_message(Req, #{type := Type, value := {message, NMessage}}) -> - NReq = Req#{message => NMessage}, - case Type of - 'CONTINUE' -> {ok, NReq}; - 'STOP_AND_RETURN' -> {stop, NReq} - end; + {ret(Type), Req#{message => NMessage}}; merge_responsed_message(_Req, Resp) -> ?LOG(warning, "Unknown responsed value ~0p to merge to callback chain", [Resp]), ignore. + +ret('CONTINUE') -> ok; +ret('STOP_AND_RETURN') -> stop. diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl index bd8b88e0a..1a11fe0ff 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mngr.erl @@ -36,6 +36,8 @@ , server/1 , put_request_failed_action/1 , get_request_failed_action/0 + , put_pool_size/1 + , get_pool_size/0 ]). %% gen_server callbacks @@ -93,11 +95,11 @@ start_link(Servers, AutoReconnect, ReqOpts, HooksOpts) -> gen_server:start_link(?MODULE, [Servers, AutoReconnect, ReqOpts, HooksOpts], []). --spec enable(pid(), atom()|string()) -> ok | {error, term()}. +-spec enable(pid(), atom() | string()) -> ok | {error, term()}. enable(Pid, Name) -> call(Pid, {load, Name}). --spec disable(pid(), atom()|string()) -> ok | {error, term()}. +-spec disable(pid(), atom() | string()) -> ok | {error, term()}. disable(Pid, Name) -> call(Pid, {unload, Name}). @@ -126,6 +128,9 @@ init([Servers, AutoReconnect, ReqOpts0, HooksOpts]) -> put_request_failed_action( maps:get(request_failed_action, ReqOpts0, deny) ), + put_pool_size( + maps:get(pool_size, ReqOpts0, erlang:system_info(schedulers)) + ), %% Load the hook servers ReqOpts = maps:without([request_failed_action], ReqOpts0), @@ -144,6 +149,7 @@ init([Servers, AutoReconnect, ReqOpts0, HooksOpts]) -> %% @private load_all_servers(Servers, ReqOpts, HooksOpts) -> load_all_servers(Servers, ReqOpts, HooksOpts, #{}, #{}). + load_all_servers([], _Request, _HooksOpts, Waiting, Running) -> {Waiting, Running}; load_all_servers([{Name, Options} | More], ReqOpts, HooksOpts, Waiting, Running) -> @@ -212,7 +218,7 @@ terminate(_Reason, State = #state{running = Running}) -> %% in the emqx_exhook:v4.3.5, we have added one new field in the state last: %% - hooks_options :: map() code_change({down, _Vsn}, State, [ToVsn]) -> - case re:run(ToVsn, "4\\.3\\.[0-4]") of + case re:run(ToVsn, "4\\.4\\.0") of {match, _} -> NState = list_to_tuple( lists:droplast( @@ -222,7 +228,7 @@ code_change({down, _Vsn}, State, [ToVsn]) -> {ok, State} end; code_change(_Vsn, State, [FromVsn]) -> - case re:run(FromVsn, "4\\.3\\.[0-4]") of + case re:run(FromVsn, "4\\.4\\.0") of {match, _} -> NState = list_to_tuple( tuple_to_list(State) ++ [?DEFAULT_HOOK_OPTS]), @@ -316,6 +322,14 @@ put_request_failed_action(Val) -> get_request_failed_action() -> persistent_term:get({?APP, request_failed_action}). +put_pool_size(Val) -> + persistent_term:put({?APP, pool_size}, Val). + +get_pool_size() -> + %% Avoid the scenario that the parameter is not set after + %% the hot upgrade completed. + persistent_term:get({?APP, pool_size}, erlang:system_info(schedulers)). + save(Name, ServerState) -> Saved = persistent_term:get(?APP, []), persistent_term:put(?APP, lists:reverse([Name | Saved])), diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 9caa166ea..7c1d0ff69 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -77,6 +77,8 @@ -dialyzer({nowarn_function, [inc_metrics/2]}). +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + %%-------------------------------------------------------------------- %% Load/Unload APIs %%-------------------------------------------------------------------- @@ -126,13 +128,18 @@ channel_opts(Opts) -> SvrAddr = format_http_uri(Scheme, Host, Port), ClientOpts = case Scheme of https -> - SslOpts = lists:keydelete(ssl, 1, proplists:get_value(ssl_options, Opts, [])), + SslOpts = lists:keydelete( + ssl, + 1, + proplists:get_value(ssl_options, Opts, []) + ), #{gun_opts => #{transport => ssl, transport_opts => SslOpts}}; _ -> #{} end, - {SvrAddr, ClientOpts}. + NClientOpts = ClientOpts#{pool_size => emqx_exhook_mngr:get_pool_size()}, + {SvrAddr, NClientOpts}. format_http_uri(Scheme, Host0, Port) -> Host = case is_tuple(Host0) of @@ -256,7 +263,7 @@ call(Hookpoint, Req, #server{name = ChannName, options = ReqOpts, %% @private inc_metrics(IncFun, Name) when is_function(IncFun) -> %% BACKW: e4.2.0-e4.2.2 - {env, [Prefix|_]} = erlang:fun_info(IncFun, env), + {env, [Prefix | _]} = erlang:fun_info(IncFun, env), inc_metrics(Prefix, Name); inc_metrics(Prefix, Name) when is_list(Prefix) -> emqx_metrics:inc(list_to_atom(Prefix ++ atom_to_list(Name))). @@ -273,8 +280,8 @@ do_call(ChannName, Fun, Req, ReqOpts) -> Options = ReqOpts#{channel => ChannName}, ?LOG(debug, "Call ~0p:~0p(~0p, ~0p)", [?PB_CLIENT_MOD, Fun, NReq, Options]), case catch apply(?PB_CLIENT_MOD, Fun, [NReq, Options]) of - {ok, Resp, _Metadata} -> - ?LOG(debug, "Response {ok, ~0p, ~0p}", [Resp, _Metadata]), + {ok, Resp, Metadata} -> + ?LOG(debug, "Response {ok, ~0p, ~0p}", [Resp, Metadata]), {ok, Resp}; {error, {Code, Msg}, _Metadata} -> ?LOG(error, "CALL ~0p:~0p(~0p, ~0p) response errcode: ~0p, errmsg: ~0p", diff --git a/apps/emqx_exhook/src/emqx_exhook_sup.erl b/apps/emqx_exhook/src/emqx_exhook_sup.erl index 73508adb7..ddd18cd2c 100644 --- a/apps/emqx_exhook/src/emqx_exhook_sup.erl +++ b/apps/emqx_exhook/src/emqx_exhook_sup.erl @@ -56,7 +56,8 @@ auto_reconnect() -> request_options() -> #{timeout => env(request_timeout, 5000), - request_failed_action => env(request_failed_action, deny) + request_failed_action => env(request_failed_action, deny), + pool_size => env(pool_size, erlang:system_info(schedulers)) }. hooks_options() -> @@ -73,7 +74,7 @@ env(Key, Def) -> -spec start_grpc_client_channel( string(), uri_string:uri_string(), - grpc_client:options()) -> {ok, pid()} | {error, term()}. + grpc_client_sup:options()) -> {ok, pid()} | {error, term()}. start_grpc_client_channel(Name, SvrAddr, Options) -> grpc_client_sup:create_channel_pool(Name, SvrAddr, Options). diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index 636de0d86..047e76ce3 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -308,21 +308,31 @@ on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> %% some cases for testing case From of <<"baduser">> -> - NMsg = Msg#{qos => 0, + NMsg = deny(Msg#{qos => 0, topic => <<"">>, payload => <<"">> - }, + }), {ok, #{type => 'STOP_AND_RETURN', value => {message, NMsg}}, Md}; <<"gooduser">> -> - NMsg = Msg#{topic => From, - payload => From}, + NMsg = allow(Msg#{topic => From, + payload => From}), {ok, #{type => 'STOP_AND_RETURN', value => {message, NMsg}}, Md}; _ -> {ok, #{type => 'IGNORE'}, Md} end. +deny(Msg) -> + NHeader = maps:put(<<"allow_publish">>, <<"false">>, + maps:get(headers, Msg, #{})), + maps:put(headers, NHeader, Msg). + +allow(Msg) -> + NHeader = maps:put(<<"allow_publish">>, <<"true">>, + maps:get(headers, Msg, #{})), + maps:put(headers, NHeader, Msg). + -spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata()) -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} | {error, grpc_cowboy_h:error_response()}. diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index 763765408..3f094077e 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -320,19 +320,24 @@ prop_message_publish() -> _ -> ExpectedOutMsg = case emqx_message:from(Msg) of <<"baduser">> -> - MsgMap = emqx_message:to_map(Msg), + MsgMap = #{headers := Headers} + = emqx_message:to_map(Msg), emqx_message:from_map( MsgMap#{qos => 0, topic => <<"">>, - payload => <<"">> + payload => <<"">>, + headers => maps:put(allow_publish, false, Headers) }); <<"gooduser">> = From -> - MsgMap = emqx_message:to_map(Msg), + MsgMap = #{headers := Headers} + = emqx_message:to_map(Msg), emqx_message:from_map( MsgMap#{topic => From, - payload => From + payload => From, + headers => maps:put(allow_publish, true, Headers) }); - _ -> Msg + _ -> + Msg end, ?assertEqual(ExpectedOutMsg, OutMsg), @@ -494,7 +499,9 @@ from_message(Msg) -> from => stringfy(emqx_message:from(Msg)), topic => emqx_message:topic(Msg), payload => emqx_message:payload(Msg), - timestamp => emqx_message:timestamp(Msg) + timestamp => emqx_message:timestamp(Msg), + headers => emqx_exhook_handler:headers( + emqx_message:get_headers(Msg)) }. %%-------------------------------------------------------------------- diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index 4ad1aa192..da868de82 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -13,7 +13,7 @@ ]}. {deps, - [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.3"}}} + [{grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.4"}}} ]}. {grpc, diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config index f190fa55e..8bb15de3c 100644 --- a/apps/emqx_lwm2m/rebar.config +++ b/apps/emqx_lwm2m/rebar.config @@ -1,10 +1,10 @@ {deps, - [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v1.1.5"}}} + [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v2.0.1"}}} ]}. {profiles, [{test, - [{deps, [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}, + [{deps, [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.4"}}}, {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.0"}}} ]} diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index fc4b0d825..e2bd9e934 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -33,4 +33,4 @@ -define(ERROR15, 115). %% bad topic -define(ERROR16, 116). %% bad QoS --define(VERSIONS, ["4.0", "4.1", "4.2", "4.3"]). \ No newline at end of file +-define(VERSIONS, ["4.0", "4.1", "4.2", "4.3", "4.4"]). diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index e203c7a56..fd941f8b8 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,9 +1,9 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.13"}, % strict semver, bump manually! + {vsn, "4.4.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, - {applications, [kernel,stdlib,minirest]}, + {applications, [kernel,stdlib,emqx_plugin_libs,minirest]}, {mod, {emqx_mgmt_app,[]}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src index 9265913e6..5121efb88 100644 --- a/apps/emqx_management/src/emqx_management.appup.src +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -1,17 +1,11 @@ %% -*- mode: erlang -*- {VSN, - [ {<<"4\\.3\\.([0-9]|1[0-2])">>, - [ {apply,{minirest,stop_http,['http:management']}}, - {apply,{minirest,stop_http,['https:management']}}, - {restart_application, emqx_management} - ]}, - {<<".*">>, []} - ], - [ {<<"4\\.3\\.([0-9]|1[0-2])">>, - [ {apply,{minirest,stop_http,['http:management']}}, - {apply,{minirest,stop_http,['https:management']}}, - {restart_application, emqx_management} - ]}, - {<<".*">>, []} - ] + [{<<".*">>, + [{apply,{minirest,stop_http,['http:management']}}, + {apply,{minirest,stop_http,['https:management']}}, + {restart_application, emqx_management}]}], + [{<<".*">>, + [{apply,{minirest,stop_http,['http:management']}}, + {apply,{minirest,stop_http,['https:management']}}, + {restart_application, emqx_management}]}] }. diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 51bc193d8..e74b1ea0a 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -22,6 +22,9 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_mgmt]}}]). +-elvis([{elvis_style, god_modules, #{ignore => [emqx_mgmt]}}]). + %% Nodes and Brokers API -export([ list_nodes/0 , lookup_node/1 @@ -49,6 +52,7 @@ , clean_acl_cache_all/1 , set_ratelimit_policy/2 , set_quota_policy/2 + , set_keepalive/2 ]). -export([ clean_pem_cache/0 @@ -145,9 +149,8 @@ node_info(Node) when Node =:= node() -> memory_used => proplists:get_value(used, Memory), process_available => erlang:system_info(process_limit), process_used => erlang:system_info(process_count), - max_fds => - proplists:get_value( max_fds - , lists:usort(lists:flatten(erlang:system_info(check_io)))), + max_fds => proplists:get_value(max_fds, + lists:usort(lists:flatten(erlang:system_info(check_io)))), connections => ets:info(emqx_channel, size), node_status => 'Running', uptime => iolist_to_binary(proplists:get_value(uptime, BrokerInfo)), @@ -202,11 +205,11 @@ get_stats(Node) -> lookup_client({clientid, ClientId}, FormatFun) -> lists:append([lookup_client(Node, {clientid, ClientId}, FormatFun) - || Node <- ekka_mnesia:running_nodes()]); + || Node <- ekka_mnesia:running_nodes()]); lookup_client({username, Username}, FormatFun) -> lists:append([lookup_client(Node, {username, Username}, FormatFun) - || Node <- ekka_mnesia:running_nodes()]). + || Node <- ekka_mnesia:running_nodes()]). lookup_client(Node, {clientid, ClientId}, {M,F}) when Node =:= node() -> lists:append(lists:map( @@ -229,7 +232,7 @@ lookup_client(Node, {username, Username}, FormatFun) -> kickout_client(ClientId) -> Results = [kickout_client(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], - check_every_ok(Results). + has_any_ok(Results). kickout_client(Node, ClientId) when Node =:= node() -> emqx_cm:kick_session(ClientId); @@ -242,7 +245,7 @@ list_acl_cache(ClientId) -> clean_acl_cache(ClientId) -> Results = [clean_acl_cache(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], - check_every_ok(Results). + has_any_ok(Results). clean_acl_cache(Node, ClientId) when Node =:= node() -> case emqx_cm:lookup_channels(ClientId) of @@ -276,6 +279,11 @@ set_ratelimit_policy(ClientId, Policy) -> set_quota_policy(ClientId, Policy) -> call_client(ClientId, {quota, Policy}). +set_keepalive(ClientId, Interval)when Interval >= 0 andalso Interval =< 65535 -> + call_client(ClientId, {keepalive, Interval}); +set_keepalive(_ClientId, _Interval) -> + {error, ?ERROR2, <<"mqtt3.1.1 specification: keepalive must between 0~65535">>}. + clean_pem_cache() -> for_nodes(fun clean_pem_cache/1). @@ -326,7 +334,8 @@ list_subscriptions(Node) -> list_subscriptions_via_topic(Topic, FormatFun) -> lists:append([list_subscriptions_via_topic(Node, Topic, FormatFun) - || Node <- ekka_mnesia:running_nodes()]). + || Node <- ekka_mnesia:running_nodes()]). + list_subscriptions_via_topic(Node, Topic, {M,F}) when Node =:= node() -> MatchSpec = [{{{'_', '$1'}, '_'}, [{'=:=','$1', Topic}], ['$_']}], @@ -451,8 +460,8 @@ list_listeners(Node) when Node =:= node() -> Http = lists:map(fun({Protocol, Opts}) -> #{protocol => Protocol, listen_on => proplists:get_value(port, Opts), - acceptors => maps:get( num_acceptors - , proplists:get_value(transport_options, Opts, #{}), 0), + acceptors => maps:get(num_acceptors, + proplists:get_value(transport_options, Opts, #{}), 0), max_conns => proplists:get_value(max_connections, Opts), current_conns => proplists:get_value(all_connections, Opts), shutdown_count => []} @@ -501,10 +510,8 @@ add_duration_field([], _Now, Acc) -> Acc; add_duration_field([Alarm = #{activated := true, activate_at := ActivateAt} | Rest], Now, Acc) -> add_duration_field(Rest, Now, [Alarm#{duration => Now - ActivateAt} | Acc]); -add_duration_field([Alarm = #{ activated := false - , activate_at := ActivateAt - , deactivate_at := DeactivateAt} - | Rest], Now, Acc) -> +add_duration_field([Alarm = #{activated := false, + activate_at := ActivateAt, deactivate_at := DeactivateAt} | Rest], Now, Acc) -> add_duration_field(Rest, Now, [Alarm#{duration => DeactivateAt - ActivateAt} | Acc]). %%-------------------------------------------------------------------- @@ -585,13 +592,13 @@ check_row_limit([Tab | Tables], Limit) -> false -> check_row_limit(Tables, Limit) end. -check_every_ok(Results) -> - case lists:any(fun(Item) -> Item =:= ok end, Results) of - true -> ok; - false -> lists:last(Results) - end. - max_row_limit() -> application:get_env(?APP, max_row_limit, ?MAX_ROW_LIMIT). table_size(Tab) -> ets:info(Tab, size). + +has_any_ok(Results) -> + case lists:any(fun(Item) -> Item =:= ok end, Results) of + true -> ok; + false -> lists:last(Results) + end. diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index c29f6bfdf..3459c2b25 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -54,17 +54,46 @@ paginate(Tables, Params, RowFun) -> query_handle(Table) when is_atom(Table) -> qlc:q([R|| R <- ets:table(Table)]); + +query_handle({Table, Opts}) when is_atom(Table) -> + qlc:q([R|| R <- ets:table(Table, Opts)]); + query_handle([Table]) when is_atom(Table) -> qlc:q([R|| R <- ets:table(Table)]); + +query_handle([{Table, Opts}]) when is_atom(Table) -> + qlc:q([R|| R <- ets:table(Table, Opts)]); + query_handle(Tables) -> - qlc:append([qlc:q([E || E <- ets:table(T)]) || T <- Tables]). + Fold = fun({Table, Opts}, Acc) -> + Handle = qlc:q([R|| R <- ets:table(Table, Opts)]), + [Handle | Acc]; + (Table, Acc) -> + Handle = qlc:q([R|| R <- ets:table(Table)]), + [Handle | Acc] + end, + Handles = lists:foldl(Fold, [], Tables), + qlc:append(lists:reverse(Handles)). count(Table) when is_atom(Table) -> ets:info(Table, size); + +count({Table, _Opts}) when is_atom(Table) -> + ets:info(Table, size); + count([Table]) when is_atom(Table) -> ets:info(Table, size); + +count([{Table, _Opts}]) when is_atom(Table) -> + ets:info(Table, size); + count(Tables) -> - lists:sum([count(T) || T <- Tables]). + Fold = fun({Table, _Opts}, Acc) -> + count(Table) ++ Acc; + (Table, Acc) -> + count(Table) ++ Acc + end, + lists:foldl(Fold, 0, Tables). count(Table, Nodes) -> lists:sum([rpc_call(Node, ets, info, [Table, size], 5000) || Node <- Nodes]). diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 91ebd55c5..e0e5af39d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -121,6 +121,12 @@ func => clean_quota, descr => "Clear the quota policy"}). +-rest_api(#{name => set_keepalive, + method => 'PUT', + path => "/clients/:bin:clientid/keepalive", + func => set_keepalive, + descr => "Set the client keepalive"}). + -import(emqx_mgmt_util, [ ntoa/1 , strftime/1 ]). @@ -134,23 +140,24 @@ , set_quota_policy/2 , clean_ratelimit/2 , clean_quota/2 + , set_keepalive/2 ]). -export([ query/3 , format_channel_info/1 ]). --define(query_fun, {?MODULE, query}). --define(format_fun, {?MODULE, format_channel_info}). +-define(QUERY_FUN, {?MODULE, query}). +-define(FORMAT_FUN, {?MODULE, format_channel_info}). list(Bindings, Params) when map_size(Bindings) == 0 -> fence(fun() -> - emqx_mgmt_api:cluster_query(Params, ?CLIENT_QS_SCHEMA, ?query_fun) + emqx_mgmt_api:cluster_query(Params, ?CLIENT_QS_SCHEMA, ?QUERY_FUN) end); list(#{node := Node}, Params) when Node =:= node() -> fence(fun() -> - emqx_mgmt_api:node_query(Node, Params, ?CLIENT_QS_SCHEMA, ?query_fun) + emqx_mgmt_api:node_query(Node, Params, ?CLIENT_QS_SCHEMA, ?QUERY_FUN) end); list(Bindings = #{node := Node}, Params) -> @@ -173,16 +180,20 @@ fence(Func) -> end. lookup(#{node := Node, clientid := ClientId}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client(Node, {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client(Node, + {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?FORMAT_FUN)}); lookup(#{clientid := ClientId}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client({clientid, emqx_mgmt_util:urldecode(ClientId)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client( + {clientid, emqx_mgmt_util:urldecode(ClientId)}, ?FORMAT_FUN)}); lookup(#{node := Node, username := Username}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client(Node, {username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}); + minirest:return({ok, emqx_mgmt:lookup_client(Node, + {username, emqx_mgmt_util:urldecode(Username)}, ?FORMAT_FUN)}); lookup(#{username := Username}, _Params) -> - minirest:return({ok, emqx_mgmt:lookup_client({username, emqx_mgmt_util:urldecode(Username)}, ?format_fun)}). + minirest:return({ok, emqx_mgmt:lookup_client({username, + emqx_mgmt_util:urldecode(Username)}, ?FORMAT_FUN)}). kickout(#{clientid := ClientId}, _Params) -> case emqx_mgmt:kickout_client(emqx_mgmt_util:urldecode(ClientId)) of @@ -208,7 +219,7 @@ list_acl_cache(#{clientid := ClientId}, _Params) -> set_ratelimit_policy(#{clientid := ClientId}, Params) -> P = [{conn_bytes_in, proplists:get_value(<<"conn_bytes_in">>, Params)}, {conn_messages_in, proplists:get_value(<<"conn_messages_in">>, Params)}], - case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of + case filter_ratelimit_params(P) of [] -> minirest:return(); Policy -> case emqx_mgmt:set_ratelimit_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of @@ -227,7 +238,7 @@ clean_ratelimit(#{clientid := ClientId}, _Params) -> set_quota_policy(#{clientid := ClientId}, Params) -> P = [{conn_messages_routing, proplists:get_value(<<"conn_messages_routing">>, Params)}], - case [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined] of + case filter_ratelimit_params(P) of [] -> minirest:return(); Policy -> case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), Policy) of @@ -237,6 +248,7 @@ set_quota_policy(#{clientid := ClientId}, Params) -> end end. + clean_quota(#{clientid := ClientId}, _Params) -> case emqx_mgmt:set_quota_policy(emqx_mgmt_util:urldecode(ClientId), []) of ok -> minirest:return(); @@ -244,6 +256,23 @@ clean_quota(#{clientid := ClientId}, _Params) -> {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) end. +set_keepalive(#{clientid := ClientId}, Params) -> + case proplists:get_value(<<"interval">>, Params) of + undefined -> + minirest:return({error, ?ERROR7, params_not_found}); + Interval0 -> + Interval = to_integer(Interval0), + case emqx_mgmt:set_keepalive(emqx_mgmt_util:urldecode(ClientId), Interval) of + ok -> minirest:return(); + {error, not_found} -> minirest:return({error, ?ERROR12, not_found}); + {error, Code, Reason} -> minirest:return({error, Code, Reason}); + {error, Reason} -> minirest:return({error, ?ERROR1, Reason}) + end + end. + +to_integer(Int)when is_integer(Int) -> Int; +to_integer(Bin) when is_binary(Bin) -> binary_to_integer(Bin). + %% @private %% S = 100,1s %% | 100KB, 1m @@ -270,7 +299,7 @@ format_channel_info({_Key, Info, Stats0}) -> ConnInfo = maps:get(conninfo, Info, #{}), Session = case maps:get(session, Info, #{}) of undefined -> #{}; - _Sess -> _Sess + Sess -> Sess end, SessCreated = maps:get(created_at, Session, maps:get(connected_at, ConnInfo)), Connected = case maps:get(conn_state, Info, connected) of @@ -292,8 +321,14 @@ format_channel_info({_Key, Info, Stats0}) -> inflight, max_inflight, awaiting_rel, max_awaiting_rel, mqueue_len, mqueue_dropped, max_mqueue, heap_size, reductions, mailbox_len, - recv_cnt, recv_msg, recv_oct, recv_pkt, send_cnt, - send_msg, send_oct, send_pkt], NStats), + recv_cnt, + recv_msg, 'recv_msg.qos0', 'recv_msg.qos1', 'recv_msg.qos2', + 'recv_msg.dropped', 'recv_msg.dropped.expired', + recv_oct, recv_pkt, send_cnt, + send_msg, 'send_msg.qos0', 'send_msg.qos1', 'send_msg.qos2', + 'send_msg.dropped', 'send_msg.dropped.expired', + 'send_msg.dropped.queue_full', 'send_msg.dropped.too_large', + send_oct, send_pkt], NStats), maps:with([clientid, username, mountpoint, is_bridge, zone], ClientInfo), maps:with([clean_start, keepalive, expiry_interval, proto_name, proto_ver, peername, connected_at, disconnected_at], ConnInfo), @@ -311,7 +346,8 @@ format(Data) when is_map(Data)-> created_at => iolist_to_binary(strftime(CreatedAt div 1000))}, case maps:get(disconnected_at, Data, undefined) of undefined -> #{}; - DisconnectedAt -> #{disconnected_at => iolist_to_binary(strftime(DisconnectedAt div 1000))} + DisconnectedAt -> #{disconnected_at => + iolist_to_binary(strftime(DisconnectedAt div 1000))} end). format_acl_cache({{PubSub, Topic}, {AclResult, Timestamp}}) -> @@ -326,7 +362,7 @@ format_acl_cache({{PubSub, Topic}, {AclResult, Timestamp}}) -> query({Qs, Fuzzy}, Start, Limit) -> case qs2ms(Qs) of - {Ms, []}when Fuzzy =:= [] -> + {Ms, []} when Fuzzy =:= [] -> emqx_mgmt_api:select_table(emqx_channel_info, Ms, Start, Limit, fun format_channel_info/1); {Ms, FuzzyStats} -> MatchFun = match_fun(Ms, Fuzzy ++ FuzzyStats), @@ -350,7 +386,7 @@ match_fun(Ms, Fuzzy) -> run_fuzzy_match(_, []) -> true; -run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr}|Fuzzy]) -> +run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr} | Fuzzy]) -> Val = case maps:get(Key, ClientInfo, undefined) of undefined -> <<>>; V -> V @@ -429,6 +465,9 @@ ms(mqueue_len, _X) -> ms(mqueue_dropped, _X) -> fuzzy_stats. +filter_ratelimit_params(P) -> + [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined]. + %%-------------------------------------------------------------------- %% EUnits %%-------------------------------------------------------------------- diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index c24f4793b..2c44fda53 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -73,39 +73,17 @@ export(_Bindings, _Params) -> case emqx_mgmt_data_backup:export() of {ok, File = #{filename := Filename}} -> - minirest:return({ok, File#{filename => filename:basename(Filename)}}); + minirest:return({ok, File#{filename => list_to_binary(filename:basename(Filename))}}); Return -> minirest:return(Return) end. list_exported(_Bindings, _Params) -> - List = [ rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes() ], + List = [rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes()], NList = lists:map(fun({_, FileInfo}) -> FileInfo end, lists:keysort(1, lists:append(List))), minirest:return({ok, NList}). get_list_exported() -> - Dir = emqx:get_env(data_dir), - {ok, Files} = file:list_dir_all(Dir), - lists:foldl( - fun(File, Acc) -> - case filename:extension(File) =:= ".json" of - true -> - FullFile = filename:join([Dir, File]), - case file:read_file_info(FullFile) of - {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} -> - CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), - Seconds = calendar:datetime_to_gregorian_seconds(CTime), - [{Seconds, [{filename, list_to_binary(File)}, - {size, Size}, - {created_at, list_to_binary(CreatedAt)}, - {node, node()} - ]} | Acc]; - {error, Reason} -> - logger:error("Read file info of ~s failed with: ~p", [File, Reason]), - Acc - end; - false -> Acc - end - end, [], Files). + emqx_mgmt_data_backup:list_backup_file(). import(_Bindings, Params) -> case proplists:get_value(<<"filename">>, Params) of @@ -121,22 +99,27 @@ import(_Bindings, Params) -> case lists:member(Node, [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ] ) of - true -> minirest:return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename])); + true -> + N = erlang:binary_to_atom(Node, utf8), + case rpc:call(N, ?MODULE, do_import, [Filename]) of + {badrpc, Reason} -> + minirest:return({error, Reason}); + Res -> + minirest:return(Res) + end; false -> minirest:return({error, no_existent_node}) end end end. do_import(Filename) -> - FullFilename = fullname(Filename), - emqx_mgmt_data_backup:import(FullFilename, "{}"). + emqx_mgmt_data_backup:import(Filename, "{}"). -download(#{filename := Filename}, _Params) -> - FullFilename = fullname(Filename), - case file:read_file(FullFilename) of - {ok, Bin} -> - {ok, #{filename => list_to_binary(Filename), - file => Bin}}; +download(#{filename := Filename0}, _Params) -> + Filename = filename_decode(Filename0), + case emqx_mgmt_data_backup:read_backup_file(Filename) of + {ok, Res} -> + {ok, Res}; {error, Reason} -> minirest:return({error, Reason}) end. @@ -146,8 +129,7 @@ upload(Bindings, Params) -> do_upload(_Bindings, #{<<"filename">> := Filename, <<"file">> := Bin}) -> - FullFilename = fullname(Filename), - case file:write_file(FullFilename, Bin) of + case emqx_mgmt_data_backup:upload_backup_file(Filename, Bin) of ok -> minirest:return({ok, [{node, node()}]}); {error, Reason} -> @@ -158,9 +140,9 @@ do_upload(Bindings, Params = #{<<"file">> := _}) -> do_upload(_Bindings, _Params) -> minirest:return({error, missing_required_params}). -delete(#{filename := Filename}, _Params) -> - FullFilename = fullname(Filename), - case file:delete(FullFilename) of +delete(#{filename := Filename0}, _Params) -> + Filename = filename_decode(Filename0), + case emqx_mgmt_data_backup:delete_backup_file(Filename) of ok -> minirest:return(); {error, Reason} -> @@ -168,20 +150,19 @@ delete(#{filename := Filename}, _Params) -> end. import_content(Content) -> - File = dump_to_tmp_file(Content), - do_import(File). - -dump_to_tmp_file(Content) -> Bin = emqx_json:encode(Content), Filename = tmp_filename(), - ok = file:write_file(fullname(Filename), Bin), - Filename. - -fullname(Name0) -> - Name = uri_string:percent_decode(Name0), - filename:join(emqx:get_env(data_dir), Name). + case emqx_mgmt_data_backup:upload_backup_file(Filename, Bin) of + ok -> + do_import(Filename); + {error, Reason} -> + {error, Reason} + end. tmp_filename() -> Seconds = erlang:system_time(second), {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), - io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]). + list_to_binary(io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S])). + +filename_decode(Filename) -> + uri_string:percent_decode(Filename). diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index 3e5ffbee5..3fe859811 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -71,19 +71,10 @@ subscribe(_Bindings, Params) -> publish(_Bindings, Params) -> logger:debug("API publish Params:~p", [Params]), - {ClientId, Topic, Qos, Retain, Payload} = parse_publish_params(Params), - case do_publish(ClientId, Topic, Qos, Retain, Payload) of - {ok, MsgIds} -> - case proplists:get_value(<<"return">>, Params, undefined) of - undefined -> minirest:return(ok); - _Val -> - case proplists:get_value(<<"topics">>, Params, undefined) of - undefined -> minirest:return({ok, #{msgid => lists:last(MsgIds)}}); - _ -> minirest:return({ok, #{msgids => MsgIds}}) - end - end; - Result -> - minirest:return(Result) + try parse_publish_params(Params) of + Result -> do_publish(Params, Result) + catch + _E : _R -> minirest:return({ok, ?ERROR8, bad_params}) end. unsubscribe(_Bindings, Params) -> @@ -114,7 +105,8 @@ loop_subscribe([Params | ParamsN], Acc) -> {_, Code0, _Reason} -> Code0 end, Result = #{clientid => ClientId, - topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + topic => resp_topic(proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_subscribe(ParamsN, [Result | Acc]). @@ -123,13 +115,19 @@ loop_publish(Params) -> loop_publish([], Result) -> lists:reverse(Result); loop_publish([Params | ParamsN], Acc) -> - {ClientId, Topic, Qos, Retain, Payload} = parse_publish_params(Params), - Code = case do_publish(ClientId, Topic, Qos, Retain, Payload) of - {ok, _} -> 0; - {_, Code0, _} -> Code0 - end, - Result = #{topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), - code => Code}, + Result = + try parse_publish_params(Params) of + Res -> + Code = case do_publish(Params, Res) of + {ok, _} -> 0; + {_, Code0, _} -> Code0 + end, + #{topic => resp_topic(proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), + code => Code} + catch + _E : _R -> #{code => ?ERROR8, message => <<"bad_params">>} + end, loop_publish(ParamsN, [Result | Acc]). loop_unsubscribe(Params) -> @@ -143,7 +141,8 @@ loop_unsubscribe([Params | ParamsN], Acc) -> {_, Code0, _} -> Code0 end, Result = #{clientid => ClientId, - topic => resp_topic(proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + topic => resp_topic(proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), code => Code}, loop_unsubscribe(ParamsN, [Result | Acc]). @@ -160,14 +159,32 @@ do_subscribe(ClientId, Topics, QoS) -> _ -> ok end. -do_publish(ClientId, _Topics, _Qos, _Retain, _Payload) when not (is_binary(ClientId) or (ClientId =:= undefined)) -> +do_publish(Params, {ClientId, Topic, Qos, Retain, Payload, UserProps}) -> + case do_publish(ClientId, Topic, Qos, Retain, Payload, UserProps) of + {ok, MsgIds} -> + case proplists:get_value(<<"return">>, Params, undefined) of + undefined -> minirest:return(ok); + _Val -> + case proplists:get_value(<<"topics">>, Params, undefined) of + undefined -> minirest:return({ok, #{msgid => lists:last(MsgIds)}}); + _ -> minirest:return({ok, #{msgids => MsgIds}}) + end + end; + Result -> + minirest:return(Result) + end. + +do_publish(ClientId, _Topics, _Qos, _Retain, _Payload, _UserProps) + when not (is_binary(ClientId) or (ClientId =:= undefined)) -> {ok, ?ERROR8, <<"bad clientid: must be string">>}; -do_publish(_ClientId, [], _Qos, _Retain, _Payload) -> +do_publish(_ClientId, [], _Qos, _Retain, _Payload, _UserProps) -> {ok, ?ERROR15, bad_topic}; -do_publish(ClientId, Topics, Qos, Retain, Payload) -> +do_publish(ClientId, Topics, Qos, Retain, Payload, UserProps) -> MsgIds = lists:map(fun(Topic) -> Msg = emqx_message:make(ClientId, Qos, Topic, Payload), - _ = emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}}), + UserProps1 = #{'User-Property' => UserProps}, + _ = emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}, + headers = #{properties => UserProps1}}), emqx_guid:to_hexstr(Msg#message.id) end, Topics), {ok, MsgIds}. @@ -187,19 +204,22 @@ do_unsubscribe(ClientId, Topic) -> parse_subscribe_params(Params) -> ClientId = proplists:get_value(<<"clientid">>, Params), - Topics = topics(filter, proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), + Topics = topics(filter, proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), QoS = proplists:get_value(<<"qos">>, Params, 0), {ClientId, Topics, QoS}. parse_publish_params(Params) -> - Topics = topics(name, proplists:get_value(<<"topic">>, Params), proplists:get_value(<<"topics">>, Params, <<"">>)), - ClientId = proplists:get_value(<<"clientid">>, Params), - Payload = decode_payload(proplists:get_value(<<"payload">>, Params, <<>>), - proplists:get_value(<<"encoding">>, Params, <<"plain">>)), - Qos = proplists:get_value(<<"qos">>, Params, 0), - Retain = proplists:get_value(<<"retain">>, Params, false), - Payload1 = maybe_maps_to_binary(Payload), - {ClientId, Topics, Qos, Retain, Payload1}. + Topics = topics(name, proplists:get_value(<<"topic">>, Params), + proplists:get_value(<<"topics">>, Params, <<"">>)), + ClientId = proplists:get_value(<<"clientid">>, Params), + Payload = decode_payload(proplists:get_value(<<"payload">>, Params, <<>>), + proplists:get_value(<<"encoding">>, Params, <<"plain">>)), + Qos = proplists:get_value(<<"qos">>, Params, 0), + Retain = proplists:get_value(<<"retain">>, Params, false), + Payload1 = maybe_maps_to_binary(Payload), + UserProps = generate_user_props(proplists:get_value(<<"user_properties">>, Params, [])), + {ClientId, Topics, Qos, Retain, Payload1, UserProps}. parse_unsubscribe_params(Params) -> ClientId = proplists:get_value(<<"clientid">>, Params), @@ -253,3 +273,24 @@ maybe_maps_to_binary(Payload) -> _C : _E : S -> error({encode_payload_fail, S}) end. + +generate_user_props(UserProps) when is_list(UserProps)-> + generate_user_props_(UserProps, []); +generate_user_props(UserProps) -> + error({user_properties_type_error, UserProps}). + +generate_user_props_([{Name, Value} | Rest], Acc) -> + generate_user_props_(Rest, [{bin(Name), bin(Value)} | Acc]); +generate_user_props_([], Acc) -> + lists:reverse(Acc). + +bin(Bin) when is_binary(Bin) -> Bin; +bin(Num) when is_number(Num) -> number_to_binary(Num); +bin(Boolean) when is_boolean(Boolean) -> atom_to_binary(Boolean); +bin(Other) -> error({user_properties_type_error, Other}). + +-define(FLOAT_PRECISION, 17). +number_to_binary(Int) when is_integer(Int) -> + integer_to_binary(Int); +number_to_binary(Float) when is_float(Float) -> + float_to_binary(Float, [{decimals, ?FLOAT_PRECISION}, compact]). diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 954a378b3..51eb5099e 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -21,7 +21,9 @@ -include("emqx_mgmt.hrl"). --define(PRINT_CMD(Cmd, Descr), io:format("~-48s# ~s~n", [Cmd, Descr])). +-elvis([{elvis_style, invalid_dynamic_call, disable}]). + +-define(PRINT_CMD(Cmd, Desc), io:format("~-48s# ~s~n", [Cmd, Desc])). -export([load/0]). @@ -36,6 +38,7 @@ , vm/1 , mnesia/1 , trace/1 + , traces/1 , log/1 , mgmt/1 , data/1 @@ -75,11 +78,8 @@ mgmt(["insert", AppId, Name]) -> mgmt(["lookup", AppId]) -> case emqx_mgmt_auth:lookup_app(list_to_binary(AppId)) of - {AppId1, AppSecret, Name, Desc, Status, Expired} -> - emqx_ctl:print("app_id: ~s~nsecret: ~s~nname: ~s~ndesc: ~s~nstatus: ~s~nexpired: ~p~n", - [AppId1, AppSecret, Name, Desc, Status, Expired]); - undefined -> - emqx_ctl:print("Not Found.~n") + undefined -> emqx_ctl:print("Not Found.~n"); + App -> print_app_info(App) end; mgmt(["update", AppId, Status]) -> @@ -100,10 +100,7 @@ mgmt(["delete", AppId]) -> end; mgmt(["list"]) -> - lists:foreach(fun({AppId, AppSecret, Name, Desc, Status, Expired}) -> - emqx_ctl:print("app_id: ~s, secret: ~s, name: ~s, desc: ~s, status: ~s, expired: ~p~n", - [AppId, AppSecret, Name, Desc, Status, Expired]) - end, emqx_mgmt_auth:list_apps()); + lists:foreach(fun print_app_info/1, emqx_mgmt_auth:list_apps()); mgmt(_) -> emqx_ctl:usage([{"mgmt list", "List Applications"}, @@ -129,10 +126,12 @@ broker([]) -> [emqx_ctl:print("~-10s: ~s~n", [Fun, emqx_sys:Fun()]) || Fun <- Funs]; broker(["stats"]) -> - [emqx_ctl:print("~-30s: ~w~n", [Stat, Val]) || {Stat, Val} <- lists:sort(emqx_stats:getstats())]; + [emqx_ctl:print("~-30s: ~w~n", [Stat, Val]) || + {Stat, Val} <- lists:sort(emqx_stats:getstats())]; broker(["metrics"]) -> - [emqx_ctl:print("~-30s: ~w~n", [Metric, Val]) || {Metric, Val} <- lists:sort(emqx_metrics:all())]; + [emqx_ctl:print("~-30s: ~w~n", [Metric, Val]) || + {Metric, Val} <- lists:sort(emqx_metrics:all())]; broker(_) -> emqx_ctl:usage([{"broker", "Show broker version, uptime and description"}, @@ -257,10 +256,12 @@ subscriptions(["del", ClientId, Topic]) -> end; subscriptions(_) -> - emqx_ctl:usage([{"subscriptions list", "List all subscriptions"}, - {"subscriptions show ", "Show subscriptions of a client"}, - {"subscriptions add ", "Add a static subscription manually"}, - {"subscriptions del ", "Delete a static subscription manually"}]). + emqx_ctl:usage([{"subscriptions list", "List all subscriptions"}, + {"subscriptions show ", "Show subscriptions of a client"}, + {"subscriptions add ", + "Add a static subscription manually"}, + {"subscriptions del ", + "Delete a static subscription manually"}]). if_valid_qos(QoS, Fun) -> try list_to_integer(QoS) of @@ -329,14 +330,20 @@ vm(["memory"]) -> [emqx_ctl:print("memory/~-17s: ~w~n", [Cat, Val]) || {Cat, Val} <- erlang:memory()]; vm(["process"]) -> - [emqx_ctl:print("process/~-16s: ~w~n", [Name, erlang:system_info(Key)]) || {Name, Key} <- [{limit, process_limit}, {count, process_count}]]; + [emqx_ctl:print("process/~-16s: ~w~n", + [Name, erlang:system_info(Key)]) || + {Name, Key} <- [{limit, process_limit}, {count, process_count}]]; vm(["io"]) -> IoInfo = lists:usort(lists:flatten(erlang:system_info(check_io))), - [emqx_ctl:print("io/~-21s: ~w~n", [Key, proplists:get_value(Key, IoInfo)]) || Key <- [max_fds, active_fds]]; + [emqx_ctl:print("io/~-21s: ~w~n", + [Key, proplists:get_value(Key, IoInfo)]) || + Key <- [max_fds, active_fds]]; vm(["ports"]) -> - [emqx_ctl:print("ports/~-16s: ~w~n", [Name, erlang:system_info(Key)]) || {Name, Key} <- [{count, port_count}, {limit, port_limit}]]; + [emqx_ctl:print("ports/~-16s: ~w~n", + [Name, erlang:system_info(Key)]) || + {Name, Key} <- [{count, port_count}, {limit, port_limit}]]; vm(_) -> emqx_ctl:usage([{"vm all", "Show info of Erlang VM"}, @@ -373,8 +380,9 @@ log(["primary-level", Level]) -> emqx_ctl:print("~s~n", [emqx_logger:get_primary_log_level()]); log(["handlers", "list"]) -> - _ = [emqx_ctl:print("LogHandler(id=~s, level=~s, destination=~s, status=~s)~n", [Id, Level, Dst, Status]) - || #{id := Id, level := Level, dst := Dst, status := Status} <- emqx_logger:get_log_handlers()], + _ = [emqx_ctl:print("LogHandler(id=~s, level=~s, destination=~s, status=~s)~n", + [Id, Level, Dst, Status]) || #{id := Id, level := Level, dst := Dst, status := Status} + <- emqx_logger:get_log_handlers()], ok; log(["handlers", "start", HandlerId]) -> @@ -407,43 +415,51 @@ log(_) -> {"log handlers list", "Show log handlers"}, {"log handlers start ", "Start a log handler"}, {"log handlers stop ", "Stop a log handler"}, - {"log handlers set-level ", "Set log level of a log handler"}]). + {"log handlers set-level ", + "Set log level of a log handler"}]). %%-------------------------------------------------------------------- %% @doc Trace Command trace(["list"]) -> - lists:foreach(fun({{Who, Name}, {Level, LogFile}}) -> - emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Who, Name, Level, LogFile]) - end, emqx_tracer:lookup_traces()); + lists:foreach(fun(Trace) -> + #{type := Type, filter := Filter, level := Level, dst := Dst} = Trace, + emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Filter, Level, Dst]) + end, emqx_trace_handler:running()); -trace(["stop", "client", ClientId]) -> - trace_off(clientid, ClientId); +trace(["stop", Operation, ClientId]) -> + case trace_type(Operation) of + {ok, Type} -> trace_off(Type, ClientId); + error -> trace([]) + end; -trace(["start", "client", ClientId, LogFile]) -> - trace_on(clientid, ClientId, all, LogFile); +trace(["start", Operation, ClientId, LogFile]) -> + trace(["start", Operation, ClientId, LogFile, "all"]); -trace(["start", "client", ClientId, LogFile, Level]) -> - trace_on(clientid, ClientId, list_to_atom(Level), LogFile); - -trace(["stop", "topic", Topic]) -> - trace_off(topic, Topic); - -trace(["start", "topic", Topic, LogFile]) -> - trace_on(topic, Topic, all, LogFile); - -trace(["start", "topic", Topic, LogFile, Level]) -> - trace_on(topic, Topic, list_to_atom(Level), LogFile); +trace(["start", Operation, ClientId, LogFile, Level]) -> + case trace_type(Operation) of + {ok, Type} -> trace_on(Type, ClientId, list_to_existing_atom(Level), LogFile); + error -> trace([]) + end; trace(_) -> - emqx_ctl:usage([{"trace list", "List all traces started"}, - {"trace start client []", "Traces for a client"}, - {"trace stop client ", "Stop tracing for a client"}, - {"trace start topic [] ", "Traces for a topic"}, - {"trace stop topic ", "Stop tracing for a topic"}]). + emqx_ctl:usage([{"trace list", "List all traces started on local node"}, + {"trace start client []", + "Traces for a client on local node"}, + {"trace stop client ", + "Stop tracing for a client on local node"}, + {"trace start topic [] ", + "Traces for a topic on local node"}, + {"trace stop topic ", + "Stop tracing for a topic on local node"}, + {"trace start ip_address [] ", + "Traces for a client ip on local node"}, + {"trace stop ip_addresss ", + "Stop tracing for a client ip on local node"} + ]). trace_on(Who, Name, Level, LogFile) -> - case emqx_tracer:start_trace({Who, iolist_to_binary(Name)}, Level, LogFile) of + case emqx_trace_handler:install(Who, Name, Level, LogFile) of ok -> emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]); {error, Error} -> @@ -451,13 +467,94 @@ trace_on(Who, Name, Level, LogFile) -> end. trace_off(Who, Name) -> - case emqx_tracer:stop_trace({Who, iolist_to_binary(Name)}) of + case emqx_trace_handler:uninstall(Who, Name) of ok -> emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]); {error, Error} -> emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Name, Error]) end. +%%-------------------------------------------------------------------- +%% @doc Trace Cluster Command +traces(["list"]) -> + {ok, List} = emqx_trace_api:list_trace(get, []), + case List of + [] -> + emqx_ctl:print("Cluster Trace is empty~n", []); + _ -> + lists:foreach(fun(Trace) -> + #{type := Type, name := Name, status := Status, + log_size := LogSize} = Trace, + emqx_ctl:print("Trace(~s: ~s=~s, ~s, LogSize:~p)~n", + [Name, Type, maps:get(Type, Trace), Status, LogSize]) + end, List) + end, + length(List); + +traces(["stop", Name]) -> + trace_cluster_off(Name); + +traces(["delete", Name]) -> + trace_cluster_del(Name); + +traces(["start", Name, Operation, Filter]) -> + traces(["start", Name, Operation, Filter, "900"]); + +traces(["start", Name, Operation, Filter, DurationS]) -> + case trace_type(Operation) of + {ok, Type} -> trace_cluster_on(Name, Type, Filter, DurationS); + error -> traces([]) + end; + +traces(_) -> + emqx_ctl:usage([{"traces list", "List all cluster traces started"}, + {"traces start client ", "Traces for a client in cluster"}, + {"traces start topic ", "Traces for a topic in cluster"}, + {"traces start ip_address ", "Traces for a IP in cluster"}, + {"traces stop ", "Stop trace in cluster"}, + {"traces delete ", "Delete trace in cluster"} + ]). + +trace_cluster_on(Name, Type, Filter, DurationS0) -> + case erlang:whereis(emqx_trace) of + undefined -> + emqx_ctl:print("[error] Tracer module not started~n" + "Please run `emqx_ctl modules start tracer` " + "or `emqx_ctl modules start emqx_mod_trace` first~n", []); + _ -> + DurationS = list_to_integer(DurationS0), + Now = erlang:system_time(second), + Trace = #{ name => list_to_binary(Name) + , type => atom_to_binary(Type) + , Type => list_to_binary(Filter) + , start_at => list_to_binary(calendar:system_time_to_rfc3339(Now)) + , end_at => list_to_binary(calendar:system_time_to_rfc3339(Now + DurationS)) + }, + case emqx_trace:create(Trace) of + ok -> + emqx_ctl:print("Cluster_trace ~p ~s ~s successfully~n", [Type, Filter, Name]); + {error, Error} -> + emqx_ctl:print("[error] Cluster_trace ~s ~s=~s ~p~n", + [Name, Type, Filter, Error]) + end + end. + +trace_cluster_del(Name) -> + case emqx_trace:delete(list_to_binary(Name)) of + ok -> emqx_ctl:print("Del cluster_trace ~s successfully~n", [Name]); + {error, Error} -> emqx_ctl:print("[error] Del cluster_trace ~s: ~p~n", [Name, Error]) + end. + +trace_cluster_off(Name) -> + case emqx_trace:update(list_to_binary(Name), false) of + ok -> emqx_ctl:print("Stop cluster_trace ~s successfully~n", [Name]); + {error, Error} -> emqx_ctl:print("[error] Stop cluster_trace ~s: ~p~n", [Name, Error]) + end. + +trace_type("client") -> {ok, clientid}; +trace_type("topic") -> {ok, topic}; +trace_type("ip_address") -> {ok, ip_address}; +trace_type(_) -> error. %%-------------------------------------------------------------------- %% @doc Listeners Command @@ -473,18 +570,20 @@ listeners([]) -> lists:foreach(fun indent_print/1, Info) end, esockd:listeners()), lists:foreach(fun({Protocol, Opts}) -> - Port = proplists:get_value(port, Opts), - Info = [{listen_on, {string, emqx_listeners:format_listen_on(Port)}}, - {acceptors, maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0)}, - {max_conns, proplists:get_value(max_connections, Opts)}, - {current_conn, proplists:get_value(all_connections, Opts)}, - {shutdown_count, []}], - emqx_ctl:print("~s~n", [listener_identifier(Protocol, Port)]), - lists:foreach(fun indent_print/1, Info) - end, ranch:info()); + Port = proplists:get_value(port, Opts), + Acceptors = maps:get(num_acceptors, proplists:get_value(transport_options, Opts, #{}), 0), + Info = [{listen_on, {string, emqx_listeners:format_listen_on(Port)}}, + {acceptors, Acceptors}, + {max_conns, proplists:get_value(max_connections, Opts)}, + {current_conn, proplists:get_value(all_connections, Opts)}, + {shutdown_count, []}], + emqx_ctl:print("~s~n", [listener_identifier(Protocol, Port)]), + lists:foreach(fun indent_print/1, Info) + end, ranch:info()); listeners(["stop", Name = "http" ++ _N | _MaybePort]) -> - %% _MaybePort is to be backward compatible, to stop http listener, there is no need for the port number + %% _MaybePort is to be backward compatible, to stop http listener, + %% there is no need for the port number case minirest:stop_http(list_to_atom(Name)) of ok -> emqx_ctl:print("Stop ~s listener successfully.~n", [Name]); @@ -565,7 +664,8 @@ data(["import", Filename, "--env", Env]) -> {error, unsupported_version} -> emqx_ctl:print("The emqx data import failed: Unsupported version.~n"); {error, Reason} -> - emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", [Reason, Filename]) + emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", + [Reason, Filename]) end; data(_) -> @@ -657,19 +757,23 @@ print({client, {ClientId, ChanPid}}) -> maps:with([created_at], Session)]), InfoKeys = [clientid, username, peername, clean_start, keepalive, expiry_interval, - subscriptions_cnt, inflight_cnt, awaiting_rel_cnt, send_msg, mqueue_len, mqueue_dropped, - connected, created_at, connected_at] ++ case maps:is_key(disconnected_at, Info) of - true -> [disconnected_at]; - false -> [] - end, + subscriptions_cnt, inflight_cnt, awaiting_rel_cnt, + send_msg, mqueue_len, mqueue_dropped, + connected, created_at, connected_at] ++ + case maps:is_key(disconnected_at, Info) of + true -> [disconnected_at]; + false -> [] + end, emqx_ctl:print("Client(~s, username=~s, peername=~s, " - "clean_start=~s, keepalive=~w, session_expiry_interval=~w, " - "subscriptions=~w, inflight=~w, awaiting_rel=~w, delivered_msgs=~w, enqueued_msgs=~w, dropped_msgs=~w, " - "connected=~s, created_at=~w, connected_at=~w" ++ case maps:is_key(disconnected_at, Info) of - true -> ", disconnected_at=~w)~n"; - false -> ")~n" - end, - [format(K, maps:get(K, Info)) || K <- InfoKeys]); + "clean_start=~s, keepalive=~w, session_expiry_interval=~w, " + "subscriptions=~w, inflight=~w, awaiting_rel=~w, " + "delivered_msgs=~w, enqueued_msgs=~w, dropped_msgs=~w, " + "connected=~s, created_at=~w, connected_at=~w" ++ + case maps:is_key(disconnected_at, Info) of + true -> ", disconnected_at=~w)~n"; + false -> ")~n" + end, + [format(K, maps:get(K, Info)) || K <- InfoKeys]); print({emqx_route, #route{topic = Topic, dest = {_, Node}}}) -> emqx_ctl:print("~s -> ~s~n", [Topic, Node]); @@ -722,6 +826,10 @@ restart_http_listener(Scheme, AppName) -> http_mod_name(emqx_management) -> emqx_mgmt_http; http_mod_name(Name) -> Name. +print_app_info({AppId, AppSecret, Name, Desc, Status, Expired}) -> + emqx_ctl:print("app_id: ~s, secret: ~s, name: ~s, desc: ~s, status: ~s, expired: ~p~n", + [AppId, AppSecret, Name, Desc, Status, Expired]). + for_node(Fun, Node) -> try list_to_existing_atom(Node) of NodeAtom -> diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 06dc0365a..f5eaaa7ab 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -31,6 +31,8 @@ ]). -endif. +-define(BACKUP_DIR, backup). + -export([ export_rules/0 , export_resources/0 , export_blacklist/0 @@ -53,8 +55,18 @@ -export([ export/0 , import/2 + , upload_backup_file/2 + , list_backup_file/0 + , read_backup_file/1 + , delete_backup_file/1 ]). +-ifdef(TEST). +-export([ backup_dir/0 + , delete_all_backup_file/0 + ]). +-endif. + %%-------------------------------------------------------------------- %% Data Export and Import %%-------------------------------------------------------------------- @@ -237,10 +249,12 @@ import_resource(#{<<"id">> := Id, config => Config, created_at => NCreatedAt, description => Desc}). + import_resources_and_rules(Resources, Rules, FromVersion) when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse - FromVersion =:= "4.2" -> + FromVersion =:= "4.2" orelse + FromVersion =:= "4.3" -> Configs = lists:foldl(fun compatible_version/2 , [], Resources), lists:foreach(fun(#{<<"actions">> := Actions} = Rule) -> NActions = apply_new_config(Actions, Configs), @@ -305,6 +319,17 @@ compatible_version(#{<<"id">> := ID, {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}), NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)), [{ID, #{headers => NHeaders, method => Method}} | Acc]; + +compatible_version(#{<<"id">> := ID, + <<"type">> := Type, + <<"config">> := Config} = Resource, Acc) + when Type =:= <<"backend_mongo_single">> + orelse Type =:= <<"backend_mongo_sharded">> + orelse Type =:= <<"backend_mongo_rs">> -> + NewConfig = maps:merge(#{<<"srv_record">> => false}, Config), + {ok, _Resource} = import_resource(Resource#{<<"config">> := NewConfig}), + [{ID, NewConfig} | Acc]; + % normal version compatible_version(Resource, Acc) -> {ok, _Resource} = import_resource(Resource), @@ -527,16 +552,39 @@ import_modules(Modules) -> undefined -> ok; _ -> - lists:foreach(fun(#{<<"id">> := Id, - <<"type">> := Type, - <<"config">> := Config, - <<"enabled">> := Enabled, - <<"created_at">> := CreatedAt, - <<"description">> := Description}) -> - _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) - end, Modules) + NModules = migrate_modules(Modules), + lists:foreach(fun(#{<<"id">> := Id, + <<"type">> := Type, + <<"config">> := Config, + <<"enabled">> := Enabled, + <<"created_at">> := CreatedAt, + <<"description">> := Description}) -> + _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) + end, NModules) end. +migrate_modules(Modules) -> + migrate_modules(Modules, []). + +migrate_modules([], Acc) -> + lists:reverse(Acc); +migrate_modules([#{<<"type">> := <<"mongo_authentication">>, + <<"config">> := Config} = Module | More], Acc) -> + WMode = case maps:get(<<"w_mode">>, Config, <<"unsafe">>) of + <<"undef">> -> <<"unsafe">>; + Other -> Other + end, + RMode = case maps:get(<<"r_mode">>, Config, <<"master">>) of + <<"undef">> -> <<"master">>; + <<"slave-ok">> -> <<"slave_ok">>; + Other0 -> Other0 + end, + NConfig = Config#{<<"srv_record">> => false, + <<"w_mode">> => WMode, + <<"r_mode">> => RMode}, + migrate_modules(More, [Module#{<<"config">> => NConfig} | Acc]); +migrate_modules([Module | More], Acc) -> + migrate_modules(More, [Module | Acc]). import_schemas(Schemas) -> case ets:info(emqx_schema) of @@ -564,19 +612,123 @@ to_version(Version) when is_binary(Version) -> to_version(Version) when is_list(Version) -> Version. +upload_backup_file(Filename0, Bin) -> + case ensure_file_name(Filename0) of + {ok, Filename} -> + case check_json(Bin) of + {ok, _} -> + logger:info("write backup file ~p", [Filename]), + file:write_file(Filename, Bin); + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. + +list_backup_file() -> + Filter = + fun(File) -> + case file:read_file_info(File) of + {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} -> + Seconds = calendar:datetime_to_gregorian_seconds(CTime), + BaseFilename = to_binary(filename:basename(File)), + CreatedAt = to_binary(io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S])), + Info = { + Seconds, + [{filename, BaseFilename}, + {size, Size}, + {created_at, CreatedAt}, + {node, node()} + ] + }, + {true, Info}; + _ -> + false + end + end, + lists:filtermap(Filter, backup_files()). + +backup_files() -> + backup_files(backup_dir()) ++ backup_files(backup_dir_old_version()). + +backup_files(Dir) -> + {ok, FilesAll} = file:list_dir_all(Dir), + Files = lists:filtermap(fun legal_filename/1, FilesAll), + [filename:join([Dir, File]) || File <- Files]. + +look_up_file(Filename) when is_binary(Filename) -> + look_up_file(binary_to_list(Filename)); +look_up_file(Filename) -> + Filter = + fun(MaybeFile) -> + filename:basename(MaybeFile) == Filename + end, + case lists:filter(Filter, backup_files()) of + [] -> + {error, not_found}; + List -> + {ok, hd(List)} + end. + +read_backup_file(Filename0) -> + case look_up_file(Filename0) of + {ok, Filename} -> + case file:read_file(Filename) of + {ok, Bin} -> + {ok, #{filename => to_binary(Filename0), + file => Bin}}; + {error, Reason} -> + logger:error("read file ~p failed ~p", [Filename, Reason]), + {error, bad_file} + end; + {error, not_found} -> + {error, not_found} + end. + +delete_backup_file(Filename0) -> + case look_up_file(Filename0) of + {ok, Filename} -> + case file:read_file_info(Filename) of + {ok, #file_info{}} -> + case file:delete(Filename) of + ok -> + logger:info("delete backup file ~p", [Filename]), + ok; + {error, Reason} -> + logger:error( + "delete backup file ~p error:~p", [Filename, Reason]), + {error, Reason} + end; + _ -> + {error, not_found} + end; + {error, not_found} -> + {error, not_found} + end. + +-ifdef(TEST). +%% clean all for test +delete_all_backup_file() -> + [begin + Filename = proplists:get_value(filename, Info), + _ = delete_backup_file(Filename) + end || {_, Info} <- list_backup_file()], + ok. +-endif. + export() -> Seconds = erlang:system_time(second), Data = do_export_data() ++ [{date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}], {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds), - Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), - NFilename = filename:join([emqx:get_env(data_dir), Filename]), - ok = filelib:ensure_dir(NFilename), - case file:write_file(NFilename, emqx_json:encode(Data)) of + BaseFilename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]), + {ok, Filename} = ensure_file_name(BaseFilename), + case file:write_file(Filename, emqx_json:encode(Data)) of ok -> - case file:read_file_info(NFilename) of + case file:read_file_info(Filename) of {ok, #file_info{size = Size, ctime = {{Y1, M1, D1}, {H1, MM1, S1}}}} -> CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y1, M1, D1, H1, MM1, S1]), - {ok, #{filename => list_to_binary(NFilename), + {ok, #{filename => Filename, size => Size, created_at => list_to_binary(CreatedAt), node => node() @@ -612,9 +764,8 @@ do_export_extra_data() -> []. -ifdef(EMQX_ENTERPRISE). import(Filename, OverridesJson) -> - case file:read_file(Filename) of - {ok, Json} -> - Imported = emqx_json:decode(Json, [return_maps]), + case check_import_json(Filename) of + {ok, Imported} -> Overrides = emqx_json:decode(OverridesJson, [return_maps]), Data = maps:merge(Imported, Overrides), Version = to_version(maps:get(<<"version">>, Data)), @@ -627,13 +778,13 @@ import(Filename, OverridesJson) -> logger:error("The emqx data import failed: ~0p", [{Class, Reason, Stack}]), {error, import_failed} end; - Error -> Error + {error, Reason} -> + {error, Reason} end. -else. import(Filename, OverridesJson) -> - case file:read_file(Filename) of - {ok, Json} -> - Imported = emqx_json:decode(Json, [return_maps]), + case check_import_json(Filename) of + {ok, Imported} -> Overrides = emqx_json:decode(OverridesJson, [return_maps]), Data = maps:merge(Imported, Overrides), Version = to_version(maps:get(<<"version">>, Data)), @@ -652,10 +803,58 @@ import(Filename, OverridesJson) -> logger:error("Unsupported version: ~p", [Version]), {error, unsupported_version, Version} end; - Error -> Error + {error, Reason} -> + {error, Reason} end. -endif. +-spec(check_import_json(binary() | string()) -> {ok, map()} | {error, term()}). +check_import_json(Filename) -> + FunList = [ + fun look_up_file/1, + fun(F) -> file:read_file(F) end, + fun check_json/1 + ], + check_import_json(Filename, FunList). + +check_import_json(Res, []) -> + {ok, Res}; +check_import_json(Acc, [Fun | FunList]) -> + case Fun(Acc) of + {ok, Next} -> + check_import_json(Next, FunList); + {error, Reason} -> + {error, Reason} + end. + +ensure_file_name(Filename) -> + case legal_filename(Filename) of + true -> + {ok, filename:join(backup_dir(), Filename)}; + false -> + {error, bad_filename} + end. + +backup_dir() -> + Dir = filename:join(emqx:get_env(data_dir), ?BACKUP_DIR), + ok = filelib:ensure_dir(filename:join([Dir, dummy])), + Dir. + +backup_dir_old_version() -> + emqx:get_env(data_dir). + +legal_filename(Filename) -> + MaybeJson = filename:extension(Filename), + MaybeJson == ".json" orelse MaybeJson == <<".json">>. + +check_json(MaybeJson) -> + case emqx_json:safe_decode(MaybeJson, [return_maps]) of + {ok, Json} -> + {ok, Json}; + {error, _} -> + {error, bad_json} + end. + do_import_data(Data, Version) -> do_import_extra_data(Data, Version), import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), @@ -697,6 +896,8 @@ is_version_supported2("4.1") -> true; is_version_supported2("4.3") -> true; +is_version_supported2("4.4") -> + true; is_version_supported2(Version) -> case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of match -> @@ -762,3 +963,6 @@ get_old_type() -> set_old_type(Type) -> application:set_env(emqx_auth_mnesia, as, Type). + +to_binary(Bin) when is_binary(Bin) -> Bin; +to_binary(Str) when is_list(Str) -> list_to_binary(Str). diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index 473cec306..cf870902e 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -89,7 +89,8 @@ listener_name(Proto) -> http_handlers() -> Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), - [{"/api/v4", minirest:handler(#{apps => Plugins ++ [emqx_modules] -- ?EXCEPT_PLUGIN, + [{"/api/v4", minirest:handler(#{apps => Plugins ++ + [emqx_plugin_libs, emqx_modules] -- ?EXCEPT_PLUGIN, except => ?EXCEPT, filter => fun ?MODULE:filter/1}), [{authorization, fun ?MODULE:authorize_appid/1}]}]. @@ -129,6 +130,7 @@ filter(_) -> true. -else. filter(#{app := emqx_modules}) -> true; +filter(#{app := emqx_plugin_libs}) -> true; filter(#{app := App}) -> case emqx_plugins:find_plugin(App) of false -> false; diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl index 6519880f8..89a261931 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl @@ -183,7 +183,10 @@ do_import(File, Config, Overrides) -> mnesia:clear_table(?ACL_TABLE2), mnesia:clear_table(emqx_user), emqx_acl_mnesia_migrator:migrate_records(), - Filename = filename:join(proplists:get_value(data_dir, Config), File), + Filename = filename:basename(File), + FilePath = filename:join([proplists:get_value(data_dir, Config), File]), + {ok, Bin} = file:read_file(FilePath), + ok = emqx_mgmt_data_backup:upload_backup_file(Filename, Bin), emqx_mgmt_data_backup:import(Filename, Overrides). test_import(username, {Username, Password}) -> diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index f01eb9631..681998012 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -34,14 +34,18 @@ init_per_suite(Cfg) -> Cfg. end_per_suite(Cfg) -> + emqx_mgmt_data_backup:delete_all_backup_file(), emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]), Cfg. get_data_path() -> emqx_ct_helpers:deps_path(emqx_management, "test/emqx_bridge_mqtt_data_export_import_SUITE_data/"). -import(FilePath, Version) -> - ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), +import(FilePath0, Version) -> + Filename = filename:basename(FilePath0), + FilePath = filename:join([get_data_path(), FilePath0]), + {ok, Bin} = file:read_file(FilePath), + ok = emqx_mgmt_data_backup:upload_backup_file(Filename, Bin), timer:sleep(500), lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> case Id of @@ -181,4 +185,4 @@ remove_resources() -> lists:foreach(fun(#resource{id = Id}) -> emqx_rule_engine:delete_resource(Id) end, emqx_rule_registry:get_resources()), - timer:sleep(500). \ No newline at end of file + timer:sleep(500). diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 66bd07405..45afbd860 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -45,6 +45,7 @@ groups() -> t_vm_cmd, t_plugins_cmd, t_trace_cmd, + t_traces_cmd, t_broker_cmd, t_router_cmd, t_subscriptions_cmd, @@ -64,6 +65,23 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_helpers:stop_apps(apps()). +init_per_testcase(t_plugins_cmd, Config) -> + meck:new(emqx_plugins, [non_strict, passthrough]), + meck:expect(emqx_plugins, load, fun(_) -> ok end), + meck:expect(emqx_plugins, unload, fun(_) -> ok end), + meck:expect(emqx_plugins, reload, fun(_) -> ok end), + mock_print(), + Config; +init_per_testcase(_Case, Config) -> + mock_print(), + Config. + +end_per_testcase(t_plugins_cmd, _Config) -> + meck:unload(emqx_plugins), + unmock_print(); +end_per_testcase(_Case, _Config) -> + unmock_print(). + t_app(_Config) -> {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>), ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret)), @@ -96,7 +114,6 @@ t_app(_Config) -> ok. t_log_cmd(_) -> - mock_print(), lists:foreach(fun(Level) -> emqx_mgmt_cli:log(["primary-level", Level]), ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["primary-level"])) @@ -109,12 +126,9 @@ t_log_cmd(_) -> ?assertEqual(Level ++ "\n", emqx_mgmt_cli:log(["handlers", "set-level", atom_to_list(Id), Level])) end, ?LOG_LEVELS) - || #{id := Id} <- emqx_logger:get_log_handlers()], - meck:unload(). + || #{id := Id} <- emqx_logger:get_log_handlers()]. t_mgmt_cmd(_) -> - % ct:pal("start testing the mgmt command"), - mock_print(), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( ["lookup", "emqx_appid"]), "Not Found.")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( @@ -127,28 +141,19 @@ t_mgmt_cmd(_) -> ["update", "emqx_appid", "ts"]), "update successfully")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:mgmt( ["delete", "emqx_appid"]), "ok")), - ok = emqx_mgmt_cli:mgmt(["list"]), - meck:unload(). + ok = emqx_mgmt_cli:mgmt(["list"]). t_status_cmd(_) -> - % ct:pal("start testing status command"), - mock_print(), %% init internal status seem to be always 'starting' when running ct tests - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([]), "Node\s.*@.*\sis\sstart(ed|ing)")), - meck:unload(). + ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([]), "Node\s.*@.*\sis\sstart(ed|ing)")). t_broker_cmd(_) -> - % ct:pal("start testing the broker command"), - mock_print(), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([]), "sysdescr")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker(["stats"]), "subscriptions.shared")), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker(["metrics"]), "bytes.sent")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([undefined]), "broker")), - meck:unload(). + ?assertMatch({match, _}, re:run(emqx_mgmt_cli:broker([undefined]), "broker")). t_clients_cmd(_) -> - % ct:pal("start testing the client command"), - mock_print(), process_flag(trap_exit, true), {ok, T} = emqtt:start_link([{clientid, <<"client12">>}, {username, <<"testuser1">>}, @@ -164,7 +169,6 @@ t_clients_cmd(_) -> receive {'EXIT', T, _} -> ok - % ct:pal("Connection closed: ~p~n", [Reason]) after 500 -> erlang:error("Client is not kick") @@ -179,10 +183,11 @@ t_clients_cmd(_) -> {ok, Connack, <<>>, _} = raw_recv_pase(Bin), timer:sleep(300), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client13"]), "client13")), - meck:unload(). + % emqx_mgmt_cli:clients(["kick", "client13"]), % timer:sleep(500), % ?assertMatch({match, _}, re:run(emqx_mgmt_cli:clients(["show", "client13"]), "Not Found")). + ok. raw_recv_pase(Packet) -> emqx_frame:parse(Packet). @@ -191,8 +196,6 @@ raw_send_serialize(Packet) -> emqx_frame:serialize(Packet). t_vm_cmd(_) -> - % ct:pal("start testing the vm command"), - mock_print(), [[?assertMatch({match, _}, re:run(Result, Name)) || Result <- emqx_mgmt_cli:vm([Name])] || Name <- ["load", "memory", "process", "io", "ports"]], @@ -205,12 +208,9 @@ t_vm_cmd(_) -> [?assertMatch({match, _}, re:run(Result, "io")) || Result <- emqx_mgmt_cli:vm(["io"])], [?assertMatch({match, _}, re:run(Result, "ports")) - || Result <- emqx_mgmt_cli:vm(["ports"])], - unmock_print(). + || Result <- emqx_mgmt_cli:vm(["ports"])]. t_trace_cmd(_) -> - % ct:pal("start testing the trace command"), - mock_print(), logger:set_primary_config(level, debug), {ok, T} = emqtt:start_link([{clientid, <<"client">>}, {username, <<"testuser">>}, @@ -237,12 +237,34 @@ t_trace_cmd(_) -> Trace7 = emqx_mgmt_cli:trace(["start", "topic", "a/b/c", "log/clientid_trace.log", "error"]), ?assertMatch({match, _}, re:run(Trace7, "successfully")), - logger:set_primary_config(level, error), - unmock_print(). + logger:set_primary_config(level, error). + +t_traces_cmd(_) -> + emqx_trace:create_table(), + Count1 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(0, Count1), + Error1 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), + ?assertMatch({match, _}, re:run(Error1, "Tracer module not started")), + emqx_trace:start_link(), + Trace1 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), + ?assertMatch({match, _}, re:run(Trace1, "successfully")), + Count2 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(1, Count2), + Error2 = emqx_mgmt_cli:traces(["start", "test-name", "client", "clientid-dev"]), + ?assertMatch({match, _}, re:run(Error2, "already_existed")), + Trace2 = emqx_mgmt_cli:traces(["stop", "test-name"]), + ?assertMatch({match, _}, re:run(Trace2, "successfully")), + Count3 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(1, Count3), + Trace3 = emqx_mgmt_cli:traces(["delete", "test-name"]), + ?assertMatch({match, _}, re:run(Trace3, "successfully")), + Count4 = emqx_mgmt_cli:traces(["list"]), + ?assertEqual(0, Count4), + Error3 = emqx_mgmt_cli:traces(["delete", "test-name"]), + ?assertMatch({match, _}, re:run(Error3, "not_found")), + ok. t_router_cmd(_) -> - % ct:pal("start testing the router command"), - mock_print(), {ok, T} = emqtt:start_link([{clientid, <<"client1">>}, {username, <<"testuser1">>}, {password, <<"pass1">>} @@ -257,12 +279,9 @@ t_router_cmd(_) -> emqtt:connect(T1), emqtt:subscribe(T1, <<"a/b/c/d">>), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["list"]), "a/b/c | a/b/c")), - ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["show", "a/b/c"]), "a/b/c")), - unmock_print(). + ?assertMatch({match, _}, re:run(emqx_mgmt_cli:routes(["show", "a/b/c"]), "a/b/c")). t_subscriptions_cmd(_) -> - % ct:pal("Start testing the subscriptions command"), - mock_print(), {ok, T3} = emqtt:start_link([{clientid, <<"client">>}, {username, <<"testuser">>}, {password, <<"pass">>} @@ -273,22 +292,18 @@ t_subscriptions_cmd(_) -> [?assertMatch({match, _} , re:run(Result, "b/b/c")) || Result <- emqx_mgmt_cli:subscriptions(["show", <<"client">>])], ?assertEqual(emqx_mgmt_cli:subscriptions(["add", "client", "b/b/c", "0"]), "ok\n"), - ?assertEqual(emqx_mgmt_cli:subscriptions(["del", "client", "b/b/c"]), "ok\n"), - unmock_print(). + ?assertEqual(emqx_mgmt_cli:subscriptions(["del", "client", "b/b/c"]), "ok\n"). t_listeners_cmd_old(_) -> ok = emqx_listeners:ensure_all_started(), - mock_print(), ?assertEqual(emqx_mgmt_cli:listeners([]), ok), ?assertEqual( "Stop mqtt:wss:external listener on 0.0.0.0:8084 successfully.\n", emqx_mgmt_cli:listeners(["stop", "wss", "8084"]) - ), - unmock_print(). + ). t_listeners_cmd_new(_) -> ok = emqx_listeners:ensure_all_started(), - mock_print(), ?assertEqual(emqx_mgmt_cli:listeners([]), ok), ?assertEqual( "Stop mqtt:wss:external listener on 0.0.0.0:8084 successfully.\n", @@ -306,15 +321,9 @@ t_listeners_cmd_new(_) -> emqx_mgmt_cli:listeners(["restart", "bad:listener:identifier"]), "Failed to restart bad:listener:identifier listener:" " {no_such_listener,\"bad:listener:identifier\"}\n" - ), - unmock_print(). + ). t_plugins_cmd(_) -> - mock_print(), - meck:new(emqx_plugins, [non_strict, passthrough]), - meck:expect(emqx_plugins, load, fun(_) -> ok end), - meck:expect(emqx_plugins, unload, fun(_) -> ok end), - meck:expect(emqx_plugins, reload, fun(_) -> ok end), ?assertEqual(emqx_mgmt_cli:plugins(["list"]), ok), ?assertEqual( emqx_mgmt_cli:plugins(["unload", "emqx_auth_mnesia"]), @@ -327,11 +336,9 @@ t_plugins_cmd(_) -> ?assertEqual( emqx_mgmt_cli:plugins(["unload", "emqx_management"]), "Plugin emqx_management can not be unloaded.\n" - ), - unmock_print(). + ). t_cli(_) -> - mock_print(), ?assertMatch({match, _}, re:run(emqx_mgmt_cli:status([""]), "status")), [?assertMatch({match, _}, re:run(Value, "broker")) || Value <- emqx_mgmt_cli:broker([""])], @@ -353,12 +360,32 @@ t_cli(_) -> || Value <- emqx_mgmt_cli:mnesia([""])], [?assertMatch({match, _}, re:run(Value, "trace")) || Value <- emqx_mgmt_cli:trace([""])], + [?assertMatch({match, _}, re:run(Value, "traces")) + || Value <- emqx_mgmt_cli:traces([""])], [?assertMatch({match, _}, re:run(Value, "mgmt")) - || Value <- emqx_mgmt_cli:mgmt([""])], - unmock_print(). + || Value <- emqx_mgmt_cli:mgmt([""])]. + +t_backup_file(_)-> + Filename = <<"test.json">>, + BadFilename = <<"bad.notjson">>, + Bin = emqx_json:encode(#{a => b}), + BadBin = <<"[bad json]">>, + + {error, bad_filename} = emqx_mgmt_data_backup:upload_backup_file(BadFilename, Bin), + {error, bad_json} = emqx_mgmt_data_backup:upload_backup_file(Filename, BadBin), + + ok = emqx_mgmt_data_backup:upload_backup_file(Filename, Bin), + {ok, #{file := <<"{\"a\":\"b\"}">>, filename := <<"test.json">>}} = + emqx_mgmt_data_backup:read_backup_file(Filename), + [{_, FileInfoList}] = emqx_mgmt_data_backup:list_backup_file(), + Filename = proplists:get_value(filename, FileInfoList), + ok = emqx_mgmt_data_backup:delete_backup_file(Filename), + + {error, not_found} = emqx_mgmt_data_backup:delete_backup_file(BadFilename), + ok. mock_print() -> - catch meck:unload(emqx_ctl), + ok = safe_unmeck(emqx_ctl), meck:new(emqx_ctl, [non_strict, passthrough]), meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg, []) end), meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end), @@ -367,3 +394,12 @@ mock_print() -> unmock_print() -> meck:unload(emqx_ctl). + +safe_unmeck(Module) -> + try + meck:unload(Module), + ok + catch + _ : _ -> + ok + end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index cfed74c16..0029b866f 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -29,6 +29,8 @@ -define(HOST, "http://127.0.0.1:8081/"). +-elvis([{elvis_style, line_length, disable}]). + -define(API_VERSION, "v4"). -define(BASE_PATH, "api"). @@ -76,30 +78,40 @@ t_alarms(_) -> ?assert(is_existing(alarm2, emqx_alarm:get_alarms(activated))), {ok, Return1} = request_api(get, api_path(["alarms/activated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return1))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return1))))), + ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return1))))), emqx_alarm:deactivate(alarm1), {ok, Return2} = request_api(get, api_path(["alarms"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return2))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return2))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return2))))), + ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return2))))), {ok, Return3} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return3))))), - ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return3))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return3))))), + ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return3))))), emqx_alarm:deactivate(alarm2), {ok, Return4} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return4))))), - ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return4))))), + ?assert(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return4))))), + ?assert(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return4))))), {ok, _} = request_api(delete, api_path(["alarms/deactivated"]), auth_header_()), {ok, Return5} = request_api(get, api_path(["alarms/deactivated"]), auth_header_()), - ?assertNot(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))), - ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, lists:nth(1, get(<<"data">>, Return5))))). + ?assertNot(lookup_alarm(<<"alarm1">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return5))))), + ?assertNot(lookup_alarm(<<"alarm2">>, maps:get(<<"alarms">>, + lists:nth(1, get(<<"data">>, Return5))))). t_apps(_) -> AppId = <<"123456">>, @@ -153,7 +165,8 @@ t_banned(_) -> [Banned] = get(<<"data">>, Result), ?assertEqual(Who, maps:get(<<"who">>, Banned)), - {ok, _} = request_api(delete, api_path(["banned", "clientid", binary_to_list(Who)]), auth_header_()), + {ok, _} = request_api(delete, api_path(["banned", "clientid", binary_to_list(Who)]), + auth_header_()), {ok, Result2} = request_api(get, api_path(["banned"]), auth_header_()), ?assertEqual([], get(<<"data">>, Result2)). @@ -205,40 +218,50 @@ t_clients(_) -> meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, kickout_client, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), + {ok, MeckRet1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), + auth_header_()), ?assertEqual(?ERROR1, get(<<"code">>, MeckRet1)), meck:expect(emqx_mgmt, clean_acl_cache, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet2} = request_api(delete, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), + {ok, MeckRet2} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR1, get(<<"code">>, MeckRet2)), meck:expect(emqx_mgmt, list_acl_cache, 1, fun(_) -> {error, undefined} end), - {ok, MeckRet3} = request_api(get, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), + {ok, MeckRet3} = request_api(get, + api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR1, get(<<"code">>, MeckRet3)), meck:unload(emqx_mgmt), - {ok, Ok} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), + {ok, Ok} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), ?assertEqual(?SUCCESS, get(<<"code">>, Ok)), timer:sleep(300), - {ok, Ok1} = request_api(delete, api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), + {ok, Ok1} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1)]), auth_header_()), ?assertEqual(?SUCCESS, get(<<"code">>, Ok1)), - {ok, Clients6} = request_api(get, api_path(["clients"]), "_limit=100&_page=1", auth_header_()), + {ok, Clients6} = request_api(get, + api_path(["clients"]), "_limit=100&_page=1", auth_header_()), ?assertEqual(1, maps:get(<<"count">>, get(<<"meta">>, Clients6))), - {ok, NotFound1} = request_api(get, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), + {ok, NotFound1} = request_api(get, + api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR12, get(<<"code">>, NotFound1)), - {ok, NotFound2} = request_api(delete, api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), + {ok, NotFound2} = request_api(delete, + api_path(["clients", binary_to_list(ClientId1), "acl_cache"]), auth_header_()), ?assertEqual(?ERROR12, get(<<"code">>, NotFound2)), - {ok, EmptyAclCache} = request_api(get, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), + {ok, EmptyAclCache} = request_api(get, + api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), ?assertEqual(0, length(get(<<"data">>, EmptyAclCache))), - {ok, Ok1} = request_api(delete, api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), + {ok, Ok1} = request_api(delete, + api_path(["clients", binary_to_list(ClientId2), "acl_cache"]), auth_header_()), ?assertEqual(?SUCCESS, get(<<"code">>, Ok1)). receive_exit(0) -> @@ -257,7 +280,8 @@ receive_exit(Count) -> t_listeners(_) -> {ok, _} = request_api(get, api_path(["listeners"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "listeners"]), auth_header_()), + {ok, _} = request_api(get, + api_path(["nodes", atom_to_list(node()), "listeners"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, list_listeners, 0, fun() -> [{node(), {error, undefined}}] end), {ok, Return} = request_api(get, api_path(["listeners"]), auth_header_()), @@ -268,10 +292,12 @@ t_listeners(_) -> t_metrics(_) -> {ok, _} = request_api(get, api_path(["metrics"]), auth_header_()), - {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), + {ok, _} = request_api(get, + api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, get_metrics, 1, fun(_) -> {error, undefined} end), - {ok, "{\"message\":\"undefined\"}"} = request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), + {ok, "{\"message\":\"undefined\"}"} = + request_api(get, api_path(["nodes", atom_to_list(node()), "metrics"]), auth_header_()), meck:unload(emqx_mgmt). t_nodes(_) -> @@ -348,7 +374,8 @@ t_acl_cache(_) -> {ok, _} = emqtt:connect(C1), {ok, _, _} = emqtt:subscribe(C1, Topic, 2), %% get acl cache, should not be empty - {ok, Result} = request_api(get, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), + {ok, Result} = request_api(get, + api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), #{<<"code">> := 0, <<"data">> := Caches} = jiffy:decode(list_to_binary(Result), [return_maps]), ?assert(length(Caches) > 0), ?assertMatch(#{<<"access">> := <<"subscribe">>, @@ -356,11 +383,14 @@ t_acl_cache(_) -> <<"result">> := <<"allow">>, <<"updated_time">> := _}, hd(Caches)), %% clear acl cache - {ok, Result2} = request_api(delete, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), + {ok, Result2} = request_api(delete, + api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), ?assertMatch(#{<<"code">> := 0}, jiffy:decode(list_to_binary(Result2), [return_maps])), %% get acl cache again, after the acl cache is cleared - {ok, Result3} = request_api(get, api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), - #{<<"code">> := 0, <<"data">> := Caches3} = jiffy:decode(list_to_binary(Result3), [return_maps]), + {ok, Result3} = request_api(get, + api_path(["clients", binary_to_list(ClientId), "acl_cache"]), [], auth_header_()), + #{<<"code">> := 0, <<"data">> := Caches3} + = jiffy:decode(list_to_binary(Result3), [return_maps]), ?assertEqual(0, length(Caches3)), ok = emqtt:disconnect(C1). @@ -371,7 +401,7 @@ t_pubsub(_) -> ClientId = <<"client1">>, Options = #{clientid => ClientId, - proto_ver => 5}, + proto_ver => v5}, Topic = <<"mytopic">>, {ok, C1} = emqtt:start_link(Options), {ok, _} = emqtt:connect(C1), @@ -417,6 +447,20 @@ t_pubsub(_) -> <<"payload">> => <<"hello">>}), ?assertEqual(?ERROR8, get(<<"code">>, BadClient2)), + {ok, BadParams} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), + #{<<"clientid">> => 1, + <<"topics">> => <<"mytopic">>, + <<"qos">> => 1, + <<"payload">> => <<"hello">>, + <<"user_properties">> => + #{<<"id">> => 10010, + <<"name">> => <<"emqx">>, + <<"foo">> => ["bad_properties1", "bad_properties2"], + <<"boolean">> => false + } + }), + ?assertEqual(?ERROR8, get(<<"code">>, BadParams)), + {ok, BadClient3} = request_api(post, api_path(["mqtt/unsubscribe"]), [], auth_header_(), #{<<"clientid">> => 1, <<"topic">> => <<"mytopic">>}), @@ -482,12 +526,15 @@ t_pubsub(_) -> Topic_list = [<<"mytopic1">>, <<"mytopic2">>], [ {ok, _, [2]} = emqtt:subscribe(C1, Topics, 2) || Topics <- Topic_list], - Body1 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2} || Topics <- Topic_list], + Body1 = [ #{<<"clientid">> => ClientId, + <<"topic">> => Topics, <<"qos">> => 2} || Topics <- Topic_list], {ok, Data1} = request_api(post, api_path(["mqtt/subscribe_batch"]), [], auth_header_(), Body1), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data1), [return_maps]))), %% tests publish_batch - Body2 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2, <<"retain">> => <<"false">>, <<"payload">> => #{body => "hello world"}} || Topics <- Topic_list ], + Body2 = [ #{<<"clientid">> => ClientId, <<"topic">> => Topics, <<"qos">> => 2, + <<"retain">> => <<"false">>, <<"payload">> => #{body => "hello world"}} + || Topics <- Topic_list ], {ok, Data2} = request_api(post, api_path(["mqtt/publish_batch"]), [], auth_header_(), Body2), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data2), [return_maps]))), [ ?assert(receive @@ -499,14 +546,33 @@ t_pubsub(_) -> %% tests unsubscribe_batch Body3 = [#{<<"clientid">> => ClientId, <<"topic">> => Topics} || Topics <- Topic_list], - {ok, Data3} = request_api(post, api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), + {ok, Data3} = request_api(post, + api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data3), [return_maps]))), + {ok, _, [1]} = emqtt:subscribe(C1, <<"mytopic">>, qos1), + timer:sleep(50), + + %% user properties + {ok, Code} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), + #{<<"clientid">> => ClientId, + <<"topic">> => <<"mytopic">>, + <<"qos">> => 1, + <<"payload">> => <<"hello world">>, + <<"user_properties">> => #{<<"porp_1">> => <<"porp_1">>}}), + ?assert(receive + {publish, #{payload := <<"hello world">>, + properties := #{'User-Property' := [{<<"porp_1">>,<<"porp_1">>}]}}} -> + true + after 100 -> + false + end), + ok = emqtt:disconnect(C1), - ?assertEqual(3, emqx_metrics:val('messages.qos1.received') - Qos1Received), + ?assertEqual(4, emqx_metrics:val('messages.qos1.received') - Qos1Received), ?assertEqual(2, emqx_metrics:val('messages.qos2.received') - Qos2Received), - ?assertEqual(5, emqx_metrics:val('messages.received') - Received). + ?assertEqual(6, emqx_metrics:val('messages.received') - Received). loop([]) -> []; @@ -523,7 +589,8 @@ t_routes_and_subscriptions(_) -> ?assertEqual([], get(<<"data">>, NonRoute)), {ok, NonSubscription} = request_api(get, api_path(["subscriptions"]), auth_header_()), ?assertEqual([], get(<<"data">>, NonSubscription)), - {ok, NonSubscription1} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), + {ok, NonSubscription1} = request_api(get, + api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), ?assertEqual([], get(<<"data">>, NonSubscription1)), {ok, NonSubscription2} = request_api(get, api_path(["subscriptions", binary_to_list(ClientId)]), @@ -554,11 +621,14 @@ t_routes_and_subscriptions(_) -> ?assertMatch(#{<<"page">> := 1, <<"limit">> := 10000, <<"hasnext">> := false, <<"count">> := 1}, get(<<"meta">>, Result3)), - {ok, Result3} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), + {ok, Result3} = request_api(get, + api_path(["nodes", atom_to_list(node()), "subscriptions"]), auth_header_()), - {ok, Result4} = request_api(get, api_path(["subscriptions", binary_to_list(ClientId)]), auth_header_()), + {ok, Result4} = request_api(get, + api_path(["subscriptions", binary_to_list(ClientId)]), auth_header_()), [Subscription] = get(<<"data">>, Result4), - {ok, Result4} = request_api(get, api_path(["nodes", atom_to_list(node()), "subscriptions", binary_to_list(ClientId)]) + {ok, Result4} = request_api(get, + api_path(["nodes", atom_to_list(node()), "subscriptions", binary_to_list(ClientId)]) , auth_header_()), ok = emqtt:disconnect(C1). @@ -623,7 +693,8 @@ t_stats(_) -> {ok, _} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), meck:new(emqx_mgmt, [passthrough, no_history]), meck:expect(emqx_mgmt, get_stats, 1, fun(_) -> {error, undefined} end), - {ok, Return} = request_api(get, api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), + {ok, Return} = request_api(get, + api_path(["nodes", atom_to_list(node()), "stats"]), auth_header_()), ?assertEqual(<<"undefined">>, get(<<"message">>, Return)), meck:unload(emqx_mgmt). @@ -632,13 +703,21 @@ t_data(_) -> ok = emqx_dashboard_admin:mnesia(boot), application:ensure_all_started(emqx_rule_engine), application:ensure_all_started(emqx_dashboard), + emqx_mgmt_data_backup:delete_all_backup_file(), {ok, Data} = request_api(post, api_path(["data","export"]), [], auth_header_(), [#{}]), #{<<"filename">> := Filename, <<"node">> := Node} = emqx_ct_http:get_http_data(Data), {ok, DataList} = request_api(get, api_path(["data","export"]), auth_header_()), - ?assertEqual(true, lists:member(emqx_ct_http:get_http_data(Data), emqx_ct_http:get_http_data(DataList))), + ?assertEqual(true, + lists:member(emqx_ct_http:get_http_data(Data), emqx_ct_http:get_http_data(DataList))), - ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename, <<"node">> => Node})), - ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename})), + ?assertMatch({ok, _}, request_api(post, + api_path(["data","import"]), [], auth_header_(), + #{<<"filename">> => Filename, <<"node">> => Node})), + ?assertMatch({ok, _}, + request_api(post, api_path(["data","import"]), [], auth_header_(), + #{<<"filename">> => Filename})), + _ = emqx_mgmt_data_backup:delete_backup_file(Filename), + emqx_mgmt_data_backup:delete_all_backup_file(), application:stop(emqx_rule_engine), application:stop(emqx_dashboard), ok. @@ -648,15 +727,45 @@ t_data_import_content(_) -> ok = emqx_dashboard_admin:mnesia(boot), application:ensure_all_started(emqx_rule_engine), application:ensure_all_started(emqx_dashboard), + emqx_mgmt_data_backup:delete_all_backup_file(), {ok, Data} = request_api(post, api_path(["data","export"]), [], auth_header_(), [#{}]), #{<<"filename">> := Filename} = emqx_ct_http:get_http_data(Data), - Dir = emqx:get_env(data_dir), + Dir = emqx_mgmt_data_backup:backup_dir(), {ok, Bin} = file:read_file(filename:join(Dir, Filename)), Content = emqx_json:decode(Bin), - ?assertMatch({ok, "{\"code\":0}"}, request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), + ct:pal("Content:::: ~p~n", [Content]), + ?assertMatch({ok, "{\"code\":0}"}, + request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), + + emqx_mgmt_data_backup:delete_all_backup_file(), application:stop(emqx_rule_engine), application:stop(emqx_dashboard). +t_keepalive(_Config) -> + application:ensure_all_started(emqx_dashboard), + Username = "user_keepalive", + ClientId = "client_keepalive", + AuthHeader = auth_header_(), + Path = api_path(["clients", ClientId, "keepalive"]), + {ok, NotFound} = request_api(put, Path, "interval=5", AuthHeader, [#{}]), + ?assertEqual("{\"message\":\"not_found\",\"code\":112}", NotFound), + {ok, C1} = emqtt:start_link(#{username => Username, clientid => ClientId}), + {ok, _} = emqtt:connect(C1), + {ok, Ok} = request_api(put, Path, "interval=5", AuthHeader, [#{}]), + ?assertEqual("{\"code\":0}", Ok), + [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), + #{conninfo := #{keepalive := Keepalive}} = emqx_connection:info(Pid), + ?assertEqual(5, Keepalive), + {ok, Error1} = request_api(put, Path, "interval=-1", AuthHeader, [#{}]), + {ok, Error2} = request_api(put, Path, "interval=65536", AuthHeader, [#{}]), + ErrMsg = #{<<"code">> => 102, + <<"message">> => <<"mqtt3.1.1 specification: keepalive must between 0~65535">>}, + ?assertEqual(ErrMsg, jiffy:decode(Error1, [return_maps])), + ?assertEqual(Error1, Error2), + emqtt:disconnect(C1), + application:stop(emqx_dashboard), + ok. + request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl new file mode 100644 index 000000000..748424a8f --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE.erl @@ -0,0 +1,71 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mongo_auth_module_migration_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-ifdef(EMQX_ENTERPRISE). +-include_lib("emqx_modules/include/emqx_modules.hrl"). +-endif. + +all() -> + emqx_ct:all(?MODULE). + +-ifdef(EMQX_ENTERPRISE). + +init_per_suite(Config) -> + application:load(emqx_modules_spec), + emqx_ct_helpers:start_apps([emqx_management, emqx_modules]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]), + application:unload(emqx_modules_spec), + ok. + +t_import_4_2(Config) -> + ?assertMatch(ok, import("e4.2.8.json", Config)), + timer:sleep(100), + + MongoAuthNModule = emqx_modules_registry:find_module_by_type(mongo_authentication), + ?assertNotEqual(not_found, MongoAuthNModule), + ?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule), + delete_modules(). + +t_import_4_3(Config) -> + ?assertMatch(ok, import("e4.3.5.json", Config)), + timer:sleep(100), + + MongoAuthNModule = emqx_modules_registry:find_module_by_type(mongo_authentication), + ?assertNotEqual(not_found, MongoAuthNModule), + ?assertMatch(#module{config = #{<<"srv_record">> := _}}, MongoAuthNModule), + delete_modules(). + +import(File, Config) -> + Filename = filename:join(proplists:get_value(data_dir, Config), File), + {ok, Content} = file:read_file(Filename), + BackupFile = filename:join(emqx:get_env(data_dir), File), + ok = file:write_file(BackupFile, Content), + emqx_mgmt_data_backup:import(File, "{}"). + +delete_modules() -> + [emqx_modules_registry:remove_module(Mod) || Mod <- emqx_modules_registry:get_modules()]. + +-endif. diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json new file mode 100644 index 000000000..0ea956a93 --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.2.8.json @@ -0,0 +1 @@ +{"version":"4.2","date":"2021-11-15 01:52:40","modules":[{"id":"module:79002e0f","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:34834081","type":"presence","config":{"qos":0},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:f6eb69d1","type":"recon","config":{},"enabled":true,"created_at":1636941076704,"description":""},{"id":"module:7ae737b2","type":"mongo_authentication","config":{"w_mode":"undef","verify":false,"type":"single","super_query_selector":"","super_query_field":"","super_query_collection":"","ssl":false,"server":"127.0.0.1:27017","r_mode":"undef","pool_size":8,"password":"public","login":"admin","keyfile":{"filename":"","file":""},"database":"mqtt","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auth_source":"admin","auth_query_selector":"username=%u","auth_query_password_hash":"sha256","auth_query_password_field":"password","auth_query_collection":"mqtt_user","acl_query_selectors":[],"acl_query_collection":"mqtt_acl"},"enabled":false,"created_at":1636941148794,"description":""},{"id":"module:e8c63201","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1636941076704,"description":""}],"rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"qP5m2iS9qnn51gHoGLbaiMo/GwE=","tags":"administrator"}],"auth_mnesia":[],"acl_mnesia":[],"schemas":[],"configs":[],"listeners_state":[]} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json new file mode 100644 index 000000000..48828a9c0 --- /dev/null +++ b/apps/emqx_management/test/emqx_mongo_auth_module_migration_SUITE_data/e4.3.5.json @@ -0,0 +1 @@ +{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"/mWV4UgV0xmVUZX4qdIXQvxXZB0=","tags":"administrator"}],"auth_mnesia":[],"acl_mnesia":[],"modules":[{"id":"module:5881add2","type":"mongo_authentication","config":{"w_mode":"undef","verify":false,"type":"single","super_query_selector":"","super_query_field":"","super_query_collection":"","ssl":false,"server":"127.0.0.1:27017","r_mode":"undef","pool_size":8,"password":"public","login":"admin","keyfile":{"filename":"","file":""},"database":"mqtt","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auth_source":"admin","auth_query_selector":"username=%u","auth_query_password_hash":"sha256","auth_query_password_field":"password","auth_query_collection":"mqtt_user","acl_query_selectors":[],"acl_query_collection":"mqtt_acl"},"enabled":false,"created_at":1636942609573,"description":""},{"id":"module:2adb6480","type":"presence","config":{"qos":0},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:24fabe8a","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:22c70ab8","type":"recon","config":{},"enabled":true,"created_at":1636942586725,"description":""},{"id":"module:a59f9a4a","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1636942586725,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2021-11-15 10:16:56"} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl index 763c34134..e289eec18 100644 --- a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl @@ -46,8 +46,11 @@ remove_resource(Id) -> emqx_rule_registry:remove_resource(Id), emqx_rule_registry:remove_resource_params(Id). -import(FilePath, Version) -> - ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), +import(FilePath0, Version) -> + Filename = filename:basename(FilePath0), + FilePath = filename:join([get_data_path(), FilePath0]), + {ok, Bin} = file:read_file(FilePath), + ok = emqx_mgmt_data_backup:upload_backup_file(Filename, Bin), lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> case Id of <<"webhook">> -> diff --git a/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl b/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl new file mode 100644 index 000000000..2bb3f9b16 --- /dev/null +++ b/apps/emqx_plugin_libs/include/emqx_slow_subs.hrl @@ -0,0 +1,38 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(TOPK_TAB, emqx_slow_subs_topk). +-define(INDEX_TAB, emqx_slow_subs_index). + +-define(ID(ClientId, Topic), {ClientId, Topic}). +-define(INDEX(TimeSpan, Id), {Id, TimeSpan}). +-define(TOPK_INDEX(TimeSpan, Id), {TimeSpan, Id}). + +-define(MAX_SIZE, 1000). + +-record(top_k, { index :: topk_index() + , last_update_time :: pos_integer() + , extra = [] + }). + +-record(index_tab, { index :: index()}). + +-type top_k() :: #top_k{}. +-type index_tab() :: #index_tab{}. + +-type id() :: {emqx_types:clientid(), emqx_types:topic()}. +-type index() :: ?INDEX(non_neg_integer(), id()). +-type topk_index() :: ?TOPK_INDEX(non_neg_integer(), id()). diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index f72ffc229..5fbd21fab 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.3.2"}, + {vsn, "4.4.3"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index 62d0ce4f0..b183c5e0a 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -1,16 +1,27 @@ -%% -*-: erlang -*- - +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {<<"4\\.3\\.[0-1]">>, [ - {load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], - [ - {<<"4\\.3\\.[0-1]">>, [ - {load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. + [{"4.4.2",[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.0", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}, + {update,emqx_slow_subs,{advanced,["4.4.0"]}}, + {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.4.2",[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.0", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}, + {update,emqx_slow_subs,{advanced,["4.4.0"]}}, + {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl new file mode 100644 index 000000000..07cab1bd8 --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs.erl @@ -0,0 +1,337 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs). + +-behaviour(gen_server). + +-include_lib("include/emqx.hrl"). +-include_lib("include/logger.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_slow_subs.hrl"). + +-logger_header("[SLOW Subs]"). + +-export([ start_link/1, on_delivery_completed/4, enable/0 + , disable/0, clear_history/0, init_tab/0 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-compile(nowarn_unused_type). + +-type state() :: #{ config := proplist:proplist() + , enable := boolean() + , last_tick_at := pos_integer() + }. + +-type message() :: #message{}. + +-import(proplists, [get_value/2, get_value/3]). + +-type stats_type() :: whole %% whole = internal + response + | internal %% timespan from message in to deliver + | response. %% timespan from delivery to client response + +-type stats_update_args() :: #{session_birth_time := pos_integer()}. + +-type stats_update_env() :: #{ threshold := non_neg_integer() + , stats_type := stats_type() + , max_size := pos_integer()}. + +-ifdef(TEST). +-define(EXPIRE_CHECK_INTERVAL, timer:seconds(1)). +-else. +-define(EXPIRE_CHECK_INTERVAL, timer:seconds(10)). +-endif. + +-define(NOW, erlang:system_time(millisecond)). +-define(DEF_CALL_TIMEOUT, timer:seconds(10)). + +%% erlang term order +%% number < atom < reference < fun < port < pid < tuple < list < bit string + +%% ets ordered_set is ascending by term order + +%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- +%% @doc Start the st_statistics +-spec(start_link(Env :: list()) -> emqx_types:startlink_ret()). +start_link(Env) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [Env], []). + +on_delivery_completed(_ClientInfo, #message{timestamp = Ts}, #{session_birth_time := BirthTime}, _Cfg) + when Ts =< BirthTime -> + ok; + +on_delivery_completed(ClientInfo, Msg, Env, Cfg) -> + on_delivery_completed(ClientInfo, Msg, Env, erlang:system_time(millisecond), Cfg). + +on_delivery_completed(#{clientid := ClientId}, + #message{topic = Topic} = Msg, + _Env, + Now, + #{threshold := Threshold, + stats_type := StatsType, + max_size := MaxSize}) -> + TimeSpan = calc_timespan(StatsType, Msg, Now), + case TimeSpan =< Threshold of + true -> ok; + _ -> + Id = ?ID(ClientId, Topic), + LastUpdateValue = find_last_update_value(Id), + case TimeSpan =< LastUpdateValue of + true -> ok; + _ -> + try_insert_to_topk(MaxSize, Now, LastUpdateValue, TimeSpan, Id) + end + end. + +clear_history() -> + gen_server:call(?MODULE, ?FUNCTION_NAME, ?DEF_CALL_TIMEOUT). + +enable() -> + gen_server:call(?MODULE, {enable, true}, ?DEF_CALL_TIMEOUT). + +disable() -> + gen_server:call(?MODULE, {enable, false}, ?DEF_CALL_TIMEOUT). + +init_tab() -> + safe_create_tab(?TOPK_TAB, [ ordered_set, public, named_table + , {keypos, #top_k.index}, {write_concurrency, true} + , {read_concurrency, true} + ]), + + safe_create_tab(?INDEX_TAB, [ ordered_set, public, named_table + , {keypos, #index_tab.index}, {write_concurrency, true} + , {read_concurrency, true} + ]). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Conf]) -> + erlang:process_flag(trap_exit, true), + expire_tick(Conf), + load(Conf), + {ok, #{config => Conf, + last_tick_at => ?NOW, + enable => true}}. + +handle_call({enable, Enable}, _From, + #{config := Cfg, enable := IsEnable} = State) -> + State2 = case Enable of + IsEnable -> + State; + true -> + load(Cfg), + State#{enable := true}; + _ -> + unload(), + State#{enable := false} + end, + {reply, ok, State2}; + +handle_call(clear_history, _, State) -> + do_clear_history(), + {reply, ok, State}; + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(expire_tick, #{config := Cfg} = State) -> + expire_tick(Cfg), + Logs = ets:tab2list(?TOPK_TAB), + do_clear(Cfg, Logs), + {noreply, State}; + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _) -> + unload(), + ok. + +code_change({down, _Vsn}, #{config := Cfg} = State, ["4.4.0"]) -> + unload(), + + MaxSize = get_value(top_k_num, Cfg), + _ = emqx:hook('message.slow_subs_stats', + {?MODULE, on_stats_update, [#{max_size => MaxSize}]}), + + erlang:send_after(?EXPIRE_CHECK_INTERVAL, self(), ?FUNCTION_NAME), + + {ok, State}; + +code_change(_OldVsn, #{config := Conf} = State, ["4.4.0"]) -> + %% clear old data + HookPoint = 'message.slow_subs_stats', + Callbacks = emqx_hooks:lookup(HookPoint), + _ = [emqx_hooks:del(HookPoint, Action) || + {callback, Action, _Filter, _Priority} <- Callbacks], + try + ets:delete_all_objects(?TOPK_TAB) + catch _:_ -> + ok + end, + + %% add new table + init_tab(), + [_Sup, SupPid] = erlang:get('$ancestors'), + ets:give_away(?INDEX_TAB, SupPid, undefined), + + %% enable + expire_tick(Conf), + load(Conf), + {ok, State}; + +code_change(_OldVsn, State, _Extras) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +expire_tick(_) -> + erlang:send_after(?EXPIRE_CHECK_INTERVAL, self(), ?FUNCTION_NAME). + +load(Cfg) -> + MaxSize = get_value(top_k_num, Cfg), + StatsType = get_value(stats_type, Cfg, whole), + Threshold = get_value(threshold, Cfg), + _ = emqx:hook('delivery.completed', + fun ?MODULE:on_delivery_completed/4, + [#{max_size => MaxSize, + stats_type => StatsType, + threshold => Threshold + }]), + ok. + +unload() -> + emqx:unhook('delivery.completed', fun ?MODULE:on_delivery_completed/4), + do_clear_history(). + +do_clear(Cfg, Logs) -> + Now = ?NOW, + Interval = get_value(expire_interval, Cfg), + Each = fun(#top_k{index = ?TOPK_INDEX(TimeSpan, Id), last_update_time = Ts}) -> + case Now - Ts >= Interval of + true -> + delete_with_index(TimeSpan, Id); + _ -> + true + end + end, + lists:foreach(Each, Logs). + +-spec calc_timespan(stats_type(), emqx_types:message(), non_neg_integer()) -> non_neg_integer(). +calc_timespan(whole, #message{timestamp = Ts}, Now) -> + Now - Ts; + +calc_timespan(internal, #message{timestamp = Ts} = Msg, Now) -> + End = emqx_message:get_header(deliver_begin_at, Msg, Now), + End - Ts; + +calc_timespan(response, Msg, Now) -> + Begin = emqx_message:get_header(deliver_begin_at, Msg, Now), + Now - Begin. + +%% update_topk is safe, because each process has a unique clientid +%% insert or delete are bind to this clientid, so there is no race condition +%% +%% but, the delete_with_index in L249 may have a race condition +%% because the data belong to other clientid will be deleted here (deleted the data written by other processes).%% so it may appear that: +%% when deleting a record, the other process is performing an update operation on this recrod +%% in order to solve this race condition problem, the index table also uses the ordered_set type, +%% so that even if the above situation occurs, it will only cause the old data to be deleted twice +%% and the correctness of the data will not be affected + +try_insert_to_topk(MaxSize, Now, LastUpdateValue, TimeSpan, Id) -> + case ets:info(?TOPK_TAB, size) of + Size when Size < MaxSize -> + update_topk(Now, LastUpdateValue, TimeSpan, Id); + _Size -> + case ets:first(?TOPK_TAB) of + '$end_of_table' -> + update_topk(Now, LastUpdateValue, TimeSpan, Id); + ?TOPK_INDEX(_, Id) -> + update_topk(Now, LastUpdateValue, TimeSpan, Id); + ?TOPK_INDEX(Min, MinId) -> + case TimeSpan =< Min of + true -> false; + _ -> + update_topk(Now, LastUpdateValue, TimeSpan, Id), + delete_with_index(Min, MinId) + end + end + end. + +-spec find_last_update_value(id()) -> non_neg_integer(). +find_last_update_value(Id) -> + case ets:next(?INDEX_TAB, ?INDEX(0, Id)) of + ?INDEX(LastUpdateValue, Id) -> + LastUpdateValue; + _ -> + 0 + end. + +-spec update_topk(pos_integer(), non_neg_integer(), non_neg_integer(), id()) -> true. +update_topk(Now, LastUpdateValue, TimeSpan, Id) -> + %% update record + ets:insert(?TOPK_TAB, #top_k{index = ?TOPK_INDEX(TimeSpan, Id), + last_update_time = Now, + extra = [] + }), + + %% update index + ets:insert(?INDEX_TAB, #index_tab{index = ?INDEX(TimeSpan, Id)}), + + %% delete the old record & index + delete_with_index(LastUpdateValue, Id). + +-spec delete_with_index(non_neg_integer(), id()) -> true. +delete_with_index(0, _) -> + true; + +delete_with_index(TimeSpan, Id) -> + ets:delete(?INDEX_TAB, ?INDEX(TimeSpan, Id)), + ets:delete(?TOPK_TAB, ?TOPK_INDEX(TimeSpan, Id)). + +safe_create_tab(Name, Opts) -> + case ets:whereis(Name) of + undefined -> + Name = ets:new(Name, Opts); + _ -> + Name + end. + +do_clear_history() -> + ets:delete_all_objects(?INDEX_TAB), + ets:delete_all_objects(?TOPK_TAB). diff --git a/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl new file mode 100644 index 000000000..9c150a9f1 --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_slow_subs/emqx_slow_subs_api.erl @@ -0,0 +1,116 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs_api). + +-rest_api(#{name => clear_history, + method => 'DELETE', + path => "/slow_subscriptions", + func => clear_history, + descr => "Clear current data and re count slow topic"}). + +-rest_api(#{name => get_history, + method => 'GET', + path => "/slow_subscriptions", + func => get_history, + descr => "Get slow topics statistics record data"}). + +-export([ clear_history/2 + , get_history/2 + , get_history/0 + ]). + +-include_lib("emqx_plugin_libs/include/emqx_slow_subs.hrl"). + +-define(DEFAULT_RPC_TIMEOUT, timer:seconds(5)). + +-import(minirest, [return/1]). + +%%-------------------------------------------------------------------- +%% HTTP API +%%-------------------------------------------------------------------- + +clear_history(_Bindings, _Params) -> + Nodes = ekka_mnesia:running_nodes(), + _ = [rpc_call(Node, emqx_slow_subs, clear_history, [], ok, ?DEFAULT_RPC_TIMEOUT) + || Node <- Nodes], + return(ok). + +get_history(_Bindings, _Params) -> + execute_when_enabled(fun do_get_history/0). + +get_history() -> + Node = node(), + RankL = ets:tab2list(?TOPK_TAB), + ConvFun = fun(#top_k{index = ?TOPK_INDEX(TimeSpan, ?ID(ClientId, Topic)), + last_update_time = LastUpdateTime + }) -> + #{ clientid => ClientId + , node => Node + , topic => Topic + , timespan => TimeSpan + , last_update_time => LastUpdateTime + } + end, + + lists:map(ConvFun, RankL). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +do_get_history() -> + Nodes = ekka_mnesia:running_nodes(), + Fun = fun(Node, Acc) -> + NodeRankL = rpc_call(Node, + ?MODULE, + get_history, + [], + [], + ?DEFAULT_RPC_TIMEOUT), + NodeRankL ++ Acc + end, + + RankL = lists:foldl(Fun, [], Nodes), + + SortFun = fun(#{timespan := A}, #{timespan := B}) -> + A > B + end, + + SortedL = lists:sort(SortFun, RankL), + SortedL2 = lists:sublist(SortedL, ?MAX_SIZE), + + return({ok, SortedL2}). + +rpc_call(Node, M, F, A, _ErrorR, _T) when Node =:= node() -> + erlang:apply(M, F, A); + +rpc_call(Node, M, F, A, ErrorR, T) -> + case rpc:call(Node, M, F, A, T) of + {badrpc, _} -> ErrorR; + Res -> Res + end. + +-ifdef(EMQX_ENTERPRISE). +execute_when_enabled(Fun) -> + Fun(). +-else. +%% this code from emqx_mod_api_topics_metrics:execute_when_enabled +execute_when_enabled(Fun) -> + case emqx_modules:find_module(emqx_mod_slow_subs) of + [{_, true}] -> Fun(); + _ -> return({error, module_not_loaded}) + end. +-endif. diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl new file mode 100644 index 000000000..84673b4a7 --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -0,0 +1,496 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_trace). + +-behaviour(gen_server). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[Tracer]"). + +-export([ publish/1 + , subscribe/3 + , unsubscribe/2 + ]). + +-export([ start_link/0 + , list/0 + , list/1 + , get_trace_filename/1 + , create/1 + , delete/1 + , clear/0 + , update/2 + ]). + +-export([ format/1 + , zip_dir/0 + , filename/2 + , trace_dir/0 + , trace_file/1 + , delete_files_after_send/2 + , is_enable/0 + ]). + + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). + +-define(TRACE, ?MODULE). +-define(MAX_SIZE, 30). + +-ifdef(TEST). +-export([ log_file/2 + , create_table/0 + , find_closest_time/2 + ]). +-endif. + +-export_type([ip_address/0]). +-type ip_address() :: string(). + +-record(?TRACE, + { name :: binary() | undefined | '_' + , type :: clientid | topic | ip_address | undefined | '_' + , filter :: emqx_types:topic() | emqx_types:clientid() | ip_address() | undefined | '_' + , enable = true :: boolean() | '_' + , start_at :: integer() | undefined | '_' + , end_at :: integer() | undefined | '_' + }). + +publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore; +publish(#message{from = From, topic = Topic, payload = Payload}) when + is_binary(From); is_atom(From) -> + emqx_logger:info( + #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "PUBLISH to ~s: ~0p", + [Topic, Payload] + ). + +subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore; +subscribe(Topic, SubId, SubOpts) -> + emqx_logger:info( + #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "~ts SUBSCRIBE ~ts: Options: ~0p", + [SubId, Topic, SubOpts] + ). + +unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; +unsubscribe(Topic, SubOpts) -> + emqx_logger:info( + #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}}, + "~ts UNSUBSCRIBE ~ts: Options: ~0p", + [maps:get(subid, SubOpts, ""), Topic, SubOpts] + ). + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec list() -> [tuple()]. +list() -> + ets:match_object(?TRACE, #?TRACE{_ = '_'}). + +-spec is_enable() -> boolean(). +is_enable() -> + undefined =/= erlang:whereis(?MODULE). + +-spec list(boolean()) -> [tuple()]. +list(Enable) -> + ets:match_object(?TRACE, #?TRACE{enable = Enable, _ = '_'}). + +-spec create([{Key :: binary(), Value :: binary()}] | #{atom() => binary()}) -> + ok | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. +create(Trace) -> + case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of + true -> + case to_trace(Trace) of + {ok, TraceRec} -> insert_new_trace(TraceRec); + {error, Reason} -> {error, Reason} + end; + false -> + {error, "The number of traces created has reache the maximum" + " please delete the useless ones first"} + end. + +-spec delete(Name :: binary()) -> ok | {error, not_found}. +delete(Name) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name) of + [_] -> mnesia:delete(?TRACE, Name, write); + [] -> mnesia:abort(not_found) + end + end, + transaction(Tran). + +-spec clear() -> ok | {error, Reason :: term()}. +clear() -> + case mnesia:clear_table(?TRACE) of + {atomic, ok} -> ok; + {aborted, Reason} -> {error, Reason} + end. + +-spec update(Name :: binary(), Enable :: boolean()) -> + ok | {error, not_found | finished}. +update(Name, Enable) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name) of + [] -> mnesia:abort(not_found); + [#?TRACE{enable = Enable}] -> ok; + [Rec] -> + case erlang:system_time(second) >= Rec#?TRACE.end_at of + false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write); + true -> mnesia:abort(finished) + end + end + end, + transaction(Tran). + +-spec get_trace_filename(Name :: binary()) -> + {ok, FileName :: string()} | {error, not_found}. +get_trace_filename(Name) -> + Tran = fun() -> + case mnesia:read(?TRACE, Name, read) of + [] -> mnesia:abort(not_found); + [#?TRACE{start_at = Start}] -> {ok, filename(Name, Start)} + end end, + transaction(Tran). + +-spec trace_file(File :: list()) -> + {ok, Node :: list(), Binary :: binary()} | + {error, Node :: list(), Reason :: term()}. +trace_file(File) -> + FileName = filename:join(trace_dir(), File), + Node = atom_to_list(node()), + case file:read_file(FileName) of + {ok, Bin} -> {ok, Node, Bin}; + {error, Reason} -> {error, Node, Reason} + end. + +delete_files_after_send(TraceLog, Zips) -> + gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}). + +-spec format(list(#?TRACE{})) -> list(map()). +format(Traces) -> + Fields = record_info(fields, ?TRACE), + lists:map(fun(Trace0 = #?TRACE{}) -> + [_ | Values] = tuple_to_list(Trace0), + maps:from_list(lists:zip(Fields, Values)) + end, Traces). + +init([]) -> + ok = create_table(), + erlang:process_flag(trap_exit, true), + OriginLogLevel = emqx_logger:get_primary_log_level(), + ok = filelib:ensure_dir(filename:join([trace_dir(), dummy])), + ok = filelib:ensure_dir(filename:join([zip_dir(), dummy])), + {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), + Traces = get_enable_trace(), + ok = update_log_primary_level(Traces, OriginLogLevel), + TRef = update_trace(Traces), + {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. + +create_table() -> + ok = ekka_mnesia:create_table(?TRACE, [ + {type, set}, + {disc_copies, [node()]}, + {record_name, ?TRACE}, + {attributes, record_info(fields, ?TRACE)}]), + ok = ekka_mnesia:copy_table(?TRACE, disc_copies). + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ok, State}. + +handle_cast({delete_tag, Pid, Files}, State = #{monitors := Monitors}) -> + erlang:monitor(process, Pid), + {noreply, State#{monitors => Monitors#{Pid => Files}}}; +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitors}) -> + case maps:take(Pid, Monitors) of + error -> {noreply, State}; + {Files, NewMonitors} -> + lists:foreach(fun file:delete/1, Files), + {noreply, State#{monitors => NewMonitors}} + end; +handle_info({timeout, TRef, update_trace}, + #{timer := TRef, primary_log_level := OriginLogLevel} = State) -> + Traces = get_enable_trace(), + ok = update_log_primary_level(Traces, OriginLogLevel), + NextTRef = update_trace(Traces), + {noreply, State#{timer => NextTRef}}; + +handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> + emqx_misc:cancel_timer(TRef), + handle_info({timeout, TRef, update_trace}, State); + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> + ok = set_log_primary_level(OriginLogLevel), + _ = mnesia:unsubscribe({table, ?TRACE, simple}), + emqx_misc:cancel_timer(TRef), + stop_all_trace_handler(), + _ = file:del_dir_r(zip_dir()), + ok. + +code_change(_, State, _Extra) -> + {ok, State}. + +insert_new_trace(Trace) -> + Tran = fun() -> + case mnesia:read(?TRACE, Trace#?TRACE.name) of + [] -> + #?TRACE{start_at = StartAt, type = Type, filter = Filter} = Trace, + Match = #?TRACE{_ = '_', start_at = StartAt, type = Type, filter = Filter}, + case mnesia:match_object(?TRACE, Match, read) of + [] -> mnesia:write(?TRACE, Trace, write); + [#?TRACE{name = Name}] -> mnesia:abort({duplicate_condition, Name}) + end; + [#?TRACE{name = Name}] -> mnesia:abort({already_existed, Name}) + end + end, + transaction(Tran). + +update_trace(Traces) -> + Now = erlang:system_time(second), + {_Waiting, Running, Finished} = classify_by_time(Traces, Now), + disable_finished(Finished), + Started = emqx_trace_handler:running(), + {NeedRunning, AllStarted} = start_trace(Running, Started), + NeedStop = AllStarted -- NeedRunning, + ok = stop_trace(NeedStop, Started), + clean_stale_trace_files(), + NextTime = find_closest_time(Traces, Now), + emqx_misc:start_timer(NextTime, update_trace). + +stop_all_trace_handler() -> + lists:foreach(fun(#{id := Id}) -> emqx_trace_handler:uninstall(Id) end, + emqx_trace_handler:running()). +get_enable_trace() -> + {atomic, Traces} = + mnesia:transaction(fun() -> + mnesia:match_object(?TRACE, #?TRACE{enable = true, _ = '_'}, read) + end), + Traces. + +find_closest_time(Traces, Now) -> + Sec = + lists:foldl( + fun(#?TRACE{start_at = Start, end_at = End, enable = true}, Closest) -> + min(closest(End, Now, Closest), closest(Start, Now, Closest)); + (_, Closest) -> Closest + end, 60 * 15, Traces), + timer:seconds(Sec). + +closest(Time, Now, Closest) when Now >= Time -> Closest; +closest(Time, Now, Closest) -> min(Time - Now, Closest). + +disable_finished([]) -> ok; +disable_finished(Traces) -> + transaction(fun() -> + lists:map(fun(#?TRACE{name = Name}) -> + case mnesia:read(?TRACE, Name, write) of + [] -> ok; + [Trace] -> mnesia:write(?TRACE, Trace#?TRACE{enable = false}, write) + end end, Traces) + end). + +start_trace(Traces, Started0) -> + Started = lists:map(fun(#{name := Name}) -> Name end, Started0), + lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> + case lists:member(Name, StartedAcc) of + true -> + {[Name | Running], StartedAcc}; + false -> + case start_trace(Trace) of + ok -> {[Name | Running], [Name | StartedAcc]}; + {error, _Reason} -> {[Name | Running], StartedAcc} + end + end + end, {[], Started}, Traces). + +start_trace(Trace) -> + #?TRACE{name = Name + , type = Type + , filter = Filter + , start_at = Start + } = Trace, + Who = #{name => Name, type => Type, filter => Filter}, + emqx_trace_handler:install(Who, debug, log_file(Name, Start)). + +stop_trace(Finished, Started) -> + lists:foreach(fun(#{name := Name, type := Type}) -> + case lists:member(Name, Finished) of + true -> emqx_trace_handler:uninstall(Type, Name); + false -> ok + end + end, Started). + +clean_stale_trace_files() -> + TraceDir = trace_dir(), + case file:list_dir(TraceDir) of + {ok, AllFiles} when AllFiles =/= ["zip"] -> + FileFun = fun(#?TRACE{name = Name, start_at = StartAt}) -> filename(Name, StartAt) end, + KeepFiles = lists:map(FileFun, list()), + case AllFiles -- ["zip" | KeepFiles] of + [] -> ok; + DeleteFiles -> + DelFun = fun(F) -> file:delete(filename:join(TraceDir, F)) end, + lists:foreach(DelFun, DeleteFiles) + end; + _ -> ok + end. + +classify_by_time(Traces, Now) -> + classify_by_time(Traces, Now, [], [], []). + +classify_by_time([], _Now, Wait, Run, Finish) -> {Wait, Run, Finish}; +classify_by_time([Trace = #?TRACE{start_at = Start} | Traces], + Now, Wait, Run, Finish) when Start > Now -> + classify_by_time(Traces, Now, [Trace | Wait], Run, Finish); +classify_by_time([Trace = #?TRACE{end_at = End} | Traces], + Now, Wait, Run, Finish) when End =< Now -> + classify_by_time(Traces, Now, Wait, Run, [Trace | Finish]); +classify_by_time([Trace | Traces], Now, Wait, Run, Finish) -> + classify_by_time(Traces, Now, Wait, [Trace | Run], Finish). + +to_trace(TraceParam) -> + case to_trace(ensure_map(TraceParam), #?TRACE{}) of + {error, Reason} -> {error, Reason}; + {ok, #?TRACE{name = undefined}} -> + {error, "name required"}; + {ok, #?TRACE{type = undefined}} -> + {error, "type=[topic,clientid,ip_address] required"}; + {ok, TraceRec0 = #?TRACE{}} -> + case fill_default(TraceRec0) of + #?TRACE{start_at = Start, end_at = End} when End =< Start -> + {error, "failed by start_at >= end_at"}; + TraceRec -> + {ok, TraceRec} + end + end. + +ensure_map(#{} = Trace) -> Trace; +ensure_map(Trace) when is_list(Trace) -> + lists:foldl( + fun({K, V}, Acc) when is_binary(K) -> Acc#{binary_to_existing_atom(K) => V}; + ({K, V}, Acc) when is_atom(K) -> Acc#{K => V}; + (_, Acc) -> Acc + end, #{}, Trace). + +fill_default(Trace = #?TRACE{start_at = undefined}) -> + fill_default(Trace#?TRACE{start_at = erlang:system_time(second)}); +fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> + fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60}); +fill_default(Trace) -> Trace. + +-define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$"). + +to_trace(#{name := Name} = Trace, Rec) -> + case re:run(Name, ?NAME_RE) of + nomatch -> {error, "Name should be " ?NAME_RE}; + _ -> to_trace(maps:remove(name, Trace), Rec#?TRACE{name = Name}) + end; +to_trace(#{type := <<"clientid">>, clientid := Filter} = Trace, Rec) -> + Trace0 = maps:without([type, clientid], Trace), + to_trace(Trace0, Rec#?TRACE{type = clientid, filter = Filter}); +to_trace(#{type := <<"topic">>, topic := Filter} = Trace, Rec) -> + case validate_topic(Filter) of + ok -> + Trace0 = maps:without([type, topic], Trace), + to_trace(Trace0, Rec#?TRACE{type = topic, filter = Filter}); + Error -> Error + end; +to_trace(#{type := <<"ip_address">>, ip_address := Filter} = Trace, Rec) -> + case validate_ip_address(Filter) of + ok -> + Trace0 = maps:without([type, ip_address], Trace), + to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = Filter}); + Error -> Error + end; +to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])}; +to_trace(#{start_at := StartAt} = Trace, Rec) -> + case to_system_second(StartAt) of + {ok, Sec} -> to_trace(maps:remove(start_at, Trace), Rec#?TRACE{start_at = Sec}); + {error, Reason} -> {error, Reason} + end; +to_trace(#{end_at := EndAt} = Trace, Rec) -> + Now = erlang:system_time(second), + case to_system_second(EndAt) of + {ok, Sec} when Sec > Now -> + to_trace(maps:remove(end_at, Trace), Rec#?TRACE{end_at = Sec}); + {ok, _Sec} -> + {error, "end_at time has already passed"}; + {error, Reason} -> + {error, Reason} + end; +to_trace(_, Rec) -> {ok, Rec}. + +validate_topic(TopicName) -> + try emqx_topic:validate(filter, TopicName) of + true -> ok + catch + error:Error -> + {error, io_lib:format("topic: ~s invalid by ~p", [TopicName, Error])} + end. +validate_ip_address(IP) -> + case inet:parse_address(binary_to_list(IP)) of + {ok, _} -> ok; + {error, Reason} -> {error, lists:flatten(io_lib:format("ip address: ~p", [Reason]))} + end. + +to_system_second(At) -> + try + Sec = calendar:rfc3339_to_system_time(binary_to_list(At), [{unit, second}]), + Now = erlang:system_time(second), + {ok, erlang:max(Now, Sec)} + catch error: {badmatch, _} -> + {error, ["The rfc3339 specification not satisfied: ", At]} + end. + +zip_dir() -> + filename:join(trace_dir(), "zip"). + +trace_dir() -> + filename:join(emqx:get_env(data_dir), "trace"). + +log_file(Name, Start) -> + filename:join(trace_dir(), filename(Name, Start)). + +filename(Name, Start) -> + [Time, _] = string:split(calendar:system_time_to_rfc3339(Start), "T", leading), + lists:flatten(["trace_", binary_to_list(Name), "_", Time, ".log"]). + +transaction(Tran) -> + case mnesia:transaction(Tran) of + {atomic, Res} -> Res; + {aborted, Reason} -> {error, Reason} + end. + +update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel); +update_log_primary_level(_, _) -> set_log_primary_level(debug). + +set_log_primary_level(NewLevel) -> + case NewLevel =/= emqx_logger:get_primary_log_level() of + true -> emqx_logger:set_primary_log_level(NewLevel); + false -> ok + end. diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl new file mode 100644 index 000000000..b8aa5c059 --- /dev/null +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -0,0 +1,217 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_api). +-include_lib("emqx/include/logger.hrl"). +-include_lib("kernel/include/file.hrl"). + +%% API +-export([ list_trace/2 + , create_trace/2 + , update_trace/2 + , delete_trace/2 + , clear_traces/2 + , download_zip_log/2 + , stream_log_file/2 +]). +-export([ read_trace_file/3 + , get_trace_size/0 + ]). + +-define(TO_BIN(_B_), iolist_to_binary(_B_)). +-define(NOT_FOUND(N), {error, 'NOT_FOUND', ?TO_BIN([N, " NOT FOUND"])}). + +list_trace(_, _Params) -> + case emqx_trace:list() of + [] -> {ok, []}; + List0 -> + List = lists:sort(fun(#{start_at := A}, #{start_at := B}) -> A > B end, + emqx_trace:format(List0)), + Nodes = ekka_mnesia:running_nodes(), + TraceSize = cluster_call(?MODULE, get_trace_size, [], 30000), + AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize), + Now = erlang:system_time(second), + Traces = + lists:map(fun(Trace = #{name := Name, start_at := Start, + end_at := End, enable := Enable, type := Type, filter := Filter}) -> + FileName = emqx_trace:filename(Name, Start), + LogSize = collect_file_size(Nodes, FileName, AllFileSize), + Trace0 = maps:without([enable, filter], Trace), + ModEnable = emqx_trace:is_enable(), + Trace0#{ log_size => LogSize + , Type => iolist_to_binary(Filter) + , start_at => list_to_binary(calendar:system_time_to_rfc3339(Start)) + , end_at => list_to_binary(calendar:system_time_to_rfc3339(End)) + , status => status(ModEnable, Enable, Start, End, Now) + } + end, List), + {ok, Traces} + end. + +create_trace(_, Param) -> + case emqx_trace:create(Param) of + ok -> ok; + {error, {already_existed, Name}} -> + {error, 'ALREADY_EXISTED', ?TO_BIN([Name, "Already Exists"])}; + {error, {duplicate_condition, Name}} -> + {error, 'DUPLICATE_CONDITION', ?TO_BIN([Name, "Duplication Condition"])}; + {error, Reason} -> + {error, 'INCORRECT_PARAMS', ?TO_BIN(Reason)} + end. + +delete_trace(#{name := Name}, _Param) -> + case emqx_trace:delete(Name) of + ok -> ok; + {error, not_found} -> ?NOT_FOUND(Name) + end. + +clear_traces(_, _) -> + emqx_trace:clear(). + +update_trace(#{name := Name, operation := Operation}, _Param) -> + Enable = case Operation of disable -> false; enable -> true end, + case emqx_trace:update(Name, Enable) of + ok -> {ok, #{enable => Enable, name => Name}}; + {error, not_found} -> ?NOT_FOUND(Name) + end. + +%% if HTTP request headers include accept-encoding: gzip and file size > 300 bytes. +%% cowboy_compress_h will auto encode gzip format. +download_zip_log(#{name := Name}, _Param) -> + case emqx_trace:get_trace_filename(Name) of + {ok, TraceLog} -> + TraceFiles = collect_trace_file(TraceLog), + ZipDir = emqx_trace:zip_dir(), + Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), + ZipFileName0 = binary_to_list(Name) ++ ".zip", + ZipFileName = filename:join([Zips, ZipFileName0]), + {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]), + emqx_trace:delete_files_after_send(ZipFileName, Zips), + {ok, ZipFile}; + {error, Reason} -> + {error, Reason} + end. + +group_trace_file(ZipDir, TraceLog, TraceFiles) -> + lists:foldl(fun(Res, Acc) -> + case Res of + {ok, Node, Bin} -> + FileName = Node ++ "-" ++ TraceLog, + ZipName = filename:join([ZipDir, FileName]), + case file:write_file(ZipName, Bin) of + ok -> [FileName | Acc]; + _ -> Acc + end; + {error, Node, Reason} -> + ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), + Acc + end + end, [], TraceFiles). + +collect_trace_file(TraceLog) -> + cluster_call(emqx_trace, trace_file, [TraceLog], 60000). + +cluster_call(Mod, Fun, Args, Timeout) -> + Nodes = ekka_mnesia:running_nodes(), + {GoodRes, BadNodes} = rpc:multicall(Nodes, Mod, Fun, Args, Timeout), + BadNodes =/= [] andalso ?LOG(error, "rpc call failed on ~p ~p", [BadNodes, {Mod, Fun, Args}]), + GoodRes. + +stream_log_file(#{name := Name}, Params) -> + Node0 = proplists:get_value(<<"node">>, Params, atom_to_binary(node())), + Position0 = proplists:get_value(<<"position">>, Params, <<"0">>), + Bytes0 = proplists:get_value(<<"bytes">>, Params, <<"1000">>), + case to_node(Node0) of + {ok, Node} -> + Position = binary_to_integer(Position0), + Bytes = binary_to_integer(Bytes0), + case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of + {ok, Bin} -> + Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes}, + {ok, #{meta => Meta, items => Bin}}; + {eof, Size} -> + Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, + {ok, #{meta => Meta, items => <<"">>}}; + {error, Reason} -> + logger:log(error, "read_file_failed ~p", [{Node, Name, Reason, Position, Bytes}]), + {error, Reason}; + {badrpc, nodedown} -> + {error, "BadRpc node down"} + end; + {error, Reason} -> {error, Reason} + end. + +get_trace_size() -> + TraceDir = emqx_trace:trace_dir(), + Node = node(), + case file:list_dir(TraceDir) of + {ok, AllFiles} -> + lists:foldl(fun(File, Acc) -> + FullFileName = filename:join(TraceDir, File), + Acc#{{Node, File} => filelib:file_size(FullFileName)} + end, #{}, lists:delete("zip", AllFiles)); + _ -> #{} + end. + +%% this is an rpc call for stream_log_file/2 +read_trace_file(Name, Position, Limit) -> + case emqx_trace:get_trace_filename(Name) of + {error, _} = Error -> Error; + {ok, TraceFile} -> + TraceDir = emqx_trace:trace_dir(), + TracePath = filename:join([TraceDir, TraceFile]), + read_file(TracePath, Position, Limit) + end. + +read_file(Path, Offset, Bytes) -> + case file:open(Path, [read, raw, binary]) of + {ok, IoDevice} -> + try + _ = case Offset of + 0 -> ok; + _ -> file:position(IoDevice, {bof, Offset}) + end, + case file:read(IoDevice, Bytes) of + {ok, Bin} -> {ok, Bin}; + {error, Reason} -> {error, Reason}; + eof -> + {ok, #file_info{size = Size}} = file:read_file_info(IoDevice), + {eof, Size} + end + after + file:close(IoDevice) + end; + {error, Reason} -> {error, Reason} + end. + +to_node(Node) -> + try {ok, binary_to_existing_atom(Node)} + catch _:_ -> + {error, "node not found"} + end. + +collect_file_size(Nodes, FileName, AllFiles) -> + lists:foldl(fun(Node, Acc) -> + Size = maps:get({Node, FileName}, AllFiles, 0), + Acc#{Node => Size} + end, #{}, Nodes). + +%% if the module is not running, it will return stopped, user can download the trace file. +status(false, _Enable, _Start, _End, _Now) -> <<"stopped">>; +status(true, false, _Start, _End, _Now) -> <<"stopped">>; +status(true, true, Start, _End, Now) when Now < Start -> <<"waiting">>; +status(true, true, _Start, End, Now) when Now >= End -> <<"stopped">>; +status(true, true, _Start, _End, _Now) -> <<"running">>. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl new file mode 100644 index 000000000..56b81424e --- /dev/null +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -0,0 +1,348 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_trace_SUITE). + +%% API +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-record(emqx_trace, {name, type, filter, enable = true, start_at, end_at}). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([]). + +init_per_testcase(_, Config) -> + load(), + ok = emqx_trace:clear(), + Config. + +end_per_testcase(_) -> + unload(), + ok. + +t_base_create_delete(_Config) -> + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + End = to_rfc3339(Now + 30 * 60), + Name = <<"name1">>, + ClientId = <<"test-device">>, + Trace = #{ + name => Name, + type => <<"clientid">>, + clientid => ClientId, + start_at => Start, + end_at => End + }, + AnotherTrace = Trace#{name => <<"anotherTrace">>}, + ok = emqx_trace:create(Trace), + ?assertEqual({error, {already_existed, Name}}, emqx_trace:create(Trace)), + ?assertEqual({error, {duplicate_condition, Name}}, emqx_trace:create(AnotherTrace)), + [TraceRec] = emqx_trace:list(), + Expect = #emqx_trace{ + name = Name, + type = clientid, + filter = ClientId, + start_at = Now, + end_at = Now + 30 * 60 + }, + ?assertEqual(Expect, TraceRec), + ExpectFormat = [ + #{ + filter => <<"test-device">>, + enable => true, + type => clientid, + name => <<"name1">>, + start_at => Now, + end_at => Now + 30 * 60 + } + ], + ?assertEqual(ExpectFormat, emqx_trace:format([TraceRec])), + ?assertEqual(ok, emqx_trace:delete(Name)), + ?assertEqual({error, not_found}, emqx_trace:delete(Name)), + ?assertEqual([], emqx_trace:list()), + ok. + +t_create_size_max(_Config) -> + lists:map(fun(Seq) -> + Name = list_to_binary("name" ++ integer_to_list(Seq)), + Trace = [{name, Name}, {type, <<"topic">>}, + {topic, list_to_binary("/x/y/" ++ integer_to_list(Seq))}], + ok = emqx_trace:create(Trace) + end, lists:seq(1, 30)), + Trace31 = [{<<"name">>, <<"name31">>}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/31">>}], + {error, _} = emqx_trace:create(Trace31), + ok = emqx_trace:delete(<<"name30">>), + ok = emqx_trace:create(Trace31), + ?assertEqual(30, erlang:length(emqx_trace:list())), + ok. + +t_create_failed(_Config) -> + Name = {<<"name">>, <<"test">>}, + UnknownField = [Name, {<<"unknown">>, 12}], + {error, Reason1} = emqx_trace:create(UnknownField), + ?assertEqual(<<"type=[topic,clientid,ip_address] required">>, iolist_to_binary(Reason1)), + + InvalidTopic = [Name, {<<"topic">>, "#/#//"}, {<<"type">>, <<"topic">>}], + {error, Reason2} = emqx_trace:create(InvalidTopic), + ?assertEqual(<<"topic: #/#// invalid by function_clause">>, iolist_to_binary(Reason2)), + + InvalidStart = [Name, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/sys/">>}, + {<<"start_at">>, <<"2021-12-3:12">>}], + {error, Reason3} = emqx_trace:create(InvalidStart), + ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, + iolist_to_binary(Reason3)), + + InvalidEnd = [Name, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/sys/">>}, + {<<"end_at">>, <<"2021-12-3:12">>}], + {error, Reason4} = emqx_trace:create(InvalidEnd), + ?assertEqual(<<"The rfc3339 specification not satisfied: 2021-12-3:12">>, + iolist_to_binary(Reason4)), + + {error, Reason7} = emqx_trace:create([Name, {<<"type">>, <<"clientid">>}]), + ?assertEqual(<<"required clientid field">>, iolist_to_binary(Reason7)), + + InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, + {<<"type">>, <<"clientid">>}], + {error, Reason9} = emqx_trace:create(InvalidPackets4), + ?assertEqual(<<"Name should be ^[A-Za-z]+[A-Za-z0-9-_]*$">>, iolist_to_binary(Reason9)), + + ?assertEqual({error, "type=[topic,clientid,ip_address] required"}, + emqx_trace:create([{<<"name">>, <<"test-name">>}, {<<"clientid">>, <<"good">>}])), + + ?assertEqual({error, "ip address: einval"}, + emqx_trace:create([Name, {<<"type">>, <<"ip_address">>}, + {<<"ip_address">>, <<"test-name">>}])), + ok. + +t_create_default(_Config) -> + {error, "name required"} = emqx_trace:create([]), + ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, <<"good">>}]), + [#emqx_trace{name = <<"test-name">>}] = emqx_trace:list(), + ok = emqx_trace:clear(), + Trace = [ + {<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"start_at">>, <<"2021-10-28T10:54:47+08:00">>}, + {<<"end_at">>, <<"2021-10-27T10:54:47+08:00">>} + ], + {error, "end_at time has already passed"} = emqx_trace:create(Trace), + Now = erlang:system_time(second), + Trace2 = [ + {<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"start_at">>, to_rfc3339(Now + 10)}, + {<<"end_at">>, to_rfc3339(Now + 3)} + ], + {error, "failed by start_at >= end_at"} = emqx_trace:create(Trace2), + ok = emqx_trace:create([{<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}]), + [#emqx_trace{start_at = Start, end_at = End}] = emqx_trace:list(), + ?assertEqual(10 * 60, End - Start), + ?assertEqual(true, Start - erlang:system_time(second) < 5), + ok. + +t_create_with_extra_fields(_Config) -> + ok = emqx_trace:clear(), + Trace = [ + {<<"name">>, <<"test-name">>}, + {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, + {<<"clientid">>, <<"dev001">>}, + {<<"ip_address">>, <<"127.0.0.1">>} + ], + ok = emqx_trace:create(Trace), + ?assertMatch([#emqx_trace{name = <<"test-name">>, filter = <<"/x/y/z">>, type = topic}], + emqx_trace:list()), + ok. + +t_update_enable(_Config) -> + Name = <<"test-name">>, + Now = erlang:system_time(second), + End = list_to_binary(calendar:system_time_to_rfc3339(Now + 2)), + ok = emqx_trace:create([{<<"name">>, Name}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>}, {<<"end_at">>, End}]), + [#emqx_trace{enable = Enable}] = emqx_trace:list(), + ?assertEqual(Enable, true), + ok = emqx_trace:update(Name, false), + [#emqx_trace{enable = false}] = emqx_trace:list(), + ok = emqx_trace:update(Name, false), + [#emqx_trace{enable = false}] = emqx_trace:list(), + ok = emqx_trace:update(Name, true), + [#emqx_trace{enable = true}] = emqx_trace:list(), + ok = emqx_trace:update(Name, false), + [#emqx_trace{enable = false}] = emqx_trace:list(), + ?assertEqual({error, not_found}, emqx_trace:update(<<"Name not found">>, true)), + ct:sleep(2100), + ?assertEqual({error, finished}, emqx_trace:update(Name, true)), + ok. + +t_load_state(_Config) -> + Now = erlang:system_time(second), + Running = #{name => <<"Running">>, type => <<"topic">>, + topic => <<"/x/y/1">>, start_at => to_rfc3339(Now - 1), + end_at => to_rfc3339(Now + 2)}, + Waiting = [{<<"name">>, <<"Waiting">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/2">>}, {<<"start_at">>, to_rfc3339(Now + 3)}, + {<<"end_at">>, to_rfc3339(Now + 8)}], + Finished = [{<<"name">>, <<"Finished">>}, {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/3">>}, {<<"start_at">>, to_rfc3339(Now - 5)}, + {<<"end_at">>, to_rfc3339(Now)}], + ok = emqx_trace:create(Running), + ok = emqx_trace:create(Waiting), + {error, "end_at time has already passed"} = emqx_trace:create(Finished), + Traces = emqx_trace:format(emqx_trace:list()), + ?assertEqual(2, erlang:length(Traces)), + Enables = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces), + ExpectEnables = [{<<"Running">>, true}, {<<"Waiting">>, true}], + ?assertEqual(ExpectEnables, lists:sort(Enables)), + ct:sleep(3500), + Traces2 = emqx_trace:format(emqx_trace:list()), + ?assertEqual(2, erlang:length(Traces2)), + Enables2 = lists:map(fun(#{name := Name, enable := Enable}) -> {Name, Enable} end, Traces2), + ExpectEnables2 = [{<<"Running">>, false}, {<<"Waiting">>, true}], + ?assertEqual(ExpectEnables2, lists:sort(Enables2)), + ok. + +t_client_event(_Config) -> + application:set_env(emqx, allow_anonymous, true), + ClientId = <<"client-test">>, + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + Name = <<"test_client_id_event">>, + ok = emqx_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + emqtt:ping(Client), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), + ok = emqx_trace:create([{<<"name">>, <<"test_topic">>}, + {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]), + ok = emqx_trace_handler_SUITE:filesync(<<"test_topic">>, topic), + {ok, Bin} = file:read_file(emqx_trace:log_file(Name, Now)), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]), + ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), + ok = emqx_trace_handler_SUITE:filesync(<<"test_topic">>, topic), + {ok, Bin2} = file:read_file(emqx_trace:log_file(Name, Now)), + {ok, Bin3} = file:read_file(emqx_trace:log_file(<<"test_topic">>, Now)), + ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), + ?assert(erlang:byte_size(Bin) > 0), + ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)), + ?assert(erlang:byte_size(Bin3) > 0), + ok. + +t_get_log_filename(_Config) -> + Now = erlang:system_time(second), + Start = calendar:system_time_to_rfc3339(Now), + End = calendar:system_time_to_rfc3339(Now + 2), + Name = <<"name1">>, + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"ip_address">>}, + {<<"ip_address">>, <<"127.0.0.1">>}, + {<<"start_at">>, list_to_binary(Start)}, + {<<"end_at">>, list_to_binary(End)} + ], + ok = emqx_trace:create(Trace), + ?assertEqual({error, not_found}, emqx_trace:get_trace_filename(<<"test">>)), + ?assertEqual(ok, element(1, emqx_trace:get_trace_filename(Name))), + ct:sleep(3000), + ?assertEqual(ok, element(1, emqx_trace:get_trace_filename(Name))), + ok. + +t_trace_file(_Config) -> + FileName = "test.log", + Content = <<"test \n test">>, + TraceDir = emqx_trace:trace_dir(), + File = filename:join(TraceDir, FileName), + ok = file:write_file(File, Content), + {ok, Node, Bin} = emqx_trace:trace_file(FileName), + ?assertEqual(Node, atom_to_list(node())), + ?assertEqual(Content, Bin), + ok = file:delete(File), + ok. + +t_download_log(_Config) -> + ClientId = <<"client-test">>, + Now = erlang:system_time(second), + Start = to_rfc3339(Now), + Name = <<"test_client_id">>, + ok = emqx_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], + ok = emqx_trace_handler_SUITE:filesync(Name, clientid), + {ok, ZipFile} = emqx_trace_api:download_zip_log(#{name => Name}, []), + ?assert(filelib:file_size(ZipFile) > 0), + ok = emqtt:disconnect(Client), + ok. + +t_find_closed_time(_Config) -> + DefaultMs = 60 * 15000, + Now = erlang:system_time(second), + Traces2 = [], + ?assertEqual(DefaultMs, emqx_trace:find_closest_time(Traces2, Now)), + Traces3 = [#emqx_trace{name = <<"disable">>, start_at = Now + 1, + end_at = Now + 2, enable = false}], + ?assertEqual(DefaultMs, emqx_trace:find_closest_time(Traces3, Now)), + Traces4 = [#emqx_trace{name = <<"running">>, start_at = Now, end_at = Now + 10, enable = true}], + ?assertEqual(10000, emqx_trace:find_closest_time(Traces4, Now)), + Traces5 = [#emqx_trace{name = <<"waiting">>, start_at = Now + 2, + end_at = Now + 10, enable = true}], + ?assertEqual(2000, emqx_trace:find_closest_time(Traces5, Now)), + Traces = [ + #emqx_trace{name = <<"waiting">>, start_at = Now + 1, end_at = Now + 2, enable = true}, + #emqx_trace{name = <<"running0">>, start_at = Now, end_at = Now + 5, enable = true}, + #emqx_trace{name = <<"running1">>, start_at = Now - 1, end_at = Now + 1, enable = true}, + #emqx_trace{name = <<"finished">>, start_at = Now - 2, end_at = Now - 1, enable = true}, + #emqx_trace{name = <<"waiting">>, start_at = Now + 1, end_at = Now + 1, enable = true}, + #emqx_trace{name = <<"stopped">>, start_at = Now, end_at = Now + 10, enable = false} + ], + ?assertEqual(1000, emqx_trace:find_closest_time(Traces, Now)), + ok. + +to_rfc3339(Second) -> + list_to_binary(calendar:system_time_to_rfc3339(Second)). + +load() -> + emqx_trace:start_link(). + +unload() -> + gen_server:stop(emqx_trace). diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 05920e985..4ef423b78 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.4.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src index 45ec6420c..82f353e6e 100644 --- a/apps/emqx_retainer/src/emqx_retainer.appup.src +++ b/apps/emqx_retainer/src/emqx_retainer.appup.src @@ -1,14 +1,7 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.2", - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-1]">>, - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]}, + [{"4.4.0",[{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.2", - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-1]">>, - [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}]}. + [{"4.4.0",[{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}] +}. diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 226ad1947..fed25ed9f 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -78,7 +78,8 @@ dispatch(Pid, Topic) -> false -> read_messages(Topic); true -> match_messages(Topic) end, - [Pid ! {deliver, Topic, Msg} || Msg <- sort_retained(Msgs)]. + Now = erlang:system_time(millisecond), + [Pid ! {deliver, Topic, refresh_timestamp_expiry(Msg, Now)} || Msg <- sort_retained(Msgs)]. %% RETAIN flag set to 1 and payload containing zero bytes on_message_publish(Msg = #message{flags = #{retain := true}, @@ -151,7 +152,7 @@ init([Env]) -> ok end, StatsFun = emqx_stats:statsfun('retained.count', 'retained.max'), - {ok, StatsTimer} = timer:send_interval(timer:seconds(1), stats), + StatsTimer = erlang:send_after(timer:seconds(1), self(), stats), State = #state{stats_fun = StatsFun, stats_timer = StatsTimer}, {ok, start_expire_timer(proplists:get_value(expiry_interval, Env, 0), State)}. @@ -160,7 +161,7 @@ start_expire_timer(0, State) -> start_expire_timer(undefined, State) -> State; start_expire_timer(Ms, State) -> - {ok, Timer} = timer:send_interval(Ms, expire), + Timer = erlang:send_after(Ms, self(), {expire, Ms}), State#state{expiry_timer = Timer}. handle_call(Req, _From, State) -> @@ -172,12 +173,14 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info(stats, State = #state{stats_fun = StatsFun}) -> + StatsTimer = erlang:send_after(timer:seconds(1), self(), stats), StatsFun(retained_count()), - {noreply, State, hibernate}; + {noreply, State#state{stats_timer = StatsTimer}, hibernate}; -handle_info(expire, State) -> +handle_info({expire, Ms} = Expire, State) -> + Timer = erlang:send_after(Ms, self(), Expire), ok = expire_messages(), - {noreply, State, hibernate}; + {noreply, State#state{expiry_timer = Timer}, hibernate}; handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), @@ -199,7 +202,7 @@ sort_retained([]) -> []; sort_retained([Msg]) -> [Msg]; sort_retained(Msgs) -> lists:sort(fun(#message{timestamp = Ts1}, #message{timestamp = Ts2}) -> - Ts1 =< Ts2 + Ts1 =< Ts2 end, Msgs). store_retained(Msg = #message{topic = Topic, payload = Payload}, Env) -> @@ -214,11 +217,13 @@ store_retained(Msg = #message{topic = Topic, payload = Payload}, Env) -> fun() -> case mnesia:read(?TAB, Topic) of [_] -> - mnesia:write(?TAB, #retained{topic = topic2tokens(Topic), - msg = Msg, - expiry_time = get_expiry_time(Msg, Env)}, write); + mnesia:write(?TAB, + #retained{topic = topic2tokens(Topic), + msg = Msg, + expiry_time = get_expiry_time(Msg, Env)}, write); [] -> - ?LOG(error, "Cannot retain message(topic=~s) for table is full!", [Topic]) + ?LOG(error, + "Cannot retain message(topic=~s) for table is full!", [Topic]) end end), ok; @@ -242,7 +247,8 @@ is_too_big(Size, Env) -> get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := 0}}}, _Env) -> 0; -get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, timestamp = Ts}, _Env) -> +get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, + timestamp = Ts}, _Env) -> Ts + Interval * 1000; get_expiry_time(#message{timestamp = Ts}, Env) -> case proplists:get_value(expiry_interval, Env, 0) of @@ -311,3 +317,18 @@ condition(Ws) -> false -> Ws1; _ -> (Ws1 -- ['#']) ++ '_' end. + +-spec(refresh_timestamp_expiry(emqx_types:message(), pos_integer()) -> emqx_types:message()). +refresh_timestamp_expiry(Msg = #message{headers = + #{properties := + #{'Message-Expiry-Interval' := Interval} = Props}, + timestamp = CreatedAt}, + Now) -> + Elapsed = max(0, Now - CreatedAt), + Interval1 = max(1, Interval - (Elapsed div 1000)), + emqx_message:set_header(properties, + Props#{'Message-Expiry-Interval' => Interval1}, + Msg#message{timestamp = Now}); + +refresh_timestamp_expiry(Msg, Now) -> + Msg#message{timestamp = Now}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 62ead0c36..d52e51316 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.10"}, % strict semver, bump manually! + {vsn, "4.4.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 4c2f653d9..754eede15 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.9", + [{"4.4.3", [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, @@ -10,143 +10,70 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.8", + {"4.4.2", [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {add_module,emqx_rule_date}, {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_date}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}]}, - {"4.3.7", - [{add_module,emqx_rule_date}, - {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_date}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, - {"4.3.6", - [{add_module,emqx_rule_date}, - {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.6"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.5", - [{add_module,emqx_rule_date}, - {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.5"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.4", - [{add_module,emqx_rule_date}, - {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.4"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.3", - [{add_module,emqx_rule_date}, - {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.3"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.2", - [{add_module,emqx_rule_date}, - {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.2"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.1", - [{add_module,emqx_rule_date}, - {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.1"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{add_module,emqx_rule_date}, - {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + {"4.4.0", + [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.0"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_date}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {update,emqx_rule_metrics,{advanced,["4.4.0"]}}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.9", + [{"4.4.3", [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {delete_module,emqx_rule_date}, {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, - {delete_module,emqx_rule_date}]}, - {"4.3.8", + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {"4.4.2", [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {delete_module,emqx_rule_date}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -154,117 +81,20 @@ {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {delete_module,emqx_rule_date}]}, - {"4.3.7", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {delete_module,emqx_rule_date}]}, - {"4.3.6", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.6"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, - {delete_module,emqx_rule_date}]}, - {"4.3.5", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.5"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, - {delete_module,emqx_rule_date}]}, - {"4.3.4", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.4"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, - {delete_module,emqx_rule_date}]}, - {"4.3.3", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.3"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, - {delete_module,emqx_rule_date}]}, - {"4.3.2", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.2"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, - {delete_module,emqx_rule_date}]}, - {"4.3.1", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.1"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, - {delete_module,emqx_rule_date}]}, - {"4.3.0", + {delete_module,emqx_rule_date}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.4.0", [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, - {update,emqx_rule_metrics,{advanced,["4.3.0"]}}, - {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {apply,{emqx_stats,cancel_update,[rule_registery_stats]}}, - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {update,emqx_rule_metrics,{advanced,["4.4.0"]}}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {delete_module,emqx_rule_date}]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 01339edbb..d829a3b14 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -31,6 +31,7 @@ -export([ on_client_connected/3 , on_client_disconnected/4 + , on_client_connack/4 , on_session_subscribed/4 , on_session_unsubscribed/4 , on_message_publish/2 @@ -38,6 +39,7 @@ , on_message_delivered/3 , on_message_acked/3 , on_delivery_dropped/4 + , on_client_check_acl_complete/6 ]). -export([ event_info/0 @@ -48,6 +50,7 @@ -define(SUPPORTED_HOOK, [ 'client.connected' , 'client.disconnected' + , 'client.connack' , 'session.subscribed' , 'session.unsubscribed' , 'message.publish' @@ -55,6 +58,7 @@ , 'message.acked' , 'message.dropped' , 'delivery.dropped' + , 'client.check_acl_complete' ]). -ifdef(TEST). @@ -106,6 +110,18 @@ on_client_disconnected(ClientInfo, Reason, ConnInfo, Env) -> may_publish_and_apply('client.disconnected', fun() -> eventmsg_disconnected(ClientInfo, ConnInfo, Reason) end, Env). +on_client_connack(ConnInfo, Reason, _, Env) -> + may_publish_and_apply('client.connack', + fun() -> eventmsg_connack(ConnInfo, Reason) end, Env). + +on_client_check_acl_complete(ClientInfo, PubSub, Topic, Result, IsCache, Env) -> + may_publish_and_apply('client.check_acl_complete', + fun() -> eventmsg_check_acl_complete(ClientInfo, + PubSub, + Topic, + Result, + IsCache) end, Env). + on_session_subscribed(ClientInfo, Topic, SubOpts, Env) -> may_publish_and_apply('session.subscribed', fun() -> eventmsg_sub_or_unsub('session.subscribed', ClientInfo, Topic, SubOpts) end, Env). @@ -224,6 +240,46 @@ eventmsg_disconnected(_ClientInfo = #{ disconnected_at => DisconnectedAt }). +eventmsg_connack(ConnInfo = #{ + clientid := ClientId, + clean_start := CleanStart, + username := Username, + peername := PeerName, + sockname := SockName, + proto_name := ProtoName, + proto_ver := ProtoVer + }, Reason) -> + Keepalive = maps:get(keepalive, ConnInfo, 0), + ConnProps = maps:get(conn_props, ConnInfo, #{}), + ExpiryInterval = maps:get(expiry_interval, ConnInfo, 0), + with_basic_columns('client.connack', + #{reason_code => reason(Reason), + clientid => ClientId, + clean_start => CleanStart, + username => Username, + peername => ntoa(PeerName), + sockname => ntoa(SockName), + proto_name => ProtoName, + proto_ver => ProtoVer, + keepalive => Keepalive, + expiry_interval => ExpiryInterval, + conn_props => printable_maps(ConnProps) + }). +eventmsg_check_acl_complete(_ClientInfo = #{ + clientid := ClientId, + username := Username, + peerhost := PeerHost + }, PubSub, Topic, Result, IsCache) -> + with_basic_columns('client.check_acl_complete', + #{clientid => ClientId, + username => Username, + peerhost => ntoa(PeerHost), + topic => Topic, + action => PubSub, + is_cache => IsCache, + result => Result + }). + eventmsg_sub_or_unsub(Event, _ClientInfo = #{ clientid := ClientId, username := Username, @@ -376,8 +432,10 @@ event_info() -> , event_info_delivery_dropped() , event_info_client_connected() , event_info_client_disconnected() + , event_info_client_connack() , event_info_session_subscribed() , event_info_session_unsubscribed() + , event_info_client_check_acl_complete() ]. event_info_message_publish() -> @@ -431,6 +489,13 @@ event_info_client_disconnected() -> {<<"client disconnected">>, <<"连接断开"/utf8>>}, <<"SELECT * FROM \"$events/client_disconnected\" WHERE topic =~ 't/#'">> ). +event_info_client_connack() -> + event_info_common( + 'client.connack', + {<<"client connack">>, <<"连接确认"/utf8>>}, + {<<"client connack">>, <<"连接确认"/utf8>>}, + <<"SELECT * FROM \"$events/client_connack\"">> + ). event_info_session_subscribed() -> event_info_common( 'session.subscribed', @@ -445,6 +510,13 @@ event_info_session_unsubscribed() -> {<<"session unsubscribed">>, <<"会话取消订阅完成"/utf8>>}, <<"SELECT * FROM \"$events/session_unsubscribed\" WHERE topic =~ 't/#'">> ). +event_info_client_check_acl_complete() -> + event_info_common( + 'client.check_acl_complete', + {<<"client check acl complete">>, <<"鉴权结果"/utf8>>}, + {<<"client check acl complete">>, <<"鉴权结果"/utf8>>}, + <<"SELECT * FROM \"$events/client_check_acl_complete\"">> + ). event_info_common(Event, {TitleEN, TitleZH}, {DescrEN, DescrZH}, SqlExam) -> #{event => event_topic(Event), @@ -489,6 +561,11 @@ test_columns('client.disconnected') -> , {<<"username">>, <<"u_emqx">>} , {<<"reason">>, <<"normal">>} ]; +test_columns('client.connack') -> + [ {<<"clientid">>, <<"c_emqx">>} + , {<<"username">>, <<"u_emqx">>} + , {<<"reason_code">>, <<"sucess">>} + ]; test_columns('session.unsubscribed') -> test_columns('session.subscribed'); test_columns('session.subscribed') -> @@ -496,6 +573,13 @@ test_columns('session.subscribed') -> , {<<"username">>, <<"u_emqx">>} , {<<"topic">>, <<"t/a">>} , {<<"qos">>, 1} + ]; +test_columns('client.check_acl_complete') -> + [ {<<"clientid">>, <<"c_emqx">>} + , {<<"username">>, <<"u_emqx">>} + , {<<"topic">>, <<"t/1">>} + , {<<"action">>, <<"publish">>} + , {<<"result">>, <<"allow">>} ]. columns_with_exam('message.publish') -> @@ -508,8 +592,8 @@ columns_with_exam('message.publish') -> , {<<"topic">>, <<"t/a">>} , {<<"qos">>, 1} , {<<"flags">>, #{}} - , {<<"headers">>, undefined} , {<<"publish_received_at">>, erlang:system_time(millisecond)} + , columns_example_props(pub_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -526,6 +610,7 @@ columns_with_exam('message.delivered') -> , {<<"qos">>, 1} , {<<"flags">>, #{}} , {<<"publish_received_at">>, erlang:system_time(millisecond)} + , columns_example_props(pub_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -542,6 +627,8 @@ columns_with_exam('message.acked') -> , {<<"qos">>, 1} , {<<"flags">>, #{}} , {<<"publish_received_at">>, erlang:system_time(millisecond)} + , columns_example_props(pub_props) + , columns_example_props(puback_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -557,6 +644,7 @@ columns_with_exam('message.dropped') -> , {<<"qos">>, 1} , {<<"flags">>, #{}} , {<<"publish_received_at">>, erlang:system_time(millisecond)} + , columns_example_props(pub_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -591,6 +679,7 @@ columns_with_exam('client.connected') -> , {<<"expiry_interval">>, 3600} , {<<"is_bridge">>, false} , {<<"connected_at">>, erlang:system_time(millisecond)} + , columns_example_props(conn_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -604,6 +693,24 @@ columns_with_exam('client.disconnected') -> , {<<"proto_name">>, <<"MQTT">>} , {<<"proto_ver">>, 5} , {<<"disconnected_at">>, erlang:system_time(millisecond)} + , columns_example_props(disconn_props) + , {<<"timestamp">>, erlang:system_time(millisecond)} + , {<<"node">>, node()} + ]; +columns_with_exam('client.connack') -> + [ {<<"event">>, 'client.connected'} + , {<<"reason_code">>, success} + , {<<"clientid">>, <<"c_emqx">>} + , {<<"username">>, <<"u_emqx">>} + , {<<"peername">>, <<"192.168.0.10:56431">>} + , {<<"sockname">>, <<"0.0.0.0:1883">>} + , {<<"proto_name">>, <<"MQTT">>} + , {<<"proto_ver">>, 5} + , {<<"keepalive">>, 60} + , {<<"clean_start">>, true} + , {<<"expiry_interval">>, 3600} + , {<<"connected_at">>, erlang:system_time(millisecond)} + , columns_example_props(conn_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -614,6 +721,7 @@ columns_with_exam('session.subscribed') -> , {<<"peerhost">>, <<"192.168.0.10">>} , {<<"topic">>, <<"t/a">>} , {<<"qos">>, 1} + , columns_example_props(sub_props) , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]; @@ -624,10 +732,54 @@ columns_with_exam('session.unsubscribed') -> , {<<"peerhost">>, <<"192.168.0.10">>} , {<<"topic">>, <<"t/a">>} , {<<"qos">>, 1} + , columns_example_props(unsub_props) + , {<<"timestamp">>, erlang:system_time(millisecond)} + , {<<"node">>, node()} + ]; +columns_with_exam('client.check_acl_complete') -> + [ {<<"event">>, 'client.check_acl_complete'} + , {<<"clientid">>, <<"c_emqx">>} + , {<<"username">>, <<"u_emqx">>} + , {<<"peerhost">>, <<"192.168.0.10">>} + , {<<"topic">>, <<"t/a">>} + , {<<"action">>, <<"publish">>} + , {<<"is_cache">>, <<"false">>} + , {<<"result">>, <<"allow">>} , {<<"timestamp">>, erlang:system_time(millisecond)} , {<<"node">>, node()} ]. +columns_example_props(PropType) -> + Props = columns_example_props_specific(PropType), + UserProps = #{ + 'User-Property' => #{<<"foo">> => <<"bar">>}, + 'User-Property-Pairs' => [ + #{key => <<"foo">>}, #{value => <<"bar">>} + ] + }, + {PropType, maps:merge(Props, UserProps)}. + +columns_example_props_specific(pub_props) -> + #{ 'Payload-Format-Indicator' => 0 + , 'Message-Expiry-Interval' => 30 + }; +columns_example_props_specific(puback_props) -> + #{ 'Reason-String' => <<"OK">> + }; +columns_example_props_specific(conn_props) -> + #{ 'Session-Expiry-Interval' => 7200 + , 'Receive-Maximum' => 32 + }; +columns_example_props_specific(disconn_props) -> + #{ 'Session-Expiry-Interval' => 7200 + , 'Reason-String' => <<"Redirect to another server">> + , 'Server Reference' => <<"192.168.22.129">> + }; +columns_example_props_specific(sub_props) -> + #{}; +columns_example_props_specific(unsub_props) -> + #{}. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- @@ -661,6 +813,7 @@ ntoa(IpAddr) -> event_name(<<"$events/client_connected", _/binary>>) -> 'client.connected'; event_name(<<"$events/client_disconnected", _/binary>>) -> 'client.disconnected'; +event_name(<<"$events/client_connack", _/binary>>) -> 'client.connack'; event_name(<<"$events/session_subscribed", _/binary>>) -> 'session.subscribed'; event_name(<<"$events/session_unsubscribed", _/binary>>) -> 'session.unsubscribed'; @@ -668,17 +821,20 @@ event_name(<<"$events/message_delivered", _/binary>>) -> 'message.delivered'; event_name(<<"$events/message_acked", _/binary>>) -> 'message.acked'; event_name(<<"$events/message_dropped", _/binary>>) -> 'message.dropped'; event_name(<<"$events/delivery_dropped", _/binary>>) -> 'delivery.dropped'; +event_name(<<"$events/client_check_acl_complete", _/binary>>) -> 'client.check_acl_complete'; event_name(_) -> 'message.publish'. event_topic('client.connected') -> <<"$events/client_connected">>; event_topic('client.disconnected') -> <<"$events/client_disconnected">>; +event_topic('client.connack') -> <<"$events/client_connack">>; event_topic('session.subscribed') -> <<"$events/session_subscribed">>; event_topic('session.unsubscribed') -> <<"$events/session_unsubscribed">>; event_topic('message.delivered') -> <<"$events/message_delivered">>; event_topic('message.acked') -> <<"$events/message_acked">>; event_topic('message.dropped') -> <<"$events/message_dropped">>; event_topic('delivery.dropped') -> <<"$events/delivery_dropped">>; -event_topic('message.publish') -> <<"$events/message_publish">>. +event_topic('message.publish') -> <<"$events/message_publish">>; +event_topic('client.check_acl_complete') -> <<"$events/client_check_acl_complete">>. printable_maps(undefined) -> #{}; printable_maps(Headers) -> @@ -687,6 +843,10 @@ printable_maps(Headers) -> AccIn#{K => ntoa(V0)}; ('User-Property', V0, AccIn) when is_list(V0) -> AccIn#{ + %% The 'User-Property' field is for the convenience of querying properties + %% using the '.' syntax, e.g. "SELECT 'User-Property'.foo as foo" + %% However, this does not allow duplicate property keys. To allow + %% duplicate keys, we have to use the 'User-Property-Pairs' field instead. 'User-Property' => maps:from_list(V0), 'User-Property-Pairs' => [#{ key => Key, diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 282c8929d..33d09c16d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -17,6 +17,9 @@ -module(emqx_rule_funcs). -include("rule_engine.hrl"). +-elvis([{elvis_style, god_modules, disable}]). +-elvis([{elvis_style, function_naming_convention, disable}]). +-elvis([{elvis_style, macro_names, disable}]). %% IoT Funcs -export([ msgid/0 @@ -460,7 +463,8 @@ subbits(Bits, Len) when is_integer(Len), is_bitstring(Bits) -> subbits(Bits, Start, Len) when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> get_subbits(Bits, Start, Len, <<"integer">>, <<"unsigned">>, <<"big">>). -subbits(Bits, Start, Len, Type, Signedness, Endianness) when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> +subbits(Bits, Start, Len, Type, Signedness, Endianness) + when is_integer(Start), is_integer(Len), is_bitstring(Bits) -> get_subbits(Bits, Start, Len, Type, Signedness, Endianness). get_subbits(Bits, Start, Len, Type, Signedness, Endianness) -> @@ -545,7 +549,7 @@ map(Data) -> emqx_rule_utils:map(Data). bin2hexstr(Bin) when is_binary(Bin) -> - emqx_misc:bin2hexstr_A_F(Bin). + emqx_misc:bin2hexstr_a_f_upper(Bin). hexstr2bin(Str) when is_binary(Str) -> emqx_misc:hexstr2bin(Str). @@ -633,7 +637,8 @@ tokens(S, Separators) -> [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators))]. tokens(S, Separators, <<"nocrlf">>) -> - [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators) ++ [$\r,$\n,[$\r,$\n]])]. + [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), + binary_to_list(Separators) ++ [$\r,$\n,[$\r,$\n]])]. concat(S1, S2) when is_binary(S1), is_binary(S2) -> unicode:characters_to_binary([S1, S2], unicode). @@ -671,7 +676,8 @@ replace(SrcStr, P, RepStr) when is_binary(SrcStr), is_binary(P), is_binary(RepSt replace(SrcStr, P, RepStr, <<"all">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> iolist_to_binary(string:replace(SrcStr, P, RepStr, all)); -replace(SrcStr, P, RepStr, <<"trailing">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> +replace(SrcStr, P, RepStr, <<"trailing">>) + when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> iolist_to_binary(string:replace(SrcStr, P, RepStr, trailing)); replace(SrcStr, P, RepStr, <<"leading">>) when is_binary(SrcStr), is_binary(P), is_binary(RepStr) -> @@ -687,7 +693,7 @@ regex_replace(SrcStr, RE, RepStr) -> re:replace(SrcStr, RE, RepStr, [global, {return,binary}]). ascii(Char) when is_binary(Char) -> - [FirstC| _] = binary_to_list(Char), + [FirstC | _] = binary_to_list(Char), FirstC. find(S, P) when is_binary(S), is_binary(P) -> @@ -807,7 +813,7 @@ sha256(S) when is_binary(S) -> hash(sha256, S). hash(Type, Data) -> - emqx_misc:bin2hexstr_a_f(crypto:hash(Type, Data)). + emqx_misc:bin2hexstr_a_f_lower(crypto:hash(Type, Data)). %%------------------------------------------------------------------------------ %% gzip Funcs @@ -967,23 +973,23 @@ convert_timestamp(MillisecondsTimestamp) -> %% the function handling to the worker module. %% @end -ifdef(EMQX_ENTERPRISE). -'$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> +'$handle_undefined_function'(schema_decode, [SchemaId, Data | MoreArgs]) -> emqx_schema_parser:decode(SchemaId, Data, MoreArgs); '$handle_undefined_function'(schema_decode, Args) -> error({args_count_error, {schema_decode, Args}}); -'$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> +'$handle_undefined_function'(schema_encode, [SchemaId, Term | MoreArgs]) -> emqx_schema_parser:encode(SchemaId, Term, MoreArgs); '$handle_undefined_function'(schema_encode, Args) -> error({args_count_error, {schema_encode, Args}}); -'$handle_undefined_function'(sprintf, [Format|Args]) -> +'$handle_undefined_function'(sprintf, [Format | Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> error({sql_function_not_supported, function_literal(Fun, Args)}). -else. -'$handle_undefined_function'(sprintf, [Format|Args]) -> +'$handle_undefined_function'(sprintf, [Format | Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index 624032056..4a532d00b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -79,6 +79,8 @@ , terminate/2 ]). +-elvis([{elvis_style, god_modules, disable}]). + -ifndef(TEST). -define(SECS_5M, 300). -define(SAMPLING, 10). @@ -368,10 +370,10 @@ handle_info(_Info, State) -> {noreply, State}. code_change({down, _Vsn}, State = #state{metric_ids = MIDs}, [Vsn]) -> - case string:tokens(Vsn, ".") of - ["4", "3", SVal] -> + case string:tokens(Vsn, ".") of + ["4", "4", SVal] -> {Val, []} = string:to_integer(SVal), - case Val =< 6 of + case Val == 0 of true -> [begin Passed = get_rules_passed(Id), @@ -381,7 +383,7 @@ code_change({down, _Vsn}, State = #state{metric_ids = MIDs}, [Vsn]) -> Exception = get_actions_exception(Id), Retry = get_actions_retry(Id), ok = delete_counters(Id), - ok = create_counters(Id, 7), + ok = create_counters(Id, max_counters_size_old()), inc_rules_matched(Id, Passed), inc_actions_taken(Id, Take), inc_actions_success(Id, Success), @@ -397,9 +399,9 @@ code_change({down, _Vsn}, State = #state{metric_ids = MIDs}, [Vsn]) -> code_change(_Vsn, State = #state{metric_ids = MIDs}, [Vsn]) -> case string:tokens(Vsn, ".") of - ["4", "3", SVal] -> + ["4", "4", SVal] -> {Val, []} = string:to_integer(SVal), - case Val =< 6 of + case Val == 0 of true -> [begin Matched = get_rules_matched(Id), @@ -471,17 +473,19 @@ calculate_speed(CurrVal, #rule_speed{max = MaxSpeed0, last_v = LastVal, %% calculate the max speed since the emqx startup MaxSpeed = - if MaxSpeed0 >= CurrSpeed -> MaxSpeed0; - true -> CurrSpeed + case MaxSpeed0 >= CurrSpeed of + true -> MaxSpeed0; + false -> CurrSpeed end, %% calculate the average speed in last 5 mins {Last5MinSamples, Acc5Min, Last5Min} = - if Tick =< ?SAMPCOUNT_5M -> + case Tick =< ?SAMPCOUNT_5M of + true -> Acc = AccSpeed5Min0 + CurrSpeed, {lists:reverse([CurrSpeed | lists:reverse(Last5MinSamples0)]), Acc, Acc / Tick}; - true -> + false -> [FirstSpeed | Speeds] = Last5MinSamples0, Acc = AccSpeed5Min0 + CurrSpeed - FirstSpeed, {lists:reverse([CurrSpeed | lists:reverse(Speeds)]), @@ -493,7 +497,7 @@ calculate_speed(CurrVal, #rule_speed{max = MaxSpeed0, last_v = LastVal, last5m_smpl = Last5MinSamples, tick = Tick + 1}. format_rule_speed(#rule_speed{max = Max, current = Current, last5m = Last5Min}) -> - #{max => Max, current => precision(Current, 2), last5m => precision(Last5Min, 2)}. + #{max => precision(Max, 2), current => precision(Current, 2), last5m => precision(Last5Min, 2)}. precision(Float, N) -> Base = math:pow(10, N), @@ -503,6 +507,9 @@ precision(Float, N) -> %% Metrics Definitions %%------------------------------------------------------------------------------ +%% for code hot upgrade +max_counters_size_old() -> 7. + max_counters_size() -> 11. metrics_idx('rules.matched') -> 1; diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 79ac0f328..3d80313a0 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -98,21 +98,8 @@ sql_test_action() -> fill_default_values(Event, Context) -> maps:merge(envs_examp(Event), Context). -envs_examp(<<"$events/", _/binary>> = EVENT_TOPIC) -> +envs_examp(EVENT_TOPIC) -> EventName = emqx_rule_events:event_name(EVENT_TOPIC), emqx_rule_maps:atom_key_map( maps:from_list( - emqx_rule_events:columns_with_exam(EventName))); -envs_examp(_) -> - #{id => emqx_guid:to_hexstr(emqx_guid:gen()), - clientid => <<"c_emqx">>, - username => <<"u_emqx">>, - payload => <<"{\"id\": 1, \"name\": \"ha\"}">>, - peerhost => <<"127.0.0.1">>, - topic => <<"t/a">>, - qos => 1, - flags => #{sys => true, event => true}, - publish_received_at => emqx_rule_utils:now_ms(), - timestamp => emqx_rule_utils:now_ms(), - node => node() - }. + emqx_rule_events:columns_with_exam(EventName))). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index f4c76f6e0..6900f76aa 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -199,6 +199,8 @@ init_per_testcase(t_events, Config) -> description = #{en => <<"Hook metrics action">>}}), SQL = "SELECT * FROM \"$events/client_connected\", " "\"$events/client_disconnected\", " + "\"$events/client_connack\", " + "\"$events/client_check_acl_complete\", " "\"$events/session_subscribed\", " "\"$events/session_unsubscribed\", " "\"$events/message_acked\", " @@ -1074,9 +1076,10 @@ t_events(_Config) -> , {proto_ver, v5} , {properties, #{'Session-Expiry-Interval' => 60}} ]), - ct:pal("====== verify $events/client_connected"), + + ct:pal("====== verify $events/client_connected, $events/client_connack"), client_connected(Client, Client2), - ct:pal("====== verify $events/session_subscribed"), + ct:pal("====== verify $events/session_subscribed, $events/client_check_acl_complete"), session_subscribed(Client2), ct:pal("====== verify t1"), message_publish(Client), @@ -1090,6 +1093,8 @@ t_events(_Config) -> message_dropped(Client), ct:pal("====== verify $events/client_disconnected"), client_disconnected(Client, Client2), + ct:pal("====== verify $events/client_connack"), + client_connack_failed(), ok. message_publish(Client) -> @@ -1097,11 +1102,33 @@ message_publish(Client) -> <<"{\"id\": 1, \"name\": \"ha\"}">>, [{qos, 1}]), verify_event('message.publish'), ok. + client_connected(Client, Client2) -> {ok, _} = emqtt:connect(Client), {ok, _} = emqtt:connect(Client2), + verify_event('client.connack'), verify_event('client.connected'), ok. + +client_connack_failed() -> + {ok, Client} = emqtt:start_link( + [ {username, <<"u_event3">>} + , {clientid, <<"c_event3">>} + , {proto_ver, v5} + , {properties, #{'Session-Expiry-Interval' => 60}} + ]), + try + meck:new(emqx_access_control, [non_strict, passthrough]), + meck:expect(emqx_access_control, authenticate, + fun(_) -> {error, bad_username_or_password} end), + process_flag(trap_exit, true), + ?assertMatch({error, _}, emqtt:connect(Client)), + timer:sleep(300), + verify_event('client.connack') + after + meck:unload(emqx_access_control) + end, + ok. client_disconnected(Client, Client2) -> ok = emqtt:disconnect(Client, 0, #{'User-Property' => {<<"reason">>, <<"normal">>}}), ok = emqtt:disconnect(Client2, 0, #{'User-Property' => {<<"reason">>, <<"normal">>}}), @@ -1114,6 +1141,7 @@ session_subscribed(Client2) -> , 1 ), verify_event('session.subscribed'), + verify_event('client.check_acl_complete'), ok. session_unsubscribed(Client2) -> {ok, _, _} = emqtt:unsubscribe( Client2 @@ -2729,6 +2757,35 @@ verify_event_fields('client.disconnected', Fields) -> ?assert(0 =< RcvdAtElapse andalso RcvdAtElapse =< 60*1000), ?assert(EventAt =< Timestamp); +verify_event_fields('client.connack', Fields) -> + #{clientid := ClientId, + clean_start := CleanStart, + username := Username, + peername := PeerName, + sockname := SockName, + proto_name := ProtoName, + proto_ver := ProtoVer, + keepalive := Keepalive, + expiry_interval := ExpiryInterval, + conn_props := Properties, + reason_code := Reason, + timestamp := Timestamp + } = Fields, + Now = erlang:system_time(millisecond), + TimestampElapse = Now - Timestamp, + ?assert(lists:member(Reason, [success, bad_username_or_password])), + ?assert(lists:member(ClientId, [<<"c_event">>, <<"c_event2">>, <<"c_event3">>])), + ?assert(lists:member(Username, [<<"u_event">>, <<"u_event2">>, <<"u_event3">>])), + verify_peername(PeerName), + verify_peername(SockName), + ?assertEqual(<<"MQTT">>, ProtoName), + ?assertEqual(5, ProtoVer), + ?assert(is_integer(Keepalive)), + ?assert(is_boolean(CleanStart)), + ?assertEqual(60, ExpiryInterval), + ?assertMatch(#{'Session-Expiry-Interval' := 60}, Properties), + ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60*1000); + verify_event_fields(SubUnsub, Fields) when SubUnsub == 'session.subscribed' ; SubUnsub == 'session.unsubscribed' -> #{clientid := ClientId, @@ -2852,7 +2909,22 @@ verify_event_fields('message.acked', Fields) -> ?assert(is_map(PubAckProps)), ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60*1000), ?assert(0 =< RcvdAtElapse andalso RcvdAtElapse =< 60*1000), - ?assert(EventAt =< Timestamp). + ?assert(EventAt =< Timestamp); + +verify_event_fields('client.check_acl_complete', Fields) -> + #{clientid := ClientId, + action := Action, + result := Result, + topic := Topic, + is_cache := IsCache, + username := Username + } = Fields, + ?assertEqual(<<"t1">>, Topic), + ?assert(lists:member(Action, [subscribe, publish])), + ?assert(lists:member(Result, [allow, deny])), + ?assert(lists:member(IsCache, [true, false])), + ?assert(lists:member(ClientId, [<<"c_event">>, <<"c_event2">>])), + ?assert(lists:member(Username, [<<"u_event">>, <<"u_event2">>])). verify_peername(PeerName) -> case string:split(PeerName, ":") of diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index 4912b4fcf..e438a4677 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -50,6 +50,9 @@ %% erlang:system_time should be unique and random enough -define(CLIENTID, iolist_to_binary([atom_to_list(?FUNCTION_NAME), "-", integer_to_list(erlang:system_time())])). + +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -66,8 +69,10 @@ end_per_suite(_) -> emqx_ct_helpers:stop_apps([emqx_sn]). set_special_confs(emqx) -> - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); + application:set_env( + emqx, + plugins_loaded_file, + emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); set_special_confs(emqx_sn) -> application:set_env(emqx_sn, enable_qos3, ?ENABLE_QOS3), application:set_env(emqx_sn, enable_stats, true), @@ -119,7 +124,8 @@ t_subscribe(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), TopicName1 = <<"abcD">>, send_register_msg(Socket, TopicName1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, @@ -151,7 +157,8 @@ t_subscribe_case01(_) -> TopicName1 = <<"abcD">>, send_register_msg(Socket, TopicName1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, @@ -172,16 +179,18 @@ t_subscribe_case02(_) -> Will = 0, CleanSession = 0, MsgId = 1, - TopicId = ?PREDEF_TOPIC_ID1, %this TopicId is the predefined topic id corresponding to ?PREDEF_TOPIC_NAME1 + TopicId = ?PREDEF_TOPIC_ID1, ReturnCode = 0, {ok, Socket} = gen_udp:open(0, [binary]), - send_connect_msg(Socket, ?CLIENTID), + ClientId = ?CLIENTID, + send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), Topic1 = ?PREDEF_TOPIC_NAME1, send_register_msg(Socket, Topic1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, @@ -211,9 +220,11 @@ t_subscribe_case03(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"te">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, + CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_short_topic(Socket, <<"te">>, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -222,8 +233,12 @@ t_subscribe_case03(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). -%%In this case We use predefined topic name to register and subcribe, and expect to receive the corresponding predefined topic id but not a new generated topic id from broker. We design this case to illustrate -%% emqx_sn_gateway's compatibility of dealing with predefined and normal topics. Once we give more restrictions to different topic id type, this case would be deleted or modified. +%% In this case We use predefined topic name to register and subcribe, and +%% expect to receive the corresponding predefined topic id but not a new +%% generated topic id from broker. We design this case to illustrate +%% emqx_sn_gateway's compatibility of dealing with predefined and normal topics. +%% Once we give more restrictions to different topic id type, this case would +%% be deleted or modified. t_subscribe_case04(_) -> Dup = 0, QoS = 0, @@ -231,7 +246,7 @@ t_subscribe_case04(_) -> Will = 0, CleanSession = 0, MsgId = 1, - TopicId = ?PREDEF_TOPIC_ID1, %this TopicId is the predefined topic id corresponding to ?PREDEF_TOPIC_NAME1 + TopicId = ?PREDEF_TOPIC_ID1, ReturnCode = 0, {ok, Socket} = gen_udp:open(0, [binary]), ClientId = ?CLIENTID, @@ -239,10 +254,14 @@ t_subscribe_case04(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), Topic1 = ?PREDEF_TOPIC_NAME1, send_register_msg(Socket, Topic1, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, + receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, Topic1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_normal_topic(Socket, Topic1, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -269,19 +288,30 @@ t_subscribe_case05(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_register_msg(Socket, <<"abcD">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"abcD">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/sport/#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/a/+/water">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/Tom/Home">>, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, @@ -311,19 +341,32 @@ t_subscribe_case06(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_register_msg(Socket, <<"abc">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/blue/#">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/blue/+/white">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/$sys/rain">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId2:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_REGACK, TopicId2:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_subscribe_msg_short_topic(Socket, QoS, <<"Q2">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_normal_topic(Socket, <<"Q2">>, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -347,8 +390,11 @@ t_subscribe_case07(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket) + ), send_unsubscribe_msg_predefined_topic(Socket, TopicId2, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -370,8 +416,11 @@ t_subscribe_case08(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_reserved_topic(Socket, QoS, TopicId2, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -395,15 +444,20 @@ t_publish_negqos_case09(_) -> send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, NegQoS, MsgId1, TopicId1, Payload1), timer:sleep(100), case ?ENABLE_QOS3 of true -> - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What) end, @@ -436,7 +490,9 @@ t_publish_qos0_case01(_) -> send_publish_msg_normal_topic(Socket, QoS, MsgId1, TopicId1, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -458,15 +514,20 @@ t_publish_qos0_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId1, PredefTopicId, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, PredefTopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, + PredefTopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -489,15 +550,20 @@ t_publish_qos0_case3(_) -> Topic = <<"/a/b/c">>, send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId1, TopicId, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -519,8 +585,11 @@ t_publish_qos0_case04(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, <<"#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 2, Payload1 = <<20, 21, 22, 23>>, @@ -528,7 +597,9 @@ t_publish_qos0_case04(_) -> send_publish_msg_short_topic(Socket, QoS, MsgId1, Topic, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, Topic/binary, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, + Topic/binary, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -549,8 +620,11 @@ t_publish_qos0_case05(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"/#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -572,15 +646,20 @@ t_publish_qos0_case06(_) -> Topic = <<"abc">>, send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, QoS, MsgId1, TopicId1, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -602,16 +681,25 @@ t_publish_qos1_case01(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), - ?assertEqual(<<7, ?SN_PUBACK, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicId1:16, + MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), timer:sleep(100), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), gen_udp:close(Socket). @@ -630,12 +718,18 @@ t_publish_qos1_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId, PredefTopicId, Payload1), - ?assertEqual(<<7, ?SN_PUBACK, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, PredefTopicId:16, + MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), timer:sleep(100), send_disconnect_msg(Socket, undefined), @@ -650,7 +744,10 @@ t_publish_qos1_case03(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_publish_msg_predefined_topic(Socket, QoS, MsgId, tid(5), <<20, 21, 22, 23>>), - ?assertEqual(<<7, ?SN_PUBACK, TopicId5:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicId5:16, + MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -669,15 +766,20 @@ t_publish_qos1_case04(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"ab">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Topic = <<"ab">>, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_short_topic(Socket, QoS, MsgId, Topic, Payload1), <> = Topic, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), timer:sleep(100), send_disconnect_msg(Socket, undefined), @@ -697,13 +799,18 @@ t_publish_qos1_case05(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, <<"ab">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/#">>, <<20, 21, 22, 23>>), <> = <<"/#">>, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -729,7 +836,10 @@ t_publish_qos1_case06(_) -> send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/+">>, <<20, 21, 22, 23>>), <> = <<"/+">>, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -756,7 +866,11 @@ t_publish_qos2_case01(_) -> send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -778,15 +892,21 @@ t_publish_qos2_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId, PredefTopicId, Payload1), ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC :2, PredefTopicId:16, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, + PredefTopicId:16, 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -817,7 +937,11 @@ t_publish_qos2_case03(_) -> ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC :2, <<"/a">>/binary, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, + "/a", 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -1167,7 +1291,11 @@ t_asleep_test03_to_awake_qos1_dl_msg(_) -> send_register_msg(Socket, TopicName1, MsgId1), ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId1:16, 0:8>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state send_disconnect_msg(Socket, 1), @@ -1193,7 +1321,10 @@ t_asleep_test03_to_awake_qos1_dl_msg(_) -> %% the broker should sent dl msgs to the awake client before sending the pingresp UdpData = receive_response(Socket), - MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId1, Payload1}, UdpData), + MsgId_udp = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicId1, Payload1}, UdpData), send_puback_msg(Socket, TopicId1, MsgId_udp), %% check the pingresp is received at last @@ -1225,8 +1356,11 @@ t_asleep_test04_to_awake_qos1_dl_msg(_) -> CleanSession = 0, ReturnCode = 0, send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1,CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state send_disconnect_msg(Socket, 1), @@ -1260,11 +1394,17 @@ t_asleep_test04_to_awake_qos1_dl_msg(_) -> send_regack_msg(Socket, TopicIdNew, MsgId3), UdpData2 = receive_response(Socket), - MsgId_udp2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2), + MsgId_udp2 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload1}, UdpData2), send_puback_msg(Socket, TopicIdNew, MsgId_udp2), UdpData3 = receive_response(Socket), - MsgId_udp3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData3), + MsgId_udp3 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload2}, UdpData3), send_puback_msg(Socket, TopicIdNew, MsgId_udp3), ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), @@ -1300,8 +1440,11 @@ t_asleep_test05_to_awake_qos1_dl_msg(_) -> CleanSession = 0, ReturnCode = 0, send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state SleepDuration = 30, @@ -1334,21 +1477,28 @@ t_asleep_test05_to_awake_qos1_dl_msg(_) -> send_regack_msg(Socket, TopicIdNew, MsgId_reg), UdpData2 = receive_response(Socket), - MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), + MsgId2 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload2}, UdpData2), send_puback_msg(Socket, TopicIdNew, MsgId2), timer:sleep(50), UdpData3 = wrap_receive_response(Socket), - MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), + MsgId3 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload3}, UdpData3), send_puback_msg(Socket, TopicIdNew, MsgId3), timer:sleep(50), case receive_response(Socket) of <<2,23>> -> ok; UdpData4 -> - MsgId4 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, - CleanSession, ?SN_NORMAL_TOPIC, - TopicIdNew, Payload4}, UdpData4), + MsgId4 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload4}, UdpData4), send_puback_msg(Socket, TopicIdNew, MsgId4) end, ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), @@ -1406,7 +1556,10 @@ t_asleep_test06_to_awake_qos2_dl_msg(_) -> send_pingreq_msg(Socket, ClientId), UdpData = wrap_receive_response(Socket), - MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId_tom, Payload1}, UdpData), + MsgId_udp = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicId_tom, Payload1}, UdpData), send_pubrec_msg(Socket, MsgId_udp), ?assertMatch(<<_:8, ?SN_PUBREL:8, _/binary>>, receive_response(Socket)), send_pubcomp_msg(Socket, MsgId_udp), @@ -1441,8 +1594,11 @@ t_asleep_test07_to_connected(_) -> send_register_msg(Socket, TopicName_tom, MsgId1), TopicId_tom = check_regack_msg_on_udp(MsgId1, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId_tom, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId_tom:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1,CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId_tom:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state send_disconnect_msg(Socket, SleepDuration), @@ -1520,8 +1676,11 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> CleanSession = 0, ReturnCode = 0, send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + WillBit:1,CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket) + ), % goto asleep state SleepDuration = 30, send_disconnect_msg(Socket, SleepDuration), @@ -1555,7 +1714,10 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> udp_receive_timeout -> ok; UdpData2 -> - MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), + MsgId2 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload2}, UdpData2), send_puback_msg(Socket, TopicIdNew, MsgId2) end, timer:sleep(100), @@ -1564,7 +1726,10 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> udp_receive_timeout -> ok; UdpData3 -> - MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), + MsgId3 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload3}, UdpData3), send_puback_msg(Socket, TopicIdNew, MsgId3) end, timer:sleep(100), @@ -1573,16 +1738,18 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> udp_receive_timeout -> ok; UdpData4 -> - MsgId4 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, - CleanSession, ?SN_NORMAL_TOPIC, - TopicIdNew, Payload4}, UdpData4), + MsgId4 = check_publish_msg_on_udp( + {Dup, QoS, Retain, WillBit, + CleanSession, ?SN_NORMAL_TOPIC, + TopicIdNew, Payload4}, UdpData4), send_puback_msg(Socket, TopicIdNew, MsgId4) end, ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), %% send PINGREQ again to enter awake state send_pingreq_msg(Socket, ClientId), - %% will not receive any buffered PUBLISH messages buffered before last awake, only receive PINGRESP here + %% will not receive any buffered PUBLISH messages buffered before last + %% awake, only receive PINGRESP here ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), gen_udp:close(Socket). @@ -2312,8 +2479,12 @@ check_dispatched_message(Dup, QoS, Retain, TopicIdType, TopicId, Payload, Socket PubMsg = receive_response(Socket), Length = 7 + byte_size(Payload), ?LOG("check_dispatched_message ~p~n", [PubMsg]), - ?LOG("expected ~p xx ~p~n", [<>, Payload]), - <> = PubMsg, + ?LOG("expected ~p xx ~p~n", + [<>, Payload]), + <> = PubMsg, case QoS of 0 -> ok; 1 -> send_puback_msg(Socket, TopicId, MsgId); @@ -2325,11 +2496,14 @@ check_dispatched_message(Dup, QoS, Retain, TopicIdType, TopicId, Payload, Socket get_udp_broadcast_address() -> "255.255.255.255". -check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, TopicType, TopicId, Payload}, UdpData) -> +check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, + CleanSession, TopicType, TopicId, Payload}, UdpData) -> <> = UdpData, ct:pal("UdpData: ~p, Payload: ~p, PayloadIn: ~p", [UdpData, Payload, PayloadIn]), Size9 = byte_size(Payload) + 7, - Eexp = <>, + Eexp = <>, ?assertEqual(Eexp, HeaderUdp), % mqtt-sn header should be same ?assertEqual(Payload, PayloadIn), % payload should be same MsgId. diff --git a/bin/emqx b/bin/emqx index a3843ec7e..180b4eb77 100755 --- a/bin/emqx +++ b/bin/emqx @@ -4,6 +4,11 @@ set -e +DEBUG="${DEBUG:-0}" +if [ "$DEBUG" -eq 1 ]; then + set -x +fi + RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 . "$RUNNER_ROOT_DIR"/releases/emqx_vars @@ -398,6 +403,43 @@ generate_config() { fi } +# check if a PID is down +is_down() { + PID="$1" + if ps -p "$PID" >/dev/null; then + # still around + # shellcheck disable=SC2009 # this grep pattern is not a part of the progra names + if ps -p "$PID" | grep -q 'defunct'; then + # zombie state, print parent pid + parent="$(ps -o ppid= -p "$PID" | tr -d ' ')" + echo "WARN: $PID is marked , parent:" + ps -p "$parent" + return 0 + fi + return 1 + fi + # it's gone + return 0 +} + +wait_for() { + local WAIT_TIME + local CMD + WAIT_TIME="$1" + shift + CMD="$*" + while true; do + if $CMD >/dev/null 2>&1; then + return 0 + fi + if [ "$WAIT_TIME" -le 0 ]; then + return 1 + fi + WAIT_TIME=$((WAIT_TIME - 1)) + sleep 1 + done +} + # Call bootstrapd for daemon commands like start/stop/console bootstrapd() { if [ -e "$RUNNER_DATA_DIR/.erlang.cookie" ]; then @@ -582,7 +624,7 @@ case "$1" in "$BINDIR/run_erl" -daemon "$PIPE_DIR" "$RUNNER_LOG_DIR" \ "$(relx_start_command)" - WAIT_TIME=${WAIT_FOR_ERLANG:-15} + WAIT_TIME=${WAIT_FOR_ERLANG:-150} while [ "$WAIT_TIME" -gt 0 ]; do if ! relx_nodetool "ping" >/dev/null 2>&1; then WAIT_TIME=$((WAIT_TIME - 1)) @@ -594,7 +636,7 @@ case "$1" in echo "$EMQX_DESCRIPTION $REL_VSN is started successfully!" exit 0 fi - done && echo "$EMQX_DESCRIPTION $REL_VSN failed to start within ${WAIT_FOR_ERLANG:-15} seconds," + done && echo "$EMQX_DESCRIPTION $REL_VSN failed to start within ${WAIT_FOR_ERLANG:-150} seconds," echo "see the output of '$0 console' for more information." echo "If you want to wait longer, set the environment variable" echo "WAIT_FOR_ERLANG to the number of seconds to wait." @@ -605,6 +647,7 @@ case "$1" in # Wait for the node to completely stop... PID="$(relx_get_pid)" if ! relx_nodetool "stop"; then + echoerr "Graceful shutdown failed PID=[$PID]" exit 1 fi WAIT_TIME="${EMQX_WAIT_FOR_STOP:-120}" diff --git a/build b/build index 44287bff5..fa6a41615 100755 --- a/build +++ b/build @@ -18,21 +18,7 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN -SYSTEM="$(./scripts/get-distro.sh)" - -ARCH="$(uname -m)" -case "$ARCH" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm - ;; -esac -export ARCH +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" ## ## Support RPM and Debian based linux systems @@ -55,6 +41,21 @@ else FIND='find' fi +UNAME="$(uname -m)" +case "$UNAME" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac +# used in -pkg Makefile +export ARCH + log() { local msg="$1" # rebar3 prints ===>, so we print ===< @@ -62,20 +63,21 @@ log() { } make_rel() { - # shellcheck disable=SC1010 - ./rebar3 as "$PROFILE" do release,tar + ./rebar3 as "$PROFILE" tar } ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup make_relup() { local lib_dir="_build/$PROFILE/rel/emqx/lib" local releases_dir="_build/$PROFILE/rel/emqx/releases" - mkdir -p "$lib_dir" "$releases_dir" + local name_pattern + name_pattern="${PROFILE}-$(./scripts/pkg-full-vsn.sh 'vsn_matcher')" + mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base' local releases=() if [ -d "$releases_dir" ]; then while read -r zip; do local base_vsn - base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-f]{8})?")" + base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-[0-9a-f]{8})?" | head -1)" if [ ! -d "$releases_dir/$base_vsn" ]; then local tmp_dir tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" @@ -86,7 +88,7 @@ make_relup() { rm -rf "$tmp_dir" fi releases+=( "$base_vsn" ) - done < <("$FIND" _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.zip" -type f) fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -97,6 +99,8 @@ make_relup() { ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" } +## try to be portable for zip packages. +## for DEB and RPM packages the dependencies are resoved by yum and apt cp_dyn_libs() { local rel_dir="$1" local target_dir="${rel_dir}/dynlibs" @@ -113,32 +117,70 @@ cp_dyn_libs() { | sort -u) } + ## make_zip turns .tar.gz into a .zip with a slightly different name. ## It assumes the .tar.gz has been built -- relies on Makefile dependency make_zip() { # build the tarball again to ensure relup is included make_rel - - tard="/tmp/emqx_untar_${PKG_VSN}" - rm -rf "${tard}" + # use relative path because abs path is tricky in windows + tard="tmp/zip-wd-${PKG_VSN}" + rm -rf "${tard}/emqx" mkdir -p "${tard}/emqx" local relpath="_build/${PROFILE}/rel/emqx" local pkgpath="_packages/${PROFILE}" + local pkgname + pkgname="${PROFILE}-$(./scripts/pkg-full-vsn.sh).zip" mkdir -p "${pkgpath}" - local tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" - if [ ! -f "$tarball" ]; then - log "ERROR: $tarball is not found" - fi - local zipball - zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" + local tarname="emqx-${PKG_VSN}.tar.gz" + local tarball="${relpath}/${tarname}" + local target_zip="${pkgpath}/${pkgname}" tar zxf "${tarball}" -C "${tard}/emqx" - ## try to be portable for zip packages. - ## for DEB and RPM packages the dependencies are resoved by yum and apt + has_relup='yes' + case "$SYSTEM" in + windows*) + # no relup support for windows + has_relup='no' + ;; + debian11) + case "$PKG_VSN" in + 4.4.2*) + # this is the first version for debian11, no relup + has_relup='no' + ;; + esac + ;; + esac + if [ "$has_relup" = 'yes' ]; then + ./scripts/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup" + fi cp_dyn_libs "${tard}/emqx" - (cd "${tard}" && zip -qr - emqx) > "${zipball}" + pushd "${tard}" >/dev/null + case "$SYSTEM" in + windows*) + 7z a "${pkgname}" emqx + ;; + *) + zip -qr "${pkgname}" emqx + ;; + esac + popd >/dev/null + mv "${tard}/${pkgname}" "${target_zip}" + case "$SYSTEM" in + macos*) + # sha256sum may not be available on macos + openssl dgst -sha256 "${target_zip}" | cut -d ' ' -f 2 > "${target_zip}.sha256" + ;; + *) + sha256sum "${target_zip}" | head -c 64 > "${target_zip}.sha256" + ;; + esac + log "Zip package successfully created: ${target_zip}" + log "Zip package sha256sum: $(cat "${target_zip}.sha256")" } -## This function builds the default docker image based on alpine:3.14 (by default) +## This function builds the default docker image +## based images is by default $EMQX_DEFAULT_BUILDER (see Makefile) make_docker() { EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}" EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}" @@ -151,6 +193,49 @@ make_docker() { -f "${DOCKERFILE}" . } +## This function accepts any base docker image, +## a emqx zip-image, and a image tag (for the image to be built), +## to build a docker image which runs EMQ X +## +## Export below variables to quickly build an image +## +## Name Default Example +## --------------------------------------------------------------------- +## EMQX_BASE_IMAGE current os centos:7 +## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp24.1.5-3-el7-amd64.zip +## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag +## +make_docker_testing() { + if [ -z "${EMQX_BASE_IMAGE:-}" ]; then + case "$SYSTEM" in + ubuntu20*) + EMQX_BASE_IMAGE="ubuntu:20.04" + ;; + el8) + EMQX_BASE_IMAGE="rockylinux:8" + ;; + *) + echo "Unsupported testing base image for $SYSTEM" + exit 1 + ;; + esac + fi + EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" + local defaultzip + defaultzip="_packages/${PROFILE}/${PROFILE}-$(./scripts/pkg-full-vsn.sh).zip" + local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}" + if [ ! -f "$zip" ]; then + log "ERROR: $zip not built?" + exit 1 + fi + set -x + docker build \ + --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \ + --build-arg EMQX_ZIP_PACKAGE="${zip}" \ + --tag "$EMQX_IMAGE_TAG" \ + -f "${DOCKERFILE_TESTING}" . +} + log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in @@ -168,13 +253,15 @@ case "$ARTIFACT" in log "Skipped making deb/rpm package for $SYSTEM" exit 0 fi - make -C "deploy/packages/${PKGERDIR}" clean + EMQX_REL="$(pwd)" make -C "deploy/packages/${PKGERDIR}" clean EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}" ;; - docker) make_docker ;; + docker-testing) + make_docker_testing + ;; *) log "Unknown artifact $ARTIFACT" exit 1 diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index ef4521ee8..fc798bbe5 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -5,7 +5,7 @@ {{ $configData := printf "%s\n%s\n%s\n%s" $cfgEnv $cfgAcl $cfgPlugins $cfgModules}} ## Compatible with previous misspellings {{ $licenseSecretName := coalesce .Values.emqxLicenseSecretName .Values.emqxLicneseSecretName }} -{{ $image := printf "%s:%s" .Values.image.repository (default .Values.image.tag .Chart.AppVersion) }} +{{ $image := printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }} apiVersion: apps/v1 kind: StatefulSet @@ -104,7 +104,9 @@ spec: secret: secretName: {{ $licenseSecretName }} {{- end }} + {{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s" }} serviceAccountName: {{ include "emqx.fullname" . }} + {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} @@ -154,18 +156,8 @@ spec: {{- if .Values.extraEnvFrom }} {{ toYaml .Values.extraEnvFrom | indent 10 }} {{- end }} - env: - - name: EMQX_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__K8S__APP_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__DISCOVERY - value: k8s - - name: EMQX_CLUSTER__K8S__SERVICE_NAME - value: {{ include "emqx.fullname" . }}-headless - - name: EMQX_CLUSTER__K8S__NAMESPACE - value: {{ .Release.Namespace }} {{- if .Values.extraEnv }} + env: {{ toYaml .Values.extraEnv | indent 10 }} {{- end }} resources: diff --git a/deploy/charts/emqx/templates/configmap.env.yaml b/deploy/charts/emqx/templates/configmap.env.yaml index ff5924c54..1167b7972 100644 --- a/deploy/charts/emqx/templates/configmap.env.yaml +++ b/deploy/charts/emqx/templates/configmap.env.yaml @@ -13,7 +13,7 @@ data: {{- range $index, $value := .Values.emqxConfig }} {{- if $value }} {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }} - {{ print "EMQX_" $key }}: {{ $value | quote }} + {{ print "EMQX_" $key }}: "{{ tpl (printf "%v" $value) $ }}" {{- end }} {{- end}} diff --git a/deploy/charts/emqx/templates/rbac.yaml b/deploy/charts/emqx/templates/rbac.yaml index 45806d698..cbd3b4f7b 100644 --- a/deploy/charts/emqx/templates/rbac.yaml +++ b/deploy/charts/emqx/templates/rbac.yaml @@ -1,3 +1,4 @@ +{{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s" }} apiVersion: v1 kind: ServiceAccount metadata: @@ -17,8 +18,8 @@ rules: - apiGroups: - "" resources: - - endpoints - verbs: + - endpoints + verbs: - get - watch - list @@ -40,3 +41,4 @@ roleRef: kind: Role name: {{ include "emqx.fullname" . }} apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 0f273671c..69ed2424e 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -70,7 +70,20 @@ initContainers: {} ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) emqxConfig: + EMQX_NAME: "{{ .Release.Name }}" + + ## Cluster discovery by dns + # EMQX_CLUSTER__DISCOVERY: "dns" + # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local" + # EMQX_CLUSTER__DNS__APP: "{{ .Release.Name }}" + # EMQX_CLUSTER__DNS__TYPE: "srv" + + ## Cluster discovery by k8s + EMQX_CLUSTER__DISCOVERY: "k8s" + EMQX_CLUSTER__K8S__APP_NAME: "{{ .Release.Name }}" EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" + EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless" + EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}" ## The address type is used to extract host from k8s service. ## Value: ip | dns | hostname ## Note：Hostname is only supported after v4.0-rc.2 @@ -114,6 +127,8 @@ emqxLoadedPlugins: > emqxLoadedModules: > {emqx_mod_acl_internal, true}. {emqx_mod_presence, true}. + {emqx_mod_trace, false}. + {emqx_mod_slow_subs, false}. {emqx_mod_delayed, false}. {emqx_mod_rewrite, false}. {emqx_mod_subscription, false}. diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index f24b4a348..d3a58ce74 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,5 +1,5 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine -ARG RUN_FROM=alpine:3.12 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-alpine3.15.1 +ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder RUN apk add --no-cache \ @@ -31,17 +31,6 @@ RUN cd /emqx \ FROM $RUN_FROM -# Basic build-time metadata as defined at http://label-schema.org -LABEL org.label-schema.docker.dockerfile="Dockerfile" \ - org.label-schema.license="GNU" \ - org.label-schema.name="emqx" \ - org.label-schema.version=${PKG_VSN} \ - org.label-schema.description="EMQ (Erlang MQTT Broker) is a distributed, massively scalable, highly extensible MQTT messaging broker written in Erlang/OTP." \ - org.label-schema.url="https://emqx.io" \ - org.label-schema.vcs-type="Git" \ - org.label-schema.vcs-url="https://github.com/emqx/emqx" \ - maintainer="EMQX Team " - ARG EMQX_NAME=emqx COPY deploy/docker/docker-entrypoint.sh /usr/bin/ diff --git a/deploy/docker/Dockerfile.testing b/deploy/docker/Dockerfile.testing new file mode 100644 index 000000000..02490272d --- /dev/null +++ b/deploy/docker/Dockerfile.testing @@ -0,0 +1,43 @@ +ARG BUILD_FROM +FROM ${BUILD_FROM} + +## all we need is the unzip command +RUN if command -v yum; then yum update -y && yum install -y unzip; fi +RUN if command -v apt-get; then apt-get update -y && apt-get install unzip; fi + +ARG EMQX_ZIP_PACKAGE +COPY ${EMQX_ZIP_PACKAGE} /opt/emqx.zip +RUN unzip -q /opt/emqx.zip -d /opt/ && rm /opt/emqx.zip + +COPY deploy/docker/docker-entrypoint.sh /usr/bin/ +RUN ln -s /opt/emqx/bin/* /usr/local/bin/ + +WORKDIR /opt/emqx + +RUN adduser -u 1000 emqx +RUN echo "emqx ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers + +RUN chgrp -Rf emqx /opt/emqx && chmod -Rf g+w /opt/emqx \ + && chown -Rf emqx /opt/emqx + +USER emqx + +VOLUME ["/opt/emqx/log", "/opt/emqx/data", "/opt/emqx/etc"] + +# emqx will occupy these port: +# - 1883 port for MQTT +# - 8081 for mgmt API +# - 8083 for WebSocket/HTTP +# - 8084 for WSS/HTTPS +# - 8883 port for MQTT(SSL) +# - 11883 port for internal MQTT/TCP +# - 18083 for dashboard +# - 4369 epmd (Erlang-distrbution port mapper daemon) listener (deprecated) +# - 4370 default Erlang distrbution port +# - 5369 for gen_rpc port mapping +# - 6369 6370 for distributed node +EXPOSE 1883 8081 8083 8084 8883 11883 18083 4369 4370 5369 6369 6370 + +ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] + +CMD ["/opt/emqx/bin/emqx", "foreground"] diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index 2f0ba5843..8d82318d7 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -28,12 +28,20 @@ if [[ -z "$EMQX_NAME" ]]; then fi if [[ -z "$EMQX_HOST" ]]; then - if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then - EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"} - EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX" - elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then - EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'} - EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX") + if [[ "$EMQX_CLUSTER__DISCOVERY" == "dns" ]] && \ + [[ "$EMQX_CLUSTER__DNS__TYPE" == "srv" ]] && \ + grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then + EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME" + elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \ + [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \ + [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then + EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"} + EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX" + elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \ + [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \ + [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then + EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'} + EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX") else EMQX_HOST="$LOCAL_IP" fi diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 1cfc4d514..15665fc07 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -1,4 +1,3 @@ -ARCH ?= amd64 TOPDIR := /tmp/emqx # Keep this short to avoid bloating beam files with long file path info SRCDIR := $(TOPDIR)/$(PKG_VSN) @@ -8,7 +7,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh) .PHONY: all all: | $(BUILT) @@ -22,6 +21,8 @@ all: | $(BUILT) cd $(SRCDIR) && dpkg-buildpackage -us -uc mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb + sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb | head -c 64 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb.sha256 $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) diff --git a/deploy/packages/deb/debian/control b/deploy/packages/deb/debian/control index e35535c12..b3794036c 100644 --- a/deploy/packages/deb/debian/control +++ b/deploy/packages/deb/debian/control @@ -4,7 +4,7 @@ Priority: optional Maintainer: emqx Build-Depends: debhelper (>=9) Standards-Version: 3.9.6 -Homepage: https://www.emqx.io +Homepage: https://www.emqx.com Package: emqx Architecture: any diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 5a6e6bee4..31c1919ff 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -5,8 +5,9 @@ BUILT := $(SRCDIR)/BUILT dash := - none := space := $(none) $(none) -RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?") -RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]") +## RPM does not allow '-' in version nubmer and release string, replace with '_' +RPM_VSN := $(subst -,_,$(PKG_VSN)) +RPM_REL := otp$(subst -,_,$(OTP_VSN)) ARCH ?= amd64 ifeq ($(ARCH),mips64) @@ -16,12 +17,8 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) -ifeq ($(RPM_REL),) - # no tail - RPM_REL := 1 -endif -SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/scripts/pkg-full-vsn.sh) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -47,11 +44,12 @@ all: | $(BUILT) --define "_service_dst $(SERVICE_DST)" \ --define "_post_addition $(POST_ADDITION)" \ --define "_preun_addition $(PREUN_ADDITION)" \ - --define "_ostype -$(SYSTEM)" \ --define "_sharedstatedir /var/lib" \ emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm + sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm | head -c 64 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm.sha256 $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index a903898bf..bde174159 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -5,7 +5,7 @@ %define _log_dir %{_var}/log/%{_name} %define _lib_home /usr/lib/%{_name} %define _var_home %{_sharedstatedir}/%{_name} -%define _build_name_fmt %{_arch}/%{_name}%{?_ostype}-%{_version}-%{_release}.%{_arch}.rpm +%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}.%{_arch}.rpm %define _build_id_links none Name: %{_package_name} diff --git a/etc/emqx.conf b/etc/emqx.conf index bc45df1fc..5210b1603 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1,4 +1,4 @@ -## EMQX Configuration 4.3 +## EMQX Configuration ## NOTE: Do not change format of CONFIG_SECTION_{BGN,END} comments! @@ -101,6 +101,11 @@ cluster.autoclean = 5m ## Value: String ## cluster.dns.app = emqx +## Type of dns record. +## +## Value: Value: a | srv +## cluster.dns.type = a + ##-------------------------------------------------------------------- ## Cluster using etcd @@ -354,7 +359,7 @@ rpc.port_discovery = stateless ## ## Value: Interger [0-256] ## Default = 1 -#rpc.tcp_client_num = 1 +#rpc.tcp_client_num = 0 ## RCP Client connect timeout. ## @@ -2213,6 +2218,29 @@ module.presence.qos = 1 ## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1 ## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 +##-------------------------------------------------------------------- +## Slow Subscribers Statistics Module + +## the expire time of the record which in topk +## +## Value: 5 minutes +#module.slow_subs.expire_interval = 5m + +## maximum number of Top-K record +## +## Defalut: 10 +#module.slow_subs.top_k_num = 10 + +## Stats Type +## +## Default: whole +#module.slow_subs.stats_type = whole + +## Stats Threshold +## +## Default: 500ms +#module.slow_subs.threshold = 500ms + ## CONFIG_SECTION_END=modules ================================================== ##------------------------------------------------------------------- diff --git a/include/emqx_mqtt.hrl b/include/emqx_mqtt.hrl index fedcaf51a..64ef1422e 100644 --- a/include/emqx_mqtt.hrl +++ b/include/emqx_mqtt.hrl @@ -542,4 +542,22 @@ -define(SHARE(Group, Topic), emqx_topic:join([<>, Group, Topic])). -define(IS_SHARE(Topic), case Topic of <> -> true; _ -> false end). +-define(TYPE_NAMES, { + 'CONNECT' + , 'CONNACK' + , 'PUBLISH' + , 'PUBACK' + , 'PUBREC' + , 'PUBREL' + , 'PUBCOMP' + , 'SUBSCRIBE' + , 'SUBACK' + , 'UNSUBSCRIBE' + , 'UNSUBACK' + , 'PINGREQ' + , 'PINGRESP' + , 'DISCONNECT' + , 'AUTH' + }). + -endif. diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 5145f7b5e..f95494ed8 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.14"}). +-define(EMQX_RELEASE, {opensource, "4.4.4-beta.1"}). -else. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index ab52c2928..95cca4e1c 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, - [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.11"}, % strict semver, bump manually! + [{description, "EMQX Web Dashboard"}, + {vsn, "4.4.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src index 902585ffb..513757418 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.appup.src @@ -1,18 +1,11 @@ %% -*- mode: erlang -*- {VSN, [ {<<".*">>, - %% load all plugins - %% NOTE: this depends on the fact that emqx_dashboard is always - %% the last application gets upgraded - [ {apply, {emqx_rule_engine, load_providers, []}} - , {restart_application, emqx_dashboard} - , {apply, {emqx_plugins, load, []}} + [ {restart_application, emqx_dashboard} ]} ], [ {<<".*">>, - [ {apply, {emqx_rule_engine, load_providers, []}} - , {restart_application, emqx_dashboard} - , {apply, {emqx_plugins, load, []}} + [ {restart_application, emqx_dashboard} ]} ] }. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl index 5d17c019e..747970637 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl @@ -41,18 +41,18 @@ start_listeners() -> lists:foreach(fun(Listener) -> start_listener(Listener) end, listeners()). -%% Start HTTP Listener -start_listener({Proto, Port, Options}) when Proto == http -> - Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, - {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, - {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_http(listener_name(Proto), ranch_opts(Port, Options), Dispatch); -start_listener({Proto, Port, Options}) when Proto == https -> +%% Start HTTP(S) Listener +start_listener({Proto, Port, Options}) -> Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, {"/api/v4/[...]", minirest, http_handlers()}], - minirest:start_https(listener_name(Proto), ranch_opts(Port, Options), Dispatch). + Server = listener_name(Proto), + RanchOpts = ranch_opts(Port, Options), + case Proto of + http -> minirest:start_http(Server, RanchOpts, Dispatch); + https -> minirest:start_https(Server, RanchOpts, Dispatch) + end. ranch_opts(Port, Options0) -> NumAcceptors = get_value(num_acceptors, Options0, 4), @@ -89,7 +89,7 @@ listener_name(Proto) -> http_handlers() -> Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), [{"/api/v4/", - minirest:handler(#{apps => Plugins ++ [emqx_modules], + minirest:handler(#{apps => Plugins ++ [emqx_modules, emqx_plugin_libs], filter => fun ?MODULE:filter/1}), [{authorization, fun ?MODULE:is_authorized/1}]}]. @@ -116,6 +116,7 @@ is_authorized(_Path, Req) -> false end. +filter(#{app := emqx_plugin_libs}) -> true; filter(#{app := emqx_modules}) -> true; filter(#{app := App}) -> case emqx_plugins:find_plugin(App) of diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 778cb7731..d6f76751d 100644 --- a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -58,6 +58,7 @@ groups() -> ]. init_per_suite(Config) -> + application:load(emqx_plugin_libs), ok = emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_dashboard]), Config. diff --git a/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl new file mode 100644 index 000000000..b9117fe8b --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_slow_subs.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_slow_subs). + +-behaviour(emqx_gen_mod). + +-include_lib("include/emqx.hrl"). +-include_lib("include/logger.hrl"). + +-logger_header("[SLOW Subs]"). + +%% emqx_gen_mod callbacks +-export([ load/1 + , unload/1 + , description/0 + ]). + +-define(LIB, emqx_slow_subs). + +%%-------------------------------------------------------------------- +%% Load/Unload +%%-------------------------------------------------------------------- + +-spec(load(list()) -> ok). +load(Env) -> + emqx_mod_sup:start_child(?LIB, worker, [Env]), + ok. + +-spec(unload(list()) -> ok). +unload(_Env) -> + _ = emqx_mod_sup:stop_child(?LIB), + ok. + +description() -> + "EMQ X Slow Subscribers Statistics Module". diff --git a/lib-ce/emqx_modules/src/emqx_mod_sup.erl b/lib-ce/emqx_modules/src/emqx_mod_sup.erl index b0f62e8f7..540cdd807 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_sup.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_sup.erl @@ -23,18 +23,22 @@ -export([ start_link/0 , start_child/1 , start_child/2 + , start_child/3 , stop_child/1 ]). -export([init/1]). %% Helper macro for declaring children of supervisor --define(CHILD(Mod, Type), #{id => Mod, - start => {Mod, start_link, []}, - restart => permanent, - shutdown => 5000, - type => Type, - modules => [Mod]}). +-define(CHILD(Mod, Type, Args), + #{id => Mod, + start => {Mod, start_link, Args}, + restart => permanent, + shutdown => 5000, + type => Type, + modules => [Mod]}). + +-define(CHILD(MOD, Type), ?CHILD(MOD, Type, [])). -spec(start_link() -> startlink_ret()). start_link() -> @@ -48,6 +52,10 @@ start_child(ChildSpec) when is_map(ChildSpec) -> start_child(Mod, Type) when is_atom(Mod) andalso is_atom(Type) -> assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Type))). +-spec start_child(atom(), atom(), list(any())) -> ok. +start_child(Mod, Type, Args) when is_atom(Mod) andalso is_atom(Type) -> + assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Type, Args))). + -spec(stop_child(any()) -> ok | {error, term()}). stop_child(ChildId) -> case supervisor:terminate_child(?MODULE, ChildId) of @@ -61,6 +69,7 @@ stop_child(ChildId) -> init([]) -> ok = emqx_tables:new(emqx_modules, [set, public, {write_concurrency, true}]), + emqx_slow_subs:init_tab(), {ok, {{one_for_one, 10, 100}, []}}. %%-------------------------------------------------------------------- @@ -69,6 +78,5 @@ init([]) -> assert_started({ok, _Pid}) -> ok; assert_started({ok, _Pid, _Info}) -> ok; -assert_started({error, {already_tarted, _Pid}}) -> ok; +assert_started({error, {already_started, _Pid}}) -> ok; assert_started({error, Reason}) -> erlang:error(Reason). - diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace.erl b/lib-ce/emqx_modules/src/emqx_mod_trace.erl new file mode 100644 index 000000000..03d468a82 --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_trace.erl @@ -0,0 +1,39 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_trace). + +-behaviour(emqx_gen_mod). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-export([ load/1 + , unload/1 + , description/0 + ]). + +-spec description() -> string(). +description() -> + "EMQ X Trace Module". + +-spec load(any()) -> ok. +load(_Env) -> + emqx_mod_sup:start_child(emqx_trace, worker). + +-spec unload(any()) -> ok. +unload(_Env) -> + emqx_mod_sup:stop_child(emqx_trace). diff --git a/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl new file mode 100644 index 000000000..5cc3f07a8 --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_mod_trace_api.erl @@ -0,0 +1,117 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mod_trace_api). + +%% API +-export([ list_trace/2 + , create_trace/2 + , disable_trace/2 + , delete_trace/2 + , clear_traces/2 + , download_zip_log/2 + , stream_log_file/2 +]). + +-import(minirest, [return/1]). + +-rest_api(#{name => list_trace, + method => 'GET', + path => "/trace/", + func => list_trace, + descr => "list all traces"}). + +-rest_api(#{name => create_trace, + method => 'POST', + path => "/trace/", + func => create_trace, + descr => "create trace"}). + +-rest_api(#{name => delete_trace, + method => 'DELETE', + path => "/trace/:bin:name", + func => delete_trace, + descr => "delete trace"}). + +-rest_api(#{name => clear_trace, + method => 'DELETE', + path => "/trace/", + func => clear_traces, + descr => "clear all traces"}). + +-rest_api(#{name => disable_trace, + method => 'PUT', + path => "/trace/:bin:name/stop", + func => disable_trace, + descr => "stop trace"}). + +-rest_api(#{name => download_zip_log, + method => 'GET', + path => "/trace/:bin:name/download", + func => download_zip_log, + descr => "download trace's log"}). + +-rest_api(#{name => stream_log_file, + method => 'GET', + path => "/trace/:bin:name/log", + func => stream_log_file, + descr => "download trace's log"}). + +-define(NOT_STARTED, {error, module_not_loaded}). +list_trace(Path, Params) -> + case is_started() of + true -> return(emqx_trace_api:list_trace(Path, Params)); + false -> return(?NOT_STARTED) + end. + +create_trace(Path, Params) -> + case is_started() of + true -> return(emqx_trace_api:create_trace(Path, Params)); + false -> return(?NOT_STARTED) + end. + +delete_trace(Path, Params) -> + case is_started() of + true -> return(emqx_trace_api:delete_trace(Path, Params)); + false -> return(?NOT_STARTED) + end. + +clear_traces(Path, Params) -> + case is_started() of + true -> return(emqx_trace_api:clear_traces(Path, Params)); + false -> return(?NOT_STARTED) + end. + +disable_trace(#{name := Name}, Params) -> + case is_started() of + true -> return(emqx_trace_api:update_trace(#{name => Name, operation => disable}, Params)); + false -> return(?NOT_STARTED) + end. + +download_zip_log(Path, Params) -> + case emqx_trace_api:download_zip_log(Path, Params) of + {ok, File} -> minirest:return_file(File); + {error, Reason} -> return({error, 'NOT_FOUND', Reason}) + end. + +stream_log_file(Path, Params) -> + case emqx_trace_api:stream_log_file(Path, Params) of + {ok, File} -> return({ok, File}); + {error, Reason} -> return({error, 'NOT_FOUND', Reason}) + end. + +is_started() -> + undefined =/= erlang:whereis(emqx_trace). diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index 361b11157..1ff39b0f8 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.6"}, + {vsn, "4.4.3"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index fdbacbfc6..51382cdfa 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,49 +1,29 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.5",[{load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, - {"4.3.4", + [{"4.4.2", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, + {"4.4.1", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]">>, - [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}]}, - {"4.3.1", + {"4.4.0", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, - {update,emqx_mod_delayed,{advanced,[]}}, - {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.5",[{load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, - {"4.3.4", + [{"4.4.2", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, + {"4.4.1", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]">>, - [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}]}, - {"4.3.1", + {"4.4.0", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, - {update,emqx_mod_delayed,{advanced,[]}}, - {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, - {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. diff --git a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl index 055ccd2cb..d2453bb2e 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl @@ -41,9 +41,8 @@ t_start_child(_) -> modules => [Mod]}, ok = emqx_mod_sup:start_child(Mod, worker), - ?assertError({already_started, _}, emqx_mod_sup:start_child(Spec)), + ?assertEqual(ok, emqx_mod_sup:start_child(Spec)), ok = emqx_mod_sup:stop_child(Mod), {error, not_found} = emqx_mod_sup:stop_child(Mod), ok. - diff --git a/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl new file mode 100644 index 000000000..36ceb8c49 --- /dev/null +++ b/lib-ce/emqx_modules/test/emqx_mod_trace_api_SUITE.erl @@ -0,0 +1,187 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mod_trace_api_SUITE). + +%% API +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-define(HOST, "http://127.0.0.1:18083/"). +-define(API_VERSION, "v4"). +-define(BASE_PATH, "api"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_plugin_libs), + emqx_ct_helpers:start_apps([emqx_modules, emqx_dashboard]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_modules, emqx_dashboard]). + +t_http_test(_Config) -> + emqx_trace:clear(), + load(), + Header = auth_header_(), + %% list + {ok, Empty} = request_api(get, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(Empty)), + %% create + ErrorTrace = #{}, + {ok, Error} = request_api(post, api_path("trace"), Header, ErrorTrace), + ?assertEqual(#{<<"message">> => <<"name required">>, + <<"code">> => <<"INCORRECT_PARAMS">>}, json(Error)), + + Name = <<"test-name">>, + Trace = [ + {<<"name">>, Name}, + {<<"type">>, <<"topic">>}, + {<<"topic">>, <<"/x/y/z">>} + ], + + {ok, Create} = request_api(post, api_path("trace"), Header, Trace), + ?assertEqual(#{<<"code">> => 0}, json(Create)), + + {ok, List} = request_api(get, api_path("trace"), Header), + #{<<"code">> := 0, <<"data">> := [Data]} = json(List), + ?assertEqual(Name, maps:get(<<"name">>, Data)), + + %% update + {ok, Update} = request_api(put, api_path("trace/test-name/stop"), Header, #{}), + ?assertEqual(#{<<"code">> => 0, + <<"data">> => #{<<"enable">> => false, + <<"name">> => <<"test-name">>}}, json(Update)), + + {ok, List1} = request_api(get, api_path("trace"), Header), + #{<<"code">> := 0, <<"data">> := [Data1]} = json(List1), + Node = atom_to_binary(node()), + ?assertMatch(#{ + <<"status">> := <<"stopped">>, + <<"name">> := <<"test-name">>, + <<"log_size">> := #{Node := _}, + <<"start_at">> := _, + <<"end_at">> := _, + <<"type">> := <<"topic">>, + <<"topic">> := <<"/x/y/z">> + }, Data1), + + %% delete + {ok, Delete} = request_api(delete, api_path("trace/test-name"), Header), + ?assertEqual(#{<<"code">> => 0}, json(Delete)), + + {ok, DeleteNotFound} = request_api(delete, api_path("trace/test-name"), Header), + ?assertEqual(#{<<"code">> => <<"NOT_FOUND">>, + <<"message">> => <<"test-name NOT FOUND">>}, json(DeleteNotFound)), + + {ok, List2} = request_api(get, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0, <<"data">> => []}, json(List2)), + + %% clear + {ok, Create1} = request_api(post, api_path("trace"), Header, Trace), + ?assertEqual(#{<<"code">> => 0}, json(Create1)), + + {ok, Clear} = request_api(delete, api_path("trace"), Header), + ?assertEqual(#{<<"code">> => 0}, json(Clear)), + + unload(), + ok. + +t_stream_log(_Config) -> + application:set_env(emqx, allow_anonymous, true), + emqx_trace:clear(), + load(), + ClientId = <<"client-stream">>, + Now = erlang:system_time(second), + Name = <<"test_stream_log">>, + Start = to_rfc3339(Now - 10), + ok = emqx_trace:create([{<<"name">>, Name}, + {<<"type">>, <<"clientid">>}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]), + ct:sleep(200), + {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(Client), + [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], + emqtt:publish(Client, <<"/good">>, #{}, <<"ghood1">>, [{qos, 0}]), + emqtt:publish(Client, <<"/good">>, #{}, <<"ghood2">>, [{qos, 0}]), + ok = emqtt:disconnect(Client), + ct:sleep(200), + File = emqx_trace:log_file(Name, Now), + ct:pal("FileName: ~p", [File]), + {ok, FileBin} = file:read_file(File), + ct:pal("FileBin: ~p ~s", [byte_size(FileBin), FileBin]), + Header = auth_header_(), + {ok, Binary} = request_api(get, api_path("trace/test_stream_log/log?bytes=10"), Header), + #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta, <<"items">> := Bin}} = json(Binary), + ?assertEqual(10, byte_size(Bin)), + ?assertEqual(#{<<"position">> => 10, <<"bytes">> => 10}, Meta), + Path = api_path("trace/test_stream_log/log?position=20&bytes=10"), + {ok, Binary1} = request_api(get, Path, Header), + #{<<"code">> := 0, <<"data">> := #{<<"meta">> := Meta1, <<"items">> := Bin1}} = json(Binary1), + ?assertEqual(#{<<"position">> => 30, <<"bytes">> => 10}, Meta1), + ?assertEqual(10, byte_size(Bin1)), + unload(), + ok. + +to_rfc3339(Second) -> + list_to_binary(calendar:system_time_to_rfc3339(Second)). + +auth_header_() -> + auth_header_("admin", "public"). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), + {"Authorization", "Basic " ++ Encoded}. + +request_api(Method, Url, Auth) -> do_request_api(Method, {Url, [Auth]}). + +request_api(Method, Url, Auth, Body) -> + Request = {Url, [Auth], "application/json", emqx_json:encode(Body)}, + do_request_api(Method, Request). + +do_request_api(Method, Request) -> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], [{body_format, binary}]) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {error,{shutdown, server_closed}} -> + {error, server_closed}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +api_path(Path) -> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION, Path]). + +json(Data) -> + {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. + +load() -> + emqx_trace:start_link(). + +unload() -> + gen_server:stop(emqx_trace). diff --git a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl index 542e8ea53..9c23a3aa3 100644 --- a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl @@ -46,7 +46,7 @@ end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]). init_per_testcase(t_ensure_default_loaded_modules_file, Config) -> - LoadedModulesFilepath = application:get_env(emqx, modules_loaded_file), + {ok, LoadedModulesFilepath} = application:get_env(emqx, modules_loaded_file), ok = application:stop(emqx_modules), TmpFilepath = filename:join(["/", "tmp", "loaded_modules_tmp"]), case file:delete(TmpFilepath) of @@ -86,8 +86,10 @@ t_ensure_default_loaded_modules_file(_Config) -> , {emqx_mod_delayed,false} , {emqx_mod_presence,true} , {emqx_mod_rewrite,false} + , {emqx_mod_slow_subs,false} , {emqx_mod_subscription,false} , {emqx_mod_topic_metrics,false} + , {emqx_mod_trace,false} ], lists:sort(emqx_modules:list())), ok. diff --git a/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl new file mode 100644 index 000000000..6434fedd0 --- /dev/null +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_SUITE.erl @@ -0,0 +1,117 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include("include/emqx_mqtt.hrl"). +-include_lib("include/emqx.hrl"). + +%-define(LOGT(Format, Args), ct:pal(Format, Args)). + +-define(TOPK_TAB, emqx_slow_subs_topk). +-define(NOW, erlang:system_time(millisecond)). + +all() -> emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx]), + Config. + +init_per_testcase(_, Config) -> + emqx_mod_slow_subs:load(base_conf()), + Config. + +end_per_testcase(_, _) -> + emqx_mod_slow_subs:unload([]), + ok. + +%%-------------------------------------------------------------------- +%% Test Cases +%%-------------------------------------------------------------------- +t_log_and_pub(_) -> + %% Sub topic first + Subs = [{<<"/test1/+">>, ?QOS_1}, {<<"/test2/+">>, ?QOS_2}], + Clients = start_client(Subs), + timer:sleep(1500), + Now = ?NOW, + + %% publish + lists:foreach(fun(I) -> + Topic = list_to_binary(io_lib:format("/test1/~p", [I])), + Msg = emqx_message:make(undefined, ?QOS_1, Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 500}) + end, + lists:seq(1, 10)), + + lists:foreach(fun(I) -> + Topic = list_to_binary(io_lib:format("/test2/~p", [I])), + Msg = emqx_message:make(undefined, ?QOS_2, Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 500}) + end, + lists:seq(1, 10)), + + timer:sleep(2000), + Size = ets:info(?TOPK_TAB, size), + %% some time record maybe delete due to it expired or the ets size exceeds 5 due to race conditions + ?assert(Size =< 8 andalso Size >= 3, + unicode:characters_to_binary(io_lib:format("size is :~p~n", [Size]))), + + timer:sleep(3000), + ?assert(ets:info(?TOPK_TAB, size) =:= 0), + [Client ! stop || Client <- Clients], + ok. +base_conf() -> + [ {threshold, 300} + , {top_k_num, 5} + , {expire_interval, timer:seconds(3)} + , {stats_type, whole} + ]. + +start_client(Subs) -> + [spawn(fun() -> client(I, Subs) end) || I <- lists:seq(1, 10)]. + +client(I, Subs) -> + {ok, C} = emqtt:start_link([{host, "localhost"}, + {clientid, io_lib:format("slow_subs_~p", [I])}, + {username, <<"plain">>}, + {password, <<"plain">>}]), + {ok, _} = emqtt:connect(C), + + Len = erlang:length(Subs), + Sub = lists:nth(I rem Len + 1, Subs), + _ = emqtt:subscribe(C, Sub), + + receive + stop -> + ok + end. + +try_receive(Acc) -> + receive + {deliver, _, #message{payload = Payload}} -> + #{<<"logs">> := Logs} = emqx_json:decode(Payload, [return_maps]), + try_receive([length(Logs) | Acc]) + after 500 -> + Acc + end. diff --git a/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl new file mode 100644 index 000000000..52694c3d9 --- /dev/null +++ b/lib-ce/emqx_modules/test/emqx_slow_subs_api_SUITE.erl @@ -0,0 +1,163 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%--------------------------------------------------------------------n + +-module(emqx_slow_subs_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx_management/include/emqx_mgmt.hrl"). +-include_lib("emqx_plugin_libs/include/emqx_slow_subs.hrl"). + +-define(CONTENT_TYPE, "application/x-www-form-urlencoded"). + +-define(HOST, "http://127.0.0.1:18083/"). + +-define(API_VERSION, "v4"). + +-define(BASE_PATH, "api"). +-define(NOW, erlang:system_time(millisecond)). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ok = meck:new([emqx_modules], [passthrough, no_history, no_link]), + ok = meck:expect(emqx_modules, find_module, fun(_) -> [{true, true}] end), + emqx_ct_helpers:boot_modules(all), + application:load(emqx_plugin_libs), + emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_dashboard]), + Config. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_management]), + ok = meck:unload(emqx_modules), + Config. + +init_per_testcase(_, Config) -> + emqx_mod_slow_subs:load(base_conf()), + Config. + +end_per_testcase(_, Config) -> + emqx_mod_slow_subs:unload([]), + Config. + +base_conf() -> + [ {threshold, 500} + , {top_k_num, 5} + , {expire_interval, timer:seconds(60)} + , {notice_interval, 0} + , {notice_qos, 0} + , {notice_batch_size, 3} + ]. + +t_get_history(_) -> + Now = ?NOW, + Each = fun(I) -> + ClientId = erlang:list_to_binary(io_lib:format("test_~p", [I])), + Topic = erlang:list_to_binary(io_lib:format("topic/~p", [I])), + ets:insert(?TOPK_TAB, #top_k{index = ?TOPK_INDEX(I, ?ID(ClientId, Topic)), + last_update_time = Now}) + end, + + lists:foreach(Each, lists:seq(1, 5)), + + {ok, Data} = request_api(get, api_path(["slow_subscriptions"]), "", + auth_header_()), + #{data := [First | _]} = decode(Data), + + RFirst = #{clientid => <<"test_5">>, + topic => <<"topic/5">>, + timespan => 5, + node => erlang:atom_to_binary(node()), + last_update_time => Now}, + + ?assertEqual(RFirst, First). + +t_clear(_) -> + ets:insert(?TOPK_TAB, #top_k{index = ?TOPK_INDEX(1, ?ID(<<"test">>, <<"test">>)), + last_update_time = ?NOW}), + + {ok, _} = request_api(delete, api_path(["slow_subscriptions"]), [], + auth_header_()), + + ?assertEqual(0, ets:info(?TOPK_TAB, size)). + +decode(Data) -> + Pairs = emqx_json:decode(Data), + to_maps(Pairs). + +to_maps([H | _] = List) when is_tuple(H) -> + to_maps(List, #{}); + +to_maps([_ | _] = List) -> + [to_maps(X) || X <- List]; + +to_maps(V) -> V. + +to_maps([{K, V} | T], Map) -> + AtomKey = erlang:binary_to_atom(K), + to_maps(T, Map#{AtomKey => to_maps(V)}); + +to_maps([], Map) -> + Map. + +request_api(Method, Url, Auth) -> + request_api(Method, Url, [], Auth, []). + +request_api(Method, Url, QueryParams, Auth) -> + request_api(Method, Url, QueryParams, Auth, []). + +request_api(Method, Url, QueryParams, Auth, []) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth]}); +request_api(Method, Url, QueryParams, Auth, Body) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + +do_request_api(Method, Request)-> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _, Return} } + when Code =:= 200 orelse Code =:= 201 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +auth_header_() -> + AppId = <<"admin">>, + AppSecret = <<"public">>, + auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User,":",Pass])), + {"Authorization","Basic " ++ Encoded}. + +api_path(Parts)-> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). diff --git a/priv/emqx.schema b/priv/emqx.schema index 4d7b1beda..d799f170d 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -96,6 +96,11 @@ {datatype, string} ]}. +{mapping, "cluster.dns.type", "ekka.cluster_discovery", [ + {datatype, {enum, [a, srv]}}, + {default, a} +]}. + %%-------------------------------------------------------------------- %% Cluster using etcd @@ -171,7 +176,8 @@ {loop, cuttlefish:conf_get("cluster.mcast.loop", Conf, true)}]; (dns) -> [{name, cuttlefish:conf_get("cluster.dns.name", Conf)}, - {app, cuttlefish:conf_get("cluster.dns.app", Conf)}]; + {app, cuttlefish:conf_get("cluster.dns.app", Conf)}, + {type, cuttlefish:conf_get("cluster.dns.type", Conf)}]; (etcd) -> SslOpts = fun(Conf) -> Options = cuttlefish_variable:filter_by_prefix("cluster.etcd.ssl", Conf), @@ -384,11 +390,40 @@ end}. ]}. %% RPC server port. +{mapping, "rpc.driver", "gen_rpc.driver", +[ {default, tcp} +, {datatype, {enum, [tcp, ssl]}} +]}. + +{mapping, "rpc.default_client_driver", "gen_rpc.default_client_driver", +[ {default, tcp} +, {datatype, {enum, [tcp, ssl]}} +]}. + {mapping, "rpc.tcp_server_port", "gen_rpc.tcp_server_port", [ {default, 5369}, {datatype, integer} ]}. +%% RPC SSL server port. +{mapping, "rpc.enable_ssl", "gen_rpc.ssl_server_port", [ + {default, 5369}, + {datatype, integer} +]}. + +%% RPC SSL certificates +{mapping, "rpc.certfile", "gen_rpc.certfile", [ + {datatype, string} +]}. + +{mapping, "rpc.keyfile", "gen_rpc.keyfile", [ + {datatype, string} +]}. + +{mapping, "rpc.cacertfile", "gen_rpc.cacertfile", [ + {datatype, string} +]}. + %% Number of tcp connections when connecting to RPC server {mapping, "rpc.tcp_client_num", "gen_rpc.tcp_client_num", [ {default, 0}, @@ -398,7 +433,7 @@ end}. {translation, "gen_rpc.tcp_client_num", fun(Conf) -> case cuttlefish:conf_get("rpc.tcp_client_num", Conf) of - 0 -> 1; %% keep allowing 0 for backward compatibility + 0 -> max(1, erlang:system_info(schedulers) div 2); V -> V end end}. @@ -1002,6 +1037,13 @@ end}. {datatype, {duration, s}} ]}. +%% @doc the number of smaples for calculate the average latency of delivery +%% @deprecated This is a obsoleted configuration, kept here only for compatibility +{mapping, "zone.$name.latency_samples", "emqx.zones", [ + {default, 10}, + {datatype, integer} +]}. + %% @doc Max Packets that Awaiting PUBREL, 0 means no limit {mapping, "zone.$name.max_awaiting_rel", "emqx.zones", [ {default, 0}, @@ -2238,6 +2280,46 @@ end}. {datatype, string} ]}. +{mapping, "module.slow_subs.threshold", "emqx.modules", [ + {default, "500ms"}, + {datatype, {duration, ms}} +]}. + +{mapping, "module.slow_subs.expire_interval", "emqx.modules", [ + {default, "300s"}, + {datatype, {duration, ms}} +]}. + +{mapping, "module.slow_subs.top_k_num", "emqx.modules", [ + {default, 10}, + {datatype, integer}, + {validators, ["range:0-1000"]} +]}. + +{mapping, "module.slow_subs.stats_type", "emqx.modules", [ + {default, whole}, + {datatype, {enum, [whole, internal, response]}} +]}. + +%% @deprecated This is a obsoleted configuration, kept here only for compatibility +{mapping, "module.slow_subs.notice_interval", "emqx.modules", [ + {default, "0s"}, + {datatype, {duration, ms}} +]}. + +%% @deprecated This is a obsoleted configuration, kept here only for compatibility +{mapping, "module.slow_subs.notice_qos", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +%% @deprecated This is a obsoleted configuration, kept here only for compatibility +{mapping, "module.slow_subs.notice_batch_size", "emqx.modules", [ + {default, 500}, + {datatype, integer} +]}. + {translation, "emqx.modules", fun(Conf, _, Conf1) -> Subscriptions = fun() -> List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), @@ -2261,12 +2343,20 @@ end}. {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} end, TotalRules) end, + + SlowSubs = fun() -> + List = cuttlefish_variable:filter_by_prefix("module.slow_subs", Conf), + [{erlang:list_to_atom(Key), Value} || {[_, _, Key], Value} <- List] + end, + lists:append([ [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}], [{emqx_mod_subscription, Subscriptions()}], [{emqx_mod_rewrite, Rewrites()}], [{emqx_mod_topic_metrics, []}], [{emqx_mod_delayed, []}], + [{emqx_mod_trace, []}], + [{emqx_mod_slow_subs, SlowSubs()}], [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] ]) end}. diff --git a/rebar.config b/rebar.config index 8ed5933fc..7835a0e2f 100644 --- a/rebar.config +++ b/rebar.config @@ -43,17 +43,17 @@ , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.2"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} - , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} + , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.10"}}} - , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}} - , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.2"}}} + , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} diff --git a/rebar.config.erl b/rebar.config.erl index 226597967..5dcfb141e 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -89,7 +89,7 @@ project_app_dirs() -> plugins(HasElixir) -> [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}} - , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}} + , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.4"}}} %% emqx main project does not require port-compiler %% pin at root level for deterministic , {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}} @@ -106,7 +106,7 @@ test_plugins() -> test_deps() -> [ {bbmustache, "1.10.0"} - , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.9"}}} + , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.11"}}} , meck ]. @@ -128,30 +128,23 @@ prod_compile_opts() -> prod_overrides() -> [{add, [ {erl_opts, [deterministic]}]}]. -relup_deps(Profile) -> - {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}. - profiles() -> Vsn = get_vsn(), [ {'emqx', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin)} , {overrides, prod_overrides()} - , relup_deps('emqx') ]} , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg)} , {overrides, prod_overrides()} - , relup_deps('emqx-pkg') ]} , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin)} , {overrides, prod_overrides()} - , relup_deps('emqx-edge') ]} , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg)} , {overrides, prod_overrides()} - , relup_deps('emqx-edge-pkg') ]} , {check, [ {erl_opts, common_compile_opts()} ]} @@ -338,6 +331,7 @@ relx_overlay(ReleaseType) -> , {mkdir, "data/configs"} , {mkdir, "data/patches"} , {mkdir, "data/scripts"} + , {mkdir, "data/backup"} , {template, "data/loaded_plugins.tmpl", "data/loaded_plugins"} , {template, "data/loaded_modules.tmpl", "data/loaded_modules"} , {template, "data/emqx_vars", "releases/emqx_vars"} diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 48cb3218f..d5c9645aa 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -1,46 +1,90 @@ #!/usr/bin/env bash set -euo pipefail -latest_release=$(git describe --abbrev=0 --tags --exclude '*rc*' --exclude '*alpha*' --exclude '*beta*') +## compare to the latest release version tag: +## match rel-e4.4.0, v4.4.*, or e4.4.* tags +## but do not include alpha, beta and rc versions +## +## NOTE: 'rel-' tags are the merge base of enterprise release in opensource repo. +## i.e. if we are to release a new enterprise without cutting a new opensource release +## we should tag rel-e4.4.X in the opensource repo, and merge this tag to enterprise +## then cut a release from the enterprise repo. +latest_release="$(git describe --abbrev=0 --tags --match 'rel-e4.4.*' --match '[v|e]4.4*' --exclude '*beta*' --exclude '*alpha*' --exclude '*rc*')" +echo "Compare base: $latest_release" bad_app_count=0 no_comment_re='(^[^\s?%])' ## TODO: c source code comments re (in $app_path/c_src dirs) -while read -r app; do - if [ "$app" != "emqx" ]; then - app_path="$app" +get_vsn() { + commit="$1" + app_src_file="$2" + if [ "$commit" = 'HEAD' ]; then + if [ -f "$app_src_file" ]; then + grep vsn "$app_src_file" | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true + fi else - app_path="." + git show "$commit":"$app_src_file" 2>/dev/null | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true fi - src_file="$app_path/src/$(basename "$app").app.src" - old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"')" - now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') - if [ "$old_app_version" = "$now_app_version" ]; then - changed_lines="$(git diff "$latest_release"...HEAD --ignore-blank-lines -G "$no_comment_re" \ +} + +check_apps() { + while read -r app_path; do + app=$(basename "$app_path") + src_file="$app_path/src/$app.app.src" + old_app_version="$(get_vsn "$latest_release" "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$old_app_version" = '' ]; then + old_app_version="$(get_vsn "$latest_release" 'src/emqx.app.src')" + fi + now_app_version="$(get_vsn 'HEAD' "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$now_app_version" = '' ]; then + now_app_version="$(get_vsn 'HEAD' 'src/emqx.app.src')" + fi + if [ -z "$now_app_version" ]; then + echo "failed_to_get_new_app_vsn for $app" + exit 1 + fi + if [ -z "${old_app_version:-}" ]; then + echo "skiped checking new app ${app}" + elif [ "$old_app_version" = "$now_app_version" ]; then + lines="$(git diff "$latest_release"...HEAD --ignore-blank-lines -G "$no_comment_re" \ -- "$app_path/src" \ - -- ":(exclude)"$app_path/src/*.appup.src"" \ + -- ":(exclude)'$app_path/src/*.appup.src'" \ -- "$app_path/priv" \ -- "$app_path/c_src" | wc -l ) " - if [ "$changed_lines" -gt 0 ]; then - echo "$src_file needs a vsn bump" - bad_app_count=$(( bad_app_count + 1)) - elif [[ ${app_path} = *emqx_dashboard* ]]; then - ## emqx_dashboard is ensured to be upgraded after all other plugins - ## at the end of its appup instructions, there is the final instruction - ## {apply, {emqx_plugins, load, []} - ## since we don't know which plugins are stopped during the upgrade - ## for safty, we just force a dashboard version bump for each and every release - ## even if there is nothing changed in the app - echo "$src_file needs a vsn bump to ensure plugins loaded after upgrade" - bad_app_count=$(( bad_app_count + 1)) + if [ "$lines" -gt 0 ]; then + echo "$src_file needs a vsn bump (old=$old_app_version)" + echo "changed: $lines" + bad_app_count=$(( bad_app_count + 1)) + elif [ "$app" = 'emqx_dashboard' ]; then + ## emqx_dashboard is ensured to be upgraded after all other plugins + ## at the end of its appup instructions, there is the final instruction + ## {apply, {emqx_plugins, load, []} + ## since we don't know which plugins are stopped during the upgrade + ## for safety, we just force a dashboard version bump for each and every release + ## even if there is nothing changed in the app + echo "$src_file needs a vsn bump to ensure plugins loaded after upgrade" + bad_app_count=$(( bad_app_count + 1)) + fi fi - fi -done < <(./scripts/find-apps.sh) + done < <(./scripts/find-apps.sh) -if [ $bad_app_count -gt 0 ]; then - exit 1 -else - echo "apps version check successfully" -fi + if [ $bad_app_count -gt 0 ]; then + exit 1 + else + echo "apps version check successfully" + fi +} + +_main() { + if echo "${latest_release}" |grep -oE '[0-9]+.[0-9]+.[0-9]+' > /dev/null 2>&1; then + check_apps + else + echo "skiped unstable tag: ${latest_release}" + fi +} + +_main diff --git a/scripts/buildx.sh b/scripts/buildx.sh new file mode 100755 index 000000000..6efc6bcc0 --- /dev/null +++ b/scripts/buildx.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +## This script helps to run docker buildx to build cross-arch/platform packages (linux only) +## It mounts (not copy) host directory to a cross-arch/platform builder container +## Make sure the source dir (specified by --src_dir option) is clean before running this script + +## NOTE: it requires $USER in docker group +## i.e. will not work if docker command has to be executed with sudo + +## example: +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-debian10 --arch arm64 + +set -euo pipefail + +help() { + echo + echo "-h|--help: To display this usage information" + echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" + echo "--pkgtype zip|pkg: Specify which package to build, zip for .zip and pkg for .rpm or .deb" + echo "--arch amd64|arm64: Target arch to build the EMQ X package for" + echo "--src_dir : EMQ X source ode in this dir, default to PWD" + echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-debian10" +} + +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + --src_dir) + SRC_DIR="$2" + shift 2 + ;; + --profile) + PROFILE="$2" + shift 2 + ;; + --pkgtype) + PKGTYPE="$2" + shift 2 + ;; + --builder) + BUILDER="$2" + shift 2 + ;; + --arch) + ARCH="$2" + shift 2 + ;; + *) + echo "WARN: Unknown arg (ignored): $1" + shift + continue + ;; + esac +done + +if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ -z "${ARCH:-}" ]; then + help + exit 1 +fi + +if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then + echo "Bad --pkgtype option, should be zip or pkg" + exit 1 +fi + +cd "${SRC_DIR:-.}" + +set -x +docker info +docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" +docker run -i --rm \ + -v "$(pwd)":/emqx \ + --workdir /emqx \ + --platform="linux/$ARCH" \ + "$BUILDER" \ + bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index e19af1283..e52a82821 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -2,7 +2,21 @@ set -euo pipefail -VERSION="$1" +## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5. +## we have to use an otp24-compiled rebar3 because the defination of record #application{} +## in systools.hrl is changed in otp24. +case ${OTP_VSN} in + 23*) + VERSION="3.14.3-emqx-8" + ;; + 24*) + VERSION="3.18.0-emqx-1" + ;; + *) + echo "Unsupporetd Erlang/OTP version $OTP_VSN" + exit 1 + ;; +esac # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." @@ -10,6 +24,7 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." DOWNLOAD_URL='https://github.com/emqx/rebar3/releases/download' download() { + echo "downloading rebar3 ${VERSION}" curl -f -L "${DOWNLOAD_URL}/${VERSION}/rebar3" -o ./rebar3 } diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index abe8e2415..16de56630 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -11,6 +11,11 @@ case "${PKG_VSN}" in EMQX_CE_DASHBOARD_VERSION='v4.3.7' EMQX_EE_DASHBOARD_VERSION='v4.3.18' ;; + 4.4*) + # keep the above 4.3 untouched, otherwise conflicts! + EMQX_CE_DASHBOARD_VERSION='v4.4.2' + EMQX_EE_DASHBOARD_VERSION='v4.4.9' + ;; *) echo "Unsupported version $PKG_VSN" >&2 exit 1 diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index 513b78e28..4bf81afb3 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -9,11 +9,13 @@ UNAME="$(uname -s)" case "$UNAME" in Darwin) - SYSTEM='macos' + DIST='macos' + VERSION_ID="$(sw_vers | grep 'ProductVersion' | cut -d ':' -f2 | cut -d'.' -f1 | tr -d ' \t')" + SYSTEM="${DIST}${VERSION_ID}" ;; Linux) if grep -q -i 'rhel' /etc/*-release; then - DIST='centos' + DIST='el' VERSION_ID="$(rpm --eval '%{rhel}')" elif grep -q -i 'centos' /etc/*-release; then DIST='centos' diff --git a/scripts/get-otp-vsn.sh b/scripts/get-otp-vsn.sh new file mode 100755 index 000000000..699a16f3f --- /dev/null +++ b/scripts/get-otp-vsn.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euo pipefail + +erl -noshell -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' diff --git a/scripts/git-hook-pre-push.sh b/scripts/git-hook-pre-push.sh index 5b3f2edf2..689604092 100755 --- a/scripts/git-hook-pre-push.sh +++ b/scripts/git-hook-pre-push.sh @@ -4,9 +4,13 @@ set -euo pipefail url="$2" +## ensure enterprise code is not pushed to public repo if [ -f 'EMQX_ENTERPRISE' ]; then if [[ "$url" != *emqx-enterprise* ]]; then echo "$(tput setaf 1)error: enterprise_code_to_non_enterprise_repo" exit 1 fi fi + +## this triggers a tag vs release version check before pushing a tag +./pkg-vsn.sh diff --git a/scripts/inject-relup.escript b/scripts/inject-relup.escript new file mode 100755 index 000000000..0ca54361b --- /dev/null +++ b/scripts/inject-relup.escript @@ -0,0 +1,139 @@ +#!/usr/bin/env escript + +%% This script injects implicit relup instructions for emqx applications. + +-mode(compile). + +-define(ERROR(FORMAT, ARGS), io:format(standard_error, "[inject-relup] " ++ FORMAT ++ "~n", ARGS)). +-define(INFO(FORMAT, ARGS), io:format(user, "[inject-relup] " ++ FORMAT ++ "~n", ARGS)). + +usage() -> + "Usage: " ++ escript:script_name() ++ " ". + +main([RelupFile]) -> + case filelib:is_regular(RelupFile) of + true -> + ok = inject_relup_file(RelupFile); + false -> + ?ERROR("not a valid file: ~p", [RelupFile]), + erlang:halt(1) + end; +main(_Args) -> + ?ERROR("~s", [usage()]), + erlang:halt(1). + +inject_relup_file(File) -> + case file:script(File) of + {ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} -> + ?INFO("injecting instructions to: ~p", [File]), + UpdatedContent = {CurrRelVsn, + inject_relup_instrs(up, UpVsnRUs), + inject_relup_instrs(down, DnVsnRUs)}, + file:write_file(File, term_to_text(UpdatedContent)); + {ok, _BadFormat} -> + ?ERROR("bad formatted relup file: ~p", [File]), + error({bad_relup_format, File}); + {error, enoent} -> + ?INFO("relup file not found: ~p", [File]), + ok; + {error, Reason} -> + ?ERROR("read relup file ~p failed: ~p", [File, Reason]), + error({read_relup_error, Reason}) + end. + +inject_relup_instrs(Type, RUs) -> + lists:map(fun({Vsn, Desc, Instrs}) -> + {Vsn, Desc, append_emqx_relup_instrs(Type, Vsn, Instrs)} + end, RUs). + +append_emqx_relup_instrs(up, FromRelVsn, Instrs0) -> + {{UpExtra, _}, Instrs1} = filter_and_check_instrs(up, Instrs0), + Instrs1 ++ + [ {load, {emqx_app, brutal_purge, soft_purge}} + , {load, {emqx_relup, brutal_purge, soft_purge}} + , {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, UpExtra]}} + ]; + +append_emqx_relup_instrs(down, ToRelVsn, Instrs0) -> + {{_, DnExtra}, Instrs1} = filter_and_check_instrs(down, Instrs0), + %% NOTE: When downgrading, we apply emqx_relup:post_release_downgrade/2 before reloading + %% or removing the emqx_relup module. + Instrs2 = Instrs1 ++ + [ {load, {emqx_app, brutal_purge, soft_purge}} + , {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}} + ], + %% emqx_relup does not exist before release "4.4.2" + LoadInsts = + case ToRelVsn of + ToRelVsn when ToRelVsn =:= "4.4.1"; ToRelVsn =:= "4.4.0" -> + [ {remove, {emqx_relup, brutal_purge, brutal_purge}} + , {purge, [emqx_relup]} + ]; + _ -> + [{load, {emqx_relup, brutal_purge, soft_purge}}] + end, + Instrs2 ++ LoadInsts. + +filter_and_check_instrs(Type, Instrs) -> + case filter_fetch_emqx_mods_and_extra(Instrs) of + {_, DnExtra, _, _} when Type =:= up, DnExtra =/= undefined -> + ?ERROR("got '{apply,{emqx_relup,post_release_downgrade,[_,Extra]}}'" + " from the upgrade instruction list, should be 'post_release_upgrade'", []), + error({instruction_not_found, load_object_code}); + {UpExtra, _, _, _} when Type =:= down, UpExtra =/= undefined -> + ?ERROR("got '{apply,{emqx_relup,post_release_upgrade,[_,Extra]}}'" + " from the downgrade instruction list, should be 'post_release_downgrade'", []), + error({instruction_not_found, load_object_code}); + {_, _, [], _} -> + ?ERROR("cannot find any 'load_object_code' instructions for app emqx", []), + error({instruction_not_found, load_object_code}); + {UpExtra, DnExtra, EmqxMods, RemainInstrs} -> + assert_mandatory_modules(Type, EmqxMods), + {{UpExtra, DnExtra}, RemainInstrs} + end. + +filter_fetch_emqx_mods_and_extra(Instrs) -> + lists:foldl(fun do_filter_and_get/2, {undefined, undefined, [], []}, Instrs). + +%% collect modules for emqx app +do_filter_and_get({load_object_code, {emqx, _AppVsn, Mods}} = Instr, + {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods ++ Mods, RemainInstrs ++ [Instr]}; +%% remove 'load' instrs for emqx_relup and emqx_app +do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) + when Mod =:= emqx_relup; Mod =:= emqx_app -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs}; +%% remove 'remove' and 'purge' instrs for emqx_relup +do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs}; +do_filter_and_get({purge, [emqx_relup]}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs}; +%% remove 'apply' instrs for upgrade, and collect the 'Extra' parameter +do_filter_and_get({apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}}, + {_, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra0, DnExtra, EmqxMods, RemainInstrs}; +%% remove 'apply' instrs for downgrade, and collect the 'Extra' parameter +do_filter_and_get({apply, {emqx_relup, post_release_downgrade, [_, DnExtra0]}}, + {UpExtra, _, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra0, EmqxMods, RemainInstrs}; +%% keep all other instrs unchanged +do_filter_and_get(Instr, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> + {UpExtra, DnExtra, EmqxMods, RemainInstrs ++ [Instr]}. + + +assert_mandatory_modules(up, Mods) -> + assert(lists:member(emqx_relup, Mods) andalso lists:member(emqx_app, Mods), + "cannot find any 'load_object_code' instructions for emqx_app and emqx_relup: ~p", [Mods]); + +assert_mandatory_modules(down, Mods) -> + assert(lists:member(emqx_app, Mods), + "cannot find any 'load_object_code' instructions for emqx_app", []). + +assert(true, _, _) -> + ok; +assert(false, Msg, Args) -> + ?ERROR(Msg, Args), + error(assert_failed). + +term_to_text(Term) -> + io_lib:format("~p.", [Term]). diff --git a/scripts/pkg-full-vsn.sh b/scripts/pkg-full-vsn.sh new file mode 100755 index 000000000..e118643c9 --- /dev/null +++ b/scripts/pkg-full-vsn.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +## This script print the package full vsn based on current build environment + +## Arg 1 is either 'vsn_exact' (default) or 'vsn_matcher' +## when 'vsn_exact' is given, the version number is the output of pkg-vsn.sh +## otherwise '*' is used for 'find' command to find old versions (as upgrade base) + +set -euo pipefail + +VSN_MATCH="${1:-vsn_exact}" + +case "${VSN_MATCH}" in + vsn_exact) + PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" + ;; + vsn_matcher) + PKG_VSN='*' + ;; + *) + echo "$0 ERROR: second arg must " + exit 1 + ;; +esac + +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" + +UNAME="$(uname -m)" +case "$UNAME" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac + +echo "${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index bedc050c7..901460a24 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -28,6 +28,7 @@ case $PROFILE in esac SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" +OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" ARCH="${ARCH:-$(uname -m)}" case "$ARCH" in @@ -42,10 +43,19 @@ case "$ARCH" in ;; esac -SHASUM="sha256sum" -if [ "$SYSTEM" = "macos" ]; then - SHASUM="shasum -a 256" -fi + +case "$SYSTEM" in + windows*) + echo "WARNING: skipped downloading relup base for windows because we do not support relup for windows yet." + exit 0 + ;; + macos*) + SHASUM="shasum -a 256" + ;; + *) + SHASUM="sha256sum" + ;; +esac # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." @@ -54,12 +64,13 @@ mkdir -p _upgrade_base pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do - filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" + filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip" url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename" - echo "downloading base package from ${url} ..." if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then + echo "downloading base package from ${url} ..." curl -L -o "${filename}" "${url}" if [ "$SYSTEM" != "centos6" ]; then + echo "downloading sha256 sum from ${url}.sha256 ..." curl -L -o "${filename}.sha256" "${url}.sha256" SUMSTR=$(cat "${filename}.sha256") echo "got sha265sum: ${SUMSTR}" diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 8f391b01b..91c26823e 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -58,6 +58,9 @@ case "${EDITION}" in ;; esac +# must not be empty for MacOS (bash 3.x) +TAGS=( 'dummy' ) +TAGS_EXCLUDE=( 'dummy' ) while read -r git_tag; do # shellcheck disable=SC2207 semver=($(parse_semver "$git_tag")) @@ -68,7 +71,25 @@ while read -r git_tag; do # because current version is not an upgrade base true else - echo "$git_tag" + TAGS+=( "$git_tag" ) fi fi done < <(git tag -l "${GIT_TAG_PREFIX}${CUR_SEMVER[0]}.${CUR_SEMVER[1]}.*") + +# debian11 is introduced since v4.4.2 and e4.4.2 +# exclude tags before them +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" +if [ "$SYSTEM" = 'debian11' ]; then + TAGS_EXCLUDE+=( 'v4.4.0' 'v4.4.1' ) + TAGS_EXCLUDE+=( 'e4.4.0' 'e4.4.1' ) +fi + +for tag_to_del in "${TAGS_EXCLUDE[@]}"; do + TAGS=( "${TAGS[@]/$tag_to_del}" ) +done + +for tag in "${TAGS[@]}"; do + if [ "$tag" != '' ]; then + echo "$tag" + fi +done diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index 9a099b025..6b15d6917 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -6,6 +6,7 @@ ## Arg1: EMQX PROFILE set -euo pipefail +set -x usage() { echo "$0 PROFILE" diff --git a/src/emqx.app.src b/src/emqx.app.src index c57bd635d..156dc57c7 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.16"}, % strict semver, bump manually! + {vsn, "4.4.4"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index b47e7b76c..81dd6c560 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.15", + [{"4.4.3", [{load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -11,933 +11,163 @@ {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.14", - [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, - {"4.3.13", + {load_module,emqx_relup}]}, + {"4.4.2", [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, - {"4.3.12", + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_relup}]}, + {"4.4.1", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.11", + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {add_module,emqx_relup}]}, + {"4.4.0", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.10", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {add_module,emqx_relup}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.9", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.8", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.7", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.6", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.5", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.4", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.3", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.2", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.1", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, - {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_trie,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.15", - [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.14", - [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, - {"4.3.13", + [{"4.4.3", [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, - {"4.3.12", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_relup}]}, + {"4.4.2", + [{load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.11", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.10", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.9", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.8", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.7", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.6", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.5", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.4", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.3", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.2", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_relup}]}, + {"4.4.1", + [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, + {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.1", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {delete_module,emqx_relup}]}, + {"4.4.0", + [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_flapping,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, - {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, - {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, - {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_vm,brutal_purge,soft_purge,[]}, - {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_trie,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_node_dump,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}, + {delete_module,emqx_relup}]}, {<<".*">>,[]}]}. diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index 93dde28c2..476fe91d4 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -53,7 +53,8 @@ check_acl(ClientInfo, PubSub, Topic) -> true -> check_acl_cache(ClientInfo, PubSub, Topic); false -> do_check_acl(ClientInfo, PubSub, Topic) end, - inc_acl_metrics(Result), Result. + inc_acl_metrics(Result), + Result. %%-------------------------------------------------------------------- %% Internal functions @@ -67,20 +68,19 @@ check_acl_cache(ClientInfo, PubSub, Topic) -> AclResult; AclResult -> inc_acl_metrics(cache_hit), + emqx:run_hook('client.check_acl_complete', [ClientInfo, PubSub, Topic, AclResult, true]), AclResult end. do_check_acl(ClientInfo = #{zone := Zone}, PubSub, Topic) -> Default = emqx_zone:get_env(Zone, acl_nomatch, deny), - case - begin - ok = emqx_metrics:inc('client.check_acl'), - emqx_hooks:run_fold('client.check_acl', [ClientInfo, PubSub, Topic], Default) - end - of - allow -> allow; - _Other -> deny - end. + ok = emqx_metrics:inc('client.check_acl'), + Result = case emqx_hooks:run_fold('client.check_acl', [ClientInfo, PubSub, Topic], Default) of + allow -> allow; + _Other -> deny + end, + emqx:run_hook('client.check_acl_complete', [ClientInfo, PubSub, Topic, Result, false]), + Result. -compile({inline, [inc_acl_metrics/1]}). inc_acl_metrics(allow) -> diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index b93227990..50a5f2288 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -82,7 +82,7 @@ -define(SUBSCRIPTION, emqx_subscription). %% Guards --define(is_subid(Id), (is_binary(Id) orelse is_atom(Id))). +-define(IS_SUBID(Id), (is_binary(Id) orelse is_atom(Id))). -spec(start_link(atom(), pos_integer()) -> startlink_ret()). start_link(Pool, Id) -> @@ -118,15 +118,17 @@ subscribe(Topic) when is_binary(Topic) -> subscribe(Topic, undefined). -spec(subscribe(emqx_topic:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok). -subscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> +subscribe(Topic, SubId) when is_binary(Topic), ?IS_SUBID(SubId) -> subscribe(Topic, SubId, ?DEFAULT_SUBOPTS); subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> subscribe(Topic, undefined, SubOpts). -spec(subscribe(emqx_topic:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok). -subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?is_subid(SubId), is_map(SubOpts0) -> +subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) -> SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0), - case ets:member(?SUBOPTION, {SubPid = self(), Topic}) of + _ = emqx_trace:subscribe(Topic, SubId, SubOpts), + SubPid = self(), + case ets:member(?SUBOPTION, {SubPid, Topic}) of false -> %% New ok = emqx_broker_helper:register_sub(SubPid, SubId), do_subscribe(Topic, SubPid, with_subid(SubId, SubOpts)); @@ -171,6 +173,7 @@ unsubscribe(Topic) when is_binary(Topic) -> SubPid = self(), case ets:lookup(?SUBOPTION, {SubPid, Topic}) of [{_, SubOpts}] -> + emqx_trace:unsubscribe(Topic, SubOpts), _ = emqx_broker_helper:reclaim_seq(Topic), do_unsubscribe(Topic, SubPid, SubOpts); [] -> ok @@ -183,13 +186,7 @@ do_unsubscribe(Topic, SubPid, SubOpts) -> do_unsubscribe(Group, Topic, SubPid, SubOpts). do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> - case maps:get(shard, SubOpts, 0) of - 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - cast(pick(Topic), {unsubscribed, Topic}); - I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end; - + clean_subscribe(SubOpts, Topic, SubPid); do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> emqx_shared_sub:unsubscribe(Group, Topic, SubPid). @@ -199,7 +196,7 @@ do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> -spec(publish(emqx_types:message()) -> emqx_types:publish_result()). publish(Msg) when is_record(Msg, message) -> - _ = emqx_tracer:trace(publish, Msg), + _ = emqx_trace:publish(Msg), emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of #message{headers = #{allow_publish := false}} -> @@ -231,8 +228,7 @@ delivery(Msg) -> #delivery{sender = self(), message = Msg}. -spec(route([emqx_types:route_entry()], emqx_types:delivery()) -> emqx_types:publish_result()). route([], #delivery{message = Msg}) -> - ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]), - ok = inc_dropped_cnt(Msg), + drop_message(Msg), []; route(Routes, Delivery) -> @@ -240,6 +236,10 @@ route(Routes, Delivery) -> [do_route(Route, Delivery) | Acc] end, [], Routes). +drop_message(Msg) -> + ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]), + ok = inc_dropped_cnt(Msg). + do_route({To, Node}, Delivery) when Node =:= node() -> {Node, To, dispatch(To, Delivery)}; do_route({To, Node}, Delivery) when is_atom(Node) -> @@ -261,7 +261,7 @@ aggre(Routes) -> end, [], Routes). %% @doc Forward message to another node. --spec(forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode::sync|async) +-spec(forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode::sync | async) -> emqx_types:deliver_result()). forward(Node, To, Delivery, async) -> case emqx_rpc:cast(To, Node, ?BROKER, dispatch, [To, Delivery]) of @@ -288,8 +288,7 @@ dispatch(Topic, #delivery{message = Msg}) -> end, 0, subscribers(Topic)), case DispN of 0 -> - ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]), - ok = inc_dropped_cnt(Msg), + drop_message(Msg), {error, no_subscribers}; _ -> {ok, DispN} @@ -336,17 +335,20 @@ subscriber_down(SubPid) -> SubOpts when is_map(SubOpts) -> _ = emqx_broker_helper:reclaim_seq(Topic), true = ets:delete(?SUBOPTION, {SubPid, Topic}), - case maps:get(shard, SubOpts, 0) of - 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), - ok = cast(pick(Topic), {unsubscribed, Topic}); - I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), - ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) - end; + clean_subscribe(SubOpts, Topic, SubPid); undefined -> ok end end, lookup_value(?SUBSCRIPTION, SubPid, [])), ets:delete(?SUBSCRIPTION, SubPid). +clean_subscribe(SubOpts, Topic, SubPid) -> + case maps:get(shard, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end. + %%-------------------------------------------------------------------- %% Management APIs %%-------------------------------------------------------------------- @@ -366,14 +368,14 @@ subscriptions(SubId) -> -spec(subscribed(pid() | emqx_types:subid(), emqx_topic:topic()) -> boolean()). subscribed(SubPid, Topic) when is_pid(SubPid) -> ets:member(?SUBOPTION, {SubPid, Topic}); -subscribed(SubId, Topic) when ?is_subid(SubId) -> +subscribed(SubId, Topic) when ?IS_SUBID(SubId) -> SubPid = emqx_broker_helper:lookup_subpid(SubId), ets:member(?SUBOPTION, {SubPid, Topic}). -spec(get_subopts(pid(), emqx_topic:topic()) -> maybe(emqx_types:subopts())). get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> lookup_value(?SUBOPTION, {SubPid, Topic}); -get_subopts(SubId, Topic) when ?is_subid(SubId) -> +get_subopts(SubId, Topic) when ?IS_SUBID(SubId) -> case emqx_broker_helper:lookup_subpid(SubId) of SubPid when is_pid(SubPid) -> get_subopts(SubPid, Topic); @@ -498,4 +500,3 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- - diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 63a67592f..5eaf88704 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -102,7 +102,7 @@ -type(reply() :: {outgoing, emqx_types:packet()} | {outgoing, [emqx_types:packet()]} - | {event, conn_state()|updated} + | {event, conn_state() | updated} | {close, Reason :: atom()}). -type(replies() :: emqx_types:packet() | reply() | [reply()]). @@ -118,7 +118,32 @@ quota_timer => expire_quota_limit }). --define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). +-define(INFO_KEYS, + [ conninfo + , conn_state + , clientinfo + , session + , will_msg + ]). + +-define(CHANNEL_METRICS, + [ recv_pkt + , recv_msg + , 'recv_msg.qos0' + , 'recv_msg.qos1' + , 'recv_msg.qos2' + , 'recv_msg.dropped' + , 'recv_msg.dropped.await_pubrel_timeout' + , send_pkt + , send_msg + , 'send_msg.qos0' + , 'send_msg.qos1' + , 'send_msg.qos2' + , 'send_msg.dropped' + , 'send_msg.dropped.expired' + , 'send_msg.dropped.queue_full' + , 'send_msg.dropped.too_large' + ]). -dialyzer({no_match, [shutdown/4, ensure_timer/2, interval/2]}). @@ -131,7 +156,7 @@ info(Channel) -> maps:from_list(info(?INFO_KEYS, Channel)). --spec(info(list(atom())|atom(), channel()) -> term()). +-spec(info(list(atom()) | atom(), channel()) -> term()). info(Keys, Channel) when is_list(Keys) -> [{Key, info(Key, Channel)} || Key <- Keys]; info(conninfo, #channel{conninfo = ConnInfo}) -> @@ -181,10 +206,9 @@ get_session(#channel{session = Session}) -> set_session(Session, Channel) -> Channel#channel{session = Session}. -%% TODO: Add more stats. -spec(stats(channel()) -> emqx_types:stats()). stats(#channel{session = Session})-> - emqx_session:stats(Session). + lists:append(emqx_session:stats(Session), emqx_pd:get_counters(?CHANNEL_METRICS)). -spec(caps(channel()) -> emqx_types:caps()). caps(#channel{clientinfo = #{zone := Zone}}) -> @@ -275,7 +299,7 @@ take_ws_cookie(ClientInfo, ConnInfo) -> handle_in(?CONNECT_PACKET(), Channel = #channel{conn_state = connected}) -> handle_out(disconnect, ?RC_PROTOCOL_ERROR, Channel); -handle_in(?CONNECT_PACKET(ConnPkt), Channel) -> +handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) -> case pipeline([fun enrich_conninfo/2, fun run_conn_hooks/2, fun check_connect/2, @@ -285,6 +309,7 @@ handle_in(?CONNECT_PACKET(ConnPkt), Channel) -> fun auth_connect/2 ], ConnPkt, Channel#channel{conn_state = connecting}) of {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} -> + ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]), NChannel1 = NChannel#channel{ will_msg = emqx_packet:will_msg(NConnPkt), alias_maximum = init_alias_maximum(NConnPkt, ClientInfo) @@ -620,7 +645,7 @@ ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> -compile({inline, [puback_reason_code/1]}). puback_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS; -puback_reason_code([_|_]) -> ?RC_SUCCESS. +puback_reason_code([_ | _]) -> ?RC_SUCCESS. -compile({inline, [after_message_acked/3]}). after_message_acked(ClientInfo, Msg, PubAckProps) -> @@ -639,7 +664,7 @@ process_subscribe(TopicFilters, SubProps, Channel) -> process_subscribe([], _SubProps, Channel, Acc) -> {lists:reverse(Acc), Channel}; -process_subscribe([Topic = {TopicFilter, SubOpts}|More], SubProps, Channel, Acc) -> +process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Acc) -> case check_sub_caps(TopicFilter, SubOpts, Channel) of ok -> {ReasonCode, NChannel} = do_subscribe(TopicFilter, @@ -677,9 +702,9 @@ process_unsubscribe(TopicFilters, UnSubProps, Channel) -> process_unsubscribe([], _UnSubProps, Channel, Acc) -> {lists:reverse(Acc), Channel}; -process_unsubscribe([{TopicFilter, SubOpts}|More], UnSubProps, Channel, Acc) -> +process_unsubscribe([{TopicFilter, SubOpts} | More], UnSubProps, Channel, Acc) -> {RC, NChannel} = do_unsubscribe(TopicFilter, SubOpts#{unsub_props => UnSubProps}, Channel), - process_unsubscribe(More, UnSubProps, NChannel, [RC|Acc]). + process_unsubscribe(More, UnSubProps, NChannel, [RC | Acc]). do_unsubscribe(TopicFilter, SubOpts, Channel = #channel{clientinfo = ClientInfo = #{mountpoint := MountPoint}, @@ -965,6 +990,19 @@ handle_call({quota, Policy}, Channel) -> Quota = emqx_limiter:init(Zone, Policy), reply(ok, Channel#channel{quota = Quota}); +handle_call({keepalive, Interval}, Channel = #channel{keepalive = KeepAlive, + conninfo = ConnInfo, timers = Timers}) -> + AliveTimer = maps:get(alive_timer, Timers, undefined), + emqx_misc:cancel_timer(AliveTimer), + ClientId = info(clientid, Channel), + NKeepalive = emqx_keepalive:set(interval, Interval * 1000, KeepAlive), + NConnInfo = maps:put(keepalive, Interval, ConnInfo), + NChannel = Channel#channel{keepalive = NKeepalive, conninfo = NConnInfo}, + SockInfo = maps:get(sockinfo, emqx_cm:get_chan_info(ClientId), #{}), + ChanInfo1 = info(NChannel), + emqx_cm:set_chan_info(ClientId, ChanInfo1#{sockinfo => SockInfo}), + reply(ok, reset_timer(alive_timer, NChannel)); + handle_call(Req, Channel) -> ?LOG(error, "Unexpected call: ~p", [Req]), reply(ignored, Channel). @@ -1655,6 +1693,8 @@ ensure_disconnected(Reason, Channel = #channel{conninfo = ConnInfo, clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.disconnected', [ClientInfo, Reason, NConnInfo]), + ChanPid = self(), + emqx_cm:mark_channel_disconnected(ChanPid), Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. %%-------------------------------------------------------------------- diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 09a2ac9a0..f9252e938 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -22,6 +22,7 @@ -include("emqx.hrl"). -include("logger.hrl"). -include("types.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[CM]"). @@ -72,7 +73,12 @@ ]). %% Internal export --export([stats_fun/0, clean_down/1]). +-export([ stats_fun/0 + , clean_down/1 + , mark_channel_connected/1 + , mark_channel_disconnected/1 + , get_connected_client_count/0 + ]). -type(chan_pid() :: pid()). @@ -80,11 +86,13 @@ -define(CHAN_TAB, emqx_channel). -define(CHAN_CONN_TAB, emqx_channel_conn). -define(CHAN_INFO_TAB, emqx_channel_info). +-define(CHAN_LIVE_TAB, emqx_channel_live). -define(CHAN_STATS, [{?CHAN_TAB, 'channels.count', 'channels.max'}, {?CHAN_TAB, 'sessions.count', 'sessions.max'}, - {?CHAN_CONN_TAB, 'connections.count', 'connections.max'} + {?CHAN_CONN_TAB, 'connections.count', 'connections.max'}, + {?CHAN_LIVE_TAB, 'live_connections.count', 'live_connections.max'} ]). %% Batch drain @@ -129,6 +137,7 @@ register_channel(ClientId, ChanPid, #{conn_mod := ConnMod}) when is_pid(ChanPid) true = ets:insert(?CHAN_TAB, Chan), true = ets:insert(?CHAN_CONN_TAB, {Chan, ConnMod}), ok = emqx_cm_registry:register_channel(Chan), + mark_channel_connected(ChanPid), cast({registered, Chan}). %% @doc Unregister a channel. @@ -233,7 +242,8 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> {ok, #{session => Session, present => false}} end, case takeover_session(ClientId) of - {ok, ConnMod, ChanPid, Session} -> + {ok, ConnMod, ChanPid, Session0} -> + Session = emqx_session:upgrade(ClientInfo, Session0), ok = emqx_session:resume(ClientInfo, Session), case request_stepdown({takeover, 'end'}, ConnMod, ChanPid) of {ok, Pendings} -> @@ -280,7 +290,7 @@ takeover_session(ClientId, ChanPid) when node(ChanPid) == node() -> ConnMod when is_atom(ConnMod) -> case request_stepdown({takeover, 'begin'}, ConnMod, ChanPid) of {ok, Session} -> - {ok, ConnMod, ChanPid, Session}; + {ok, ConnMod, ChanPid, emqx_session:downgrade(Session)}; {error, Reason} -> {error, Reason} end @@ -468,8 +478,10 @@ init([]) -> ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true}|TabOpts]), ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]), ok = emqx_tables:new(?CHAN_INFO_TAB, [set, compressed | TabOpts]), + ok = emqx_tables:new(?CHAN_LIVE_TAB, [set, {write_concurrency, true} | TabOpts]), ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0), - {ok, #{chan_pmon => emqx_pmon:new()}}. + State = #{chan_pmon => emqx_pmon:new()}, + {ok, State}. handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), @@ -478,17 +490,17 @@ handle_call(Req, _From, State) -> handle_cast({registered, {ClientId, ChanPid}}, State = #{chan_pmon := PMon}) -> PMon1 = emqx_pmon:monitor(ChanPid, ClientId, PMon), {noreply, State#{chan_pmon := PMon1}}; - handle_cast(Msg, State) -> ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> + ?tp(emqx_cm_process_down, #{pid => Pid, reason => _Reason}), ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), + lists:foreach(fun mark_channel_disconnected/1, ChanPids), ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]), {noreply, State#{chan_pmon := PMon1}}; - handle_info(Info, State) -> ?LOG(error, "Unexpected info: ~p", [Info]), {noreply, State}. @@ -524,3 +536,18 @@ get_chann_conn_mod(ClientId, ChanPid) when node(ChanPid) == node() -> get_chann_conn_mod(ClientId, ChanPid) -> rpc_call(node(ChanPid), get_chann_conn_mod, [ClientId, ChanPid], ?T_GET_INFO). +mark_channel_connected(ChanPid) -> + ?tp(emqx_cm_connected_client_count_inc, #{}), + ets:insert_new(?CHAN_LIVE_TAB, {ChanPid, true}), + ok. + +mark_channel_disconnected(ChanPid) -> + ?tp(emqx_cm_connected_client_count_dec, #{}), + ets:delete(?CHAN_LIVE_TAB, ChanPid), + ok. + +get_connected_client_count() -> + case ets:info(?CHAN_LIVE_TAB, size) of + undefined -> 0; + Size -> Size + end. diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 14dd76211..44f3f9301 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -106,9 +106,6 @@ -type(state() :: #state{}). -define(ACTIVE_N, 100). --define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]). --define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). --define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). -define(ENABLED(X), (X =/= undefined)). @@ -117,12 +114,48 @@ [emqx_channel:info(clientid, Channel), emqx_channel:info(username, Channel)]))). --define(ALARM_CONN_INFO_KEYS, [ - socktype, sockname, peername, - clientid, username, proto_name, proto_ver, connected_at -]). --define(ALARM_SOCK_STATS_KEYS, [send_pend, recv_cnt, recv_oct, send_cnt, send_oct]). --define(ALARM_SOCK_OPTS_KEYS, [high_watermark, high_msgq_watermark, sndbuf, recbuf, buffer]). +-define(INFO_KEYS, + [ socktype + , peername + , sockname + , sockstate + , active_n + ]). + +-define(SOCK_STATS, + [ recv_oct + , recv_cnt + , send_oct + , send_cnt + , send_pend + ]). + +-define(ALARM_CONN_INFO_KEYS, + [ socktype + , sockname + , peername + , clientid + , username + , proto_name + , proto_ver + , connected_at + ]). + +-define(ALARM_SOCK_STATS_KEYS, + [ send_pend + , recv_cnt + , recv_oct + , send_cnt + , send_oct + ]). + +-define(ALARM_SOCK_OPTS_KEYS, + [ high_watermark + , high_msgq_watermark + , sndbuf + , recbuf + , buffer + ]). -dialyzer({no_match, [info/2]}). -dialyzer({nowarn_function, [ init/4 @@ -144,7 +177,7 @@ start_link(Transport, Socket, Options) -> %%-------------------------------------------------------------------- %% @doc Get infos of the connection/channel. --spec(info(pid()|state()) -> emqx_types:infos()). +-spec(info(pid() | state()) -> emqx_types:infos()). info(CPid) when is_pid(CPid) -> call(CPid, info); info(State = #state{channel = Channel}) -> @@ -173,7 +206,7 @@ info(limiter, #state{limiter = Limiter}) -> maybe_apply(fun emqx_limiter:info/1, Limiter). %% @doc Get stats of the connection/channel. --spec(stats(pid()|state()) -> emqx_types:stats()). +-spec(stats(pid() | state()) -> emqx_types:stats()). stats(CPid) when is_pid(CPid) -> call(CPid, stats); stats(#state{transport = Transport, @@ -183,10 +216,9 @@ stats(#state{transport = Transport, {ok, Ss} -> Ss; {error, _} -> [] end, - ConnStats = emqx_pd:get_counters(?CONN_STATS), ChanStats = emqx_channel:stats(Channel), ProcStats = emqx_misc:proc_stats(), - lists:append([SockStats, ConnStats, ChanStats, ProcStats]). + lists:append([SockStats, ChanStats, ProcStats]). %% @doc Set TCP keepalive socket options to override system defaults. %% Idle: The number of seconds a connection needs to be idle before @@ -357,7 +389,7 @@ cancel_stats_timer(State) -> State. process_msg([], State) -> {ok, State}; -process_msg([Msg|More], State) -> +process_msg([Msg | More], State) -> try case handle_msg(Msg, State) of ok -> @@ -451,7 +483,7 @@ handle_msg({Passive, _Sock}, State) handle_msg(Deliver = {deliver, _Topic, _Msg}, #state{active_n = ActiveN} = State) -> - Delivers = [Deliver|emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent @@ -515,8 +547,8 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport, E : C : S -> ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S}) end, - ?tp(debug, terminate, #{reason => Reason}), - maybe_raise_excption(Reason). + ?tp(info, terminate, #{reason => Reason}), + maybe_raise_exception(Reason). %% close socket, discard new state, always return ok. close_socket_ok(State) -> @@ -524,12 +556,12 @@ close_socket_ok(State) -> ok. %% tell truth about the original exception -maybe_raise_excption(#{exception := Exception, +maybe_raise_exception(#{exception := Exception, context := Context, stacktrace := Stacktrace }) -> erlang:raise(Exception, Context, Stacktrace); -maybe_raise_excption(Reason) -> +maybe_raise_exception(Reason) -> exit(Reason). %%-------------------------------------------------------------------- @@ -625,7 +657,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> {Packets, State#state{parse_state = NParseState}}; {ok, Packet, Rest, NParseState} -> NState = State#state{parse_state = NParseState}, - parse_incoming(Rest, [Packet|Packets], NState) + parse_incoming(Rest, [Packet | Packets], NState) catch error:proxy_protocol_config_disabled:_Stk -> ?LOG(error, @@ -689,6 +721,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> [emqx_packet:format(Packet)]), ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), + ok = inc_outgoing_stats({error, message_too_large}), <<>>; Data -> ?LOG(debug, "SEND ~s", [emqx_packet:format(Packet)]), ok = inc_outgoing_stats(Packet), @@ -826,6 +859,7 @@ inc_incoming_stats(Packet = ?PACKET(Type)) -> case Type =:= ?PUBLISH of true -> inc_counter(recv_msg, 1), + inc_qos_stats(recv_msg, Packet), inc_counter(incoming_pubs, 1); false -> ok @@ -833,17 +867,32 @@ inc_incoming_stats(Packet = ?PACKET(Type)) -> emqx_metrics:inc_recv(Packet). -compile({inline, [inc_outgoing_stats/1]}). +inc_outgoing_stats({error, message_too_large}) -> + inc_counter('send_msg.dropped', 1), + inc_counter('send_msg.dropped.too_large', 1); inc_outgoing_stats(Packet = ?PACKET(Type)) -> inc_counter(send_pkt, 1), case Type =:= ?PUBLISH of true -> inc_counter(send_msg, 1), - inc_counter(outgoing_pubs, 1); + inc_counter(outgoing_pubs, 1), + inc_qos_stats(send_msg, Packet); false -> ok end, emqx_metrics:inc_sent(Packet). +inc_qos_stats(Type, #mqtt_packet{header = #mqtt_packet_header{qos = QoS}}) when ?IS_QOS(QoS) -> + inc_counter(inc_qos_stats_key(Type, QoS), 1); +inc_qos_stats(_, _) -> ok. + +inc_qos_stats_key(send_msg, ?QOS_0) -> 'send_msg.qos0'; +inc_qos_stats_key(send_msg, ?QOS_1) -> 'send_msg.qos1'; +inc_qos_stats_key(send_msg, ?QOS_2) -> 'send_msg.qos2'; + +inc_qos_stats_key(recv_msg, ?QOS_0) -> 'recv_msg.qos0'; +inc_qos_stats_key(recv_msg, ?QOS_1) -> 'recv_msg.qos1'; +inc_qos_stats_key(recv_msg, ?QOS_2) -> 'recv_msg.qos2'. %%-------------------------------------------------------------------- %% Helper functions diff --git a/src/emqx_guid.erl b/src/emqx_guid.erl index 5482de2a7..adcf7abd1 100644 --- a/src/emqx_guid.erl +++ b/src/emqx_guid.erl @@ -39,6 +39,8 @@ , from_base62/1 ]). +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + -define(TAG_VERSION, 131). -define(PID_EXT, 103). -define(NEW_PID_EXT, 88). @@ -137,7 +139,7 @@ npid() -> NPid. to_hexstr(I) when byte_size(I) =:= 16 -> - emqx_misc:bin2hexstr_A_F(I). + emqx_misc:bin2hexstr_a_f_upper(I). from_hexstr(S) when byte_size(S) =:= 32 -> emqx_misc:hexstr2bin(S). diff --git a/src/emqx_keepalive.erl b/src/emqx_keepalive.erl index a7b0e72f4..be37644c0 100644 --- a/src/emqx_keepalive.erl +++ b/src/emqx_keepalive.erl @@ -20,9 +20,11 @@ , info/1 , info/2 , check/2 + , set/3 ]). -export_type([keepalive/0]). +-elvis([{elvis_style, no_if_expression, disable}]). -record(keepalive, { interval :: pos_integer(), @@ -49,7 +51,7 @@ info(#keepalive{interval = Interval, repeat => Repeat }. --spec(info(interval|statval|repeat, keepalive()) +-spec(info(interval | statval | repeat, keepalive()) -> non_neg_integer()). info(interval, #keepalive{interval = Interval}) -> Interval; @@ -71,3 +73,19 @@ check(NewVal, KeepAlive = #keepalive{statval = OldVal, true -> {error, timeout} end. +%% from mqtt-v3.1.1 specific +%% A Keep Alive value of zero (0) has the effect of turning off the keep alive mechanism. +%% This means that, in this case, the Server is not required +%% to disconnect the Client on the grounds of inactivity. +%% Note that a Server is permitted to disconnect a Client that it determines +%% to be inactive or non-responsive at any time, +%% regardless of the Keep Alive value provided by that Client. +%% Non normative comment +%%The actual value of the Keep Alive is application specific; +%% typically this is a few minutes. +%% The maximum value is (65535s) 18 hours 12 minutes and 15 seconds. + +%% @doc Update keepalive's interval +-spec(set(interval, non_neg_integer(), keepalive()) -> keepalive()). +set(interval, Interval, KeepAlive) when Interval >= 0 andalso Interval =< 65535000 -> + KeepAlive#keepalive{interval = Interval}. diff --git a/src/emqx_logger.erl b/src/emqx_logger.erl index a01f028d8..d16b3edf1 100644 --- a/src/emqx_logger.erl +++ b/src/emqx_logger.erl @@ -18,6 +18,8 @@ -compile({no_auto_import, [error/1]}). +-elvis([{elvis_style, god_modules, disable}]). + %% Logs -export([ debug/1 , debug/2 @@ -64,10 +66,11 @@ id := logger:handler_id(), level := logger:level(), dst := logger_dst(), + filters := [{logger:filter_id(), logger:filter()}], status := started | stopped }). --define(stopped_handlers, {?MODULE, stopped_handlers}). +-define(STOPPED_HANDLERS, {?MODULE, stopped_handlers}). %%-------------------------------------------------------------------- %% APIs @@ -171,19 +174,19 @@ get_log_handlers() -> -spec(get_log_handlers(started | stopped) -> [logger_handler_info()]). get_log_handlers(started) -> - [log_hanlder_info(Conf, started) || Conf <- logger:get_handler_config()]; + [log_handler_info(Conf, started) || Conf <- logger:get_handler_config()]; get_log_handlers(stopped) -> - [log_hanlder_info(Conf, stopped) || Conf <- list_stopped_handler_config()]. + [log_handler_info(Conf, stopped) || Conf <- list_stopped_handler_config()]. -spec(get_log_handler(logger:handler_id()) -> logger_handler_info()). get_log_handler(HandlerId) -> case logger:get_handler_config(HandlerId) of {ok, Conf} -> - log_hanlder_info(Conf, started); + log_handler_info(Conf, started); {error, _} -> case read_stopped_handler_config(HandlerId) of error -> {error, {not_found, HandlerId}}; - {ok, Conf} -> log_hanlder_info(Conf, stopped) + {ok, Conf} -> log_handler_info(Conf, stopped) end end. @@ -245,21 +248,21 @@ parse_transform(AST, _Opts) -> %% Internal Functions %%-------------------------------------------------------------------- -log_hanlder_info(#{id := Id, level := Level, module := logger_std_h, - config := #{type := Type}}, Status) when +log_handler_info(#{id := Id, level := Level, module := logger_std_h, + filters := Filters, config := #{type := Type}}, Status) when Type =:= standard_io; Type =:= standard_error -> - #{id => Id, level => Level, dst => console, status => Status}; -log_hanlder_info(#{id := Id, level := Level, module := logger_std_h, - config := Config = #{type := file}}, Status) -> - #{id => Id, level => Level, status => Status, + #{id => Id, level => Level, dst => console, status => Status, filters => Filters}; +log_handler_info(#{id := Id, level := Level, module := logger_std_h, + filters := Filters, config := Config = #{type := file}}, Status) -> + #{id => Id, level => Level, status => Status, filters => Filters, dst => maps:get(file, Config, atom_to_list(Id))}; -log_hanlder_info(#{id := Id, level := Level, module := logger_disk_log_h, - config := #{file := Filename}}, Status) -> - #{id => Id, level => Level, dst => Filename, status => Status}; -log_hanlder_info(#{id := Id, level := Level, module := _OtherModule}, Status) -> - #{id => Id, level => Level, dst => unknown, status => Status}. +log_handler_info(#{id := Id, level := Level, module := logger_disk_log_h, + filters := Filters, config := #{file := Filename}}, Status) -> + #{id => Id, level => Level, dst => Filename, status => Status, filters => Filters}; +log_handler_info(#{id := Id, level := Level, filters := Filters}, Status) -> + #{id => Id, level => Level, dst => unknown, status => Status, filters => Filters}. %% set level for all log handlers in one command set_all_log_handlers_level(Level) -> @@ -281,29 +284,29 @@ rollback([{ID, Level} | List]) -> rollback([]) -> ok. save_stopped_handler_config(HandlerId, Config) -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> - persistent_term:put(?stopped_handlers, #{HandlerId => Config}); + persistent_term:put(?STOPPED_HANDLERS, #{HandlerId => Config}); ConfList -> - persistent_term:put(?stopped_handlers, ConfList#{HandlerId => Config}) + persistent_term:put(?STOPPED_HANDLERS, ConfList#{HandlerId => Config}) end. read_stopped_handler_config(HandlerId) -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> error; ConfList -> maps:find(HandlerId, ConfList) end. remove_stopped_handler_config(HandlerId) -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> ok; ConfList -> case maps:find(HandlerId, ConfList) of error -> ok; {ok, _} -> - persistent_term:put(?stopped_handlers, maps:remove(HandlerId, ConfList)) + persistent_term:put(?STOPPED_HANDLERS, maps:remove(HandlerId, ConfList)) end end. list_stopped_handler_config() -> - case persistent_term:get(?stopped_handlers, undefined) of + case persistent_term:get(?STOPPED_HANDLERS, undefined) of undefined -> []; ConfList -> maps:values(ConfList) end. diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index 36a3386e3..c366b878d 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -21,6 +21,8 @@ -include("types.hrl"). -include("logger.hrl"). +-elvis([{elvis_style, god_modules, disable}]). + -export([ merge_opts/2 , maybe_apply/2 , compose/1 @@ -47,8 +49,8 @@ , ipv6_probe/1 ]). --export([ bin2hexstr_A_F/1 - , bin2hexstr_a_f/1 +-export([ bin2hexstr_a_f_upper/1 + , bin2hexstr_a_f_lower/1 , hexstr2bin/1 ]). @@ -84,16 +86,9 @@ maybe_parse_ip(Host) -> %% @doc Add `ipv6_probe' socket option if it's supported. ipv6_probe(Opts) -> - case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of - unknown -> - %% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64 - OtpVsn = emqx_vm:get_otp_version(), - Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])), - _ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool), - ipv6_probe(Bool, Opts); - Bool -> - ipv6_probe(Bool, Opts) - end. + Bool = try gen_tcp:ipv6_probe() + catch _ : _ -> false end, + ipv6_probe(Bool, Opts). ipv6_probe(false, Opts) -> Opts; ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts]. @@ -118,9 +113,9 @@ maybe_apply(Fun, Arg) when is_function(Fun) -> -spec(compose(list(F)) -> G when F :: fun((any()) -> any()), G :: fun((any()) -> any())). -compose([F|More]) -> compose(F, More). +compose([F | More]) -> compose(F, More). --spec(compose(F, G|[Gs]) -> C +-spec(compose(F, G | [Gs]) -> C when F :: fun((X1) -> X2), G :: fun((X2) -> X3), Gs :: [fun((Xn) -> Xn1)], @@ -128,19 +123,19 @@ compose([F|More]) -> compose(F, More). X3 :: any(), Xn :: any(), Xn1 :: any(), Xm :: any()). compose(F, G) when is_function(G) -> fun(X) -> G(F(X)) end; compose(F, [G]) -> compose(F, G); -compose(F, [G|More]) -> compose(compose(F, G), More). +compose(F, [G | More]) -> compose(compose(F, G), More). %% @doc RunFold run_fold([], Acc, _State) -> Acc; -run_fold([Fun|More], Acc, State) -> +run_fold([Fun | More], Acc, State) -> run_fold(More, Fun(Acc, State), State). %% @doc Pipeline pipeline([], Input, State) -> {ok, Input, State}; -pipeline([Fun|More], Input, State) -> +pipeline([Fun | More], Input, State) -> case apply_fun(Fun, Input, State) of ok -> pipeline(More, Input, State); {ok, NState} -> @@ -189,7 +184,7 @@ drain_deliver(0, Acc) -> drain_deliver(N, Acc) -> receive Deliver = {deliver, _Topic, _Msg} -> - drain_deliver(N-1, [Deliver|Acc]) + drain_deliver(N-1, [Deliver | Acc]) after 0 -> lists:reverse(Acc) end. @@ -204,7 +199,7 @@ drain_down(0, Acc) -> drain_down(Cnt, Acc) -> receive {'DOWN', _MRef, process, Pid, _Reason} -> - drain_down(Cnt-1, [Pid|Acc]) + drain_down(Cnt-1, [Pid | Acc]) after 0 -> lists:reverse(Acc) end. @@ -230,7 +225,7 @@ check_oom(Pid, #{message_queue_len := MaxQLen, end. do_check_oom([]) -> ok; -do_check_oom([{Val, Max, Reason}|Rest]) -> +do_check_oom([{Val, Max, Reason} | Rest]) -> case is_integer(Max) andalso (0 < Max) andalso (Max < Val) of true -> {shutdown, Reason}; false -> do_check_oom(Rest) @@ -277,8 +272,8 @@ proc_stats(Pid) -> reductions, memory]) of undefined -> []; - [{message_queue_len, Len}|ProcStats] -> - [{mailbox_len, Len}|ProcStats] + [{message_queue_len, Len} | ProcStats] -> + [{mailbox_len, Len} | ProcStats] end. rand_seed() -> @@ -298,17 +293,17 @@ index_of(E, L) -> index_of(_E, _I, []) -> error(badarg); -index_of(E, I, [E|_]) -> +index_of(E, I, [E | _]) -> I; -index_of(E, I, [_|L]) -> +index_of(E, I, [_ | L]) -> index_of(E, I+1, L). --spec(bin2hexstr_A_F(binary()) -> binary()). -bin2hexstr_A_F(B) when is_binary(B) -> +-spec(bin2hexstr_a_f_upper(binary()) -> binary()). +bin2hexstr_a_f_upper(B) when is_binary(B) -> << <<(int2hexchar(H, upper)), (int2hexchar(L, upper))>> || <> <= B>>. --spec(bin2hexstr_a_f(binary()) -> binary()). -bin2hexstr_a_f(B) when is_binary(B) -> +-spec(bin2hexstr_a_f_lower(binary()) -> binary()). +bin2hexstr_a_f_lower(B) when is_binary(B) -> << <<(int2hexchar(H, lower)), (int2hexchar(L, lower))>> || <> <= B>>. int2hexchar(I, _) when I >= 0 andalso I < 10 -> I + $0; diff --git a/src/emqx_packet.erl b/src/emqx_packet.erl index 4665e7733..c60c06811 100644 --- a/src/emqx_packet.erl +++ b/src/emqx_packet.erl @@ -46,24 +46,6 @@ -export([format/1]). --define(TYPE_NAMES, - { 'CONNECT' - , 'CONNACK' - , 'PUBLISH' - , 'PUBACK' - , 'PUBREC' - , 'PUBREL' - , 'PUBCOMP' - , 'SUBSCRIBE' - , 'SUBACK' - , 'UNSUBSCRIBE' - , 'UNSUBACK' - , 'PINGREQ' - , 'PINGRESP' - , 'DISCONNECT' - , 'AUTH' - }). - -type(connect() :: #mqtt_packet_connect{}). -type(publish() :: #mqtt_packet_publish{}). -type(subscribe() :: #mqtt_packet_subscribe{}). @@ -107,14 +89,14 @@ retain(#mqtt_packet{header = #mqtt_packet_header{retain = Retain}}) -> %%-------------------------------------------------------------------- %% @doc Protocol name of the CONNECT Packet. --spec(proto_name(emqx_types:packet()|connect()) -> binary()). +-spec(proto_name(emqx_types:packet() | connect()) -> binary()). proto_name(?CONNECT_PACKET(ConnPkt)) -> proto_name(ConnPkt); proto_name(#mqtt_packet_connect{proto_name = Name}) -> Name. %% @doc Protocol version of the CONNECT Packet. --spec(proto_ver(emqx_types:packet()|connect()) -> emqx_types:version()). +-spec(proto_ver(emqx_types:packet() | connect()) -> emqx_types:version()). proto_ver(?CONNECT_PACKET(ConnPkt)) -> proto_ver(ConnPkt); proto_ver(#mqtt_packet_connect{proto_ver = Ver}) -> @@ -249,7 +231,7 @@ set_props(Props, #mqtt_packet_auth{} = Pkt) -> %%-------------------------------------------------------------------- %% @doc Check PubSub Packet. --spec(check(emqx_types:packet()|publish()|subscribe()|unsubscribe()) +-spec(check(emqx_types:packet() | publish() | subscribe() | unsubscribe()) -> ok | {error, emqx_types:reason_code()}). check(#mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH}, variable = PubPkt}) when not is_tuple(PubPkt) -> @@ -318,7 +300,7 @@ check_pub_props(#{'Response-Topic' := ResponseTopic}) -> check_pub_props(_Props) -> ok. %% @doc Check CONNECT Packet. --spec(check(emqx_types:packet()|connect(), Opts :: map()) +-spec(check(emqx_types:packet() | connect(), Opts :: map()) -> ok | {error, emqx_types:reason_code()}). check(?CONNECT_PACKET(ConnPkt), Opts) -> check(ConnPkt, Opts); @@ -357,11 +339,13 @@ check_conn_props(#mqtt_packet_connect{properties = undefined}, _Opts) -> ok; check_conn_props(#mqtt_packet_connect{properties = #{'Receive-Maximum' := 0}}, _Opts) -> {error, ?RC_PROTOCOL_ERROR}; -check_conn_props(#mqtt_packet_connect{properties = #{'Request-Response-Information' := ReqRespInfo}}, _Opts) - when ReqRespInfo =/= 0, ReqRespInfo =/= 1 -> +check_conn_props(#mqtt_packet_connect{properties = + #{'Request-Response-Information' := ReqRespInfo}}, _Opts) + when ReqRespInfo =/= 0, ReqRespInfo =/= 1 -> {error, ?RC_PROTOCOL_ERROR}; -check_conn_props(#mqtt_packet_connect{properties = #{'Request-Problem-Information' := ReqProInfo}}, _Opts) - when ReqProInfo =/= 0, ReqProInfo =/= 1 -> +check_conn_props(#mqtt_packet_connect{properties = + #{'Request-Problem-Information' := ReqProInfo}}, _Opts) + when ReqProInfo =/= 0, ReqProInfo =/= 1 -> {error, ?RC_PROTOCOL_ERROR}; check_conn_props(_ConnPkt, _Opts) -> ok. @@ -382,7 +366,7 @@ check_will_msg(#mqtt_packet_connect{will_topic = WillTopic}, _Opts) -> run_checks([], _Packet, _Options) -> ok; -run_checks([Check|More], Packet, Options) -> +run_checks([Check | More], Packet, Options) -> case Check(Packet, Options) of ok -> run_checks(More, Packet, Options); Error = {error, _Reason} -> Error @@ -419,7 +403,8 @@ to_message(#mqtt_packet{ Msg#message{flags = #{dup => Dup, retain => Retain}, headers = Headers#{properties => Props}}. --spec(will_msg(#mqtt_packet_connect{}) -> emqx_types:message()). +-type(connectPacket() :: #mqtt_packet_connect{}). +-spec(will_msg(connectPacket()) -> emqx_types:message()). will_msg(#mqtt_packet_connect{will_flag = false}) -> undefined; will_msg(#mqtt_packet_connect{clientid = ClientId, @@ -468,13 +453,16 @@ format_variable(#mqtt_packet_connect{ will_payload = WillPayload, username = Username, password = Password}) -> - Format = "ClientId=~s, ProtoName=~s, ProtoVsn=~p, CleanStart=~s, KeepAlive=~p, Username=~s, Password=~s", - Args = [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)], - {Format1, Args1} = if - WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~s, Payload=~0p)", - Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]}; - true -> {Format, Args} - end, + Format = "ClientId=~s, ProtoName=~s, ProtoVsn=~p, CleanStart=~s," + " KeepAlive=~p, Username=~s, Password=~s", + Args = [ClientId, ProtoName, ProtoVer, CleanStart, + KeepAlive, Username, format_password(Password)], + {Format1, Args1} = + case WillFlag of + true -> {Format ++ ", Will(Q~p, R~p, Topic=~s, Payload=~0p)", + Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]}; + false -> {Format, Args} + end, io_lib:format(Format1, Args1); format_variable(#mqtt_packet_disconnect @@ -520,4 +508,3 @@ format_password(_Password) -> '******'. i(true) -> 1; i(false) -> 0; i(I) when is_integer(I) -> I. - diff --git a/src/emqx_relup.erl b/src/emqx_relup.erl new file mode 100644 index 000000000..fcc90d088 --- /dev/null +++ b/src/emqx_relup.erl @@ -0,0 +1,56 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_relup). + +%% NOTE: DO NOT remove this `-include`. +%% We use this to force this module to be upgraded every release. +-include("emqx_release.hrl"). + +-export([ post_release_upgrade/2 + , post_release_downgrade/2 + ]). + +-define(INFO(FORMAT), io:format("[emqx_relup] " ++ FORMAT ++ "~n")). +-define(INFO(FORMAT, ARGS), io:format("[emqx_relup] " ++ FORMAT ++ "~n", ARGS)). + +%% What to do after upgraded from an old release vsn. +post_release_upgrade(FromRelVsn, _) -> + {_, CurrRelVsn} = ?EMQX_RELEASE, + ?INFO("emqx has been upgraded to from ~s to ~s!", [FromRelVsn, CurrRelVsn]), + reload_components(). + +%% What to do after downgraded to an old release vsn. +post_release_downgrade(ToRelVsn, _) -> + {_, CurrRelVsn} = ?EMQX_RELEASE, + ?INFO("emqx has been downgraded to from ~s to ~s!", [CurrRelVsn, ToRelVsn]), + reload_components(). + +-ifdef(EMQX_ENTERPRISE). +reload_components() -> + ?INFO("reloading resource providers ..."), + emqx_rule_engine:load_providers(), + ?INFO("reloading module providers ..."), + emqx_modules:load_providers(), + ?INFO("loading plugins ..."), + emqx_plugins:load(). +-else. +reload_components() -> + ?INFO("reloading resource providers ..."), + emqx_rule_engine:load_providers(), + ?INFO("loading plugins ..."), + emqx_plugins:load(). +-endif. diff --git a/src/emqx_session.erl b/src/emqx_session.erl index d01f33e6a..5e22fd297 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -83,6 +83,8 @@ -export([ takeover/1 , resume/2 , replay/2 + , upgrade/2 + , downgrade/1 ]). -export([expire/2]). @@ -95,6 +97,8 @@ -import(emqx_zone, [get_env/3]). -record(session, { + %% Client's id + clientid :: emqx_types:clientid(), %% Client’s Subscriptions. subscriptions :: map(), %% Max subscriptions allowed @@ -121,8 +125,14 @@ %% Awaiting PUBREL Timeout (Unit: millsecond) await_rel_timeout :: timeout(), %% Created at - created_at :: pos_integer() - }). + created_at :: pos_integer(), + + extras :: map() + }). + +%% in the previous code, we will replace the message record with the pubrel atom +%% in the pubrec function, this will lose the creation time of the message, +-record(pubrel_await, {message :: emqx_types:message()}). -type(session() :: #session{}). @@ -153,14 +163,21 @@ -define(DEFAULT_BATCH_N, 1000). +-ifdef(TEST). +-define(GET_CLIENT_ID(C), maps:get(clientid, C, <<>>)). +-export([dummy/0]). +-else. +-define(GET_CLIENT_ID(C), maps:get(clientid, C)). +-endif. %%-------------------------------------------------------------------- %% Init a Session %%-------------------------------------------------------------------- -spec(init(emqx_types:clientinfo(), emqx_types:conninfo()) -> session()). -init(#{zone := Zone}, #{receive_maximum := MaxInflight}) -> - #session{max_subscriptions = get_env(Zone, max_subscriptions, 0), +init(#{zone := Zone} = CInfo, #{receive_maximum := MaxInflight}) -> + #session{clientid = ?GET_CLIENT_ID(CInfo), + max_subscriptions = get_env(Zone, max_subscriptions, 0), subscriptions = #{}, upgrade_qos = get_env(Zone, upgrade_qos, false), inflight = emqx_inflight:new(MaxInflight), @@ -170,7 +187,8 @@ init(#{zone := Zone}, #{receive_maximum := MaxInflight}) -> awaiting_rel = #{}, max_awaiting_rel = get_env(Zone, max_awaiting_rel, 100), await_rel_timeout = timer:seconds(get_env(Zone, await_rel_timeout, 300)), - created_at = erlang:system_time(millisecond) + created_at = erlang:system_time(millisecond), + extras = #{} }. %% @private init mq @@ -181,6 +199,20 @@ init_mqueue(Zone) -> default_priority => get_env(Zone, mqueue_default_priority, lowest) }). +%% @doc uprade from 4.3 +upgrade(CInfo, S) -> + [session | Fields] = tuple_to_list(S), + #session{} = list_to_tuple([session, ?GET_CLIENT_ID(CInfo) | Fields] ++ [#{}]). + +%% @doc Downgrade to 4.3 +downgrade(#session{} = S) -> + [session, _ClientID | Fields] = tuple_to_list(S), + list_to_tuple([session | lists:reverse(tl(lists:reverse(Fields)))]). + +-ifdef(TEST). +dummy() -> + #session{}. +-endif. %%-------------------------------------------------------------------- %% Info, Stats %%-------------------------------------------------------------------- @@ -269,7 +301,8 @@ unsubscribe(ClientInfo, TopicFilter, UnSubOpts, Session = #session{subscriptions case maps:find(TopicFilter, Subs) of {ok, SubOpts} -> ok = emqx_broker:unsubscribe(TopicFilter), - ok = emqx_hooks:run('session.unsubscribed', [ClientInfo, TopicFilter, maps:merge(SubOpts, UnSubOpts)]), + ok = emqx_hooks:run('session.unsubscribed', + [ClientInfo, TopicFilter, maps:merge(SubOpts, UnSubOpts)]), {ok, Session#session{subscriptions = maps:remove(TopicFilter, Subs)}}; error -> {error, ?RC_NO_SUBSCRIPTION_EXISTED} @@ -319,9 +352,10 @@ is_awaiting_full(#session{awaiting_rel = AwaitingRel, puback(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> + on_delivery_completed(ClientInfo, Msg, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), return_with(Msg, dequeue(ClientInfo, Session#session{inflight = Inflight1})); - {value, {_Pubrel, _Ts}} -> + {value, _Other} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} @@ -343,9 +377,10 @@ return_with(Msg, {ok, Publishes, Session}) -> pubrec(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight), + Update = with_ts(#pubrel_await{message = Msg}), + Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; - {value, {pubrel, _Ts}} -> + {value, _Other} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} @@ -374,7 +409,8 @@ pubrel(PacketId, Session = #session{awaiting_rel = AwaitingRel}) -> | {error, emqx_types:reason_code()}). pubcomp(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of - {value, {pubrel, _Ts}} -> + {value, {#pubrel_await{message = Msg}, _Ts}} -> + on_delivery_completed(ClientInfo, Msg, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), dequeue(ClientInfo, Session#session{inflight = Inflight1}); {value, _Other} -> @@ -403,11 +439,11 @@ dequeue(ClientInfo, Cnt, Msgs, Q) -> {empty, _Q} -> dequeue(ClientInfo, 0, Msgs, Q); {{value, Msg}, Q1} -> case emqx_message:is_expired(Msg) of - true -> - ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, expired]), + true -> ok = inc_delivery_expired_cnt(), + ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, expired]), dequeue(ClientInfo, Cnt, Msgs, Q1); - false -> dequeue(ClientInfo, acc_cnt(Msg, Cnt), [Msg|Msgs], Q1) + false -> dequeue(ClientInfo, acc_cnt(Msg, Cnt), [Msg | Msgs], Q1) end end. @@ -437,15 +473,16 @@ do_deliver(ClientInfo, [Msg | More], Acc, Session) -> {ok, Session1} -> do_deliver(ClientInfo, More, Acc, Session1); {ok, [Publish], Session1} -> - do_deliver(ClientInfo, More, [Publish|Acc], Session1) + do_deliver(ClientInfo, More, [Publish | Acc], Session1) end. -deliver_msg(_ClientInfo, Msg = #message{qos = ?QOS_0}, Session) -> +deliver_msg(ClientInfo, Msg = #message{qos = ?QOS_0}, Session) -> + on_delivery_completed(ClientInfo, Msg, Session), {ok, [{undefined, maybe_ack(Msg)}], Session}; deliver_msg(ClientInfo, Msg = #message{qos = QoS}, Session = - #session{next_pkt_id = PacketId, inflight = Inflight}) - when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 -> + #session{next_pkt_id = PacketId, inflight = Inflight}) + when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 -> case emqx_inflight:is_full(Inflight) of true -> Session1 = case maybe_nack(Msg) of @@ -458,11 +495,12 @@ deliver_msg(ClientInfo, Msg = #message{qos = QoS}, Session = %% But add to inflight with ack headers %% This ack header is required for redispatch-on-terminate feature to work Publish = {PacketId, maybe_ack(Msg)}, + Msg2 = mark_begin_deliver(Msg), Inflight1 = emqx_inflight:insert(PacketId, with_ts(Msg), Inflight), {ok, [Publish], next_pkt_id(Session#session{inflight = Inflight1})} end. --spec(enqueue(emqx_types:clientinfo(), list(emqx_types:deliver())|emqx_types:message(), +-spec(enqueue(emqx_types:clientinfo(), list(emqx_types:deliver()) | emqx_types:message(), session()) -> session()). enqueue(ClientInfo, Delivers, Session) when is_list(Delivers) -> Msgs = [enrich_delivers(D, Session) || D <- Delivers], @@ -481,11 +519,14 @@ log_dropped(ClientInfo, Msg = #message{qos = QoS}, #session{mqueue = Q}) -> ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, qos0_msg]), ok = emqx_metrics:inc('delivery.dropped'), ok = emqx_metrics:inc('delivery.dropped.qos0_msg'), + ok = inc_pd('send_msg.dropped'), ?LOG(warning, "Dropped qos0 msg: ~s", [emqx_message:format(Msg)]); false -> ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, queue_full]), ok = emqx_metrics:inc('delivery.dropped'), ok = emqx_metrics:inc('delivery.dropped.queue_full'), + ok = inc_pd('send_msg.dropped'), + ok = inc_pd('send_msg.dropped.queue_full'), ?LOG(warning, "Dropped msg due to mqueue is full: ~s", [emqx_message:format(Msg)]) end. @@ -509,23 +550,23 @@ get_subopts(Topic, SubMap) -> end. enrich_subopts([], Msg, _Session) -> Msg; -enrich_subopts([{nl, 1}|Opts], Msg, Session) -> +enrich_subopts([{nl, 1} | Opts], Msg, Session) -> enrich_subopts(Opts, emqx_message:set_flag(nl, Msg), Session); -enrich_subopts([{nl, 0}|Opts], Msg, Session) -> +enrich_subopts([{nl, 0} | Opts], Msg, Session) -> enrich_subopts(Opts, Msg, Session); -enrich_subopts([{qos, SubQoS}|Opts], Msg = #message{qos = PubQoS}, +enrich_subopts([{qos, SubQoS} | Opts], Msg = #message{qos = PubQoS}, Session = #session{upgrade_qos = true}) -> enrich_subopts(Opts, Msg#message{qos = max(SubQoS, PubQoS)}, Session); -enrich_subopts([{qos, SubQoS}|Opts], Msg = #message{qos = PubQoS}, +enrich_subopts([{qos, SubQoS} | Opts], Msg = #message{qos = PubQoS}, Session = #session{upgrade_qos = false}) -> enrich_subopts(Opts, Msg#message{qos = min(SubQoS, PubQoS)}, Session); -enrich_subopts([{rap, 1}|Opts], Msg, Session) -> +enrich_subopts([{rap, 1} | Opts], Msg, Session) -> enrich_subopts(Opts, Msg, Session); -enrich_subopts([{rap, 0}|Opts], Msg = #message{headers = #{retained := true}}, Session) -> +enrich_subopts([{rap, 0} | Opts], Msg = #message{headers = #{retained := true}}, Session) -> enrich_subopts(Opts, Msg, Session); -enrich_subopts([{rap, 0}|Opts], Msg, Session) -> +enrich_subopts([{rap, 0} | Opts], Msg, Session) -> enrich_subopts(Opts, emqx_message:set_flag(retain, false, Msg), Session); -enrich_subopts([{subid, SubId}|Opts], Msg, Session) -> +enrich_subopts([{subid, SubId} | Opts], Msg, Session) -> Props = emqx_message:get_header(properties, Msg, #{}), Msg1 = emqx_message:set_header(properties, Props#{'Subscription-Identifier' => SubId}, Msg), enrich_subopts(Opts, Msg1, Session). @@ -534,18 +575,22 @@ enrich_subopts([{subid, SubId}|Opts], Msg, Session) -> %% Retry Delivery %%-------------------------------------------------------------------- --spec(retry(emqx_types:clientinfo(), session()) -> {ok, session()} | {ok, replies(), timeout(), session()}). +-spec(retry(emqx_types:clientinfo(), session()) + -> {ok, session()} + | {ok, replies(), timeout(), session()}). retry(ClientInfo, Session = #session{inflight = Inflight}) -> case emqx_inflight:is_empty(Inflight) of true -> {ok, Session}; - false -> retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight), - [], erlang:system_time(millisecond), Session, ClientInfo) + false -> + Now = erlang:system_time(millisecond), + retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight), + [], Now, Session, ClientInfo) end. retry_delivery([], Acc, _Now, Session = #session{retry_interval = Interval}, _ClientInfo) -> {ok, lists:reverse(Acc), Interval, Session}; -retry_delivery([{PacketId, {Msg, Ts}}|More], Acc, Now, Session = +retry_delivery([{PacketId, {Msg, Ts}} | More], Acc, Now, Session = #session{retry_interval = Interval, inflight = Inflight}, ClientInfo) -> case (Age = age(Now, Ts)) >= Interval of true -> @@ -565,12 +610,12 @@ do_retry_delivery(PacketId, Msg, Now, Acc, Inflight, ClientInfo) false -> Msg1 = emqx_message:set_flag(dup, true, Msg), Inflight1 = emqx_inflight:update(PacketId, {Msg1, Now}, Inflight), - {[{PacketId, Msg1}|Acc], Inflight1} + {[{PacketId, Msg1} | Acc], Inflight1} end; -do_retry_delivery(PacketId, pubrel, Now, Acc, Inflight, _) -> - Inflight1 = emqx_inflight:update(PacketId, {pubrel, Now}, Inflight), - {[{pubrel, PacketId}|Acc], Inflight1}. +do_retry_delivery(PacketId, Pubrel, Now, Acc, Inflight, _) -> + Inflight1 = emqx_inflight:update(PacketId, {Pubrel, Now}, Inflight), + {[{pubrel, PacketId} | Acc], Inflight1}. %%-------------------------------------------------------------------- %% Expire Awaiting Rel @@ -613,10 +658,10 @@ resume(ClientInfo = #{clientid := ClientId}, Session = #session{subscriptions = -spec(replay(emqx_types:clientinfo(), session()) -> {ok, replies(), session()}). replay(ClientInfo, Session = #session{inflight = Inflight}) -> - Pubs = lists:map(fun({PacketId, {pubrel, _Ts}}) -> - {pubrel, PacketId}; + Pubs = lists:map(fun({PacketId, {Pubrel, _Msg}}) when is_record(Pubrel, pubrel_await) -> + {pubrel, PacketId}; ({PacketId, {Msg, _Ts}}) -> - {PacketId, emqx_message:set_flag(dup, true, Msg)} + {PacketId, emqx_message:set_flag(dup, true, Msg)} end, emqx_inflight:to_list(Inflight)), case dequeue(ClientInfo, Session) of {ok, NSession} -> {ok, Pubs, NSession}; @@ -670,13 +715,23 @@ inc_delivery_expired_cnt() -> inc_delivery_expired_cnt(1). inc_delivery_expired_cnt(N) -> + ok = inc_pd('send_msg.dropped', N), + ok = inc_pd('send_msg.dropped.expired', N), ok = emqx_metrics:inc('delivery.dropped', N), emqx_metrics:inc('delivery.dropped.expired', N). inc_await_pubrel_timeout(N) -> + ok = inc_pd('recv_msg.dropped', N), + ok = inc_pd('recv_msg.dropped.await_pubrel_timeout', N), ok = emqx_metrics:inc('messages.dropped', N), emqx_metrics:inc('messages.dropped.await_pubrel_timeout', N). +inc_pd(Key) -> + inc_pd(Key, 1). +inc_pd(Key, Inc) -> + _ = emqx_pd:inc_counter(Key, Inc), + ok. + %%-------------------------------------------------------------------- %% Next Packet Id %%-------------------------------------------------------------------- @@ -689,6 +744,24 @@ next_pkt_id(Session = #session{next_pkt_id = ?MAX_PACKET_ID}) -> next_pkt_id(Session = #session{next_pkt_id = Id}) -> Session#session{next_pkt_id = Id + 1}. +%%-------------------------------------------------------------------- +%% Message Latency Stats +%%-------------------------------------------------------------------- +on_delivery_completed(ClientInfo, + Msg, + #session{created_at = CreateAt}) when is_record(Msg, message) -> + emqx:run_hook('delivery.completed', + [ClientInfo, Msg, #{session_birth_time => CreateAt}]); + +%% Hot upgrade compatibility clause. +%% In the 4.4.0, timestamp are stored in pubrel_await, not a message record. +%% This clause should be kept in all 4.4.x versions. +on_delivery_completed(_ClientInfo, _Ts, _Session) -> + ok. + +mark_begin_deliver(Msg) -> + emqx_message:set_header(deliver_begin_at, erlang:system_time(second), Msg). + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- @@ -716,4 +789,3 @@ age(Now, Ts) -> Now - Ts. set_field(Name, Value, Session) -> Pos = emqx_misc:index_of(Name, record_info(fields, session)), setelement(Pos+1, Session, Value). - diff --git a/src/emqx_slow_subs/emqx_moving_average.erl b/src/emqx_slow_subs/emqx_moving_average.erl new file mode 100644 index 000000000..64c73f987 --- /dev/null +++ b/src/emqx_slow_subs/emqx_moving_average.erl @@ -0,0 +1,90 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @see https://en.wikipedia.org/wiki/Moving_average + +-module(emqx_moving_average). + +%% API +-export([new/0, new/1, new/2, update/2]). + +-type type() :: cumulative + | exponential. + +-type ema() :: #{ type := exponential + , average := 0 | float() + , coefficient := float() + }. + +-type cma() :: #{ type := cumulative + , average := 0 | float() + , count := non_neg_integer() + }. + +-type moving_average() :: ema() + | cma(). + +-define(DEF_EMA_ARG, #{period => 10}). +-define(DEF_AVG_TYPE, exponential). + +-export_type([type/0, moving_average/0, ema/0, cma/0]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new() -> moving_average(). +new() -> + new(?DEF_AVG_TYPE, #{}). + +-spec new(type()) -> moving_average(). +new(Type) -> + new(Type, #{}). + +-spec new(type(), Args :: map()) -> moving_average(). +new(cumulative, _) -> + #{ type => cumulative + , average => 0 + , count => 0 + }; + +new(exponential, Arg) -> + #{period := Period} = maps:merge(?DEF_EMA_ARG, Arg), + #{ type => exponential + , average => 0 + %% coefficient = 2/(N+1) is a common convention, see the wiki link for details + , coefficient => 2 / (Period + 1) + }. + +-spec update(number(), moving_average()) -> moving_average(). + +update(Val, #{average := 0} = Avg) -> + Avg#{average := Val}; + +update(Val, #{ type := cumulative + , average := Average + , count := Count} = CMA) -> + NewCount = Count + 1, + CMA#{average := (Count * Average + Val) / NewCount, + count := NewCount}; + +update(Val, #{ type := exponential + , average := Average + , coefficient := Coefficient} = EMA) -> + EMA#{average := Coefficient * Val + (1 - Coefficient) * Average}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- diff --git a/src/emqx_stats.erl b/src/emqx_stats.erl index 1d8fb2d33..59b3d32d8 100644 --- a/src/emqx_stats.erl +++ b/src/emqx_stats.erl @@ -21,6 +21,7 @@ -include("emqx.hrl"). -include("logger.hrl"). -include("types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[Stats]"). @@ -67,8 +68,10 @@ %% Connection stats -define(CONNECTION_STATS, - ['connections.count', %% Count of Concurrent Connections - 'connections.max' %% Maximum Number of Concurrent Connections + [ 'connections.count' %% Count of Concurrent Connections + , 'connections.max' %% Maximum Number of Concurrent Connections + , 'live_connections.count' %% Count of connected clients + , 'live_connections.max' %% Maximum number of connected clients ]). %% Channel stats @@ -216,6 +219,11 @@ handle_cast({setstat, Stat, MaxStat, Val}, State) -> ets:insert(?TAB, {MaxStat, Val}) end, safe_update_element(Stat, Val), + ?tp(emqx_stats_setstat, + #{ count_stat => Stat + , max_stat => MaxStat + , value => Val + }), {noreply, State}; handle_cast({update_interval, Update = #update{name = Name}}, @@ -274,4 +282,3 @@ safe_update_element(Key, Val) -> error:badarg -> ?LOG(warning, "Failed to update ~0p to ~0p", [Key, Val]) end. - diff --git a/src/emqx_trace_handler.erl b/src/emqx_trace_handler.erl new file mode 100644 index 000000000..49f3157cc --- /dev/null +++ b/src/emqx_trace_handler.erl @@ -0,0 +1,259 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_handler). + +-include("emqx.hrl"). +-include("logger.hrl"). + +-logger_header("[Tracer]"). + +%% APIs +-export([ running/0 + , install/3 + , install/4 + , uninstall/1 + , uninstall/2 + ]). + +%% For logger handler filters callbacks +-export([ filter_clientid/2 + , filter_topic/2 + , filter_ip_address/2 + ]). + +-export([handler_id/2]). + +-type tracer() :: #{ + name := binary(), + type := clientid | topic | ip_address, + filter := emqx_types:clientid() | emqx_types:topic() | emqx_trace:ip_address() + }. + +-define(FORMAT, + {logger_formatter, #{ + template => [ + time, " [", level, "] ", + {clientid, + [{peername, [clientid, "@", peername, " "], [clientid, " "]}], + [{peername, [peername, " "], []}] + }, + msg, "\n" + ], + single_line => false, + max_size => unlimited, + depth => unlimited + }} +). + +-define(CLIENT_FORMAT, + {logger_formatter, #{ + template => [ + time, " [", level, "] ", + {clientid, + [{peername, [clientid, "@", peername, " "], [clientid, " "]}], + [{peername, [peername, " "], []}] + }, + msg, "\n" + ], + single_line => false, + max_size => unlimited, + depth => unlimited + }} +). + +-define(CONFIG(_LogFile_), #{ + type => halt, + file => _LogFile_, + max_no_bytes => 512 * 1024 * 1024, + overload_kill_enable => true, + overload_kill_mem_size => 50 * 1024 * 1024, + overload_kill_qlen => 20000, + %% disable restart + overload_kill_restart_after => infinity + }). + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +-spec install(Name :: binary() | list(), + Type :: clientid | topic | ip_address, + Filter ::emqx_types:clientid() | emqx_types:topic() | string(), + Level :: logger:level() | all, + LogFilePath :: string()) -> ok | {error, term()}. +install(Name, Type, Filter, Level, LogFile) -> + Who = #{type => Type, filter => ensure_bin(Filter), name => ensure_bin(Name)}, + install(Who, Level, LogFile). + +-spec install(Type :: clientid | topic | ip_address, + Filter ::emqx_types:clientid() | emqx_types:topic() | string(), + Level :: logger:level() | all, + LogFilePath :: string()) -> ok | {error, term()}. +install(Type, Filter, Level, LogFile) -> + install(Filter, Type, Filter, Level, LogFile). + +-spec install(tracer(), logger:level() | all, string()) -> ok | {error, term()}. +install(Who, all, LogFile) -> + install(Who, debug, LogFile); +install(Who, Level, LogFile) -> + PrimaryLevel = emqx_logger:get_primary_log_level(), + try logger:compare_levels(Level, PrimaryLevel) of + lt -> + {error, + io_lib:format( + "Cannot trace at a log level (~s) " + "lower than the primary log level (~s)", + [Level, PrimaryLevel] + )}; + _GtOrEq -> + install_handler(Who, Level, LogFile) + catch + error:badarg -> + {error, {invalid_log_level, Level}} + end. + +-spec uninstall(Type :: clientid | topic | ip_address, + Name :: binary() | list()) -> ok | {error, term()}. +uninstall(Type, Name) -> + HandlerId = handler_id(ensure_bin(Name), Type), + uninstall(HandlerId). + +-spec uninstall(HandlerId :: atom()) -> ok | {error, term()}. +uninstall(HandlerId) -> + Res = logger:remove_handler(HandlerId), + show_prompts(Res, HandlerId, "Stop trace"), + Res. + +%% @doc Return all running trace handlers information. +-spec running() -> + [ + #{ + name => binary(), + type => topic | clientid | ip_address, + id => atom(), + filter => emqx_types:topic() | emqx_types:clienetid() | emqx_trace:ip_address(), + level => logger:level(), + dst => file:filename() | console | unknown + } + ]. +running() -> + lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). + +-spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log; +filter_clientid(_Log, _ExpectId) -> ignore. + +-spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) -> + case emqx_topic:match(Topic, TopicFilter) of + true -> Log; + false -> ignore + end; +filter_topic(_Log, _ExpectId) -> ignore. + +-spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. +filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) -> + case lists:prefix(IP, Peername) of + true -> Log; + false -> ignore + end; +filter_ip_address(_Log, _ExpectId) -> ignore. + +install_handler(Who = #{name := Name, type := Type}, Level, LogFile) -> + HandlerId = handler_id(Name, Type), + Config = #{ level => Level, + formatter => formatter(Who), + filter_default => stop, + filters => filters(Who), + config => ?CONFIG(LogFile) + }, + Res = logger:add_handler(HandlerId, logger_disk_log_h, Config), + show_prompts(Res, Who, "Start trace"), + Res. + +filters(#{type := clientid, filter := Filter, name := Name}) -> + [{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}]; +filters(#{type := topic, filter := Filter, name := Name}) -> + [{topic, {fun ?MODULE:filter_topic/2, {ensure_bin(Filter), Name}}}]; +filters(#{type := ip_address, filter := Filter, name := Name}) -> + [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}]. + +formatter(#{type := Type}) -> + {logger_formatter, + #{ + template => template(Type), + single_line => false, + max_size => unlimited, + depth => unlimited + } + }. + +%% Don't log clientid since clientid only supports exact match, all client ids are the same. +%% if clientid is not latin characters. the logger_formatter restricts the output must be `~tp` +%% (actually should use `~ts`), the utf8 characters clientid will become very difficult to read. +template(clientid) -> + [time, " [", level, "] ", {peername, [peername, " "], []}, msg, "\n"]; +%% TODO better format when clientid is utf8. +template(_) -> + [ time, " [", level, "] ", + {clientid, + [{peername, [clientid, "@", peername, " "], [clientid, " "]}], + [{peername, [peername, " "], []}] + }, + msg, "\n" + ]. + +filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> + Init = #{id => Id, level => Level, dst => Dst}, + case Filters of + [{Type, {_FilterFun, {Filter, Name}}}] when + Type =:= topic orelse + Type =:= clientid orelse + Type =:= ip_address -> + [Init#{type => Type, filter => Filter, name => Name} | Acc]; + _ -> + Acc + end. + +handler_id(Name, Type) -> + try + do_handler_id(Name, Type) + catch + _ : _ -> + Hash = emqx_misc:bin2hexstr_a_f_lower(crypto:hash(md5, Name)), + do_handler_id(Hash, Type) + end. + +%% Handler ID must be an atom. +do_handler_id(Name, Type) -> + TypeStr = atom_to_list(Type), + NameStr = unicode:characters_to_list(Name, utf8), + FullNameStr = "trace_" ++ TypeStr ++ "_" ++ NameStr, + true = io_lib:printable_unicode_list(FullNameStr), + FullNameBin = unicode:characters_to_binary(FullNameStr, utf8), + binary_to_atom(FullNameBin, utf8). + +ensure_bin(List) when is_list(List) -> iolist_to_binary(List); +ensure_bin(Bin) when is_binary(Bin) -> Bin. + +ensure_list(Bin) when is_binary(Bin) -> unicode:characters_to_list(Bin, utf8); +ensure_list(List) when is_list(List) -> List. + +show_prompts(ok, Who, Msg) -> + ?LOG(info, Msg ++ " ~p " ++ "successfully~n", [Who]); +show_prompts({error, Reason}, Who, Msg) -> + ?LOG(error, Msg ++ " ~p " ++ "failed with ~p~n", [Who, Reason]). diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl deleted file mode 100644 index 8f789faa7..000000000 --- a/src/emqx_tracer.erl +++ /dev/null @@ -1,168 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_tracer). - --include("emqx.hrl"). --include("logger.hrl"). - --logger_header("[Tracer]"). - -%% APIs --export([ trace/2 - , start_trace/3 - , lookup_traces/0 - , stop_trace/1 - ]). - --type(trace_who() :: {clientid | topic, binary()}). - --define(TRACER, ?MODULE). --define(FORMAT, {logger_formatter, - #{template => - [time, " [", level, "] ", - {clientid, - [{peername, - [clientid, "@", peername, " "], - [clientid, " "]}], - [{peername, - [peername, " "], - []}]}, - msg, "\n"], - single_line => false - }}). --define(TOPIC_TRACE_ID(T), "trace_topic_"++T). --define(CLIENT_TRACE_ID(C), "trace_clientid_"++C). --define(TOPIC_TRACE(T), {topic, T}). --define(CLIENT_TRACE(C), {clientid, C}). - --define(IS_LOG_LEVEL(L), - L =:= emergency orelse - L =:= alert orelse - L =:= critical orelse - L =:= error orelse - L =:= warning orelse - L =:= notice orelse - L =:= info orelse - L =:= debug). - --dialyzer({nowarn_function, [install_trace_handler/3]}). - -%%------------------------------------------------------------------------------ -%% APIs -%%------------------------------------------------------------------------------ -trace(publish, #message{topic = <<"$SYS/", _/binary>>}) -> - %% Do not trace '$SYS' publish - ignore; -trace(publish, #message{from = From, topic = Topic, payload = Payload}) - when is_binary(From); is_atom(From) -> - emqx_logger:info(#{topic => Topic, - mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} }, - "PUBLISH to ~s: ~0p", [Topic, Payload]). - -%% @doc Start to trace clientid or topic. --spec(start_trace(trace_who(), logger:level() | all, string()) -> ok | {error, term()}). -start_trace(Who, all, LogFile) -> - start_trace(Who, debug, LogFile); -start_trace(Who, Level, LogFile) -> - case ?IS_LOG_LEVEL(Level) of - true -> - #{level := PrimaryLevel} = logger:get_primary_config(), - try logger:compare_levels(Level, PrimaryLevel) of - lt -> - {error, - io_lib:format("Cannot trace at a log level (~s) " - "lower than the primary log level (~s)", - [Level, PrimaryLevel])}; - _GtOrEq -> - install_trace_handler(Who, Level, LogFile) - catch - _:Error -> - {error, Error} - end; - false -> {error, {invalid_log_level, Level}} - end. - -%% @doc Stop tracing clientid or topic. --spec(stop_trace(trace_who()) -> ok | {error, term()}). -stop_trace(Who) -> - uninstall_trance_handler(Who). - -%% @doc Lookup all traces --spec(lookup_traces() -> [{Who :: trace_who(), LogFile :: string()}]). -lookup_traces() -> - lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). - -install_trace_handler(Who, Level, LogFile) -> - case logger:add_handler(handler_id(Who), logger_disk_log_h, - #{level => Level, - formatter => ?FORMAT, - config => #{type => halt, file => LogFile}, - filter_default => stop, - filters => [{meta_key_filter, - {fun filter_by_meta_key/2, Who}}]}) - of - ok -> - ?LOG(info, "Start trace for ~p", [Who]); - {error, Reason} -> - ?LOG(error, "Start trace for ~p failed, error: ~p", [Who, Reason]), - {error, Reason} - end. - -uninstall_trance_handler(Who) -> - case logger:remove_handler(handler_id(Who)) of - ok -> - ?LOG(info, "Stop trace for ~p", [Who]); - {error, Reason} -> - ?LOG(error, "Stop trace for ~p failed, error: ~p", [Who, Reason]), - {error, Reason} - end. - -filter_traces(#{id := Id, level := Level, dst := Dst}, Acc) -> - case atom_to_list(Id) of - ?TOPIC_TRACE_ID(T)-> - [{?TOPIC_TRACE(T), {Level, Dst}} | Acc]; - ?CLIENT_TRACE_ID(C) -> - [{?CLIENT_TRACE(C), {Level, Dst}} | Acc]; - _ -> Acc - end. - -handler_id(?TOPIC_TRACE(Topic)) -> - list_to_atom(?TOPIC_TRACE_ID(handler_name(Topic))); -handler_id(?CLIENT_TRACE(ClientId)) -> - list_to_atom(?CLIENT_TRACE_ID(handler_name(ClientId))). - -filter_by_meta_key(#{meta := Meta} = Log, {Key, Value}) -> - case is_meta_match(Key, Value, Meta) of - true -> Log; - false -> ignore - end. - -is_meta_match(clientid, ClientId, #{clientid := ClientIdStr}) -> - ClientId =:= iolist_to_binary(ClientIdStr); -is_meta_match(topic, TopicFilter, #{topic := TopicMeta}) -> - emqx_topic:match(TopicMeta, TopicFilter); -is_meta_match(_, _, _) -> - false. - -handler_name(Bin) -> - case byte_size(Bin) of - Size when Size =< 200 -> binary_to_list(Bin); - _ -> hashstr(Bin) - end. - -hashstr(Bin) -> - binary_to_list(emqx_misc:bin2hexstr_A_F(Bin)). diff --git a/src/emqx_types.erl b/src/emqx_types.erl index 4a0db262c..aa1fe3b0b 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -147,7 +147,7 @@ dn => binary(), atom() => term() }). --type(clientid() :: binary()|atom()). +-type(clientid() :: binary() | atom()). -type(username() :: maybe(binary())). -type(password() :: maybe(binary())). -type(peerhost() :: inet:ip_address()). @@ -193,6 +193,7 @@ username => username(), peerhost => peerhost(), properties => properties(), + allow_publish => boolean(), atom() => term()}). -type(banned() :: #banned{}). diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index 6cde8eb01..cd1abc781 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -93,9 +93,21 @@ -type(ws_cmd() :: {active, boolean()}|close). -define(ACTIVE_N, 100). --define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]). --define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]). --define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). + +-define(INFO_KEYS, + [ socktype + , peername + , sockname + , sockstate + , active_n + ]). + +-define(SOCK_STATS, + [ recv_oct + , recv_cnt + , send_oct + , send_cnt + ]). -define(ENABLED(X), (X =/= undefined)). @@ -146,10 +158,9 @@ stats(WsPid) when is_pid(WsPid) -> call(WsPid, stats); stats(#state{channel = Channel}) -> SockStats = emqx_pd:get_counters(?SOCK_STATS), - ConnStats = emqx_pd:get_counters(?CONN_STATS), ChanStats = emqx_channel:stats(Channel), ProcStats = emqx_misc:proc_stats(), - lists:append([SockStats, ConnStats, ChanStats, ProcStats]). + lists:append([SockStats, ChanStats, ProcStats]). %% kick|discard|takeover -spec(call(pid(), Req :: term()) -> Reply :: term()). @@ -615,6 +626,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> [emqx_packet:format(Packet)]), ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), + ok = inc_outgoing_stats({error, message_too_large}), <<>>; Data -> ?LOG(debug, "SEND ~s", [emqx_packet:format(Packet)]), ok = inc_outgoing_stats(Packet), @@ -641,19 +653,28 @@ inc_recv_stats(Cnt, Oct) -> inc_incoming_stats(Packet = ?PACKET(Type)) -> _ = emqx_pd:inc_counter(recv_pkt, 1), - if Type == ?PUBLISH -> - inc_counter(recv_msg, 1), - inc_counter(incoming_pubs, 1); - true -> ok + case Type of + ?PUBLISH -> + inc_counter(recv_msg, 1), + inc_qos_stats(recv_msg, Packet), + inc_counter(incoming_pubs, 1); + _ -> + ok end, emqx_metrics:inc_recv(Packet). +inc_outgoing_stats({error, message_too_large}) -> + inc_counter('send_msg.dropped', 1), + inc_counter('send_msg.dropped.too_large', 1); inc_outgoing_stats(Packet = ?PACKET(Type)) -> _ = emqx_pd:inc_counter(send_pkt, 1), - if Type == ?PUBLISH -> - inc_counter(send_msg, 1), - inc_counter(outgoing_pubs, 1); - true -> ok + case Type of + ?PUBLISH -> + inc_counter(send_msg, 1), + inc_qos_stats(send_msg, Packet), + inc_counter(outgoing_pubs, 1); + _ -> + ok end, emqx_metrics:inc_sent(Packet). @@ -666,6 +687,20 @@ inc_sent_stats(Cnt, Oct) -> inc_counter(Name, Value) -> _ = emqx_pd:inc_counter(Name, Value), ok. + +inc_qos_stats(Type, #mqtt_packet{header = #mqtt_packet_header{qos = QoS}}) when ?IS_QOS(QoS) -> + inc_counter(inc_qos_stats_key(Type, QoS), 1); +inc_qos_stats(_, _) -> + ok. + +inc_qos_stats_key(send_msg, ?QOS_0) -> 'send_msg.qos0'; +inc_qos_stats_key(send_msg, ?QOS_1) -> 'send_msg.qos1'; +inc_qos_stats_key(send_msg, ?QOS_2) -> 'send_msg.qos2'; + +inc_qos_stats_key(recv_msg, ?QOS_0) -> 'recv_msg.qos0'; +inc_qos_stats_key(recv_msg, ?QOS_1) -> 'recv_msg.qos1'; +inc_qos_stats_key(recv_msg, ?QOS_2) -> 'recv_msg.qos2'. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl index e8d19c5c0..9d393bb44 100644 --- a/test/emqx_broker_SUITE.erl +++ b/test/emqx_broker_SUITE.erl @@ -23,20 +23,64 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> + [ {group, all_cases} + , {group, connected_client_count_group} + ]. -init_per_suite(Config) -> +groups() -> + TCs = emqx_ct:all(?MODULE), + ConnClientTCs = [ t_connected_client_count_persistent + , t_connected_client_count_anonymous + , t_connected_client_count_transient_takeover + , t_connected_client_stats + ], + OtherTCs = TCs -- ConnClientTCs, + [ {all_cases, [], OtherTCs} + , {connected_client_count_group, [ {group, tcp} + , {group, ws} + ]} + , {tcp, [], ConnClientTCs} + , {ws, [], ConnClientTCs} + ]. + +init_per_group(connected_client_count_group, Config) -> + Config; +init_per_group(tcp, Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([]), + [{conn_fun, connect} | Config]; +init_per_group(ws, Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([]), + [ {ssl, false} + , {enable_websocket, true} + , {conn_fun, ws_connect} + , {port, 8083} + , {host, "localhost"} + | Config + ]; +init_per_group(_Group, Config) -> emqx_ct_helpers:boot_modules(all), emqx_ct_helpers:start_apps([]), Config. -end_per_suite(_Config) -> +end_per_group(connected_client_count_group, _Config) -> + ok; +end_per_group(_Group, _Config) -> emqx_ct_helpers:stop_apps([]). +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + init_per_testcase(Case, Config) -> ?MODULE:Case({init, Config}). @@ -277,6 +321,322 @@ t_stats_fun({'end', _Config}) -> ok = emqx_broker:unsubscribe(<<"topic">>), ok = emqx_broker:unsubscribe(<<"topic2">>). +%% persistent sessions, when gone, do not contribute to connected +%% client count +t_connected_client_count_persistent({init, Config}) -> + ok = snabbkaffe:start_trace(), + process_flag(trap_exit, true), + Config; +t_connected_client_count_persistent(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ClientID = <<"clientid">>, + ?assertEqual(0, emqx_cm:get_connected_client_count()), + {ok, ConnPid0} = emqtt:start_link([ {clean_start, false} + , {clientid, ClientID} + | Config]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid0) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + {ok, {ok, [_]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid0) end, + [emqx_cm_connected_client_count_dec] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% reconnecting + {ok, ConnPid1} = emqtt:start_link([ {clean_start, false} + , {clientid, ClientID} + | Config + ]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid1) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% taking over + {ok, ConnPid2} = emqtt:start_link([ {clean_start, false} + , {clientid, ClientID} + | Config + ]), + {{ok, _}, {ok, [_, _]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid2) end, + [ emqx_cm_connected_client_count_inc + , emqx_cm_connected_client_count_dec + ], + 500 + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% abnormal exit of channel process + ChanPids = emqx_cm:all_channels(), + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> + lists:foreach( + fun(ChanPid) -> exit(ChanPid, kill) end, + ChanPids) + end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ok; +t_connected_client_count_persistent({'end', _Config}) -> + snabbkaffe:stop(), + ok. + +%% connections without client_id also contribute to connected client +%% count +t_connected_client_count_anonymous({init, Config}) -> + ok = snabbkaffe:start_trace(), + process_flag(trap_exit, true), + Config; +t_connected_client_count_anonymous(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% first client + {ok, ConnPid0} = emqtt:start_link([ {clean_start, true} + | Config]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid0) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% second client + {ok, ConnPid1} = emqtt:start_link([ {clean_start, true} + | Config]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid1) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(2, emqx_cm:get_connected_client_count()), + %% when first client disconnects, shouldn't affect the second + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid0) end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% reconnecting + {ok, ConnPid2} = emqtt:start_link([ {clean_start, true} + | Config + ]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid2) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(2, emqx_cm:get_connected_client_count()), + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid1) end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% abnormal exit of channel process + Chans = emqx_cm:all_channels(), + {ok, {ok, [_, _]}} = wait_for_events( + fun() -> + lists:foreach( + fun(ChanPid) -> exit(ChanPid, kill) end, + Chans) + end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ok; +t_connected_client_count_anonymous({'end', _Config}) -> + snabbkaffe:stop(), + ok. + +t_connected_client_count_transient_takeover({init, Config}) -> + ok = snabbkaffe:start_trace(), + process_flag(trap_exit, true), + Config; +t_connected_client_count_transient_takeover(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ClientID = <<"clientid">>, + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% we spawn several clients simultaneously to cause the race + %% condition for the client id lock + NumClients = 20, + {ok, {ok, [_, _]}} = + wait_for_events( + fun() -> + lists:foreach( + fun(_) -> + spawn( + fun() -> + {ok, ConnPid} = + emqtt:start_link([ {clean_start, true} + , {clientid, ClientID} + | Config]), + %% don't assert the result: most of them fail + %% during the race + emqtt:ConnFun(ConnPid), + ok + end), + ok + end, + lists:seq(1, NumClients)) + end, + %% there can be only one channel that wins the race for the + %% lock for this client id. we also expect a decrement + %% event because the client dies along with the ephemeral + %% process. + [ emqx_cm_connected_client_count_inc + , emqx_cm_connected_client_count_dec + ], + 1000), + %% Since more than one pair of inc/dec may be emitted, we need to + %% wait for full stabilization + timer:sleep(100), + %% It must be 0 again because we spawn-linked the clients in + %% ephemeral processes above, and all should be dead now. + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% connecting again + {ok, ConnPid1} = emqtt:start_link([ {clean_start, true} + , {clientid, ClientID} + | Config + ]), + {{ok, _}, {ok, [_]}} = + wait_for_events( + fun() -> emqtt:ConnFun(ConnPid1) end, + [emqx_cm_connected_client_count_inc] + ), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + %% abnormal exit of channel process + [ChanPid] = emqx_cm:all_channels(), + {ok, {ok, [_, _]}} = + wait_for_events( + fun() -> + exit(ChanPid, kill), + ok + end, + [ emqx_cm_connected_client_count_dec + , emqx_cm_process_down + ] + ), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ok; +t_connected_client_count_transient_takeover({'end', _Config}) -> + snabbkaffe:stop(), + ok. + +t_connected_client_stats({init, Config}) -> + ok = supervisor:terminate_child(emqx_kernel_sup, emqx_stats), + {ok, _} = supervisor:restart_child(emqx_kernel_sup, emqx_stats), + ok = snabbkaffe:start_trace(), + Config; +t_connected_client_stats(Config) when is_list(Config) -> + ConnFun = ?config(conn_fun, Config), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + ?assertEqual(0, emqx_stats:getstat('live_connections.count')), + ?assertEqual(0, emqx_stats:getstat('live_connections.max')), + {ok, ConnPid} = emqtt:start_link([ {clean_start, true} + , {clientid, <<"clientid">>} + | Config + ]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid) end, + [emqx_cm_connected_client_count_inc] + ), + %% ensure stats are synchronized + {_, {ok, [_]}} = wait_for_stats( + fun emqx_cm:stats_fun/0, + [#{count_stat => 'live_connections.count', + max_stat => 'live_connections.max'}] + ), + ?assertEqual(1, emqx_stats:getstat('live_connections.count')), + ?assertEqual(1, emqx_stats:getstat('live_connections.max')), + {ok, {ok, [_]}} = wait_for_events( + fun() -> emqtt:disconnect(ConnPid) end, + [emqx_cm_connected_client_count_dec] + ), + %% ensure stats are synchronized + {_, {ok, [_]}} = wait_for_stats( + fun emqx_cm:stats_fun/0, + [#{count_stat => 'live_connections.count', + max_stat => 'live_connections.max'}] + ), + ?assertEqual(0, emqx_stats:getstat('live_connections.count')), + ?assertEqual(1, emqx_stats:getstat('live_connections.max')), + ok; +t_connected_client_stats({'end', _Config}) -> + ok = snabbkaffe:stop(), + ok = supervisor:terminate_child(emqx_kernel_sup, emqx_stats), + {ok, _} = supervisor:restart_child(emqx_kernel_sup, emqx_stats), + ok. + +%% the count must be always non negative +t_connect_client_never_negative({init, Config}) -> + Config; +t_connect_client_never_negative(Config) when is_list(Config) -> + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% would go to -1 + ChanPid = list_to_pid("<0.0.1>"), + emqx_cm:mark_channel_disconnected(ChanPid), + ?assertEqual(0, emqx_cm:get_connected_client_count()), + %% would be 0, if really went to -1 + emqx_cm:mark_channel_connected(ChanPid), + ?assertEqual(1, emqx_cm:get_connected_client_count()), + ok; +t_connect_client_never_negative({'end', _Config}) -> + ok. + +wait_for_events(Action, Kinds) -> + wait_for_events(Action, Kinds, 500). + +wait_for_events(Action, Kinds, Timeout) -> + Predicate = fun(#{?snk_kind := K}) -> + lists:member(K, Kinds) + end, + N = length(Kinds), + {ok, Sub} = snabbkaffe_collector:subscribe(Predicate, N, Timeout, 0), + Res = Action(), + case snabbkaffe_collector:receive_events(Sub) of + {timeout, _} -> + {Res, timeout}; + {ok, Events} -> + {Res, {ok, Events}} + end. + +wait_for_stats(Action, Stats) -> + Predicate = fun(Event = #{?snk_kind := emqx_stats_setstat}) -> + Stat = maps:with( + [ count_stat + , max_stat + ], Event), + lists:member(Stat, Stats); + (_) -> + false + end, + N = length(Stats), + Timeout = 500, + {ok, Sub} = snabbkaffe_collector:subscribe(Predicate, N, Timeout, 0), + Res = Action(), + case snabbkaffe_collector:receive_events(Sub) of + {timeout, _} -> + {Res, timeout}; + {ok, Events} -> + {Res, {ok, Events}} + end. + +insert_fake_channels() -> + %% Insert copies to simulate missed counts + Tab = emqx_channel_info, + Key = ets:first(Tab), + [{_Chan, ChanInfo = #{conn_state := connected}, Stats}] = ets:lookup(Tab, Key), + ets:insert(Tab, [ {{"fake" ++ integer_to_list(N), undefined}, ChanInfo, Stats} + || N <- lists:seq(1, 9)]), + %% these should not be counted + ets:insert(Tab, [ { {"fake" ++ integer_to_list(N), undefined} + , ChanInfo#{conn_state := disconnected}, Stats} + || N <- lists:seq(10, 20)]). + recv_msgs(Count) -> recv_msgs(Count, []). diff --git a/test/emqx_channel_SUITE.erl b/test/emqx_channel_SUITE.erl index 1422f07b6..8d3d03b87 100644 --- a/test/emqx_channel_SUITE.erl +++ b/test/emqx_channel_SUITE.erl @@ -33,6 +33,8 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> %% CM Meck ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), + ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Access Control Meck ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), ok = meck:expect(emqx_access_control, authenticate, diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl index 909536e94..6906c647a 100644 --- a/test/emqx_cm_SUITE.erl +++ b/test/emqx_cm_SUITE.erl @@ -332,15 +332,17 @@ t_discard_session_race(_) -> t_takeover_session(_) -> #{conninfo := ConnInfo} = ?ChanInfo, {error, not_found} = emqx_cm:takeover_session(<<"clientid">>), + Dummy = emqx_session:dummy(), + Downgraded = emqx_session:downgrade(Dummy), erlang:spawn_link(fun() -> ok = emqx_cm:register_channel(<<"clientid">>, self(), ConnInfo), receive {'$gen_call', From, {takeover, 'begin'}} -> - gen_server:reply(From, test), ok + gen_server:reply(From, Dummy), ok end end), timer:sleep(100), - {ok, emqx_connection, _, test} = emqx_cm:takeover_session(<<"clientid">>), + {ok, emqx_connection, _, Downgraded} = emqx_cm:takeover_session(<<"clientid">>), emqx_cm:unregister_channel(<<"clientid">>). t_all_channels(_) -> diff --git a/test/emqx_connection_SUITE.erl b/test/emqx_connection_SUITE.erl index b8233f38c..ed070aa98 100644 --- a/test/emqx_connection_SUITE.erl +++ b/test/emqx_connection_SUITE.erl @@ -36,6 +36,8 @@ init_per_suite(Config) -> ok = meck:new(emqx_channel, [passthrough, no_history, no_link]), %% Meck Cm ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), + ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Meck Limiter ok = meck:new(emqx_limiter, [passthrough, no_history, no_link]), %% Meck Pd @@ -112,7 +114,7 @@ t_ws_pingreq_before_connected(_) -> t_info(_) -> CPid = spawn(fun() -> - receive + receive {'$gen_call', From, info} -> gen_server:reply(From, emqx_connection:info(st())) after @@ -132,7 +134,7 @@ t_info_limiter(_) -> t_stats(_) -> CPid = spawn(fun() -> - receive + receive {'$gen_call', From, stats} -> gen_server:reply(From, emqx_connection:stats(st())) after @@ -147,10 +149,10 @@ t_stats(_) -> {send_pend,0}| _] , Stats). t_process_msg(_) -> - with_conn(fun(CPid) -> - ok = meck:expect(emqx_channel, handle_in, - fun(_Packet, Channel) -> - {ok, Channel} + with_conn(fun(CPid) -> + ok = meck:expect(emqx_channel, handle_in, + fun(_Packet, Channel) -> + {ok, Channel} end), CPid ! {incoming, ?PACKET(?PINGREQ)}, CPid ! {incoming, undefined}, @@ -318,7 +320,7 @@ t_with_channel(_) -> t_handle_outgoing(_) -> ?assertEqual(ok, emqx_connection:handle_outgoing(?PACKET(?PINGRESP), st())), ?assertEqual(ok, emqx_connection:handle_outgoing([?PACKET(?PINGRESP)], st())). - + t_handle_info(_) -> ?assertMatch({ok, {event,running}, _NState}, emqx_connection:handle_info(activate_socket, st())), @@ -345,7 +347,7 @@ t_activate_socket(_) -> State = st(), {ok, NStats} = emqx_connection:activate_socket(State), ?assertEqual(running, emqx_connection:info(sockstate, NStats)), - + State1 = st(#{sockstate => blocked}), ?assertEqual({ok, State1}, emqx_connection:activate_socket(State1)), diff --git a/test/emqx_listeners_SUITE.erl b/test/emqx_listeners_SUITE.erl index ee64e1ad3..5f498d4f0 100644 --- a/test/emqx_listeners_SUITE.erl +++ b/test/emqx_listeners_SUITE.erl @@ -47,6 +47,12 @@ t_restart_listeners(_) -> ok = emqx_listeners:restart(), ok = emqx_listeners:stop(). +t_wss_conn(_) -> + ok = emqx_listeners:start(), + {ok, Socket} = ssl:connect({127, 0, 0, 1}, 8084, [{verify, verify_none}], 1000), + ok = ssl:close(Socket), + ok = emqx_listeners:stop(). + render_config_file() -> Path = local_path(["..", "..", "..", "..", "etc", "emqx.conf"]), {ok, Temp} = file:read_file(Path), @@ -91,4 +97,3 @@ get_base_dir(Module) -> get_base_dir() -> get_base_dir(?MODULE). - diff --git a/test/emqx_session_SUITE.erl b/test/emqx_session_SUITE.erl index 462c1af29..f15fadb82 100644 --- a/test/emqx_session_SUITE.erl +++ b/test/emqx_session_SUITE.erl @@ -180,7 +180,8 @@ t_puback_with_dequeue(_) -> ?assertEqual(<<"t2">>, emqx_message:topic(Msg3)). t_puback_error_packet_id_in_use(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Now = ts(millisecond), + Inflight = emqx_inflight:insert(1, {{pubrel_await, Now}, Now}, emqx_inflight:new()), {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:puback(clientinfo(), 1, session(#{inflight => Inflight})). @@ -191,11 +192,13 @@ t_pubrec(_) -> Msg = emqx_message:make(test, ?QOS_2, <<"t">>, <<>>), Inflight = emqx_inflight:insert(2, {Msg, ts(millisecond)}, emqx_inflight:new()), Session = session(#{inflight => Inflight}), - {ok, Msg, Session1} = emqx_session:pubrec(2, Session), - ?assertMatch([{pubrel, _}], emqx_inflight:values(emqx_session:info(inflight, Session1))). + {ok, MsgWithTime, Session1} = emqx_session:pubrec(2, Session), + ?assertEqual(Msg, emqx_message:remove_header(deliver_begin_at, MsgWithTime)), + ?assertMatch([{{pubrel_await, _}, _}], emqx_inflight:values(emqx_session:info(inflight, Session1))). t_pubrec_packet_id_in_use_error(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Now = ts(millisecond), + Inflight = emqx_inflight:insert(1, {{pubrel_await, Now}, Now}, emqx_inflight:new()), {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:pubrec(1, session(#{inflight => Inflight})). @@ -211,7 +214,8 @@ t_pubrel_error_packetid_not_found(_) -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubrel(1, session()). t_pubcomp(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Now = ts(millisecond), + Inflight = emqx_inflight:insert(1, {{pubrel_await, undefined}, Now}, emqx_inflight:new()), Session = session(#{inflight => Inflight}), {ok, Session1} = emqx_session:pubcomp(clientinfo(), 1, Session), ?assertEqual(0, emqx_session:info(inflight_cnt, Session1)). @@ -262,15 +266,19 @@ t_deliver_qos0(_) -> t_deliver_qos1(_) -> ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), {ok, Session} = emqx_session:subscribe( - clientinfo(), <<"t1">>, subopts(#{qos => ?QOS_1}), session()), + clientinfo(), <<"t1">>, subopts(#{qos => ?QOS_1}), session()), Delivers = [delivery(?QOS_1, T) || T <- [<<"t1">>, <<"t2">>]], {ok, [{1, Msg1}, {2, Msg2}], Session1} = emqx_session:deliver(clientinfo(), Delivers, Session), ?assertEqual(2, emqx_session:info(inflight_cnt, Session1)), ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)), ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)), - {ok, Msg1, Session2} = emqx_session:puback(clientinfo(), 1, Session1), + + {ok, Msg1WithTime, Session2} = emqx_session:puback(clientinfo(), 1, Session1), + ?assertEqual(Msg1, emqx_message:remove_header(deliver_begin_at, Msg1WithTime)), ?assertEqual(1, emqx_session:info(inflight_cnt, Session2)), - {ok, Msg2, Session3} = emqx_session:puback(clientinfo(), 2, Session2), + + {ok, Msg2WithTime, Session3} = emqx_session:puback(clientinfo(), 2, Session2), + ?assertEqual(Msg2, emqx_message:remove_header(deliver_begin_at, Msg2WithTime)), ?assertEqual(0, emqx_session:info(inflight_cnt, Session3)). t_deliver_qos2(_) -> @@ -314,7 +322,11 @@ t_retry(_) -> {ok, Pubs, Session1} = emqx_session:deliver(clientinfo(), Delivers, Session), ok = timer:sleep(200), Msgs1 = [{I, emqx_message:set_flag(dup, Msg)} || {I, Msg} <- Pubs], - {ok, Msgs1, 100, Session2} = emqx_session:retry(clientinfo(), Session1), + {ok, Msgs1WithTime, 100, Session2} = emqx_session:retry(clientinfo(), Session1), + ?assertEqual(Msgs1, + lists:map(fun({Id, M}) -> + {Id, emqx_message:remove_header(deliver_begin_at, M)} + end, Msgs1WithTime)), ?assertEqual(2, emqx_session:info(inflight_cnt, Session2)). %%-------------------------------------------------------------------- @@ -338,7 +350,10 @@ t_replay(_) -> Session2 = emqx_session:enqueue(clientinfo(), Msg, Session1), Pubs1 = [{I, emqx_message:set_flag(dup, M)} || {I, M} <- Pubs], {ok, ReplayPubs, Session3} = emqx_session:replay(clientinfo(), Session2), - ?assertEqual(Pubs1 ++ [{3, Msg}], ReplayPubs), + ?assertEqual(Pubs1 ++ [{3, Msg}], + lists:map(fun({Id, M}) -> + {Id, emqx_message:remove_header(deliver_begin_at, M)} + end, ReplayPubs)), ?assertEqual(3, emqx_session:info(inflight_cnt, Session3)). t_expire_awaiting_rel(_) -> @@ -375,7 +390,7 @@ mqueue(Opts) -> session() -> session(#{}). session(InitFields) when is_map(InitFields) -> maps:fold(fun(Field, Value, Session) -> - emqx_session:set_field(Field, Value, Session) + emqx_session:set_field(Field, Value, Session) end, emqx_session:init(#{zone => channel}, #{receive_maximum => 0}), InitFields). @@ -398,4 +413,3 @@ ts(second) -> erlang:system_time(second); ts(millisecond) -> erlang:system_time(millisecond). - diff --git a/test/emqx_trace_handler_SUITE.erl b/test/emqx_trace_handler_SUITE.erl new file mode 100644 index 000000000..d46c18a70 --- /dev/null +++ b/test/emqx_trace_handler_SUITE.erl @@ -0,0 +1,233 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_handler_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). +-define(CLIENT, [{host, "localhost"}, + {clientid, <<"client">>}, + {username, <<"testuser">>}, + {password, <<"pass">>} + ]). + +all() -> [t_trace_clientid, t_trace_topic, t_trace_ip_address, t_trace_clientid_utf8]. + +init_per_suite(Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([]). + +init_per_testcase(t_trace_clientid, Config) -> + Config; +init_per_testcase(_Case, Config) -> + ok = emqx_logger:set_log_level(debug), + _ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()], + Config. + +end_per_testcase(_Case, _Config) -> + ok = emqx_logger:set_log_level(warning), + ok. + +t_trace_clientid(_Config) -> + %% Start tracing + emqx_logger:set_log_level(error), + {error, _} = emqx_trace_handler:install(clientid, <<"client">>, debug, "tmp/client.log"), + emqx_logger:set_log_level(debug), + %% add list clientid + ok = emqx_trace_handler:install(clientid, "client", debug, "tmp/client.log"), + ok = emqx_trace_handler:install(clientid, <<"client2">>, all, "tmp/client2.log"), + ok = emqx_trace_handler:install(clientid, <<"client3">>, all, "tmp/client3.log"), + {error, {invalid_log_level, bad_level}} = + emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"), + {error, {handler_not_added, {file_error, ".", eisdir}}} = + emqx_trace_handler:install(clientid, <<"client5">>, debug, "."), + ok = filesync(<<"client">>, clientid), + ok = filesync(<<"client2">>, clientid), + ok = filesync(<<"client3">>, clientid), + + %% Verify the tracing file exits + ?assert(filelib:is_regular("tmp/client.log")), + ?assert(filelib:is_regular("tmp/client2.log")), + ?assert(filelib:is_regular("tmp/client3.log")), + + %% Get current traces + ?assertMatch([#{type := clientid, filter := "client", name := <<"client">>, + level := debug, dst := "tmp/client.log"}, + #{type := clientid, filter := "client2", name := <<"client2">> + , level := debug, dst := "tmp/client2.log"}, + #{type := clientid, filter := "client3", name := <<"client3">>, + level := debug, dst := "tmp/client3.log"} + ], emqx_trace_handler:running()), + + %% Client with clientid = "client" publishes a "hi" message to "a/b/c". + {ok, T} = emqtt:start_link(?CLIENT), + emqtt:connect(T), + emqtt:publish(T, <<"a/b/c">>, <<"hi">>), + emqtt:ping(T), + ok = filesync(<<"client">>, clientid), + ok = filesync(<<"client2">>, clientid), + ok = filesync(<<"client3">>, clientid), + + %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". + {ok, Bin} = file:read_file("tmp/client.log"), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"CONNECT">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"CONNACK">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PUBLISH">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PINGREQ">>])), + ?assert(filelib:file_size("tmp/client2.log") == 0), + + %% Stop tracing + ok = emqx_trace_handler:uninstall(clientid, <<"client">>), + ok = emqx_trace_handler:uninstall(clientid, <<"client2">>), + ok = emqx_trace_handler:uninstall(clientid, <<"client3">>), + + emqtt:disconnect(T), + ?assertEqual([], emqx_trace_handler:running()). + +t_trace_clientid_utf8(_) -> + emqx_logger:set_log_level(debug), + + Utf8Id = <<"client 漢字編碼"/utf8>>, + ok = emqx_trace_handler:install(clientid, Utf8Id, debug, "tmp/client-utf8.log"), + {ok, T} = emqtt:start_link([{clientid, Utf8Id}]), + emqtt:connect(T), + [begin emqtt:publish(T, <<"a/b/c">>, <<"hi">>) end|| _ <- lists:seq(1, 10)], + emqtt:ping(T), + + ok = filesync(Utf8Id, clientid), + ok = emqx_trace_handler:uninstall(clientid, Utf8Id), + emqtt:disconnect(T), + ?assertEqual([], emqx_trace_handler:running()), + ok. + +t_trace_topic(_Config) -> + {ok, T} = emqtt:start_link(?CLIENT), + emqtt:connect(T), + + %% Start tracing + emqx_logger:set_log_level(debug), + ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), + ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), + ok = filesync(<<"x/#">>, topic), + ok = filesync(<<"y/#">>, topic), + + %% Verify the tracing file exits + ?assert(filelib:is_regular("tmp/topic_trace_x.log")), + ?assert(filelib:is_regular("tmp/topic_trace_y.log")), + + %% Get current traces + ?assertMatch([#{type := topic, filter := <<"x/#">>, + level := debug, dst := "tmp/topic_trace_x.log", name := <<"x/#">>}, + #{type := topic, filter := <<"y/#">>, + name := <<"y/#">>, level := debug, dst := "tmp/topic_trace_y.log"} + ], + emqx_trace_handler:running()), + + %% Client with clientid = "client" publishes a "hi" message to "x/y/z". + emqtt:publish(T, <<"x/y/z">>, <<"hi1">>), + emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), + emqtt:subscribe(T, <<"x/y/z">>), + emqtt:unsubscribe(T, <<"x/y/z">>), + ok = filesync(<<"x/#">>, topic), + ok = filesync(<<"y/#">>, topic), + + {ok, Bin} = file:read_file("tmp/topic_trace_x.log"), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi2">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PUBLISH">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"SUBSCRIBE">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"UNSUBSCRIBE">>])), + ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0), + + %% Stop tracing + ok = emqx_trace_handler:uninstall(topic, <<"x/#">>), + ok = emqx_trace_handler:uninstall(topic, <<"y/#">>), + {error, _Reason} = emqx_trace_handler:uninstall(topic, <<"z/#">>), + ?assertEqual([], emqx_trace_handler:running()), + emqtt:disconnect(T). + +t_trace_ip_address(_Config) -> + {ok, T} = emqtt:start_link(?CLIENT), + emqtt:connect(T), + + %% Start tracing + ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"), + ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"), + ok = filesync(<<"127.0.0.1">>, ip_address), + ok = filesync(<<"192.168.1.1">>, ip_address), + + %% Verify the tracing file exits + ?assert(filelib:is_regular("tmp/ip_trace_x.log")), + ?assert(filelib:is_regular("tmp/ip_trace_y.log")), + + %% Get current traces + ?assertMatch([#{type := ip_address, filter := "127.0.0.1", + name := <<"127.0.0.1">>, + level := debug, dst := "tmp/ip_trace_x.log"}, + #{type := ip_address, filter := "192.168.1.1", + name := <<"192.168.1.1">>, + level := debug, dst := "tmp/ip_trace_y.log"} + ], + emqx_trace_handler:running()), + + %% Client with clientid = "client" publishes a "hi" message to "x/y/z". + emqtt:publish(T, <<"x/y/z">>, <<"hi1">>), + emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), + emqtt:subscribe(T, <<"x/y/z">>), + emqtt:unsubscribe(T, <<"x/y/z">>), + ok = filesync(<<"127.0.0.1">>, ip_address), + ok = filesync(<<"192.168.1.1">>, ip_address), + + {ok, Bin} = file:read_file("tmp/ip_trace_x.log"), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi2">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"PUBLISH">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"SUBSCRIBE">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"UNSUBSCRIBE">>])), + ?assert(filelib:file_size("tmp/ip_trace_y.log") =:= 0), + + %% Stop tracing + ok = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.1">>), + ok = emqx_trace_handler:uninstall(ip_address, <<"192.168.1.1">>), + {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>), + emqtt:disconnect(T), + ?assertEqual([], emqx_trace_handler:running()). + + +filesync(Name, Type) -> + ct:sleep(50), + filesync(Name, Type, 3). + +%% sometime the handler process is not started yet. +filesync(_Name, _Type, 0) -> ok; +filesync(Name, Type, Retry) -> + try + Handler = binary_to_atom(<<"trace_", + (atom_to_binary(Type))/binary, "_", Name/binary>>), + ok = logger_disk_log_h:filesync(Handler) + catch E:R -> + ct:pal("Filesync error:~p ~p~n", [{Name, Type, Retry}, {E, R}]), + ct:sleep(100), + filesync(Name, Type, Retry - 1) + end. diff --git a/test/emqx_trace_handler_tests.erl b/test/emqx_trace_handler_tests.erl new file mode 100644 index 000000000..40586810e --- /dev/null +++ b/test/emqx_trace_handler_tests.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_trace_handler_tests). + +-include_lib("eunit/include/eunit.hrl"). + +handler_id_test_() -> + [{"normal_printable", + fun() -> + ?assertEqual(trace_topic_t1, emqx_trace_handler:handler_id("t1", topic)) + end}, + {"normal_unicode", + fun() -> + ?assertEqual('trace_topic_主题', emqx_trace_handler:handler_id("主题", topic)) + end + }, + {"not_printable", + fun() -> + ?assertEqual('trace_topic_93b885adfe0da089cdf634904fd59f71', + emqx_trace_handler:handler_id("\0", topic)) + end + }, + {"too_long", + fun() -> + T = lists:duplicate(250, $a), + ?assertEqual('trace_topic_1bdbdf1c9087c796394bcda5789f7206', + emqx_trace_handler:handler_id(T, topic)) + end + } + ]. diff --git a/test/emqx_tracer_SUITE.erl b/test/emqx_tracer_SUITE.erl deleted file mode 100644 index 8044beb84..000000000 --- a/test/emqx_tracer_SUITE.erl +++ /dev/null @@ -1,120 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_tracer_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --include_lib("common_test/include/ct.hrl"). - -all() -> [t_trace_clientid, t_trace_topic]. - -init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), - Config. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). - -t_trace_clientid(_Config) -> - {ok, T} = emqtt:start_link([{host, "localhost"}, - {clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]), - emqtt:connect(T), - - %% Start tracing - emqx_logger:set_log_level(error), - {error, _} = emqx_tracer:start_trace({clientid, <<"client">>}, debug, "tmp/client.log"), - emqx_logger:set_log_level(debug), - ok = emqx_tracer:start_trace({clientid, <<"client">>}, debug, "tmp/client.log"), - ok = emqx_tracer:start_trace({clientid, <<"client2">>}, all, "tmp/client2.log"), - ok = emqx_tracer:start_trace({clientid, <<"client3">>}, all, "tmp/client3.log"), - {error, {invalid_log_level, bad_level}} = emqx_tracer:start_trace({clientid, <<"client4">>}, bad_level, "tmp/client4.log"), - {error, {handler_not_added, {file_error,".",eisdir}}} = emqx_tracer:start_trace({clientid, <<"client5">>}, debug, "."), - ct:sleep(100), - - %% Verify the tracing file exits - ?assert(filelib:is_regular("tmp/client.log")), - ?assert(filelib:is_regular("tmp/client2.log")), - - %% Get current traces - ?assertEqual([{{clientid,"client"},{debug,"tmp/client.log"}}, - {{clientid,"client2"},{debug,"tmp/client2.log"}}, - {{clientid,"client3"},{debug,"tmp/client3.log"}} - ], emqx_tracer:lookup_traces()), - - %% set the overall log level to debug - emqx_logger:set_log_level(debug), - - %% Client with clientid = "client" publishes a "hi" message to "a/b/c". - emqtt:publish(T, <<"a/b/c">>, <<"hi">>), - ct:sleep(200), - - %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". - ?assert(filelib:file_size("tmp/client.log") > 0), - ?assert(filelib:file_size("tmp/client2.log") == 0), - - %% Stop tracing - ok = emqx_tracer:stop_trace({clientid, <<"client">>}), - ok = emqx_tracer:stop_trace({clientid, <<"client2">>}), - ok = emqx_tracer:stop_trace({clientid, <<"client3">>}), - emqtt:disconnect(T), - - emqx_logger:set_log_level(warning). - -t_trace_topic(_Config) -> - {ok, T} = emqtt:start_link([{host, "localhost"}, - {clientid, <<"client">>}, - {username, <<"testuser">>}, - {password, <<"pass">>} - ]), - emqtt:connect(T), - - %% Start tracing - emqx_logger:set_log_level(debug), - ok = emqx_tracer:start_trace({topic, <<"x/#">>}, all, "tmp/topic_trace.log"), - ok = emqx_tracer:start_trace({topic, <<"y/#">>}, all, "tmp/topic_trace.log"), - ct:sleep(100), - - %% Verify the tracing file exits - ?assert(filelib:is_regular("tmp/topic_trace.log")), - - %% Get current traces - ?assertEqual([{{topic,"x/#"},{debug,"tmp/topic_trace.log"}}, - {{topic,"y/#"},{debug,"tmp/topic_trace.log"}}], emqx_tracer:lookup_traces()), - - %% set the overall log level to debug - emqx_logger:set_log_level(debug), - - %% Client with clientid = "client" publishes a "hi" message to "x/y/z". - emqtt:publish(T, <<"x/y/z">>, <<"hi">>), - ct:sleep(200), - - ?assert(filelib:file_size("tmp/topic_trace.log") > 0), - - %% Stop tracing - ok = emqx_tracer:stop_trace({topic, <<"x/#">>}), - ok = emqx_tracer:stop_trace({topic, <<"y/#">>}), - {error, _Reason} = emqx_tracer:stop_trace({topic, <<"z/#">>}), - emqtt:disconnect(T), - - emqx_logger:set_log_level(warning). diff --git a/test/emqx_ws_connection_SUITE.erl b/test/emqx_ws_connection_SUITE.erl index 7affdae93..52d01e068 100644 --- a/test/emqx_ws_connection_SUITE.erl +++ b/test/emqx_ws_connection_SUITE.erl @@ -48,6 +48,10 @@ init_per_testcase(TestCase, Config) when TestCase =/= t_ws_pingreq_before_connected, TestCase =/= t_ws_non_check_origin -> + %% Meck Cm + ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), + ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), + ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Mock cowboy_req ok = meck:new(cowboy_req, [passthrough, no_history, no_link]), ok = meck:expect(cowboy_req, header, fun(_, _, _) -> <<>> end), @@ -95,7 +99,8 @@ end_per_testcase(TestCase, _Config) when TestCase =/= t_ws_pingreq_before_connected -> lists:foreach(fun meck:unload/1, - [cowboy_req, + [emqx_cm, + cowboy_req, emqx_zone, emqx_access_control, emqx_broker, @@ -164,8 +169,9 @@ t_stats(_) -> end end), Stats = ?ws_conn:call(WsPid, stats), - [{recv_oct, 0}, {recv_cnt, 0}, {send_oct, 0}, {send_cnt, 0}, - {recv_pkt, 0}, {recv_msg, 0}, {send_pkt, 0}, {send_msg, 0}|_] = Stats. + [lists:member(V, Stats) || + V <- [{recv_oct, 0}, {recv_cnt, 0}, {send_oct, 0}, {send_cnt, 0}, + {recv_pkt, 0}, {recv_msg, 0}, {send_pkt, 0}, {send_msg, 0}]]. t_call(_) -> Info = ?ws_conn:info(st()), @@ -389,14 +395,12 @@ t_handle_info_close(_) -> {[{close, _}], _St} = ?ws_conn:handle_info({close, protocol_error}, st()). t_handle_info_event(_) -> - ok = meck:new(emqx_cm, [passthrough, no_history]), ok = meck:expect(emqx_cm, register_channel, fun(_,_,_) -> ok end), ok = meck:expect(emqx_cm, insert_channel_info, fun(_,_,_) -> ok end), ok = meck:expect(emqx_cm, connection_closed, fun(_) -> true end), {ok, _} = ?ws_conn:handle_info({event, connected}, st()), {ok, _} = ?ws_conn:handle_info({event, disconnected}, st()), - {ok, _} = ?ws_conn:handle_info({event, updated}, st()), - ok = meck:unload(emqx_cm). + {ok, _} = ?ws_conn:handle_info({event, updated}, st()). t_handle_timeout_idle_timeout(_) -> TRef = make_ref(), From 98a06f25b0ce189a17185e4e317147bf41cc7bda Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 23 May 2022 12:48:38 +0800 Subject: [PATCH 319/363] fix: remove downgrade emqx_metrics instrs from emqx.appup.src --- src/emqx.appup.src | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 81dd6c560..147b37903 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -95,7 +95,6 @@ {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, @@ -104,10 +103,10 @@ {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, @@ -117,7 +116,6 @@ {"4.4.1", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -160,7 +158,6 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, From 10b01be2648fa35bd0e7fd4f62cffd653ba6b8eb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 23 May 2022 11:19:58 +0200 Subject: [PATCH 320/363] docs: update changelog for 4.4 --- CHANGES-4.4.md | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 601585858..88da5fc88 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,10 +1,81 @@ # EMQX 4.4 Changes + +## v4.4.4 + +### Enhancements (synced from v4.3.15) + +* Refactored `bin/emqx` help messages. +* Upgrade script refuses upgrade from incompatible versions. (e.g. hot upgrade from 4.3 to 4.4 will fail fast). +* Made possible for EMQX to boot from a Linux directory which has white spaces in its path. +* Add support for JWT authorization [#7596] + Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists. +* Better randomisation of app screts (changed from timestamp seeded sha hash (uuid) to crypto:strong_rand_bytes) +* Return a client_identifier_not_valid error when username is empty and username_as_clientid is set to true [#7862] +* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/4 [#7894] +* Add proto_name and proto_ver fields for $event/client_disconnected event. +* Mnesia auth/acl http api support multiple condition queries. +* Inflight QoS1 Messages for shared topics are now redispatched to another alive subscribers upon chosen subscriber session termination. +* Make auth metrics name more understandable. + +### Bug fixes (synced from v4.3.15) + +* List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. +* SSL closed error bug fixed for redis client. +* Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message +* Rule-engine function hexstr2bin/1 support half byte [#7977] +* Add rule-engine function float2str/2, user can specify the float output precision [#7991] + +* Improved resilience against autocluster partitioning during cluster + startup. [#7876] + [ekka-158](https://github.com/emqx/ekka/pull/158) +* Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979] + +## v4.4.3 + +** NOTE**: v4.4.3 is in sync with v4.3.14 + ### Enhancements * Add rule events: client.connack, client.check_acl_complete - client.connack The rule event is triggered when the server sends a CONNACK packet to the client. reason_code contains the error reason code. - client.check_acl_complete The rule event is triggered when the client check acl complete. +### Enhancements (synced from v4.3.14) + +* Add `RequestMeta` for exhook.proto in order to expose `cluster_name` of emqx in each gRPC request. [#7524] +* Support customize emqx_exhook execution priority. [#7408] +* add api: PUT /rules/{id}/reset_metrics. + This api reset the metrics of the rule engine of a rule, and reset the metrics of the action related to this rule. [#7474] +* Enhanced rule engine error handling when json parsing error. +* Add support for `RSA-PSK-AES256-GCM-SHA384`, `RSA-PSK-AES256-CBC-SHA384`, + `RSA-PSK-AES128-GCM-SHA256`, `RSA-PSK-AES128-CBC-SHA256` PSK ciphers, and remove `PSK-3DES-EDE-CBC-SHA`, + `PSK-RC4-SHA` from the default configuration. [#7427] +* Diagnostic logging for mnesia `wait_for_table` + - prints check points of mnesia internal stats + - prints check points of per table loading stats + Help to locate the problem of long table loading time. +* Add `local` strategy for Shared Subscription. + That will preferentially dispatch messages to a shared subscriber at the same + node. It will improves the efficiency of shared messages dispatching in certain + scenarios, especially when the emqx-bridge-mqtt plugin is configured as shared + subscription. [#7462] +* Add some compression functions to rule-engine: gzip, gunzip, zip, unzip, zip_compress, zip_uncompress + +### Bug Fixes (synced from v4.3.14) + +* Prohibit empty topics in strict mode +* Make sure ehttpc delete useless pool always succeed. +* Update mongodb driver to fix potential process leak. +* Fix a potential security issue #3155 with emqx-dashboard plugin. + In the earlier implementation, the Dashboard password is reset back to the + default value of emqx_dashboard.conf after the node left cluster. + Now we persist changed password to protect against reset. [#7518] +* Silence grep/sed warnings in docker-entrypoint.sh. [#7520] +* Generate `loaded_modules` and `loaded_plugins` files with default values when no such files exists. [#7520] +* Fix the configuration `server_name_indication` set to disable does not take effect. +* Fix backup files are not deleted and downloaded correctly when the API path has ISO8859-1 escape characters. + + ## v4.4.2 **NOTE**: v4.4.2 is in sync with: v4.3.13 From 90aa3018300aef6dc17c37375828d080bd3f9a8e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 23 May 2022 21:44:05 +0800 Subject: [PATCH 321/363] chore: enhance emqx_auth_mongo appup.src --- .../src/emqx_auth_mongo.appup.src | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src index 67fc5c1db..7a23082aa 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src @@ -1,15 +1,29 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4.4.[0-3]">>, - [{load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + [{<<"4\\.4\\.[2-3]">>, + [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, + {"4.4.0", + [{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4.4.[0-3]">>, - [{load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + [{<<"4\\.4\\.[2-3]">>, + [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, + {"4.4.0", + [{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. From 3689977e26f2798a6c5c096c8bef093521895852 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 23 May 2022 11:00:24 -0300 Subject: [PATCH 322/363] chore: update changelog --- CHANGES-4.4.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 88da5fc88..c62e82032 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -30,6 +30,8 @@ startup. [#7876] [ekka-158](https://github.com/emqx/ekka/pull/158) * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979] +* Allow uploading or referencing a backup file outside the + `data/backup` directory when restoring a backup. [#7996] ## v4.4.3 From 433d74254e4d42f1dfacc5029d04a45aa3815520 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 23 May 2022 23:24:32 +0800 Subject: [PATCH 323/363] chore: dashboard v4.4.10 --- scripts/get-dashboard.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index f2e8e0a28..6d94bae58 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -14,7 +14,7 @@ case "${PKG_VSN}" in 4.4*) # keep the above 4.3 untouched, otherwise conflicts! EMQX_CE_DASHBOARD_VERSION='v4.4.2' - EMQX_EE_DASHBOARD_VERSION='v4.4.9' + EMQX_EE_DASHBOARD_VERSION='v4.4.10' ;; *) echo "Unsupported version $PKG_VSN" >&2 From c839df539ba472d0027be28921ebf25b1507f168 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 25 May 2022 17:03:48 +0800 Subject: [PATCH 324/363] chore: rearrange CHANGS-4.4.md --- CHANGES-4.4.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index c62e82032..5b2e134b1 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,6 +1,5 @@ # EMQX 4.4 Changes - ## v4.4.4 ### Enhancements (synced from v4.3.15) @@ -17,6 +16,13 @@ * Mnesia auth/acl http api support multiple condition queries. * Inflight QoS1 Messages for shared topics are now redispatched to another alive subscribers upon chosen subscriber session termination. * Make auth metrics name more understandable. +* Allow emqx_management http listener binding to specific interface [#8005] +* Add rule-engine function float2str/2, user can specify the float output precision [#7991] + +### Bug fixes + +- Allow uploading or referencing a backup file outside the + `data/backup` directory when restoring a backup. [#7996] ### Bug fixes (synced from v4.3.15) @@ -24,14 +30,11 @@ * SSL closed error bug fixed for redis client. * Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message * Rule-engine function hexstr2bin/1 support half byte [#7977] -* Add rule-engine function float2str/2, user can specify the float output precision [#7991] - * Improved resilience against autocluster partitioning during cluster startup. [#7876] [ekka-158](https://github.com/emqx/ekka/pull/158) * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979] -* Allow uploading or referencing a backup file outside the - `data/backup` directory when restoring a backup. [#7996] +* Fix `node_dump` variable sourcing. [#8026] ## v4.4.3 From 4b36d9b08b129c9a8f3c99d62176a708793753db Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 26 May 2022 13:57:04 +0800 Subject: [PATCH 325/363] chore: bump version to 4.4.4-rc.2 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 15cb64b79..b5765abf3 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.4-rc.1"}). +-define(EMQX_RELEASE, {opensource, "4.4.4-rc.2"}). -else. From 29622558c586d0d19cd8b37cc833f5d2d7bfb3e0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 25 May 2022 11:31:50 +0200 Subject: [PATCH 326/363] ci: pin emqx-buider 4.4-12 Centos/Rockylinux had troulbe booting because OpenSSL version was too new (1.1.1n). builder 4.4-12 has * Centos7: only one version (1.1.1k) OpenSSL installed * Centos8: go back to use default OpenSSL (1.1.1k) --- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/apps_version_check.yaml | 2 +- .github/workflows/build_packages.yaml | 13 +++++++------ .github/workflows/build_slim_packages.yaml | 5 +++-- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 2 +- Makefile | 2 +- deploy/docker/Dockerfile | 2 +- scripts/buildx.sh | 6 +++--- 12 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 819522dbd..a95acc772 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 16dc5c218..9ab6fa5ad 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -13,7 +13,7 @@ jobs: os: - ubuntu20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 09e9aaa6a..5c861a88f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -34,6 +34,7 @@ jobs: shell: bash working-directory: source run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" if make emqx-ee --dry-run > /dev/null 2>&1; then echo "::set-output name=profiles::[\"emqx-ee\"]" else @@ -217,8 +218,8 @@ jobs: - debian11 - debian10 - debian9 - - rockylinux8 - - centos7 + - el8 + - el7 - raspbian10 exclude: - package: pkg @@ -265,7 +266,7 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/4.4-8:${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/4.4-12:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: @@ -342,7 +343,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -358,7 +359,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index dab02d1ec..2ac23d19c 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -21,14 +21,15 @@ jobs: - 24.1.5-3 os: - ubuntu20.04 - - rockylinux8 + - el8 - container: ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 - name: prepare run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" if make emqx-ee --dry-run > /dev/null 2>&1; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 2b49d4d77..e50738914 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ffcb69247..b03ac2077 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 3b11a9368..b491ef47b 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index dbc79a8f5..db1a351c4 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -224,7 +224,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -275,7 +275,7 @@ jobs: otp: - 24.1.5-3 runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index efca499ea..d58486cda 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 0ba09b68c..417423f4e 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-alpine3.15.1 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-alpine3.15.1 export EMQX_DEFAULT_RUNNER = alpine:3.15.1 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index d3a58ce74..63702e2c4 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-alpine3.15.1 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-alpine3.15.1 ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 6efc6bcc0..749ca565e 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-debian10 --arch arm64 +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-debian10 --arch arm64 set -euo pipefail @@ -20,7 +20,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-debian10" } while [ "$#" -gt 0 ]; do @@ -77,4 +77,4 @@ docker run -i --rm \ --workdir /emqx \ --platform="linux/$ARCH" \ "$BUILDER" \ - bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" + bash -euc "git config --global --add safe.directory /emqx && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" From a403c9ff916667c5f8f694874f7bae4fa1d10596 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 25 May 2022 11:31:50 +0200 Subject: [PATCH 327/363] ci: pin emqx-buider 4.4-12 Centos/Rockylinux had troulbe booting because OpenSSL version was too new (1.1.1n). builder 4.4-12 has * Centos7: only one version (1.1.1k) OpenSSL installed * Centos8: go back to use default OpenSSL (1.1.1k) --- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/apps_version_check.yaml | 2 +- .github/workflows/build_packages.yaml | 13 +++++++------ .github/workflows/build_slim_packages.yaml | 5 +++-- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 2 +- Makefile | 2 +- deploy/docker/Dockerfile | 2 +- scripts/buildx.sh | 6 +++--- 12 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 819522dbd..a95acc772 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 16dc5c218..9ab6fa5ad 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -13,7 +13,7 @@ jobs: os: - ubuntu20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 09e9aaa6a..5c861a88f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -34,6 +34,7 @@ jobs: shell: bash working-directory: source run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" if make emqx-ee --dry-run > /dev/null 2>&1; then echo "::set-output name=profiles::[\"emqx-ee\"]" else @@ -217,8 +218,8 @@ jobs: - debian11 - debian10 - debian9 - - rockylinux8 - - centos7 + - el8 + - el7 - raspbian10 exclude: - package: pkg @@ -265,7 +266,7 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/4.4-8:${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/4.4-12:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: @@ -342,7 +343,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -358,7 +359,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index dab02d1ec..2ac23d19c 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -21,14 +21,15 @@ jobs: - 24.1.5-3 os: - ubuntu20.04 - - rockylinux8 + - el8 - container: ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 - name: prepare run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" if make emqx-ee --dry-run > /dev/null 2>&1; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 2b49d4d77..e50738914 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ffcb69247..b03ac2077 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 3b11a9368..b491ef47b 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index dbc79a8f5..db1a351c4 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -224,7 +224,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -275,7 +275,7 @@ jobs: otp: - 24.1.5-3 runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index efca499ea..d58486cda 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 5cfccf879..1a2729794 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-alpine3.15.1 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-alpine3.15.1 export EMQX_DEFAULT_RUNNER = alpine:3.15.1 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index d3a58ce74..63702e2c4 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-alpine3.15.1 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-alpine3.15.1 ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 6efc6bcc0..749ca565e 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-debian10 --arch arm64 +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-debian10 --arch arm64 set -euo pipefail @@ -20,7 +20,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-debian10" } while [ "$#" -gt 0 ]; do @@ -77,4 +77,4 @@ docker run -i --rm \ --workdir /emqx \ --platform="linux/$ARCH" \ "$BUILDER" \ - bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" + bash -euc "git config --global --add safe.directory /emqx && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" From 94092bc52e353e31574f48703b2ca6efb4dbfb5c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 26 May 2022 23:57:27 +0200 Subject: [PATCH 328/363] ci: ensure _build dir is owned by root so the git command can run without safe.directory config for cloned dependencies --- scripts/buildx.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 749ca565e..f15069f86 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -76,5 +76,6 @@ docker run -i --rm \ -v "$(pwd)":/emqx \ --workdir /emqx \ --platform="linux/$ARCH" \ + --user root \ "$BUILDER" \ - bash -euc "git config --global --add safe.directory /emqx && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" + bash -euc "git config --global --add safe.directory /emqx && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" From e9905d706331c6d1b267551cbe4e4fef035c02be Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 27 May 2022 01:25:21 +0200 Subject: [PATCH 329/363] build: ensure _build dir is owned by root when build in docker so the dependency clones will not complain safe.directory etc. --- scripts/buildx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 749ca565e..3337383eb 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -77,4 +77,4 @@ docker run -i --rm \ --workdir /emqx \ --platform="linux/$ARCH" \ "$BUILDER" \ - bash -euc "git config --global --add safe.directory /emqx && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" + bash -euc "git config --global --add safe.directory /emqx && chown -R root:root ./_build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" From 3029e2b24384e4fb37f84fe7467116e6cf17b3af Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 27 May 2022 01:31:42 +0200 Subject: [PATCH 330/363] ci: For 4.4, release package on OTP 24 only --- .github/workflows/build_packages.yaml | 4 ---- scripts/buildx.sh | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 09e9aaa6a..ab7a0eb32 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -205,7 +205,6 @@ jobs: - zip - pkg otp: - - 23.3.4.9-3 - 24.1.5-3 arch: - amd64 @@ -221,8 +220,6 @@ jobs: - centos7 - raspbian10 exclude: - - package: pkg - otp: 23.3.4.9-3 - os: raspbian9 arch: amd64 - os: raspbian10 @@ -376,7 +373,6 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.9-3 - 24.1.5-3 include: - profile: emqx diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 6efc6bcc0..f50315a22 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -77,4 +77,4 @@ docker run -i --rm \ --workdir /emqx \ --platform="linux/$ARCH" \ "$BUILDER" \ - bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" + bash -euc "git config --global --add safe.directory /emqx && chown -R root:root ./_build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" From 918e2fc58fc66fe65f25a31f0eb409811474a50a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 27 May 2022 18:14:01 +0800 Subject: [PATCH 331/363] chore: bump version 4.4.4-rc.3 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index b5765abf3..85b37e4c1 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.4-rc.2"}). +-define(EMQX_RELEASE, {opensource, "4.4.4-rc.3"}). -else. From 2ca952bbe6d6e97b502c048dae4d109adb44f511 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 27 May 2022 18:22:39 +0800 Subject: [PATCH 332/363] chore: update emqx.appup.src --- src/emqx.appup.src | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index cbb9e8e66..932a30755 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -42,7 +42,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, {"4.4.1", - [{add_module,emqx_calendar}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, @@ -72,7 +73,8 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}]}, {"4.4.0", - [{add_module,emqx_calendar}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, @@ -107,7 +109,8 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.3", - [{add_module,emqx_calendar}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -123,7 +126,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, {"4.4.2", - [{add_module,emqx_calendar}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -173,7 +177,8 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}]}, {"4.4.0", - [{add_module,emqx_calendar}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, From ce1dd88676d34a48381050946c3ab37c207a7fe6 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 27 May 2022 21:54:57 +0800 Subject: [PATCH 333/363] chore: fix bad `add_module` instrs --- src/emqx.appup.src | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 932a30755..c7cdd3e60 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -110,7 +110,7 @@ {<<".*">>,[]}], [{"4.4.3", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {add_module,emqx_calendar}, + {delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -127,7 +127,7 @@ {load_module,emqx_relup}]}, {"4.4.2", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {add_module,emqx_calendar}, + {delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -147,7 +147,7 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, {"4.4.1", - [{add_module,emqx_calendar}, + [{delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, @@ -178,7 +178,7 @@ {delete_module,emqx_relup}]}, {"4.4.0", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {add_module,emqx_calendar}, + {delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, From a96984d280c02d6a7ff5d1bf84d77f9df7eb24f0 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 30 May 2022 11:32:55 +0800 Subject: [PATCH 334/363] feat: dashboard v4.4.11 for ee 4.4.4 --- scripts/get-dashboard.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index 6d94bae58..b5e637568 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -14,7 +14,7 @@ case "${PKG_VSN}" in 4.4*) # keep the above 4.3 untouched, otherwise conflicts! EMQX_CE_DASHBOARD_VERSION='v4.4.2' - EMQX_EE_DASHBOARD_VERSION='v4.4.10' + EMQX_EE_DASHBOARD_VERSION='v4.4.11' ;; *) echo "Unsupported version $PKG_VSN" >&2 From f033cd1a8738fc2450957595f28119f42d85791f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 31 May 2022 22:09:53 +0800 Subject: [PATCH 335/363] chore: sync changes --- CHANGES-4.4.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 5b2e134b1..6862d0f6a 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -30,11 +30,14 @@ * SSL closed error bug fixed for redis client. * Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message * Rule-engine function hexstr2bin/1 support half byte [#7977] +* Shared message delivery when all alive shared subs have full inflight [#7984] * Improved resilience against autocluster partitioning during cluster startup. [#7876] [ekka-158](https://github.com/emqx/ekka/pull/158) * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979] * Fix `node_dump` variable sourcing. [#8026] +* Fix heap size is growing too fast when trace large message. +* Support customized timestamp format of the log messages. ## v4.4.3 From 983a6995922d03fdc692d50b1c78897950023f82 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Wed, 1 Jun 2022 14:39:50 +0800 Subject: [PATCH 336/363] fix: better release up log --- src/emqx_relup.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/emqx_relup.erl b/src/emqx_relup.erl index fcc90d088..3ad2eb250 100644 --- a/src/emqx_relup.erl +++ b/src/emqx_relup.erl @@ -30,13 +30,13 @@ %% What to do after upgraded from an old release vsn. post_release_upgrade(FromRelVsn, _) -> {_, CurrRelVsn} = ?EMQX_RELEASE, - ?INFO("emqx has been upgraded to from ~s to ~s!", [FromRelVsn, CurrRelVsn]), + ?INFO("emqx has been upgraded from ~s to ~s!", [FromRelVsn, CurrRelVsn]), reload_components(). %% What to do after downgraded to an old release vsn. post_release_downgrade(ToRelVsn, _) -> {_, CurrRelVsn} = ?EMQX_RELEASE, - ?INFO("emqx has been downgraded to from ~s to ~s!", [CurrRelVsn, ToRelVsn]), + ?INFO("emqx has been downgraded from ~s to ~s!", [CurrRelVsn, ToRelVsn]), reload_components(). -ifdef(EMQX_ENTERPRISE). From e2b659d3210795fd6f2b633675892c7cd7914105 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 2 Jun 2022 11:35:09 +0200 Subject: [PATCH 337/363] ci: ensure otp 24.1.5 in run_fvt_tests.yaml --- .github/workflows/run_fvt_tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index db1a351c4..0fa30a54e 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -13,6 +13,9 @@ jobs: steps: - uses: actions/checkout@v1 + - uses: erlef/setup-beam@v1 + with: + otp-version: "24.1.5" - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then From 266354cf15896e2ab20824237776f0ff2cfb5a1a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 2 Jun 2022 13:45:00 +0200 Subject: [PATCH 338/363] ci: use erlef/setup-beam@v1 to install erlang --- .github/workflows/run_fvt_tests.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 1eb189720..ef76e8108 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -13,10 +13,9 @@ jobs: steps: - uses: actions/checkout@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang + - uses: erlef/setup-beam@v1 with: - otp-version: 23.2 + otp-version: "23.2" - name: make docker run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -66,10 +65,9 @@ jobs: steps: - uses: actions/checkout@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang + - uses: erlef/setup-beam@v1 with: - otp-version: 23.2 + otp-version: "23.2" - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then From 13f575bc84bc54f7f43814f9054d009d7e23544e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 15 Jun 2022 13:45:05 -0300 Subject: [PATCH 339/363] chore(relup): bump release and app vsns, update appups --- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 46 +++++++++++++++---- include/emqx_release.hrl | 2 +- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- .../emqx_modules/src/emqx_modules.appup.src | 22 ++++++--- src/emqx.app.src | 2 +- src/emqx.appup.src | 17 +++++-- 7 files changed, 70 insertions(+), 23 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index d52e51316..b8f53f7d6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.4.4"}, % strict semver, bump manually! + {vsn, "4.4.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 754eede15..49157894f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,8 +1,15 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.3", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{"4.4.4", + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, + {"4.4.3", + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {add_module,emqx_rule_date}, @@ -11,7 +18,9 @@ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.4.2", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, @@ -24,7 +33,9 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.1", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -37,7 +48,9 @@ {add_module,emqx_rule_date}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -50,8 +63,15 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.3", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{"4.4.4", + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, + {"4.4.3", + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {delete_module,emqx_rule_date}, @@ -60,7 +80,9 @@ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.4.2", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, @@ -73,7 +95,9 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.1", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -86,7 +110,9 @@ {delete_module,emqx_rule_date}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index ea8d74f17..d40240579 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.4"}). +-define(EMQX_RELEASE, {opensource, "4.4.5"}). -else. diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index 1ff39b0f8..854d54e4e 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.4.3"}, + {vsn, "4.4.4"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index 51382cdfa..91ae657da 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,26 +1,36 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.2", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, + [{"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + {"4.4.2", + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, {"4.4.1", - [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.2", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, + [{"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + {"4.4.2", + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, {"4.4.1", - [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, diff --git a/src/emqx.app.src b/src/emqx.app.src index 156dc57c7..5efc0de67 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.4.4"}, % strict semver, bump manually! + {vsn, "4.4.5"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index fc38c25da..272b5b58d 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.3", + [{"4.4.4", + [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}]}, + {"4.4.3", [{add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_packet,brutal_purge,soft_purge,[]}, @@ -109,8 +114,14 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.3", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + [{"4.4.4", + [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}]}, + {"4.4.3", + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, From a55d583de6b45631b23c441d7d2f8f463d90e0f2 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 15 Jun 2022 14:14:34 -0300 Subject: [PATCH 340/363] chore: bump more app versions --- apps/emqx_management/src/emqx_management.app.src | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index fd941f8b8..28be88159 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.4.4"}, % strict semver, bump manually! + {vsn, "4.4.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,emqx_plugin_libs,minirest]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 95cca4e1c..4e9770dab 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQX Web Dashboard"}, - {vsn, "4.4.5"}, % strict semver, bump manually! + {vsn, "4.4.6"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, From 5220cdd1f68cbe2bd89e1c7af9ac5d76451f2fab Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 17 Jun 2022 20:26:56 +0200 Subject: [PATCH 341/363] ci: no more 'latest' docker tag for 4.4 --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index c7e9ed1e7..7b9a99620 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -319,7 +319,7 @@ jobs: images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} ## only stable tag is latest flavor: | - latest=${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'rc') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }} + latest=false # latest is now 5.0 tags: | type=ref,event=branch type=ref,event=pr From fd11e02639d1cbdd233b49c281e1ea270923cd79 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 22 Jun 2022 10:00:25 +0800 Subject: [PATCH 342/363] feat: mqtt/publish support to publish with properties and user_properties --- CHANGES-4.4.md | 6 ++ .../src/emqx_mgmt_api_pubsub.erl | 63 +++++++++++++------ .../test/emqx_mgmt_api_SUITE.erl | 28 ++++++++- 3 files changed, 77 insertions(+), 20 deletions(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 6862d0f6a..dbc42fc3f 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,5 +1,11 @@ # EMQX 4.4 Changes +## v4.4.5 + +### Enhancements (synced from v4.3.16) +* HTTP API `mqtt/publish` support to publish with properties and user_properties. + + ## v4.4.4 ### Enhancements (synced from v4.3.15) diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index 3fe859811..b2a050d17 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -74,7 +74,9 @@ publish(_Bindings, Params) -> try parse_publish_params(Params) of Result -> do_publish(Params, Result) catch - _E : _R -> minirest:return({ok, ?ERROR8, bad_params}) + _E : _R -> + logger:debug("API publish result:~p ~p", [_E, _R]), + minirest:return({ok, ?ERROR8, bad_params}) end. unsubscribe(_Bindings, Params) -> @@ -159,8 +161,8 @@ do_subscribe(ClientId, Topics, QoS) -> _ -> ok end. -do_publish(Params, {ClientId, Topic, Qos, Retain, Payload, UserProps}) -> - case do_publish(ClientId, Topic, Qos, Retain, Payload, UserProps) of +do_publish(Params, {ClientId, Topic, Qos, Retain, Payload, Props}) -> + case do_publish(ClientId, Topic, Qos, Retain, Payload, Props) of {ok, MsgIds} -> case proplists:get_value(<<"return">>, Params, undefined) of undefined -> minirest:return(ok); @@ -174,17 +176,16 @@ do_publish(Params, {ClientId, Topic, Qos, Retain, Payload, UserProps}) -> minirest:return(Result) end. -do_publish(ClientId, _Topics, _Qos, _Retain, _Payload, _UserProps) +do_publish(ClientId, _Topics, _Qos, _Retain, _Payload, _Props) when not (is_binary(ClientId) or (ClientId =:= undefined)) -> {ok, ?ERROR8, <<"bad clientid: must be string">>}; -do_publish(_ClientId, [], _Qos, _Retain, _Payload, _UserProps) -> +do_publish(_ClientId, [], _Qos, _Retain, _Payload, _Props) -> {ok, ?ERROR15, bad_topic}; -do_publish(ClientId, Topics, Qos, Retain, Payload, UserProps) -> +do_publish(ClientId, Topics, Qos, Retain, Payload, Props) -> MsgIds = lists:map(fun(Topic) -> - Msg = emqx_message:make(ClientId, Qos, Topic, Payload), - UserProps1 = #{'User-Property' => UserProps}, - _ = emqx_mgmt:publish(Msg#message{flags = #{retain => Retain}, - headers = #{properties => UserProps1}}), + Msg = emqx_message:make(ClientId, Qos, Topic, Payload, + #{retain => Retain}, Props), + _ = emqx_mgmt:publish(Msg), emqx_guid:to_hexstr(Msg#message.id) end, Topics), {ok, MsgIds}. @@ -218,8 +219,8 @@ parse_publish_params(Params) -> Qos = proplists:get_value(<<"qos">>, Params, 0), Retain = proplists:get_value(<<"retain">>, Params, false), Payload1 = maybe_maps_to_binary(Payload), - UserProps = generate_user_props(proplists:get_value(<<"user_properties">>, Params, [])), - {ClientId, Topics, Qos, Retain, Payload1, UserProps}. + Props = parse_props(Params), + {ClientId, Topics, Qos, Retain, Payload1, Props}. parse_unsubscribe_params(Params) -> ClientId = proplists:get_value(<<"clientid">>, Params), @@ -274,16 +275,42 @@ maybe_maps_to_binary(Payload) -> error({encode_payload_fail, S}) end. +-define(PROP_MAPPING, + #{<<"payload_format_indicator">> => 'Payload-Format-Indicator', + <<"message_expiry_interval">> => 'Message-Expiry-Interval', + <<"response_topic">> => 'Response-Topic', + <<"correlation_data">> => 'Correlation-Data', + <<"user_properties">> => 'User-Property', + <<"subscription_identifier">> => 'Subscription-Identifier', + <<"content_type">> => 'Content-Type' + }). + +parse_props(Params) -> + Properties0 = proplists:get_value(<<"properties">>, Params, []), + Properties1 = lists:foldl(fun({Name, Value}, Acc) -> + case maps:find(Name, ?PROP_MAPPING) of + {ok, Key} -> Acc#{Key => Value}; + error -> error({invalid_property, Name}) + end + end, #{}, Properties0), + %% Compatible with older API + UserProp1 = generate_user_props(proplists:get_value(<<"user_properties">>, Params, [])), + UserProp2 = + case Properties1 of + #{'User-Property' := UserProp1List} -> generate_user_props(UserProp1List); + _ -> [] + end, + #{properties => Properties1#{'User-Property' => UserProp1 ++ UserProp2}}. + generate_user_props(UserProps) when is_list(UserProps)-> - generate_user_props_(UserProps, []); + lists:map(fun + ({Name, Value}) -> {bin(Name), bin(Value)}; + (Invalid) -> error({invalid_user_property, Invalid}) + end + , UserProps); generate_user_props(UserProps) -> error({user_properties_type_error, UserProps}). -generate_user_props_([{Name, Value} | Rest], Acc) -> - generate_user_props_(Rest, [{bin(Name), bin(Value)} | Acc]); -generate_user_props_([], Acc) -> - lists:reverse(Acc). - bin(Bin) when is_binary(Bin) -> Bin; bin(Num) when is_number(Num) -> number_to_binary(Num); bin(Boolean) when is_boolean(Boolean) -> atom_to_binary(Boolean); diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index 0029b866f..f80128eb3 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -568,11 +568,35 @@ t_pubsub(_) -> false end), + %% properties + {ok, Code} = request_api(post, api_path(["mqtt/publish"]), [], auth_header_(), + #{<<"clientid">> => ClientId, + <<"topic">> => <<"mytopic">>, + <<"qos">> => 1, + <<"payload">> => <<"hello properties">>, + <<"user_properties">> => #{<<"prop_key1">> => <<"prop_val1">>}, + <<"properties">> => #{ + <<"message_expiry_interval">> => 1000, + <<"user_properties">> => #{<<"prop_key2">> => <<"prop_val2">>}} + }), + Msg = receive + {publish, MsgTmp} -> + MsgTmp + after 150 -> + false + end, + ?assertMatch(#{payload := <<"hello properties">>, + qos := 1, + properties := #{ + 'Message-Expiry-Interval' := 1000, + 'User-Property' := [{<<"prop_key1">>,<<"prop_val1">>}, + {<<"prop_key2">>,<<"prop_val2">>}]}}, Msg), + ok = emqtt:disconnect(C1), - ?assertEqual(4, emqx_metrics:val('messages.qos1.received') - Qos1Received), + ?assertEqual(5, emqx_metrics:val('messages.qos1.received') - Qos1Received), ?assertEqual(2, emqx_metrics:val('messages.qos2.received') - Qos2Received), - ?assertEqual(6, emqx_metrics:val('messages.received') - Received). + ?assertEqual(7, emqx_metrics:val('messages.received') - Received). loop([]) -> []; From 773c4645729e57d44eb4b8388ec087f29a63a992 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 20 Jun 2022 11:38:51 +0800 Subject: [PATCH 343/363] fix: clean trace zip file after download --- apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src | 2 +- apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src | 12 ++++++++++-- apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl | 5 +++-- .../src/emqx_trace/emqx_trace_api.erl | 4 ++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 5fbd21fab..9315f8c53 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.4.3"}, + {vsn, "4.4.4"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index b183c5e0a..5a8a90d95 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -1,7 +1,11 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.2",[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + [ + {"4.4.3", + [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.2",[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, {"4.4.1", [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, {load_module,emqx_trace,brutal_purge,soft_purge,[]}, @@ -13,7 +17,11 @@ {update,emqx_slow_subs,{advanced,["4.4.0"]}}, {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.2",[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + [ + {"4.4.3", + [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {"4.4.2",[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, {"4.4.1", [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, {load_module,emqx_trace,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index 84673b4a7..d2c02d06d 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -226,7 +226,8 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitor case maps:take(Pid, Monitors) of error -> {noreply, State}; {Files, NewMonitors} -> - lists:foreach(fun file:delete/1, Files), + ZipDir = emqx_trace:zip_dir(), + lists:foreach(fun(F) -> file:delete(filename:join([ZipDir, F])) end, Files), {noreply, State#{monitors => NewMonitors}} end; handle_info({timeout, TRef, update_trace}, @@ -403,7 +404,7 @@ fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) -> fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60}); fill_default(Trace) -> Trace. --define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$"). +-define(NAME_RE, "^[0-9A-Za-z]+[A-Za-z0-9-_]*$"). to_trace(#{name := Name} = Trace, Rec) -> case re:run(Name, ?NAME_RE) of diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index b8aa5c059..49321086b 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -97,9 +97,9 @@ download_zip_log(#{name := Name}, _Param) -> ZipDir = emqx_trace:zip_dir(), Zips = group_trace_file(ZipDir, TraceLog, TraceFiles), ZipFileName0 = binary_to_list(Name) ++ ".zip", - ZipFileName = filename:join([Zips, ZipFileName0]), + ZipFileName = filename:join([ZipDir, ZipFileName0]), {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]), - emqx_trace:delete_files_after_send(ZipFileName, Zips), + emqx_trace:delete_files_after_send(ZipFileName0, Zips), {ok, ZipFile}; {error, Reason} -> {error, Reason} From df40e3aaa1144a92ed9cc6edbd8b22a2f35a70e7 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 20 Jun 2022 11:41:57 +0800 Subject: [PATCH 344/363] chore: update trace changelog --- CHANGES-4.4.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index dbc42fc3f..cca548ae0 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -5,6 +5,8 @@ ### Enhancements (synced from v4.3.16) * HTTP API `mqtt/publish` support to publish with properties and user_properties. +### Bug fixes +- Clean trace zip files when file has been downloaded. ## v4.4.4 From 040e964956de304994161cf382898df462ef5a5c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 20 Jun 2022 14:36:15 +0800 Subject: [PATCH 345/363] fix: appup failed --- apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index 5a8a90d95..3d12e8b8a 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -5,7 +5,10 @@ {"4.4.3", [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, - {"4.4.2",[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + {"4.4.2",[ + {load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, {"4.4.1", [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, {load_module,emqx_trace,brutal_purge,soft_purge,[]}, @@ -21,7 +24,10 @@ {"4.4.3", [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, - {"4.4.2",[{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + {"4.4.2", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, {"4.4.1", [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, {load_module,emqx_trace,brutal_purge,soft_purge,[]}, From d345590206ef3a515c4f40550623730ce102550b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 22 Jun 2022 17:35:57 +0800 Subject: [PATCH 346/363] fix: CT failed --- apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index 56b81424e..e444633a7 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -133,7 +133,7 @@ t_create_failed(_Config) -> InvalidPackets4 = [{<<"name">>, <<"/test">>}, {<<"clientid">>, <<"t">>}, {<<"type">>, <<"clientid">>}], {error, Reason9} = emqx_trace:create(InvalidPackets4), - ?assertEqual(<<"Name should be ^[A-Za-z]+[A-Za-z0-9-_]*$">>, iolist_to_binary(Reason9)), + ?assertEqual(<<"Name should be ^[0-9A-Za-z]+[A-Za-z0-9-_]*$">>, iolist_to_binary(Reason9)), ?assertEqual({error, "type=[topic,clientid,ip_address] required"}, emqx_trace:create([{<<"name">>, <<"test-name">>}, {<<"clientid">>, <<"good">>}])), From ddc06e8bcde681bf7b6c68236184e2598c203fef Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 27 Jun 2022 18:42:44 +0300 Subject: [PATCH 347/363] fix: remove unnecessary error messages generated by JWT auth --- CHANGES-4.4.md | 1 + apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- .../emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 19 ++++++------------- apps/emqx_auth_jwt/src/emqx_auth_jwt.erl | 8 +++++++- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index cca548ae0..b2b331240 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -7,6 +7,7 @@ ### Bug fixes - Clean trace zip files when file has been downloaded. +- Remove unnecessary error messages generated by JWT auth. ## v4.4.4 diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index cc0029e0f..7e6361bef 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.4.2"}, % strict semver, bump manually! + {vsn, "4.4.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index 34e964e65..d8b60f0a3 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,16 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {<<"4\\.4\\.[0-1]">>, [ - {restart_application,emqx_auth_jwt} - ]}, - {<<".*">>, []} - ], - [ - {<<"4\\.4\\.[0-1]">>, [ - {restart_application,emqx_auth_jwt} - ]}, - {<<".*">>, []} - ] -}. + [{"4.4.2",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {<<"4\\.4\\.[0-1]">>,[{restart_application,emqx_auth_jwt}]}, + {<<".*">>,[]}], + [{"4.4.2",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {<<"4\\.4\\.[0-1]">>,[{restart_application,emqx_auth_jwt}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl index 040f9b629..725207d9a 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl @@ -72,7 +72,13 @@ check_acl(ClientInfo = #{jwt_claims := Claims}, _ -> ?DEBUG("no_acl_jwt_claim", []), ignore - end. + end; +check_acl(_ClientInfo, + _PubSub, + _Topic, + _NoMatchAction, + _AclEnv) -> + ignore. is_expired(Exp) when is_binary(Exp) -> ExpInt = binary_to_integer(Exp), From a365c40765dfb09b90dd0f0e58c04fc635e3c095 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 28 Jun 2022 10:39:37 +0800 Subject: [PATCH 348/363] chore: update dashboard version --- scripts/get-dashboard.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index c7e5bc303..32ee0ac49 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -13,8 +13,8 @@ case "${PKG_VSN}" in ;; 4.4*) # keep the above 4.3 untouched, otherwise conflicts! - EMQX_CE_DASHBOARD_VERSION='v4.4.2' - EMQX_EE_DASHBOARD_VERSION='v4.4.11' + EMQX_CE_DASHBOARD_VERSION='v4.4.3' + EMQX_EE_DASHBOARD_VERSION='v4.4.12' ;; *) echo "Unsupported version $PKG_VSN" >&2 From b10f8b892897c905af44a62af65ad16382df536e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 30 Jun 2022 06:08:12 -0300 Subject: [PATCH 349/363] test(mgmt): run all tests in suite --- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 72 ++++++++++++++----- 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 48a771d7a..edf241315 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -29,29 +29,35 @@ -define(LOG_HANDLER_ID, [file, default]). all() -> + OtherTCs = (emqx_ct:all(?MODULE) -- manage_apps_tests()) -- check_cli_tests(), [{group, manage_apps}, - {group, check_cli}]. + {group, check_cli}] ++ OtherTCs. + +manage_apps_tests() -> + [t_app]. + +check_cli_tests() -> + [t_cli, + t_log_cmd, + t_mgmt_cmd, + t_status_cmd, + t_clients_cmd, + t_vm_cmd, + t_plugins_cmd, + t_trace_cmd, + t_traces_cmd, + t_broker_cmd, + t_router_cmd, + t_subscriptions_cmd, + t_listeners_cmd_old, + t_listeners_cmd_new + ]. groups() -> [{manage_apps, [sequence], - [t_app - ]}, - {check_cli, [sequence], - [t_cli, - t_log_cmd, - t_mgmt_cmd, - t_status_cmd, - t_clients_cmd, - t_vm_cmd, - t_plugins_cmd, - t_trace_cmd, - t_traces_cmd, - t_broker_cmd, - t_router_cmd, - t_subscriptions_cmd, - t_listeners_cmd_old, - t_listeners_cmd_new - ]}]. + manage_apps_tests()}, + {check_cli, [sequence], + check_cli_tests()}]. apps() -> [emqx_management, emqx_auth_mnesia, emqx_modules]. @@ -73,12 +79,26 @@ init_per_testcase(t_plugins_cmd, Config) -> mock_print(), Config; init_per_testcase(t_import_outside_backup_dir, Config) -> + BackupDir = emqx_mgmt_data_backup:backup_dir(), + {ok, Files} = file:list_dir(BackupDir), + lists:foreach( + fun(F) -> + file:delete(filename:join(BackupDir, F)) + end, Files), RandomName = emqx_guid:to_hexstr(emqx_guid:gen()), Filepath = "/tmp/" ++ binary_to_list(RandomName) ++ ".json", FakeData = #{version => "4.4"}, ok = file:write_file(Filepath, emqx_json:encode(FakeData)), [ {tmp_file, Filepath} | Config]; +init_per_testcase(t_backup_file, Config) -> + BackupDir = emqx_mgmt_data_backup:backup_dir(), + {ok, Files} = file:list_dir(BackupDir), + lists:foreach( + fun(F) -> + file:delete(filename:join(BackupDir, F)) + end, Files), + Config; init_per_testcase(_Case, Config) -> mock_print(), Config. @@ -89,6 +109,20 @@ end_per_testcase(t_plugins_cmd, _Config) -> end_per_testcase(t_import_outside_backup_dir, Config) -> Filepath = ?config(tmp_file, Config), file:delete(Filepath), + BackupDir = emqx_mgmt_data_backup:backup_dir(), + {ok, Files} = file:list_dir(BackupDir), + lists:foreach( + fun(F) -> + file:delete(filename:join(BackupDir, F)) + end, Files), + ok; +end_per_testcase(t_backup_file, _Config) -> + BackupDir = emqx_mgmt_data_backup:backup_dir(), + {ok, Files} = file:list_dir(BackupDir), + lists:foreach( + fun(F) -> + file:delete(filename:join(BackupDir, F)) + end, Files), ok; end_per_testcase(_Case, _Config) -> unmock_print(). From 7415e2dbc700d35fd46eff329e119dd30612d5a3 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 30 Jun 2022 21:40:40 +0800 Subject: [PATCH 350/363] chore: bump version --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index e02e076e9..d40240579 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.5-beta.4"}). +-define(EMQX_RELEASE, {opensource, "4.4.5"}). -else. From 92d1c376623e90bd7606c5bdb7d852e225d8407e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 1 Jul 2022 22:40:45 +0800 Subject: [PATCH 351/363] Merge v4.3.16 into hotfix/v4.4.5 --- src/emqx.appup.src | 20 +++++++++++++++----- src/emqx_exclusive_subscription.erl | 20 +++++++++++++++++++- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 7d2ffc119..9fe801743 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -6,9 +6,10 @@ {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {add_module,emqx_exclusive_subscription}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {add_module,emqx_exclusive_subscription}, {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -17,9 +18,10 @@ [{add_module,emqx_calendar}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {add_module,emqx_exclusive_subscription}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {add_module,emqx_exclusive_subscription}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -41,9 +43,10 @@ [{add_module,emqx_calendar}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {add_module,emqx_exclusive_subscription}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {add_module,emqx_exclusive_subscription}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -67,9 +70,10 @@ {"4.4.1", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, + {add_module,emqx_exclusive_subscription}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {add_module,emqx_exclusive_subscription}, {add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, @@ -103,9 +107,10 @@ [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, {add_module,emqx_calendar}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, + {add_module,emqx_exclusive_subscription}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {add_module,emqx_exclusive_subscription}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, @@ -146,6 +151,7 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, + {apply,{emqx_exclusive_subscription,on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -156,6 +162,7 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, + {apply,{emqx_exclusive_subscription,on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_packet,brutal_purge,soft_purge,[]}, @@ -179,6 +186,7 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, + {apply,{emqx_exclusive_subscription,on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {delete_module,emqx_calendar}, @@ -204,6 +212,7 @@ [{load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, + {apply,{emqx_exclusive_subscription,on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -238,6 +247,7 @@ [{load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, + {apply,{emqx_exclusive_subscription,on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {delete_module,emqx_calendar}, diff --git a/src/emqx_exclusive_subscription.erl b/src/emqx_exclusive_subscription.erl index ab26a6bd6..487256903 100644 --- a/src/emqx_exclusive_subscription.erl +++ b/src/emqx_exclusive_subscription.erl @@ -27,6 +27,9 @@ -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). +%% For upgrade +-export([on_add_module/0, on_delete_module/0]). + -export([ check_subscribe/2, unsubscribe/2 @@ -58,7 +61,22 @@ mnesia(boot) -> {storage_properties, StoreProps} ]); mnesia(copy) -> - ok = ekka_mnesia:copy_table(?TAB, ram_copies). + case ekka_mnesia:copy_table(?TAB, ram_copies) of + ok -> + ok; + {no_exists, _} -> + mnesia(boot) + end. + +%%-------------------------------------------------------------------- +%% Upgrade +%%-------------------------------------------------------------------- + +on_add_module() -> + mnesia(boot). + +on_delete_module() -> + mnesia:clear_table(?TAB). %%-------------------------------------------------------------------- %% APIs From 1180eca04dc48ade5b8fd72884391d76e985dff9 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 18 Jul 2022 14:11:02 +0800 Subject: [PATCH 352/363] chore: update dashboard version --- scripts/get-dashboard.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index 32ee0ac49..b014f5531 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -13,8 +13,8 @@ case "${PKG_VSN}" in ;; 4.4*) # keep the above 4.3 untouched, otherwise conflicts! - EMQX_CE_DASHBOARD_VERSION='v4.4.3' - EMQX_EE_DASHBOARD_VERSION='v4.4.12' + EMQX_CE_DASHBOARD_VERSION='v4.4.4' + EMQX_EE_DASHBOARD_VERSION='v4.4.13' ;; *) echo "Unsupported version $PKG_VSN" >&2 From 368fe40c34c1edbc23b13f562f69a8bd2f738af1 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 19 Jul 2022 11:15:06 +0800 Subject: [PATCH 353/363] fix: merge appup.src --- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 20 ++++++------ lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- .../emqx_modules/src/emqx_modules.appup.src | 6 ++-- src/emqx.app.src | 2 +- src/emqx.appup.src | 31 ++++++++----------- 6 files changed, 31 insertions(+), 32 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index b8f53f7d6..c798ae8bc 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.4.5"}, % strict semver, bump manually! + {vsn, "4.4.6"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index cdf2a970b..e56b1bd75 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,17 +1,19 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.4", + [{"4.4.5", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {"4.4.4", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, {"4.4.3", - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -68,7 +70,10 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.4", + [ {"4.4.5", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {"4.4.4", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -76,10 +81,7 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, {"4.4.3", - {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index 854d54e4e..e4f8383dd 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.4.4"}, + {vsn, "4.4.5"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index 91ae657da..dbad1c4ef 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + [{"4.4.4",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + {"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, {"4.4.2", [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, {load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, @@ -19,7 +20,8 @@ {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + [{"4.4.4",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + {"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, {"4.4.2", [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, {load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, diff --git a/src/emqx.app.src b/src/emqx.app.src index 5efc0de67..4a360ca06 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.4.5"}, % strict semver, bump manually! + {vsn, "4.4.6"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index da289db26..7d22e4579 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,10 @@ -%% -*- mode: erlang -*- +% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.4", + [{"4.4.5", [ + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]}, + {"4.4.4", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -13,6 +16,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}]}, {"4.4.3", [{add_module,emqx_calendar}, @@ -144,7 +149,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.4", + [{"4.4.5", + [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]}, + {"4.4.4", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -156,6 +164,8 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}]}, {"4.4.3", [{load_module,emqx_broker,brutal_purge,soft_purge,[]}, @@ -210,9 +220,6 @@ {load_module,emqx_relup}]}, {"4.4.1", [{load_module,emqx_broker,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {apply,{emqx_exclusive_subscription,on_delete_module, []}}, @@ -279,18 +286,6 @@ {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, - {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {update,emqx_os_mon,{advanced,[]}}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, From b400ccdeb4fecb81428cf510fbc138a51ff4a3a9 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 19 Jul 2022 15:09:04 +0800 Subject: [PATCH 354/363] chore(relup): download relup base version packages from s3 --- scripts/relup-base-packages.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 3b0c2aee1..9505a68f0 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -14,15 +14,15 @@ fi case $PROFILE in "emqx") - DIR='broker' + DIR='emqx-ce' EDITION='community' ;; "emqx-ee") - DIR='enterprise' + DIR='emqx-ee' EDITION='enterprise' ;; "emqx-edge") - DIR='edge' + DIR='emqx-edge' EDITION='edge' ;; esac @@ -65,7 +65,7 @@ pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip" - url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename" + url="https://packages.emqx.io/$DIR/$tag/$filename" if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then echo "downloading base package from ${url} ..." curl -L -o "${filename}" "${url}" From 28e553964535ed642bc6b0ebac9a06dc9d5715a1 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 19 Jul 2022 17:45:25 +0800 Subject: [PATCH 355/363] chore: revert emqx.appup.src --- .../src/emqx_rule_engine.appup.src | 11 ++--------- src/emqx.appup.src | 16 +++------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index e56b1bd75..f8a82bdf6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,16 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.5", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.4.4", + [{"4.4.4", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, {"4.4.3", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, @@ -70,10 +66,7 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [ {"4.4.5", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, - {"4.4.4", + [{"4.4.4", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 7d22e4579..9fe801743 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,10 +1,7 @@ -% -*- mode: erlang -*- +%% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.5", [ - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]}, - {"4.4.4", + [{"4.4.4", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -16,8 +13,6 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}]}, {"4.4.3", [{add_module,emqx_calendar}, @@ -149,10 +144,7 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.5", - [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]}, - {"4.4.4", + [{"4.4.4", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -164,8 +156,6 @@ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}]}, {"4.4.3", [{load_module,emqx_broker,brutal_purge,soft_purge,[]}, From b6b4ad9b70395710bc02cde7752939e46d38c9c3 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 19 Jul 2022 18:44:04 +0800 Subject: [PATCH 356/363] chore: update appup.src --- .../src/emqx_rule_engine.appup.src | 16 ++++++++--- src/emqx.appup.src | 28 +++++++++++++------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index f8a82bdf6..7d059ce96 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,8 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.4", - [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + [{"4.4.5", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {"4.4.4", + [{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -66,8 +70,12 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.4", - [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + [{"4.4.5", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {"4.4.4", + [{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 9fe801743..720390f7e 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,8 +1,13 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.4", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{"4.4.5", + [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]}, + {"4.4.4", + [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, @@ -144,14 +149,19 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.4", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{"4.4.5", + [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]}, + {"4.4.4", + [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {apply,{emqx_exclusive_subscription,on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -162,7 +172,7 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {apply,{emqx_exclusive_subscription,on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_packet,brutal_purge,soft_purge,[]}, @@ -186,7 +196,7 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {apply,{emqx_exclusive_subscription,on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {delete_module,emqx_calendar}, @@ -212,7 +222,7 @@ [{load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {apply,{emqx_exclusive_subscription,on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -247,7 +257,7 @@ [{load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {apply,{emqx_exclusive_subscription,on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {delete_module,emqx_calendar}, From 347f071079f7277cdfd37dca5bcf3dd805981795 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 19 Jul 2022 19:01:40 +0800 Subject: [PATCH 357/363] chore: make sure emqx_app/emqx_relup in emqx's appup.src --- src/emqx.appup.src | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 720390f7e..b4c7917fb 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,10 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.5", - [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + [ + {load_module,emqx_relup,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]}, {"4.4.4", [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -150,7 +153,10 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.5", - [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + [ + {load_module,emqx_relup,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]}, {"4.4.4", [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, From 2a4ded6f4fb53f95529f680c3f4f4d5e2ccfc228 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 19 Jul 2022 19:23:58 +0800 Subject: [PATCH 358/363] chore: bump to 4.4.6.beta.1 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index d40240579..4af432057 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.5"}). +-define(EMQX_RELEASE, {opensource, "4.4.6-beta.1"}). -else. From 2ab2c39fdc625fa1ace2c42711acf4028bf6cc80 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 20 Jul 2022 15:21:00 +0800 Subject: [PATCH 359/363] chore: update emqx-builder to 4.4-18 --- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/apps_version_check.yaml | 2 +- .github/workflows/build_packages.yaml | 8 ++++---- .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 2 +- Makefile | 2 +- deploy/docker/Dockerfile | 2 +- scripts/buildx.sh | 4 ++-- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index a95acc772..d5e490b0c 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 44882dc5c..524ba8926 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -13,7 +13,7 @@ jobs: os: - ubuntu20.04 - container: ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-18:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 7b9a99620..51fc7dd35 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -261,7 +261,7 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/4.4-12:${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/4.4-18:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 with: name: ${{ matrix.profile }}-${{ matrix.otp }} @@ -337,7 +337,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-18:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -353,7 +353,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-18:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 73bc9dbb4..07fcbc35b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -23,7 +23,7 @@ jobs: - ubuntu20.04 - el8 - container: ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-18:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index e50738914..9ad140a81 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b03ac2077..411092160 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index b491ef47b..0b3bf1c83 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index cbd341d33..cbbab69a0 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -226,7 +226,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -277,7 +277,7 @@ jobs: otp: - 24.1.5-3 runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 239b8184e..2d5a4e613 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 417423f4e..76d318024 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-alpine3.15.1 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-alpine3.15.1 export EMQX_DEFAULT_RUNNER = alpine:3.15.1 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 63702e2c4..aab61175d 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-alpine3.15.1 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-alpine3.15.1 ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder diff --git a/scripts/buildx.sh b/scripts/buildx.sh index f15069f86..4e7960547 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-debian10 --arch arm64 +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-debian10 --arch arm64 set -euo pipefail @@ -20,7 +20,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-18:24.1.5-3-debian10" } while [ "$#" -gt 0 ]; do From 276c735b2c9cac2dea5c9937a8405dda0601c520 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 20 Jul 2022 23:22:58 +0200 Subject: [PATCH 360/363] chore: Update emqx.appup.src --- src/emqx.appup.src | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 994ca8c62..fb9fdd961 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -7,8 +7,8 @@ {update,emqx_broker_sup,supervisor}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, - {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}] - {load_module,emqx_session,brutal_purge,soft_purge,[]}}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}]}, {"4.4.4", [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {update,emqx_broker_sup,supervisor}, From 81be04456b5b51c1cf10119c7540ad700e04a951 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 25 Jul 2022 07:48:23 +0800 Subject: [PATCH 361/363] chore: update version to beta.3 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 9b4c0e6f2..a1bfff8e9 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.6-beta.2"}). +-define(EMQX_RELEASE, {opensource, "4.4.6-beta.3"}). -else. From 60e0b644342c1184eb903df212f18035a43a0cec Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 25 Jul 2022 08:54:33 +0800 Subject: [PATCH 362/363] fix: emqx.appup.src format --- src/emqx.appup.src | 1 - 1 file changed, 1 deletion(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 31bdd5a58..d6d20aa0f 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,6 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, -<<<<<<< HEAD [{"4.4.5", [ {load_module,emqx_relup,brutal_purge,soft_purge,[]}, From 0ba6ab7feb5c45fc0180ea11983255b6a0bbf60f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 29 Jul 2022 14:59:58 +0800 Subject: [PATCH 363/363] chore: bump to 4.4.6 --- deploy/charts/emqx/Chart.yaml | 4 ++-- include/emqx_release.hrl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 3f24cc699..1440b3666 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -13,8 +13,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 4.3.0 +version: 4.4.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: latest +appVersion: 4.4.6 diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 6c2904ce9..5d66c1299 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.6-beta.4"}). +-define(EMQX_RELEASE, {opensource, "4.4.6"}). -else.