fix: monitor bad mnesia write & rpc call (#6060)

This commit is contained in:
DDDHuang 2021-11-15 15:28:19 +08:00 committed by GitHub
parent 01f29ab052
commit 1476accd63
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 32 additions and 18 deletions

View File

@ -33,6 +33,8 @@
extra = [] :: term() %% not used so far, fur future extension extra = [] :: term() %% not used so far, fur future extension
}). }).
-define(TAB_COLLECT, emqx_collect).
-define(EMPTY_KEY(Key), ((Key == undefined) orelse (Key == <<>>))). -define(EMPTY_KEY(Key), ((Key == undefined) orelse (Key == <<>>))).
-define(DASHBOARD_SHARD, emqx_dashboard_shard). -define(DASHBOARD_SHARD, emqx_dashboard_shard).

View File

@ -40,10 +40,10 @@
-define(EXPIRE_INTERVAL, 86400000 * 7). -define(EXPIRE_INTERVAL, 86400000 * 7).
mnesia(boot) -> mnesia(boot) ->
ok = mria:create_table(emqx_collect, [ ok = mria:create_table(?TAB_COLLECT, [
{type, set}, {type, set},
{local_content, true}, {local_content, true},
{storage, disc_only_copies}, {storage, disc_copies},
{record_name, mqtt_collect}, {record_name, mqtt_collect},
{attributes, record_info(fields, mqtt_collect)}]). {attributes, record_info(fields, mqtt_collect)}]).
@ -87,7 +87,10 @@ handle_call(_Req, _From, State) ->
handle_cast(_Req, State) -> handle_cast(_Req, State) ->
{noreply, State}. {noreply, State}.
handle_info(collect, State = #{collect := Collect, count := 1, temp_collect := TempCollect, last_collects := LastCollect}) -> handle_info(collect, State = #{ collect := Collect
, count := 1
, temp_collect := TempCollect
, last_collects := LastCollect}) ->
timer(next_interval(), collect), timer(next_interval(), collect),
NewLastCollect = flush(collect_all(Collect), LastCollect), NewLastCollect = flush(collect_all(Collect), LastCollect),
TempCollect1 = temp_collect(TempCollect), TempCollect1 = temp_collect(TempCollect),
@ -107,9 +110,9 @@ handle_info(clear_expire_data, State = #{expire_interval := ExpireInterval}) ->
timer(?CLEAR_INTERVAL, clear_expire_data), timer(?CLEAR_INTERVAL, clear_expire_data),
T1 = get_local_time(), T1 = get_local_time(),
Spec = ets:fun2ms(fun({_, T, _C} = Data) when (T1 - T) > ExpireInterval -> Data end), Spec = ets:fun2ms(fun({_, T, _C} = Data) when (T1 - T) > ExpireInterval -> Data end),
Collects = dets:select(emqx_collect, Spec), Collects = ets:select(?TAB_COLLECT, Spec),
lists:foreach(fun(Collect) -> lists:foreach(fun(Collect) ->
dets:delete_object(emqx_collect, Collect) true = ets:delete_object(?TAB_COLLECT, Collect)
end, Collects), end, Collects),
{noreply, State, hibernate}; {noreply, State, hibernate};
@ -131,9 +134,9 @@ temp_collect({_, _, Received, Sent}) ->
Sent1}. Sent1}.
collect_all({Connection, Route, Subscription}) -> collect_all({Connection, Route, Subscription}) ->
{[collect(connections)| Connection], {[collect(connections) | Connection],
[collect(routes)| Route], [collect(routes) | Route],
[collect(subscriptions)| Subscription]}. [collect(subscriptions) | Subscription]}.
collect(connections) -> collect(connections) ->
emqx_stats:getstat('connections.count'); emqx_stats:getstat('connections.count');
@ -159,8 +162,11 @@ flush({Connection, Route, Subscription}, {Received0, Sent0, Dropped0}) ->
diff(Sent, Sent0), diff(Sent, Sent0),
diff(Dropped, Dropped0)}, diff(Dropped, Dropped0)},
Ts = get_local_time(), Ts = get_local_time(),
_ = mria:transaction(mria:local_content_shard(), {atomic, ok} = mria:transaction(mria:local_content_shard(),
fun mnesia:write/1, [#mqtt_collect{timestamp = Ts, collect = Collect}]), fun mnesia:write/3,
[ ?TAB_COLLECT
, #mqtt_collect{timestamp = Ts, collect = Collect}
, write]),
{Received, Sent, Dropped}. {Received, Sent, Dropped}.
avg(Items) -> avg(Items) ->

View File

@ -18,6 +18,10 @@
, current_counters/2 , current_counters/2
]). ]).
-export([ sampling/1
, sampling/2
]).
-define(COUNTERS, [ connection -define(COUNTERS, [ connection
, route , route
, subscriptions , subscriptions
@ -174,8 +178,10 @@ format_current_metrics(Collects) ->
format_current_metrics(Collects, {0,0,0,0}). format_current_metrics(Collects, {0,0,0,0}).
format_current_metrics([], Acc) -> format_current_metrics([], Acc) ->
Acc; Acc;
format_current_metrics([{Received, Sent, Sub, Conn} | Collects], {Received1, Sent1, Sub1, Conn1}) -> format_current_metrics([{Received, Sent, Sub, Conn} | Collects],
format_current_metrics(Collects, {Received1 + Received, Sent1 + Sent, Sub1 + Sub, Conn1 + Conn}). {Received1, Sent1, Sub1, Conn1}) ->
format_current_metrics(Collects,
{Received1 + Received, Sent1 + Sent, Sub1 + Sub, Conn1 + Conn}).
%%%============================================================================================== %%%==============================================================================================
@ -260,19 +266,19 @@ key_replace([Term | List], All, Comparison, Default) ->
end. end.
sampling(Node) when Node =:= node() -> sampling(Node) when Node =:= node() ->
Time = emqx_dashboard_collection:get_local_time() - 7200000, format(lists:sort(select_data()));
All = dets:select(emqx_collect, [{{mqtt_collect,'$1','$2'}, [{'>', '$1', Time}], ['$_']}]),
format(lists:sort(All));
sampling(Node) -> sampling(Node) ->
rpc:call(Node, ?MODULE, sampling, [Node]). rpc:call(Node, ?MODULE, sampling, [Node]).
sampling(Node, Counter) when Node =:= node() -> sampling(Node, Counter) when Node =:= node() ->
Time = emqx_dashboard_collection:get_local_time() - 7200000, format_single(lists:sort(select_data()), Counter);
All = dets:select(emqx_collect, [{{mqtt_collect,'$1','$2'}, [{'>', '$1', Time}], ['$_']}]),
format_single(lists:sort(All), Counter);
sampling(Node, Counter) -> sampling(Node, Counter) ->
rpc:call(Node, ?MODULE, sampling, [Node, Counter]). rpc:call(Node, ?MODULE, sampling, [Node, Counter]).
select_data() ->
Time = emqx_dashboard_collection:get_local_time() - 7200000,
ets:select(?TAB_COLLECT, [{{mqtt_collect,'$1','$2'}, [{'>', '$1', Time}], ['$_']}]).
format(Collects) -> format(Collects) ->
format(Collects, {[],[],[],[],[],[]}). format(Collects, {[],[],[],[],[],[]}).
format([], {Connection, Route, Subscription, Received, Sent, Dropped}) -> format([], {Connection, Route, Subscription, Received, Sent, Dropped}) ->