fix: monitor crash by ets badarg

This commit is contained in:
Zhongwen Deng 2022-04-24 11:13:33 +08:00
parent 2a25767112
commit b7a6bd973f
1 changed files with 47 additions and 35 deletions

View File

@ -26,21 +26,23 @@
-export([start_link/0]).
-export([ init/1
, handle_call/3
, handle_cast/2
, handle_info/2
, terminate/2
, code_change/3
-export([
init/1,
handle_call/3,
handle_cast/2,
handle_info/2,
terminate/2,
code_change/3
]).
-export([mnesia/1]).
-export([ samplers/0
, samplers/2
, current_rate/0
, current_rate/1
, granularity_adapter/1
-export([
samplers/0,
samplers/2,
current_rate/0,
current_rate/1,
granularity_adapter/1
]).
%% for rpc
@ -68,7 +70,8 @@ mnesia(boot) ->
{local_content, true},
{storage, disc_copies},
{record_name, emqx_monit},
{attributes, record_info(fields, emqx_monit)}]).
{attributes, record_info(fields, emqx_monit)}
]).
%% -------------------------------------------------------------------------------------------------
%% API
@ -133,11 +136,15 @@ current_rate(Node) when Node == node() ->
try
{ok, Rate} = do_call(current_rate),
{ok, Rate}
catch _E:R ->
catch
_E:R ->
?SLOG(warning, #{msg => "Dashboard monitor error", reason => R}),
%% Rate map 0, ensure api will not crash.
%% When joining cluster, dashboard monitor restart.
Rate0 = [{Key, 0} || Key <- ?GAUGE_SAMPLER_LIST ++ maps:values(?DELTA_SAMPLER_RATE_MAP)],
Rate0 = [
{Key, 0}
|| Key <- ?GAUGE_SAMPLER_LIST ++ maps:values(?DELTA_SAMPLER_RATE_MAP)
],
{ok, maps:from_list(Rate0)}
end;
current_rate(Node) ->
@ -175,12 +182,10 @@ handle_info({sample, Time}, State = #state{last = Last}) ->
{atomic, ok} = flush(Last, Now),
sample_timer(),
{noreply, State#state{last = Now}};
handle_info(clean_expired, State) ->
clean(),
clean_timer(),
{noreply, State};
handle_info(_Info, State = #state{}) ->
{noreply, State}.
@ -256,8 +261,10 @@ format(TimeStamp, Data, All) ->
cal_rate(_Now, undefined) ->
AllSamples = ?GAUGE_SAMPLER_LIST ++ maps:values(?DELTA_SAMPLER_RATE_MAP),
lists:foldl(fun(Key, Acc) -> Acc#{Key => 0} end, #{}, AllSamples);
cal_rate( #emqx_monit{data = NowData, time = NowTime}
, #emqx_monit{data = LastData, time = LastTime}) ->
cal_rate(
#emqx_monit{data = NowData, time = NowTime},
#emqx_monit{data = LastData, time = LastTime}
) ->
TimeDelta = NowTime - LastTime,
Filter = fun(Key, _) -> lists:member(Key, ?GAUGE_SAMPLER_LIST) end,
Gauge = maps:filter(Filter, NowData),
@ -340,9 +347,12 @@ clean() ->
Now = erlang:system_time(millisecond),
ExpiredMS = [{{'_', '$1', '_'}, [{'>', {'-', Now, '$1'}, ?RETENTION_TIME}], ['$_']}],
Expired = ets:select(?TAB, ExpiredMS),
lists:foreach(fun(Data) ->
lists:foreach(
fun(Data) ->
true = ets:delete_object(?TAB, Data)
end, Expired),
end,
Expired
),
ok.
%% To make it easier to do data aggregation
@ -364,8 +374,10 @@ count_map(M1, M2) ->
getstats(Key) ->
%% Stats ets maybe not exist when ekka join.
try stats(Key)
catch _: _ -> 0
try
stats(Key)
catch
_:_ -> 0
end.
stats(connections) -> emqx_stats:getstat('connections.count');