yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #6867 from zhongwencool/sys-mem-alarm-4.2.11 

fix: sys_mem alarm is not triggered after reboot.
This commit is contained in:
zhongwencool 2022-01-26 16:39:53 +08:00 committed by GitHub
parent 8cdfb531a7 8bb9ad7206
commit ec40bac20f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 86 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/emqx.appup.src
View File

@ -25,6 +25,7 @@
{load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module, emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_misc,brutal_purge,soft_purge,[]}
]},
{"4.2.1", [
@ -49,6 +50,7 @@
{load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module, emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_misc,brutal_purge,soft_purge,[]}
]},
{<<"4.2.[23]">>, [
@ -71,6 +73,7 @@
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module, emqx_app, brutal_purge, soft_purge, []},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
]},
{<<"4.2.4">>, [
@ -80,6 +83,7 @@
{load_module, emqx_congestion, brutal_purge, soft_purge, []},
{update, emqx_connection, {advanced, []}},
{load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
{load_module, emqx_os_mon, brutal_purge, soft_purge, []},
{load_module, emqx_alarm, brutal_purge, soft_purge, []},
{load_module, emqx_router_sup, soft_purge, soft_purge, []},
{load_module, emqx_broker, soft_purge, soft_purge, []},
@ -91,6 +95,7 @@
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module, emqx_app, brutal_purge, soft_purge, []},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
]},
{<<"4.2.5">>, [
@ -100,6 +105,7 @@
{load_module, emqx_congestion, brutal_purge, soft_purge, []},
{update, emqx_connection, {advanced, []}},
{load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
{load_module, emqx_os_mon, brutal_purge, soft_purge, []},
{load_module, emqx_alarm, brutal_purge, soft_purge, []},
{load_module, emqx_router_sup, soft_purge, soft_purge, []},
{load_module, emqx_broker, soft_purge, soft_purge, []},
@ -111,6 +117,7 @@
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module, emqx_app, brutal_purge, soft_purge, []},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
]},
{<<"4.2.[6-7]">>, [
@ -124,6 +131,9 @@
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module, emqx_app, brutal_purge, soft_purge, []},
{load_module, emqx_alarm, brutal_purge, soft_purge, []},
{load_module, emqx_os_mon, brutal_purge, soft_purge, []},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
]},
{<<"4.2.8">>, [
@ -132,13 +142,24 @@
{load_module, emqx_connection, brutal_purge, soft_purge, []},
{load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
{load_module, emqx_cm, brutal_purge, soft_purge, []},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
{load_module, emqx_limiter, brutal_purge, soft_purge, []},
{load_module, emqx_app, brutal_purge, soft_purge, []},
{load_module, emqx_alarm, brutal_purge, soft_purge, []},
{load_module, emqx_os_mon, brutal_purge, soft_purge, []}
]},
{<<"4.2.9">>, [
{load_module, emqx_frame, brutal_purge, soft_purge, []},
{load_module, emqx_connection, brutal_purge, soft_purge, []},
{load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
{load_module, emqx_limiter, brutal_purge, soft_purge, []},
{load_module, emqx_app, brutal_purge, soft_purge, []},
{load_module, emqx_alarm, brutal_purge, soft_purge, []},
{load_module, emqx_os_mon, brutal_purge, soft_purge, []}
]},
{<<"4.2.10">>, [
{load_module,emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_os_mon, brutal_purge, soft_purge, []},
{load_module,emqx_alarm, brutal_purge, soft_purge, []}
]},
{<<".*">>, []}
],
@ -166,6 +187,7 @@
{load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module,emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_misc,brutal_purge,soft_purge,[]}
]},
{"4.2.1", [
@ -190,6 +212,7 @@
{load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module,emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_misc,brutal_purge,soft_purge,[]}
]},
{<<"4.2.[23]">>, [
@ -212,6 +235,7 @@
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module,emqx_app, brutal_purge, soft_purge, []},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
]},
{<<"4.2.4">>, [
@ -231,6 +255,8 @@
{load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module,emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_os_mon, brutal_purge, soft_purge, []},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
]},
@ -251,6 +277,8 @@
{load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module,emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_os_mon, brutal_purge, soft_purge, []},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
]},
@ -264,6 +292,9 @@
{load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
{load_module,emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_os_mon, brutal_purge, soft_purge, []},
{load_module,emqx_alarm, brutal_purge, soft_purge, []},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
]},
@ -273,13 +304,24 @@
{load_module, emqx_connection, brutal_purge, soft_purge, []},
{load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
{load_module, emqx_cm, brutal_purge, soft_purge, []},
{load_module,emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_os_mon, brutal_purge, soft_purge, []},
{load_module,emqx_alarm, brutal_purge, soft_purge, []},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
]},
{<<"4.2.9">>, [
{load_module, emqx_frame, brutal_purge, soft_purge, []},
{load_module, emqx_connection, brutal_purge, soft_purge, []},
{load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
{load_module, emqx_limiter, brutal_purge, soft_purge, []}
{load_module, emqx_limiter, brutal_purge, soft_purge, []},
{load_module,emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_os_mon, brutal_purge, soft_purge, []},
{load_module,emqx_alarm, brutal_purge, soft_purge, []}
]},
{<<"4.2.10">>, [
{load_module,emqx_app, brutal_purge, soft_purge, []},
{load_module,emqx_os_mon, brutal_purge, soft_purge, []},
{load_module,emqx_alarm, brutal_purge, soft_purge, []}
]},
{<<".*">>, []}
]

3
src/emqx_alarm.erl
View File

@ -165,6 +165,8 @@ init([Opts]) ->
Actions = proplists:get_value(actions, Opts),
SizeLimit = proplists:get_value(size_limit, Opts),
ValidityPeriod = timer:seconds(proplists:get_value(validity_period, Opts)),
emqx_alarm_handler:load(),
process_flag(trap_exit, true),
{ok, ensure_delete_timer(#state{actions = Actions,
size_limit = SizeLimit,
validity_period = ValidityPeriod})}.
@ -228,6 +230,7 @@ handle_info(Info, State) ->
{noreply, State}.
terminate(_Reason, _State) ->
emqx_alarm_handler:unload(),
ok.
code_change(_OldVsn, State, _Extra) ->

3
src/emqx_app.erl
View File

@ -38,13 +38,11 @@ start(_Type, _Args) ->
ok = emqx_plugins:init(),
emqx_plugins:load(),
register(emqx, self()),
emqx_alarm_handler:load(),
print_vsn(),
{ok, Sup}.
-spec(stop(State :: term()) -> term()).
stop(_State) ->
emqx_alarm_handler:unload(),
emqx_boot:is_enabled(listeners)
andalso emqx_listeners:stop().
@ -68,4 +66,3 @@ start_autocluster() ->
ekka:callback(prepare, fun emqx:shutdown/1),
ekka:callback(reboot, fun emqx:reboot/0),
ekka:autocluster(?APP).

21
src/emqx_os_mon.erl
View File

@ -105,8 +105,10 @@ call(Req) ->
init([Opts]) ->
set_mem_check_interval(proplists:get_value(mem_check_interval, Opts)),
set_sysmem_high_watermark(proplists:get_value(sysmem_high_watermark, Opts)),
HW = proplists:get_value(sysmem_high_watermark, Opts),
set_sysmem_high_watermark(HW),
set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts)),
ensure_system_memory_alarm(HW),
{ok, ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts),
cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts),
cpu_check_interval => proplists:get_value(cpu_check_interval, Opts),
@ -177,3 +179,20 @@ ensure_check_timer(State = #{cpu_check_interval := Interval}) ->
"x86_64-pc-linux-musl" -> State;
_ -> State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)}
end.
%% At startup, memsup starts first and checks for memory alarms,
%% but emqx_alarm_handler is not yet used instead of alarm_handler,
%% so alarm_handler is used directly for notification (normally emqx_alarm_handler should be used).
%%The internal memsup will no longer trigger events that have been alerted,
%% and there is no exported function to remove the alerted flag,
%% so it can only be checked again at startup.
ensure_system_memory_alarm(HW) ->
case erlang:whereis(memsup) of
undefined -> ok;
_Pid ->
{Allocated, Total, _Worst} = memsup:get_memory_data(),
case Total =/= 0 andalso Allocated/Total * 100 >= HW of
true -> emqx_alarm:activate(high_system_memory_usage, #{high_watermark => HW});
false -> ok
end
end.

21
test/emqx_os_mon_SUITE.erl
View File

@ -24,10 +24,23 @@
all() -> emqx_ct:all(?MODULE).
init_per_suite(Config) ->
emqx_ct_helpers:boot_modules(all),
emqx_ct_helpers:start_apps([],
fun(emqx) ->
application:set_env(emqx, os_mon, [
{cpu_check_interval, 1},
{cpu_high_watermark, 5},
{cpu_low_watermark, 80},
{mem_check_interval, 60},
{sysmem_high_watermark, 70},
{procmem_high_watermark, 5}]);
(_) -> ok
end),
application:ensure_all_started(os_mon),
Config.
end_per_suite(_Config) ->
emqx_ct_helpers:stop_apps([]),
application:stop(os_mon).
% t_set_mem_check_interval(_) ->
@ -40,13 +53,6 @@ end_per_suite(_Config) ->
% error('TODO').
t_api(_) ->
gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}),
{ok, _} = emqx_os_mon:start_link([{cpu_check_interval, 1},
{cpu_high_watermark, 5},
{cpu_low_watermark, 80},
{mem_check_interval, 60},
{sysmem_high_watermark, 70},
{procmem_high_watermark, 5}]),
?assertEqual(1, emqx_os_mon:get_cpu_check_interval()),
?assertEqual(5, emqx_os_mon:get_cpu_high_watermark()),
?assertEqual(80, emqx_os_mon:get_cpu_low_watermark()),
@ -69,4 +75,3 @@ t_api(_) ->
emqx_os_mon ! ignored,
gen_server:stop(emqx_os_mon),
ok.