Merge pull request #6867 from zhongwencool/sys-mem-alarm-4.2.11

fix: sys_mem alarm is not triggered after reboot.

src/emqx.appup.src

@@ -25,6 +25,7 @@
      {load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
+     {load_module, emqx_app, brutal_purge, soft_purge, []},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]}
    ]},
    {"4.2.1", [
@@ -49,6 +50,7 @@
      {load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
+     {load_module, emqx_app, brutal_purge, soft_purge, []},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]}
    ]},
    {<<"4.2.[23]">>, [
@@ -71,6 +73,7 @@
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module, emqx_app, brutal_purge, soft_purge, []},
      {load_module, emqx_limiter, brutal_purge, soft_purge, []}
    ]},
    {<<"4.2.4">>, [
@@ -80,6 +83,7 @@
      {load_module, emqx_congestion, brutal_purge, soft_purge, []},
      {update, emqx_connection, {advanced, []}},
      {load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
+     {load_module, emqx_os_mon, brutal_purge, soft_purge, []},
      {load_module, emqx_alarm, brutal_purge, soft_purge, []},
      {load_module, emqx_router_sup, soft_purge, soft_purge, []},
      {load_module, emqx_broker, soft_purge, soft_purge, []},
@@ -91,6 +95,7 @@
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module, emqx_app, brutal_purge, soft_purge, []},
      {load_module, emqx_limiter, brutal_purge, soft_purge, []}
    ]},
    {<<"4.2.5">>, [
@@ -100,6 +105,7 @@
      {load_module, emqx_congestion, brutal_purge, soft_purge, []},
      {update, emqx_connection, {advanced, []}},
      {load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
+     {load_module, emqx_os_mon, brutal_purge, soft_purge, []},
      {load_module, emqx_alarm, brutal_purge, soft_purge, []},
      {load_module, emqx_router_sup, soft_purge, soft_purge, []},
      {load_module, emqx_broker, soft_purge, soft_purge, []},
@@ -111,6 +117,7 @@
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module, emqx_app, brutal_purge, soft_purge, []},
      {load_module, emqx_limiter, brutal_purge, soft_purge, []}
    ]},
    {<<"4.2.[6-7]">>, [
@@ -124,6 +131,9 @@
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module, emqx_app, brutal_purge, soft_purge, []},
+     {load_module, emqx_alarm, brutal_purge, soft_purge, []},
+     {load_module, emqx_os_mon, brutal_purge, soft_purge, []},
      {load_module, emqx_limiter, brutal_purge, soft_purge, []}
    ]},
    {<<"4.2.8">>, [
@@ -132,13 +142,24 @@
      {load_module, emqx_connection, brutal_purge, soft_purge, []},
      {load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
      {load_module, emqx_cm, brutal_purge, soft_purge, []},
-     {load_module, emqx_limiter, brutal_purge, soft_purge, []}
+     {load_module, emqx_limiter, brutal_purge, soft_purge, []},
+     {load_module, emqx_app, brutal_purge, soft_purge, []},
+     {load_module, emqx_alarm, brutal_purge, soft_purge, []},
+     {load_module, emqx_os_mon, brutal_purge, soft_purge, []}
    ]},
    {<<"4.2.9">>, [
      {load_module, emqx_frame, brutal_purge, soft_purge, []},
      {load_module, emqx_connection, brutal_purge, soft_purge, []},
      {load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
-     {load_module, emqx_limiter, brutal_purge, soft_purge, []}
+     {load_module, emqx_limiter, brutal_purge, soft_purge, []},
+     {load_module, emqx_app, brutal_purge, soft_purge, []},
+     {load_module, emqx_alarm, brutal_purge, soft_purge, []},
+     {load_module, emqx_os_mon, brutal_purge, soft_purge, []}
+   ]},
+   {<<"4.2.10">>, [
+     {load_module,emqx_app, brutal_purge, soft_purge, []},
+     {load_module,emqx_os_mon, brutal_purge, soft_purge, []},
+     {load_module,emqx_alarm, brutal_purge, soft_purge, []}
    ]},
    {<<".*">>, []}
  ],
@@ -166,6 +187,7 @@
      {load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app, brutal_purge, soft_purge, []},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]}
    ]},
    {"4.2.1", [
@@ -190,6 +212,7 @@
      {load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app, brutal_purge, soft_purge, []},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]}
    ]},
    {<<"4.2.[23]">>, [
@@ -212,6 +235,7 @@
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app, brutal_purge, soft_purge, []},
      {load_module, emqx_limiter, brutal_purge, soft_purge, []}
    ]},
    {<<"4.2.4">>, [
@@ -231,6 +255,8 @@
      {load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app, brutal_purge, soft_purge, []},
+     {load_module,emqx_os_mon, brutal_purge, soft_purge, []},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module, emqx_limiter, brutal_purge, soft_purge, []}
    ]},
@@ -251,6 +277,8 @@
      {load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app, brutal_purge, soft_purge, []},
+     {load_module,emqx_os_mon, brutal_purge, soft_purge, []},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module, emqx_limiter, brutal_purge, soft_purge, []}
    ]},
@@ -264,6 +292,9 @@
      {load_module,emqx_pqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_mqueue,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app, brutal_purge, soft_purge, []},
+     {load_module,emqx_os_mon, brutal_purge, soft_purge, []},
+     {load_module,emqx_alarm, brutal_purge, soft_purge, []},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module, emqx_limiter, brutal_purge, soft_purge, []}
    ]},
@@ -273,13 +304,24 @@
      {load_module, emqx_connection, brutal_purge, soft_purge, []},
      {load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
      {load_module, emqx_cm, brutal_purge, soft_purge, []},
+     {load_module,emqx_app, brutal_purge, soft_purge, []},
+     {load_module,emqx_os_mon, brutal_purge, soft_purge, []},
+     {load_module,emqx_alarm, brutal_purge, soft_purge, []},
      {load_module, emqx_limiter, brutal_purge, soft_purge, []}
    ]},
    {<<"4.2.9">>, [
      {load_module, emqx_frame, brutal_purge, soft_purge, []},
      {load_module, emqx_connection, brutal_purge, soft_purge, []},
      {load_module, emqx_ws_connection, brutal_purge, soft_purge, []},
-     {load_module, emqx_limiter, brutal_purge, soft_purge, []}
+     {load_module, emqx_limiter, brutal_purge, soft_purge, []},
+     {load_module,emqx_app, brutal_purge, soft_purge, []},
+     {load_module,emqx_os_mon, brutal_purge, soft_purge, []},
+     {load_module,emqx_alarm, brutal_purge, soft_purge, []}
+   ]},
+   {<<"4.2.10">>, [
+     {load_module,emqx_app, brutal_purge, soft_purge, []},
+     {load_module,emqx_os_mon, brutal_purge, soft_purge, []},
+     {load_module,emqx_alarm, brutal_purge, soft_purge, []}
    ]},
    {<<".*">>, []}
  ]

src/emqx_alarm.erl

@@ -56,9 +56,9 @@
           name :: binary() | atom(),
 
           details :: map() | list(),
-    
+
           message :: binary(),
-    
+
           activate_at :: integer()
         }).
 
@@ -68,9 +68,9 @@
           name :: binary() | atom(),
 
           details :: map() | list(),
-    
+
           message :: binary(),
-    
+
           deactivate_at :: integer() | infinity
         }).
 
@@ -165,6 +165,8 @@ init([Opts]) ->
     Actions = proplists:get_value(actions, Opts),
     SizeLimit = proplists:get_value(size_limit, Opts),
     ValidityPeriod = timer:seconds(proplists:get_value(validity_period, Opts)),
+    emqx_alarm_handler:load(),
+    process_flag(trap_exit, true),
     {ok, ensure_delete_timer(#state{actions = Actions,
                                     size_limit = SizeLimit,
                                     validity_period = ValidityPeriod})}.
@@ -228,6 +230,7 @@ handle_info(Info, State) ->
     {noreply, State}.
 
 terminate(_Reason, _State) ->
+    emqx_alarm_handler:unload(),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->

src/emqx_app.erl

@@ -38,13 +38,11 @@ start(_Type, _Args) ->
     ok = emqx_plugins:init(),
     emqx_plugins:load(),
     register(emqx, self()),
-    emqx_alarm_handler:load(),
     print_vsn(),
     {ok, Sup}.
 
 -spec(stop(State :: term()) -> term()).
 stop(_State) ->
-    emqx_alarm_handler:unload(),
     emqx_boot:is_enabled(listeners)
       andalso emqx_listeners:stop().
 
@@ -68,4 +66,3 @@ start_autocluster() ->
     ekka:callback(prepare, fun emqx:shutdown/1),
     ekka:callback(reboot,  fun emqx:reboot/0),
     ekka:autocluster(?APP).
-

src/emqx_os_mon.erl

@@ -81,7 +81,7 @@ get_mem_check_interval() ->
 
 set_mem_check_interval(Seconds) when Seconds < 60 ->
     memsup:set_check_interval(1);
-set_mem_check_interval(Seconds) -> 
+set_mem_check_interval(Seconds) ->
     memsup:set_check_interval(Seconds div 60).
 
 get_sysmem_high_watermark() ->
@@ -105,8 +105,10 @@ call(Req) ->
 
 init([Opts]) ->
     set_mem_check_interval(proplists:get_value(mem_check_interval, Opts)),
-    set_sysmem_high_watermark(proplists:get_value(sysmem_high_watermark, Opts)),
+    HW = proplists:get_value(sysmem_high_watermark, Opts),
+    set_sysmem_high_watermark(HW),
     set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts)),
+    ensure_system_memory_alarm(HW),
     {ok, ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts),
                               cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts),
                               cpu_check_interval => proplists:get_value(cpu_check_interval, Opts),
@@ -177,3 +179,20 @@ ensure_check_timer(State = #{cpu_check_interval := Interval}) ->
         "x86_64-pc-linux-musl" -> State;
         _ -> State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)}
     end.
+
+%% At startup, memsup starts first and checks for memory alarms,
+%% but emqx_alarm_handler is not yet used instead of alarm_handler,
+%% so alarm_handler is used directly for notification (normally emqx_alarm_handler should be used).
+%%The internal memsup will no longer trigger events that have been alerted,
+%% and there is no exported function to remove the alerted flag,
+%% so it can only be checked again at startup.
+ensure_system_memory_alarm(HW) ->
+    case erlang:whereis(memsup) of
+        undefined -> ok;
+        _Pid ->
+            {Allocated, Total, _Worst} = memsup:get_memory_data(),
+            case Total =/= 0 andalso Allocated/Total * 100 >= HW of
+                true -> emqx_alarm:activate(high_system_memory_usage, #{high_watermark => HW});
+                false -> ok
+            end
+    end.

test/emqx_os_mon_SUITE.erl

@@ -24,10 +24,23 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
+    emqx_ct_helpers:boot_modules(all),
+    emqx_ct_helpers:start_apps([],
+        fun(emqx) ->
+            application:set_env(emqx, os_mon, [
+                {cpu_check_interval, 1},
+                {cpu_high_watermark, 5},
+                {cpu_low_watermark, 80},
+                {mem_check_interval, 60},
+                {sysmem_high_watermark, 70},
+                {procmem_high_watermark, 5}]);
+            (_) -> ok
+        end),
     application:ensure_all_started(os_mon),
     Config.
 
 end_per_suite(_Config) ->
+    emqx_ct_helpers:stop_apps([]),
     application:stop(os_mon).
 
 % t_set_mem_check_interval(_) ->
@@ -40,13 +53,6 @@ end_per_suite(_Config) ->
 %     error('TODO').
 
 t_api(_) ->
-    gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}),
-    {ok, _} = emqx_os_mon:start_link([{cpu_check_interval, 1},
-                                      {cpu_high_watermark, 5},
-                                      {cpu_low_watermark, 80},
-                                      {mem_check_interval, 60},
-                                      {sysmem_high_watermark, 70},
-                                      {procmem_high_watermark, 5}]),
     ?assertEqual(1, emqx_os_mon:get_cpu_check_interval()),
     ?assertEqual(5, emqx_os_mon:get_cpu_high_watermark()),
     ?assertEqual(80, emqx_os_mon:get_cpu_low_watermark()),
@@ -69,4 +75,3 @@ t_api(_) ->
     emqx_os_mon ! ignored,
     gen_server:stop(emqx_os_mon),
     ok.
-