fix(alarm): fix the issue that memory watermark were not set correctly
This commit is contained in:
zhouzb
tigercl
parent
1ba4743213
commit
aaf29ee164
|
|
@ -323,8 +323,8 @@ normalize_message(high_process_memory_usage, #{high_watermark := HighWatermark})
|
|||
list_to_binary(io_lib:format("Process memory usage is higher than ~p%", [HighWatermark]));
|
||||
normalize_message(high_cpu_usage, #{usage := Usage}) ->
|
||||
list_to_binary(io_lib:format("~p% cpu usage", [Usage]));
|
||||
normalize_message(too_many_processes, #{high_watermark := HighWatermark}) ->
|
||||
list_to_binary(io_lib:format("High watermark: ~p%", [HighWatermark]));
|
||||
normalize_message(too_many_processes, #{usage := Usage}) ->
|
||||
list_to_binary(io_lib:format("~p% process usage", [Usage]));
|
||||
normalize_message(partition, #{occurred := Node}) ->
|
||||
list_to_binary(io_lib:format("Partition occurs at node ~s", [Node]));
|
||||
normalize_message(_Name, _UnknownDetails) ->
|
||||
|
|
|
|||
|
|
@ -79,20 +79,22 @@ set_cpu_low_watermark(Float) ->
|
|||
get_mem_check_interval() ->
|
||||
memsup:get_check_interval() div 1000.
|
||||
|
||||
set_mem_check_interval(Seconds) ->
|
||||
set_mem_check_interval(Seconds) when Seconds < 60 ->
|
||||
memsup:set_check_interval(1);
|
||||
set_mem_check_interval(Seconds) ->
|
||||
memsup:set_check_interval(Seconds div 60).
|
||||
|
||||
get_sysmem_high_watermark() ->
|
||||
memsup:get_sysmem_high_watermark().
|
||||
|
||||
set_sysmem_high_watermark(Float) ->
|
||||
memsup:set_sysmem_high_watermark(Float).
|
||||
memsup:set_sysmem_high_watermark(Float / 100).
|
||||
|
||||
get_procmem_high_watermark() ->
|
||||
memsup:get_procmem_high_watermark().
|
||||
|
||||
set_procmem_high_watermark(Float) ->
|
||||
memsup:set_procmem_high_watermark(Float).
|
||||
memsup:set_procmem_high_watermark(Float / 100).
|
||||
|
||||
call(Req) ->
|
||||
gen_server:call(?OS_MON, Req, infinity).
|
||||
|
|
@ -102,6 +104,9 @@ call(Req) ->
|
|||
%%--------------------------------------------------------------------
|
||||
|
||||
init([Opts]) ->
|
||||
set_mem_check_interval(proplists:get_value(mem_check_interval, Opts)),
|
||||
set_sysmem_high_watermark(proplists:get_value(sysmem_high_watermark, Opts)),
|
||||
set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts)),
|
||||
{ok, ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts),
|
||||
cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts),
|
||||
cpu_check_interval => proplists:get_value(cpu_check_interval, Opts),
|
||||
|
|
|
|||
|
|
@ -25,14 +25,6 @@
|
|||
|
||||
all() -> emqx_ct:all(?MODULE).
|
||||
|
||||
init_per_suite(Config) ->
|
||||
emqx_ct_helpers:boot_modules(all),
|
||||
emqx_ct_helpers:start_apps([]),
|
||||
Config.
|
||||
|
||||
end_per_suite(_Config) ->
|
||||
emqx_ct_helpers:stop_apps([]).
|
||||
|
||||
init_per_testcase(t_size_limit, Config) ->
|
||||
emqx_ct_helpers:boot_modules(all),
|
||||
emqx_ct_helpers:start_apps([],
|
||||
|
|
@ -107,9 +99,6 @@ t_size_limit(_) ->
|
|||
t_validity_period(_) ->
|
||||
ok = emqx_alarm:activate(a),
|
||||
ok = emqx_alarm:deactivate(a),
|
||||
dbg:tracer(),
|
||||
dbg:p(all, c),
|
||||
dbg:tpl(emqx_alarm, delete_expired_deactivated_alarms, cx),
|
||||
?assertNotEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))),
|
||||
ct:sleep(2000),
|
||||
?assertEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))).
|
||||
|
|
|
|||
|
|
@ -70,30 +70,3 @@ t_api(_) ->
|
|||
gen_server:stop(emqx_os_mon),
|
||||
ok.
|
||||
|
||||
t_timeout(_) ->
|
||||
ok = meck:new(emqx_vm),
|
||||
|
||||
ok = meck:expect(emqx_vm, cpu_util, fun() -> 0 end),
|
||||
{ok, _} = emqx_os_mon:start_link([{cpu_check_interval, 1}]),
|
||||
timer:sleep(1500),
|
||||
gen_server:stop(emqx_os_mon),
|
||||
|
||||
ok = meck:expect(emqx_vm, cpu_util, fun() -> {error, test_case} end),
|
||||
{ok, _} = emqx_os_mon:start_link([{cpu_check_interval, 1}]),
|
||||
timer:sleep(1500),
|
||||
gen_server:stop(emqx_os_mon),
|
||||
|
||||
ok = meck:expect(emqx_vm, cpu_util, fun() -> 90 end),
|
||||
{ok, _} = emqx_os_mon:start_link([{cpu_check_interval, 1},
|
||||
{cpu_high_watermark, 0.80},
|
||||
{cpu_low_watermark, 0.60}]),
|
||||
timer:sleep(1500),
|
||||
|
||||
emqx_os_mon:set_cpu_high_watermark(1.00),
|
||||
timer:sleep(1500),
|
||||
|
||||
emqx_os_mon:set_cpu_low_watermark(0.95),
|
||||
timer:sleep(1500),
|
||||
|
||||
gen_server:stop(emqx_os_mon),
|
||||
ok = meck:unload(emqx_vm).
|
||||
|
|
|
|||
|
|
@ -21,64 +21,45 @@
|
|||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
-define(WAIT(PATTERN, TIMEOUT),
|
||||
receive
|
||||
PATTERN ->
|
||||
ok
|
||||
after
|
||||
TIMEOUT ->
|
||||
error(timeout)
|
||||
end).
|
||||
|
||||
all() -> emqx_ct:all(?MODULE).
|
||||
|
||||
init_per_suite(Config) ->
|
||||
application:ensure_all_started(sasl),
|
||||
init_per_testcase(t_api, Config) ->
|
||||
emqx_ct_helpers:boot_modules(all),
|
||||
emqx_ct_helpers:start_apps([],
|
||||
fun(emqx) ->
|
||||
application:set_env(emqx, vm_mon, [{check_interval, 1},
|
||||
{process_high_watermark, 80},
|
||||
{process_low_watermark, 75}]),
|
||||
ok;
|
||||
(_) ->
|
||||
ok
|
||||
end),
|
||||
Config;
|
||||
init_per_testcase(_, Config) ->
|
||||
emqx_ct_helpers:boot_modules(all),
|
||||
emqx_ct_helpers:start_apps([]),
|
||||
Config.
|
||||
|
||||
end_per_suite(_Config) ->
|
||||
application:stop(sasl).
|
||||
|
||||
t_api(_) ->
|
||||
meck:new(alarm_handler, [passthrough, no_history]),
|
||||
Tester = self(),
|
||||
Ref = make_ref(),
|
||||
try
|
||||
meck:expect(alarm_handler, set_alarm,
|
||||
fun(What) ->
|
||||
Res = meck:passthrough([What]),
|
||||
Tester ! {Ref, set_alarm, What},
|
||||
Res
|
||||
end),
|
||||
meck:expect(alarm_handler, clear_alarm,
|
||||
fun(What) ->
|
||||
Res = meck:passthrough([What]),
|
||||
Tester ! {Ref, clear_alarm, What},
|
||||
Res
|
||||
end),
|
||||
gen_event:swap_handler(alarm_handler, {emqx_alarm_handler, swap}, {alarm_handler, []}),
|
||||
{ok, _} = emqx_vm_mon:start_link([{check_interval, 1},
|
||||
{process_high_watermark, 80},
|
||||
{process_low_watermark, 75}]),
|
||||
timer:sleep(emqx_vm_mon:get_check_interval() * 1000),
|
||||
emqx_vm_mon:set_process_high_watermark(0),
|
||||
emqx_vm_mon:set_process_low_watermark(60),
|
||||
?assertEqual(0, emqx_vm_mon:get_process_high_watermark()),
|
||||
?assertEqual(60, emqx_vm_mon:get_process_low_watermark()),
|
||||
?WAIT({Ref, set_alarm, {too_many_processes, _Count}}, 2000),
|
||||
?assertEqual(true, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
|
||||
emqx_vm_mon:set_process_high_watermark(80),
|
||||
emqx_vm_mon:set_process_low_watermark(75),
|
||||
?assertEqual(80, emqx_vm_mon:get_process_high_watermark()),
|
||||
?assertEqual(75, emqx_vm_mon:get_process_low_watermark()),
|
||||
?WAIT({Ref, clear_alarm, too_many_processes}, 3000),
|
||||
?assertEqual(false, lists:keymember(too_many_processes, 1, alarm_handler:get_alarms())),
|
||||
emqx_vm_mon:set_check_interval(20),
|
||||
?assertEqual(20, emqx_vm_mon:get_check_interval()),
|
||||
?assertEqual(ignored, gen_server:call(emqx_vm_mon, ignored)),
|
||||
?assertEqual(ok, gen_server:cast(emqx_vm_mon, ignored)),
|
||||
?assertEqual(ignored, emqx_vm_mon ! ignored)
|
||||
after
|
||||
meck:unload(alarm_handler)
|
||||
end.
|
||||
end_per_testcase(_, _Config) ->
|
||||
emqx_ct_helpers:stop_apps([]).
|
||||
|
||||
t_api(_) ->
|
||||
?assertEqual(1, emqx_vm_mon:get_check_interval()),
|
||||
?assertEqual(80, emqx_vm_mon:get_process_high_watermark()),
|
||||
?assertEqual(75, emqx_vm_mon:get_process_low_watermark()),
|
||||
emqx_vm_mon:set_process_high_watermark(0),
|
||||
emqx_vm_mon:set_process_low_watermark(60),
|
||||
?assertEqual(0, emqx_vm_mon:get_process_high_watermark()),
|
||||
?assertEqual(60, emqx_vm_mon:get_process_low_watermark()),
|
||||
timer:sleep(emqx_vm_mon:get_check_interval() * 1000 * 2),
|
||||
?assert(is_existing(too_many_processes, emqx_alarm:get_alarms(activated))),
|
||||
emqx_vm_mon:set_process_high_watermark(70),
|
||||
timer:sleep(emqx_vm_mon:get_check_interval() * 1000 * 2),
|
||||
?assertNot(is_existing(too_many_processes, emqx_alarm:get_alarms(activated))).
|
||||
|
||||
is_existing(Name, [#{name := Name} | _More]) ->
|
||||
true;
|
||||
is_existing(Name, [_Alarm | More]) ->
|
||||
is_existing(Name, More);
|
||||
is_existing(_Name, []) ->
|
||||
false.
|
||||
|
|
|
|||
Loading…
Reference in New Issue