feat: add sysmon config handler to reload conf
This commit is contained in:
parent
9c094a8611
commit
116c664df1
|
@ -556,10 +556,12 @@ save_to_override_conf(RawConf, Opts) ->
|
||||||
|
|
||||||
add_handlers() ->
|
add_handlers() ->
|
||||||
ok = emqx_config_logger:add_handler(),
|
ok = emqx_config_logger:add_handler(),
|
||||||
|
emqx_sys_mon:add_handler(),
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
remove_handlers() ->
|
remove_handlers() ->
|
||||||
ok = emqx_config_logger:remove_handler(),
|
ok = emqx_config_logger:remove_handler(),
|
||||||
|
emqx_sys_mon:remove_handler(),
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
load_hocon_file(FileName, LoadType) ->
|
load_hocon_file(FileName, LoadType) ->
|
||||||
|
|
|
@ -35,6 +35,8 @@
|
||||||
current_sysmem_percent/0
|
current_sysmem_percent/0
|
||||||
]).
|
]).
|
||||||
|
|
||||||
|
-export([update/1]).
|
||||||
|
|
||||||
%% gen_server callbacks
|
%% gen_server callbacks
|
||||||
-export([
|
-export([
|
||||||
init/1,
|
init/1,
|
||||||
|
@ -52,6 +54,9 @@
|
||||||
start_link() ->
|
start_link() ->
|
||||||
gen_server:start_link({local, ?OS_MON}, ?MODULE, [], []).
|
gen_server:start_link({local, ?OS_MON}, ?MODULE, [], []).
|
||||||
|
|
||||||
|
update(OS) ->
|
||||||
|
erlang:send(?MODULE, {monitor_conf_update, OS}).
|
||||||
|
|
||||||
%%--------------------------------------------------------------------
|
%%--------------------------------------------------------------------
|
||||||
%% API
|
%% API
|
||||||
%%--------------------------------------------------------------------
|
%%--------------------------------------------------------------------
|
||||||
|
@ -87,18 +92,24 @@ current_sysmem_percent() ->
|
||||||
init([]) ->
|
init([]) ->
|
||||||
%% memsup is not reliable, ignore
|
%% memsup is not reliable, ignore
|
||||||
memsup:set_sysmem_high_watermark(1.0),
|
memsup:set_sysmem_high_watermark(1.0),
|
||||||
|
SysHW = init_os_monitor(),
|
||||||
|
_ = start_mem_check_timer(),
|
||||||
|
_ = start_cpu_check_timer(),
|
||||||
|
{ok, #{sysmem_high_watermark => SysHW}}.
|
||||||
|
|
||||||
|
init_os_monitor() ->
|
||||||
|
init_os_monitor(emqx:get_config([sysmon, os])).
|
||||||
|
|
||||||
|
init_os_monitor(OS) ->
|
||||||
#{
|
#{
|
||||||
sysmem_high_watermark := SysHW,
|
sysmem_high_watermark := SysHW,
|
||||||
procmem_high_watermark := PHW,
|
procmem_high_watermark := PHW,
|
||||||
mem_check_interval := MCI
|
mem_check_interval := MCI
|
||||||
} = emqx:get_config([sysmon, os]),
|
} = OS,
|
||||||
|
|
||||||
set_procmem_high_watermark(PHW),
|
set_procmem_high_watermark(PHW),
|
||||||
set_mem_check_interval(MCI),
|
set_mem_check_interval(MCI),
|
||||||
ok = update_mem_alarm_status(SysHW),
|
ok = update_mem_alarm_status(SysHW),
|
||||||
_ = start_mem_check_timer(),
|
SysHW.
|
||||||
_ = start_cpu_check_timer(),
|
|
||||||
{ok, #{sysmem_high_watermark => SysHW}}.
|
|
||||||
|
|
||||||
handle_call(get_sysmem_high_watermark, _From, #{sysmem_high_watermark := HWM} = State) ->
|
handle_call(get_sysmem_high_watermark, _From, #{sysmem_high_watermark := HWM} = State) ->
|
||||||
{reply, HWM, State};
|
{reply, HWM, State};
|
||||||
|
@ -147,6 +158,9 @@ handle_info({timeout, _Timer, cpu_check}, State) ->
|
||||||
end,
|
end,
|
||||||
ok = start_cpu_check_timer(),
|
ok = start_cpu_check_timer(),
|
||||||
{noreply, State};
|
{noreply, State};
|
||||||
|
handle_info({monitor_conf_update, OS}, _State) ->
|
||||||
|
SysHW = init_os_monitor(OS),
|
||||||
|
{noreply, #{sysmem_high_watermark => SysHW}};
|
||||||
handle_info(Info, State) ->
|
handle_info(Info, State) ->
|
||||||
?SLOG(error, #{msg => "unexpected_info", info => Info}),
|
?SLOG(error, #{msg => "unexpected_info", info => Info}),
|
||||||
{noreply, State}.
|
{noreply, State}.
|
||||||
|
|
|
@ -35,32 +35,52 @@
|
||||||
terminate/2,
|
terminate/2,
|
||||||
code_change/3
|
code_change/3
|
||||||
]).
|
]).
|
||||||
|
-export([add_handler/0, remove_handler/0, post_config_update/5]).
|
||||||
|
-export([update/1]).
|
||||||
|
|
||||||
-define(SYSMON, ?MODULE).
|
-define(SYSMON, ?MODULE).
|
||||||
|
-define(SYSMON_CONF_ROOT, [sysmon]).
|
||||||
|
|
||||||
%% @doc Start the system monitor.
|
%% @doc Start the system monitor.
|
||||||
-spec start_link() -> startlink_ret().
|
-spec start_link() -> startlink_ret().
|
||||||
start_link() ->
|
start_link() ->
|
||||||
gen_server:start_link({local, ?SYSMON}, ?MODULE, [], []).
|
gen_server:start_link({local, ?SYSMON}, ?MODULE, [], []).
|
||||||
|
|
||||||
|
add_handler() ->
|
||||||
|
ok = emqx_config_handler:add_handler(?SYSMON_CONF_ROOT, ?MODULE),
|
||||||
|
ok.
|
||||||
|
|
||||||
|
remove_handler() ->
|
||||||
|
ok = emqx_config_handler:remove_handler(?SYSMON_CONF_ROOT),
|
||||||
|
ok.
|
||||||
|
|
||||||
|
post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) ->
|
||||||
|
#{os := OS1, vm := VM1} = OldConf,
|
||||||
|
#{os := OS2, vm := VM2} = NewConf,
|
||||||
|
VM1 =/= VM2 andalso ?MODULE:update(VM2),
|
||||||
|
OS1 =/= OS2 andalso emqx_os_mon:update(OS2),
|
||||||
|
ok.
|
||||||
|
|
||||||
|
update(VM) ->
|
||||||
|
erlang:send(?MODULE, {monitor_conf_update, VM}).
|
||||||
|
|
||||||
%%--------------------------------------------------------------------
|
%%--------------------------------------------------------------------
|
||||||
%% gen_server callbacks
|
%% gen_server callbacks
|
||||||
%%--------------------------------------------------------------------
|
%%--------------------------------------------------------------------
|
||||||
|
|
||||||
init([]) ->
|
init([]) ->
|
||||||
_ = erlang:system_monitor(self(), sysm_opts()),
|
|
||||||
emqx_logger:set_proc_metadata(#{sysmon => true}),
|
emqx_logger:set_proc_metadata(#{sysmon => true}),
|
||||||
|
init_system_monitor(),
|
||||||
|
|
||||||
%% Monitor cluster partition event
|
%% Monitor cluster partition event
|
||||||
ekka:monitor(partition, fun handle_partition_event/1),
|
ekka:monitor(partition, fun handle_partition_event/1),
|
||||||
|
|
||||||
{ok, start_timer(#{timer => undefined, events => []})}.
|
{ok, start_timer(#{timer => undefined, events => []})}.
|
||||||
|
|
||||||
start_timer(State) ->
|
start_timer(State) ->
|
||||||
State#{timer := emqx_misc:start_timer(timer:seconds(2), reset)}.
|
State#{timer := emqx_misc:start_timer(timer:seconds(2), reset)}.
|
||||||
|
|
||||||
sysm_opts() ->
|
sysm_opts(VM) ->
|
||||||
sysm_opts(maps:to_list(emqx:get_config([sysmon, vm])), []).
|
sysm_opts(maps:to_list(VM), []).
|
||||||
sysm_opts([], Acc) ->
|
sysm_opts([], Acc) ->
|
||||||
Acc;
|
Acc;
|
||||||
sysm_opts([{_, disabled} | Opts], Acc) ->
|
sysm_opts([{_, disabled} | Opts], Acc) ->
|
||||||
|
@ -176,12 +196,16 @@ handle_info({monitor, SusPid, busy_dist_port, Port}, State) ->
|
||||||
);
|
);
|
||||||
handle_info({timeout, _Ref, reset}, State) ->
|
handle_info({timeout, _Ref, reset}, State) ->
|
||||||
{noreply, State#{events := []}, hibernate};
|
{noreply, State#{events := []}, hibernate};
|
||||||
|
handle_info({monitor_conf_update, VM}, State) ->
|
||||||
|
init_system_monitor(VM),
|
||||||
|
{noreply, State#{events := []}, hibernate};
|
||||||
handle_info(Info, State) ->
|
handle_info(Info, State) ->
|
||||||
?SLOG(error, #{msg => "unexpected_info", info => Info}),
|
?SLOG(error, #{msg => "unexpected_info", info => Info}),
|
||||||
{noreply, State}.
|
{noreply, State}.
|
||||||
|
|
||||||
terminate(_Reason, #{timer := TRef}) ->
|
terminate(_Reason, #{timer := TRef}) ->
|
||||||
emqx_misc:cancel_timer(TRef).
|
emqx_misc:cancel_timer(TRef),
|
||||||
|
ok.
|
||||||
|
|
||||||
code_change(_OldVsn, State, _Extra) ->
|
code_change(_OldVsn, State, _Extra) ->
|
||||||
{ok, State}.
|
{ok, State}.
|
||||||
|
@ -237,3 +261,11 @@ safe_publish(Event, WarnMsg) ->
|
||||||
sysmon_msg(Topic, Payload) ->
|
sysmon_msg(Topic, Payload) ->
|
||||||
Msg = emqx_message:make(?SYSMON, Topic, Payload),
|
Msg = emqx_message:make(?SYSMON, Topic, Payload),
|
||||||
emqx_message:set_flag(sys, Msg).
|
emqx_message:set_flag(sys, Msg).
|
||||||
|
|
||||||
|
init_system_monitor() ->
|
||||||
|
VM = emqx:get_config([sysmon, vm]),
|
||||||
|
init_system_monitor(VM).
|
||||||
|
|
||||||
|
init_system_monitor(VM) ->
|
||||||
|
_ = erlang:system_monitor(self(), sysm_opts(VM)),
|
||||||
|
ok.
|
||||||
|
|
|
@ -86,7 +86,7 @@ handle_info({timeout, _Timer, check}, State) ->
|
||||||
},
|
},
|
||||||
Message
|
Message
|
||||||
);
|
);
|
||||||
_Precent ->
|
_Percent ->
|
||||||
ok
|
ok
|
||||||
end,
|
end,
|
||||||
_ = start_check_timer(),
|
_ = start_check_timer(),
|
||||||
|
|
|
@ -50,6 +50,7 @@ t_update(_Config) ->
|
||||||
{ok, SysMon1} = get_config(<<"sysmon">>),
|
{ok, SysMon1} = get_config(<<"sysmon">>),
|
||||||
#{<<"vm">> := #{<<"busy_port">> := BusyPort1}} = SysMon1,
|
#{<<"vm">> := #{<<"busy_port">> := BusyPort1}} = SysMon1,
|
||||||
?assertEqual(BusyPort, not BusyPort1),
|
?assertEqual(BusyPort, not BusyPort1),
|
||||||
|
assert_busy_port(BusyPort1),
|
||||||
|
|
||||||
%% update failed
|
%% update failed
|
||||||
ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"),
|
ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"),
|
||||||
|
@ -64,6 +65,7 @@ t_update(_Config) ->
|
||||||
ok = reset_config(<<"sysmon">>, "conf_path=vm.busy_port"),
|
ok = reset_config(<<"sysmon">>, "conf_path=vm.busy_port"),
|
||||||
{ok, SysMon3} = get_config(<<"sysmon">>),
|
{ok, SysMon3} = get_config(<<"sysmon">>),
|
||||||
?assertMatch(#{<<"vm">> := #{<<"busy_port">> := true}}, SysMon3),
|
?assertMatch(#{<<"vm">> := #{<<"busy_port">> := true}}, SysMon3),
|
||||||
|
assert_busy_port(true),
|
||||||
|
|
||||||
%% reset no_default_value config
|
%% reset no_default_value config
|
||||||
NewSysMon1 = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false),
|
NewSysMon1 = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false),
|
||||||
|
@ -73,6 +75,11 @@ t_update(_Config) ->
|
||||||
?assertMatch(#{<<"vm">> := #{<<"busy_port">> := false}}, SysMon4),
|
?assertMatch(#{<<"vm">> := #{<<"busy_port">> := false}}, SysMon4),
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
|
assert_busy_port(BusyPort) ->
|
||||||
|
{_Pid, Monitors} = erlang:system_monitor(),
|
||||||
|
RealBusyPort = proplists:get_value(busy_port, Monitors, false),
|
||||||
|
?assertEqual(BusyPort, RealBusyPort).
|
||||||
|
|
||||||
t_log(_Config) ->
|
t_log(_Config) ->
|
||||||
{ok, Log} = get_config("log"),
|
{ok, Log} = get_config("log"),
|
||||||
File = "log/emqx-test.log",
|
File = "log/emqx-test.log",
|
||||||
|
|
Loading…
Reference in New Issue