feat(logger): update configs for logger at runtime

apps/emqx/src/emqx_alarm.erl

@@ -28,7 +28,7 @@
 -boot_mnesia({mnesia, [boot]}).
 -copy_mnesia({mnesia, [copy]}).
 
--export([post_config_update/3]).
+-export([post_config_update/4]).
 
 -export([ start_link/0
         , stop/0
@@ -148,7 +148,7 @@ get_alarms(activated) ->
 get_alarms(deactivated) ->
     gen_server:call(?MODULE, {get_alarms, deactivated}).
 
-post_config_update(_, #{validity_period := Period0}, _OldConf) ->
+post_config_update(_, #{validity_period := Period0}, _OldConf, _AppEnv) ->
     ?MODULE ! {update_timer, Period0},
     ok.
 
@@ -179,7 +179,7 @@ format(_) ->
 
 init([]) ->
     deactivate_all_alarms(),
-    emqx_config_handler:add_handler([alarm], ?MODULE),
+    ok = emqx_config_handler:add_handler([alarm], ?MODULE),
     {ok, #state{timer = ensure_timer(undefined, get_validity_period())}}.
 
 %% suppress dialyzer warning due to dirty read/write race condition.
@@ -255,6 +255,7 @@ handle_info(Info, State) ->
     {noreply, State}.
 
 terminate(_Reason, _State) ->
+    ok = emqx_config_handler:remove_handler([alarm]),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->

apps/emqx/src/emqx_config.erl

@@ -80,7 +80,7 @@
         error:badarg -> EXP_ON_FAIL
     end).
 
--export_type([update_request/0, raw_config/0, config/0,
+-export_type([update_request/0, raw_config/0, config/0, app_envs/0,
               update_opts/0, update_cmd/0, update_args/0,
               update_error/0, update_result/0]).
 

apps/emqx/src/emqx_config_handler.erl

@@ -24,6 +24,7 @@
 %% API functions
 -export([ start_link/0
         , add_handler/2
+        , remove_handler/1
         , update_config/3
         , merge_to_old_config/2
         ]).
@@ -49,14 +50,15 @@
 -type handlers() :: #{emqx_config:config_key() => handlers(), ?MOD => handler_name()}.
 
 -optional_callbacks([ pre_config_update/2
-                    , post_config_update/3
+                    , post_config_update/4
                     ]).
 
 -callback pre_config_update(emqx_config:update_request(), emqx_config:raw_config()) ->
     {ok, emqx_config:update_request()} | {error, term()}.
 
 -callback post_config_update(emqx_config:update_request(), emqx_config:config(),
-    emqx_config:config()) -> ok | {ok, Result::any()} | {error, Reason::term()}.
+    emqx_config:config(), emqx_config:app_envs()) ->
+        ok | {ok, Result::any()} | {error, Reason::term()}.
 
 -type state() :: #{
     handlers := handlers(),
@@ -76,6 +78,10 @@ update_config(SchemaModule, ConfKeyPath, UpdateArgs) ->
 add_handler(ConfKeyPath, HandlerName) ->
     gen_server:call(?MODULE, {add_child, ConfKeyPath, HandlerName}).
 
+-spec remove_handler(emqx_config:config_key_path()) -> ok.
+remove_handler(ConfKeyPath) ->
+    gen_server:call(?MODULE, {remove_child, ConfKeyPath}).
+
 %%============================================================================
 
 -spec init(term()) -> {ok, state()}.
@@ -87,6 +93,11 @@ handle_call({add_child, ConfKeyPath, HandlerName}, _From,
     {reply, ok, State#{handlers =>
         emqx_map_lib:deep_put(ConfKeyPath, Handlers, #{?MOD => HandlerName})}};
 
+handle_call({remove_child, ConfKeyPath}, _From,
+            State = #{handlers := Handlers}) ->
+    {reply, ok, State#{handlers =>
+        emqx_map_lib:deep_remove(ConfKeyPath, Handlers)}};
+
 handle_call({change_config, SchemaModule, ConfKeyPath, UpdateArgs}, _From,
             #{handlers := Handlers} = State) ->
     Reply = try
@@ -152,7 +163,7 @@ check_and_save_configs(SchemaModule, ConfKeyPath, Handlers, NewRawConf, Override
     FullRawConf = with_full_raw_confs(NewRawConf),
     {AppEnvs, CheckedConf} = emqx_config:check_config(SchemaModule, FullRawConf),
     NewConf = maps:with(maps:keys(OldConf), CheckedConf),
-    case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, UpdateArgs, #{}) of
+    case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of
         {ok, Result0} ->
             case save_configs(ConfKeyPath, AppEnvs, NewConf, NewRawConf, OverrideConf,
                     UpdateArgs) of
@@ -163,16 +174,18 @@ check_and_save_configs(SchemaModule, ConfKeyPath, Handlers, NewRawConf, Override
         Error -> Error
     end.
 
-do_post_config_update([], Handlers, OldConf, NewConf, UpdateArgs, Result) ->
+do_post_config_update([], Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, Result) ->
-    call_post_config_update(Handlers, OldConf, NewConf, up_req(UpdateArgs), Result);
+    call_post_config_update(Handlers, OldConf, NewConf, AppEnvs, up_req(UpdateArgs), Result);
-do_post_config_update([ConfKey | ConfKeyPath], Handlers, OldConf, NewConf, UpdateArgs, Result) ->
+do_post_config_update([ConfKey | ConfKeyPath], Handlers, OldConf, NewConf, AppEnvs, UpdateArgs,
+        Result) ->
     SubOldConf = get_sub_config(ConfKey, OldConf),
     SubNewConf = get_sub_config(ConfKey, NewConf),
     SubHandlers = maps:get(ConfKey, Handlers, #{}),
-    case do_post_config_update(ConfKeyPath, SubHandlers, SubOldConf, SubNewConf, UpdateArgs,
+    case do_post_config_update(ConfKeyPath, SubHandlers, SubOldConf, SubNewConf, AppEnvs,
-            Result) of
+            UpdateArgs, Result) of
         {ok, Result1} ->
-            call_post_config_update(Handlers, OldConf, NewConf, up_req(UpdateArgs), Result1);
+            call_post_config_update(Handlers, OldConf, NewConf, AppEnvs, up_req(UpdateArgs),
+                Result1);
         Error -> Error
     end.
 
@@ -192,11 +205,11 @@ call_pre_config_update(Handlers, OldRawConf, UpdateReq) ->
         false -> merge_to_old_config(UpdateReq, OldRawConf)
     end.
 
-call_post_config_update(Handlers, OldConf, NewConf, UpdateReq, Result) ->
+call_post_config_update(Handlers, OldConf, NewConf, AppEnvs, UpdateReq, Result) ->
     HandlerName = maps:get(?MOD, Handlers, undefined),
-    case erlang:function_exported(HandlerName, post_config_update, 3) of
+    case erlang:function_exported(HandlerName, post_config_update, 4) of
         true ->
-            case HandlerName:post_config_update(UpdateReq, NewConf, OldConf) of
+            case HandlerName:post_config_update(UpdateReq, NewConf, OldConf, AppEnvs) of
                 ok -> {ok, Result};
                 {ok, Result1} -> {ok, Result#{HandlerName => Result1}};
                 {error, Reason} -> {error, {post_config_update, HandlerName, Reason}}

apps/emqx/src/emqx_kernel_sup.erl

@@ -34,6 +34,7 @@ init([]) ->
           , child_spec(emqx_stats, worker)
           , child_spec(emqx_metrics, worker)
           , child_spec(emqx_ctl, worker)
+          , child_spec(emqx_logger, worker)
           ]}}.
 
 child_spec(M, Type) ->

apps/emqx/src/emqx_logger.erl

@@ -18,6 +18,19 @@
 
 -compile({no_auto_import, [error/1]}).
 
+-behaviour(gen_server).
+-behaviour(emqx_config_handler).
+
+%% gen_server callbacks
+-export([ start_link/0
+        , init/1
+        , handle_call/3
+        , handle_cast/2
+        , handle_info/2
+        , terminate/2
+        , code_change/3
+        ]).
+
 %% Logs
 -export([ debug/1
         , debug/2
@@ -47,6 +60,7 @@
         ]).
 
 -export([ get_primary_log_level/0
+        , tune_primary_log_level/0
         , get_log_handlers/0
         , get_log_handlers/1
         , get_log_handler/1
@@ -56,6 +70,8 @@
         , stop_log_handler/1
         ]).
 
+-export([post_config_update/4]).
+
 -type(peername_str() :: list()).
 -type(logger_dst() :: file:filename() | console | unknown).
 -type(logger_handler_info() :: #{
@@ -66,6 +82,54 @@
       }).
 
 -define(stopped_handlers, {?MODULE, stopped_handlers}).
+-define(CONF_PATH, [log]).
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+%%--------------------------------------------------------------------
+%% gen_server callbacks
+%%--------------------------------------------------------------------
+init([]) ->
+    ok = emqx_config_handler:add_handler(?CONF_PATH, ?MODULE),
+    {ok, #{}}.
+
+handle_call({update_config, AppEnvs}, _From, State) ->
+    io:format("----new appenvs: ~p~n", [AppEnvs]),
+    OldEnvs = application:get_env(kernel, logger, []),
+    NewEnvs = proplists:get_value(logger, proplists:get_value(kernel, AppEnvs, []), []),
+    io:format("----new logger configs: ~p~n", [NewEnvs]),
+    ok = application:set_env(kernel, logger, NewEnvs),
+    io:format("----1~n", []),
+    _ = [logger:remove_handler(HandlerId) || {handler, HandlerId, _Mod, _Conf} <- OldEnvs],
+    io:format("----2~n", []),
+    _ = [logger:add_handler(HandlerId, Mod, Conf) || {handler, HandlerId, Mod, Conf} <- NewEnvs],
+    io:format("----3~n", []),
+    ok = tune_primary_log_level(),
+    {reply, ok, State};
+
+handle_call(_Req, _From, State) ->
+    {reply, ignored, State}.
+
+handle_cast(_Msg, State) ->
+    {noreply, State}.
+
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok = emqx_config_handler:remove_handler(?CONF_PATH),
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+
+%%--------------------------------------------------------------------
+%% emqx_config_handler callbacks
+%%--------------------------------------------------------------------
+post_config_update(_Req, _NewConf, _OldConf, AppEnvs) ->
+    gen_server:call(?MODULE, {update_config, AppEnvs}, 5000).
 
 %%--------------------------------------------------------------------
 %% APIs
@@ -159,6 +223,16 @@ get_primary_log_level() ->
     #{level := Level} = logger:get_primary_config(),
     Level.
 
+-spec tune_primary_log_level() -> ok.
+tune_primary_log_level() ->
+    LowestLevel = lists:foldl(fun(#{level := Level}, OldLevel) ->
+            case logger:compare_levels(Level, OldLevel) of
+                lt -> Level;
+                _ -> OldLevel
+            end
+        end, get_primary_log_level(), get_log_handlers()),
+    set_primary_log_level(LowestLevel).
+
 -spec(set_primary_log_level(logger:level()) -> ok | {error, term()}).
 set_primary_log_level(Level) ->
     logger:set_primary_config(level, Level).

apps/emqx/src/emqx_map_lib.erl

@@ -62,12 +62,12 @@ deep_find(_KeyPath, Data) ->
     {not_found, _KeyPath, Data}.
 
 -spec deep_put(config_key_path(), map(), term()) -> map().
-deep_put([], Map, Config) when is_map(Map) ->
+deep_put([], Map, Data) when is_map(Map) ->
-    Config;
+    Data;
-deep_put([], _Map, Config) -> %% not map, replace it
+deep_put([], _Map, Data) -> %% not map, replace it
-    Config;
+    Data;
-deep_put([Key | KeyPath], Map, Config) ->
+deep_put([Key | KeyPath], Map, Data) ->
-    SubMap = deep_put(KeyPath, maps:get(Key, Map, #{}), Config),
+    SubMap = deep_put(KeyPath, maps:get(Key, Map, #{}), Data),
     Map#{Key => SubMap}.
 
 -spec deep_remove(config_key_path(), map()) -> map().

apps/emqx_authn/src/emqx_authn.erl

@@ -24,7 +24,7 @@
 -include_lib("emqx/include/logger.hrl").
 
 -export([ pre_config_update/2
-        , post_config_update/3
+        , post_config_update/4
         , update_config/2
         ]).
 
@@ -137,11 +137,11 @@ pre_config_update({move_authenticator, ID, Position}, OldConfig) ->
             end
     end.
 
-post_config_update({enable, true}, _NewConfig, _OldConfig) ->
+post_config_update({enable, true}, _NewConfig, _OldConfig, _AppEnvs) ->
     emqx_authn:enable();
-post_config_update({enable, false}, _NewConfig, _OldConfig) ->
+post_config_update({enable, false}, _NewConfig, _OldConfig, _AppEnvs) ->
     emqx_authn:disable();
-post_config_update({create_authenticator, #{<<"name">> := Name}}, NewConfig, _OldConfig) ->
+post_config_update({create_authenticator, #{<<"name">> := Name}}, NewConfig, _OldConfig, _AppEnvs) ->
     case lists:filter(
              fun(#{name := N}) ->
                  N =:= Name
@@ -151,12 +151,12 @@ post_config_update({create_authenticator, #{<<"name">> := Name}}, NewConfig, _Ol
         [_Config | _] ->
             {error, name_has_be_used}
     end;
-post_config_update({delete_authenticator, ID}, _NewConfig, _OldConfig) ->
+post_config_update({delete_authenticator, ID}, _NewConfig, _OldConfig, _AppEnvs) ->
     case delete_authenticator(?CHAIN, ID) of
         ok -> ok;
         {error, Reason} -> throw(Reason)
     end;
-post_config_update({update_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig) ->
+post_config_update({update_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig, _AppEnvs) ->
     case lists:filter(
              fun(#{name := N}) ->
                  N =:= Name
@@ -166,7 +166,7 @@ post_config_update({update_authenticator, ID, #{<<"name">> := Name}}, NewConfig,
         [_Config | _] ->
             {error, name_has_be_used}
     end;
-post_config_update({update_or_create_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig) ->
+post_config_update({update_or_create_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig, _AppEnvs) ->
     case lists:filter(
              fun(#{name := N}) ->
                  N =:= Name
@@ -176,7 +176,7 @@ post_config_update({update_or_create_authenticator, ID, #{<<"name">> := Name}},
         [_Config | _] ->
             {error, name_has_be_used}
     end;
-post_config_update({move_authenticator, ID, Position}, _NewConfig, _OldConfig) ->
+post_config_update({move_authenticator, ID, Position}, _NewConfig, _OldConfig, _AppEnvs) ->
     NPosition = case Position of
                     <<"top">> -> top;
                     <<"bottom">> -> bottom;

apps/emqx_authz/src/emqx_authz.erl

@@ -34,7 +34,7 @@
         , authorize/5
         ]).
 
--export([post_config_update/3, pre_config_update/2]).
+-export([post_config_update/4, pre_config_update/2]).
 
 -define(CONF_KEY_PATH, [authorization_rules, rules]).
 
@@ -107,23 +107,23 @@ pre_config_update({_, Rules}, _Conf) when is_list(Rules)->
     %% overwrite the entire config!
     {ok, Rules}.
 
-post_config_update(_, undefined, _Conf) ->
+post_config_update(_, undefined, _Conf, _AppEnvs) ->
     ok;
-post_config_update({move, Id, <<"top">>}, _NewRules, _OldRules) ->
+post_config_update({move, Id, <<"top">>}, _NewRules, _OldRules, _AppEnvs) ->
     InitedRules = lookup(),
     {Index, Rule} = find_rule_by_id(Id, InitedRules),
     {Rules1, Rules2 } = lists:split(Index, InitedRules),
     Rules3 = [Rule] ++ lists:droplast(Rules1) ++ Rules2,
     ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Rules3]}, -1),
     ok = emqx_authz_cache:drain_cache();
-post_config_update({move, Id, <<"bottom">>}, _NewRules, _OldRules) ->
+post_config_update({move, Id, <<"bottom">>}, _NewRules, _OldRules, _AppEnvs) ->
     InitedRules = lookup(),
     {Index, Rule} = find_rule_by_id(Id, InitedRules),
     {Rules1, Rules2 } = lists:split(Index, InitedRules),
     Rules3 = lists:droplast(Rules1) ++ Rules2 ++ [Rule],
     ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Rules3]}, -1),
     ok = emqx_authz_cache:drain_cache();
-post_config_update({move, Id, #{<<"before">> := BeforeId}}, _NewRules, _OldRules) ->
+post_config_update({move, Id, #{<<"before">> := BeforeId}}, _NewRules, _OldRules, _AppEnvs) ->
     InitedRules = lookup(),
     {_, Rule0} = find_rule_by_id(Id, InitedRules),
     {Index, Rule1} = find_rule_by_id(BeforeId, InitedRules),
@@ -134,7 +134,7 @@ post_config_update({move, Id, #{<<"before">> := BeforeId}}, _NewRules, _OldRules
     ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Rules3]}, -1),
     ok = emqx_authz_cache:drain_cache();
 
-post_config_update({move, Id, #{<<"after">> := AfterId}}, _NewRules, _OldRules) ->
+post_config_update({move, Id, #{<<"after">> := AfterId}}, _NewRules, _OldRules, _AppEnvs) ->
     InitedRules = lookup(),
     {_, Rule} = find_rule_by_id(Id, InitedRules),
     {Index, _} = find_rule_by_id(AfterId, InitedRules),
@@ -145,17 +145,17 @@ post_config_update({move, Id, #{<<"after">> := AfterId}}, _NewRules, _OldRules)
     ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Rules3]}, -1),
     ok = emqx_authz_cache:drain_cache();
 
-post_config_update({head, Rules}, _NewRules, _OldConf) ->
+post_config_update({head, Rules}, _NewRules, _OldConf, _AppEnvs) ->
     InitedRules = [init_provider(R) || R <- check_rules(Rules)],
     ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [InitedRules ++ lookup()]}, -1),
     ok = emqx_authz_cache:drain_cache();
 
-post_config_update({tail, Rules}, _NewRules, _OldConf) ->
+post_config_update({tail, Rules}, _NewRules, _OldConf, _AppEnvs) ->
     InitedRules = [init_provider(R) || R <- check_rules(Rules)],
     emqx_hooks:put('client.authorize', {?MODULE, authorize, [lookup() ++ InitedRules]}, -1),
     ok = emqx_authz_cache:drain_cache();
 
-post_config_update({{replace_once, Id}, Rule}, _NewRules, _OldConf) when is_map(Rule) ->
+post_config_update({{replace_once, Id}, Rule}, _NewRules, _OldConf, _AppEnvs) when is_map(Rule) ->
     OldInitedRules = lookup(),
     {Index, OldRule} = find_rule_by_id(Id, OldInitedRules),
     case maps:get(type, OldRule, undefined) of
@@ -169,7 +169,7 @@ post_config_update({{replace_once, Id}, Rule}, _NewRules, _OldConf) when is_map(
     ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [lists:droplast(OldRules1) ++ InitedRules ++ OldRules2]}, -1),
     ok = emqx_authz_cache:drain_cache();
 
-post_config_update(_, NewRules, _OldConf) ->
+post_config_update(_, NewRules, _OldConf, _AppEnvs) ->
     %% overwrite the entire config!
     OldInitedRules = lookup(),
     InitedRules = [init_provider(Rule) || Rule <- NewRules],

apps/emqx_management/src/emqx_mgmt_api_configs.erl

@@ -46,7 +46,13 @@
 
 -define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))).
 
--define(CORE_CONFS, [node, log, alarm, zones, cluster, rpc, broker, sysmon,
+-define(CORE_CONFS, [
+    %% from emqx_machine_schema
+    log, rpc,
+    %% from emqx_schema
+    zones, mqtt, flapping_detect, force_shutdown, force_gc, conn_congestion, rate_limit, quota,
+    broker, alarm, sysmon,
+    %% from other apps
     emqx_dashboard, emqx_management]).
 
 api_spec() ->
@@ -109,9 +115,9 @@ config(get, _Params, Req) ->
             {404, #{code => 'NOT_FOUND', message => <<"Config cannot found">>}}
     end;
 
-config(put, _Params, Req) ->
+config(put, #{body := Body}, Req) ->
     Path = conf_path(Req),
-    {ok, #{raw_config := RawConf}} = emqx:update_config(Path, http_body(Req),
+    {ok, #{raw_config := RawConf}} = emqx:update_config(Path, Body,
         #{rawconf_with_defaults => true}),
     {200, emqx_map_lib:jsonable_map(RawConf)}.
 
@@ -142,12 +148,6 @@ conf_path_reset(Req) ->
     <<"/api/v5", ?PREFIX_RESET, Path/binary>> = cowboy_req:path(Req),
     string:lexemes(Path, "/ ").
 
-http_body(Req) ->
-    {ok, Body, _} = cowboy_req:read_body(Req),
-    try jsx:decode(Body, [{return_maps, true}])
-    catch error:badarg -> Body
-    end.
-
 get_conf_schema(Conf, MaxDepth) ->
     get_conf_schema([], maps:to_list(Conf), [], MaxDepth).