refactor(config): update the conf struct for logger

This commit is contained in:
Shawn 2021-08-20 18:46:47 +08:00
parent 31f588671d
commit 985dce786c
2 changed files with 414 additions and 258 deletions

View File

@ -312,45 +312,183 @@ cluster {
## Log
##==================================================================
log {
## The primary log level
##
## - all the log messages with levels lower than this level will
## be dropped.
## - all the log messages with levels higher than this level will
## go into the log handlers. The handlers then decide to log it
## out or drop it according to the level setting of the handler.
##
## Note: Only the messages with severity level higher than or
## equal to this level will be logged.
##
## @doc log.primary_level
## ValueType: debug | info | notice | warning | error | critical | alert | emergency
## Default: warning
primary_level = warning
##----------------------------------------------------------------
## The console log handler send log messages to emqx console
##----------------------------------------------------------------
console_handler {
## Log to single line
## @doc log.console_handler.enable
## @doc log.console_handler.<name>.enable
## ValueType: Boolean
## Default: false
console_handler.enable = false
enable = false
## The log level of this handler
## All the log messages with levels lower than this level will
## be dropped.
##
## @doc log.console_handler.level
## @doc log.console_handler.<name>.level
## ValueType: debug | info | notice | warning | error | critical | alert | emergency
## Default: warning
console_handler.level = warning
level = warning
## Timezone offset to display in logs
##
## @doc log.console_handler.<name>.time_offset
## ValueType: system | utc | String
## - "system" use system zone
## - "utc" for Universal Coordinated Time (UTC)
## - "+hh:mm" or "-hh:mm" for a specified offset
## Default: system
time_offset = system
## Limits the total number of characters printed for each log event.
##
## @doc log.console_handler.<name>.chars_limit
## ValueType: unlimited | Integer
## Range: [0, +Inf)
## Default: unlimited
chars_limit = unlimited
## Maximum depth for Erlang term log formatting
## and Erlang process message queue inspection.
##
## @doc log.console_handler.<name>.max_depth
## ValueType: unlimited | Integer
## Default: 100
max_depth = 100
## Log formatter
## @doc log.console_handler.<name>.formatter
## ValueType: text | json
## Default: text
formatter = text
## Log to single line
## @doc log.console_handler.<name>.single_line
## ValueType: Boolean
## Default: true
single_line = true
## The max allowed queue length before switching to sync mode.
##
## Log overload protection parameter. If the message queue grows
## larger than this value the handler switches from anync to sync mode.
##
## @doc log.console_handler.<name>.sync_mode_qlen
## ValueType: Integer
## Range: [0, ${log.console_handler.<name>.drop_mode_qlen}]
## Default: 100
sync_mode_qlen = 100
## The max allowed queue length before switching to drop mode.
##
## Log overload protection parameter. When the message queue grows
## larger than this threshold, the handler switches to a mode in which
## it drops all new events that senders want to log.
##
## @doc log.console_handler.<name>.drop_mode_qlen
## ValueType: Integer
## Range: [${log.console_handler.<name>.sync_mode_qlen}, ${log.console_handler.<name>.flush_qlen}]
## Default: 3000
drop_mode_qlen = 3000
## The max allowed queue length before switching to flush mode.
##
## Log overload protection parameter. If the length of the message queue
## grows larger than this threshold, a flush (delete) operation takes place.
## To flush events, the handler discards the messages in the message queue
## by receiving them in a loop without logging.
##
## @doc log.console_handler.<name>.flush_qlen
## ValueType: Integer
## Range: [${log.console_handler.<name>.drop_mode_qlen}, infinity)
## Default: 8000
flush_qlen = 8000
## Kill the log handler when it gets overloaded.
##
## Log overload protection parameter. It is possible that a handler,
## even if it can successfully manage peaks of high load without crashing,
## can build up a large message queue, or use a large amount of memory.
## We could kill the log handler in these cases and restart it after a
## few seconds.
##
## @doc log.console_handler.<name>.overload_kill.enable
## ValueType: Boolean
## Default: true
overload_kill.enable = true
## The max allowed queue length before killing the log hanlder.
##
## Log overload protection parameter. This is the maximum allowed queue
## length. If the message queue grows larger than this, the handler
## process is terminated.
##
## @doc log.console_handler.<name>.overload_kill.qlen
## ValueType: Integer
## Range: [0, 1048576]
## Default: 20000
overload_kill.qlen = 20000
## The max allowed memory size before killing the log hanlder.
##
## Log overload protection parameter. This is the maximum memory size
## that the handler process is allowed to use. If the handler grows
## larger than this, the process is terminated.
##
## @doc log.console_handler.<name>.overload_kill.mem_size
## ValueType: Size
## Default: 30MB
overload_kill.mem_size = 30MB
## Restart the log hanlder after some seconds.
##
## Log overload protection parameter. If the handler is terminated,
## it restarts automatically after a delay specified in seconds.
##
## @doc log.console_handler.<name>.overload_kill.restart_after
## ValueType: Duration
## Default: 5s
overload_kill.restart_after = 5s
## Controlling Bursts of Log Requests.
##
## Log overload protection parameter. Large bursts of log events - many
## events received by the handler under a short period of time - can
## potentially cause problems. By specifying the maximum number of events
## to be handled within a certain time frame, the handler can avoid
## choking the log with massive amounts of printouts.
##
## Note that there would be no warning if any messages were
## dropped because of burst control.
##
## @doc log.console_handler.<name>.burst_limit.enable
## ValueType: Boolean
## Default: false
burst_limit.enable = false
## This config controls the maximum number of events to handle within
## a time frame. After the limit is reached, successive events are
## dropped until the end of the time frame defined by `window_time`.
##
## @doc log.console_handler.<name>.burst_limit.max_count
## ValueType: Integer
## Default: 10000
burst_limit.max_count = 10000
## See the previous description of burst_limit_max_count.
##
## @doc log.console_handler.<name>.burst_limit.window_time
## ValueType: duration
## Default: 1s
burst_limit.window_time = 1s
}
##----------------------------------------------------------------
## The file log handlers send log messages to files
##----------------------------------------------------------------
## file_handlers.<name>
file_handlers.emqx_log {
file_handlers.default {
## The log level filter of this handler
## All the log messages with levels lower than this level will
## be dropped.
@ -402,20 +540,10 @@ log {
## ValueType: Size | infinity
## Default: 10MB
max_size = 10MB
}
## file_handlers.<name>
##
## You could also create multiple file handlers for different
## log level for example:
file_handlers.emqx_error_log {
level = error
file = "{{ platform_log_dir }}/error.log"
}
## Timezone offset to display in logs
##
## @doc log.time_offset
## @doc log.file_handlers.<name>.time_offset
## ValueType: system | utc | String
## - "system" use system zone
## - "utc" for Universal Coordinated Time (UTC)
@ -425,7 +553,7 @@ log {
## Limits the total number of characters printed for each log event.
##
## @doc log.chars_limit
## @doc log.file_handlers.<name>.chars_limit
## ValueType: unlimited | Integer
## Range: [0, +Inf)
## Default: unlimited
@ -434,19 +562,19 @@ log {
## Maximum depth for Erlang term log formatting
## and Erlang process message queue inspection.
##
## @doc log.max_depth
## @doc log.file_handlers.<name>.max_depth
## ValueType: unlimited | Integer
## Default: 80
max_depth = 80
## Default: 100
max_depth = 100
## Log formatter
## @doc log.formatter
## @doc log.file_handlers.<name>.formatter
## ValueType: text | json
## Default: text
formatter = text
## Log to single line
## @doc log.single_line
## @doc log.file_handlers.<name>.single_line
## ValueType: Boolean
## Default: true
single_line = true
@ -456,9 +584,9 @@ log {
## Log overload protection parameter. If the message queue grows
## larger than this value the handler switches from anync to sync mode.
##
## @doc log.sync_mode_qlen
## @doc log.file_handlers.<name>.sync_mode_qlen
## ValueType: Integer
## Range: [0, ${log.drop_mode_qlen}]
## Range: [0, ${log.file_handlers.<name>.drop_mode_qlen}]
## Default: 100
sync_mode_qlen = 100
@ -468,9 +596,9 @@ log {
## larger than this threshold, the handler switches to a mode in which
## it drops all new events that senders want to log.
##
## @doc log.drop_mode_qlen
## @doc log.file_handlers.<name>.drop_mode_qlen
## ValueType: Integer
## Range: [${log.sync_mode_qlen}, ${log.flush_qlen}]
## Range: [${log.file_handlers.<name>.sync_mode_qlen}, ${log.file_handlers.<name>.flush_qlen}]
## Default: 3000
drop_mode_qlen = 3000
@ -481,9 +609,9 @@ log {
## To flush events, the handler discards the messages in the message queue
## by receiving them in a loop without logging.
##
## @doc log.flush_qlen
## @doc log.file_handlers.<name>.flush_qlen
## ValueType: Integer
## Range: [${log.drop_mode_qlen}, infinity)
## Range: [${log.file_handlers.<name>.drop_mode_qlen}, infinity)
## Default: 8000
flush_qlen = 8000
@ -495,7 +623,7 @@ log {
## We could kill the log handler in these cases and restart it after a
## few seconds.
##
## @doc log.overload_kill.enable
## @doc log.file_handlers.<name>.overload_kill.enable
## ValueType: Boolean
## Default: true
overload_kill.enable = true
@ -506,7 +634,7 @@ log {
## length. If the message queue grows larger than this, the handler
## process is terminated.
##
## @doc log.overload_kill.qlen
## @doc log.file_handlers.<name>.overload_kill.qlen
## ValueType: Integer
## Range: [0, 1048576]
## Default: 20000
@ -518,7 +646,7 @@ log {
## that the handler process is allowed to use. If the handler grows
## larger than this, the process is terminated.
##
## @doc log.overload_kill.mem_size
## @doc log.file_handlers.<name>.overload_kill.mem_size
## ValueType: Size
## Default: 30MB
overload_kill.mem_size = 30MB
@ -528,7 +656,7 @@ log {
## Log overload protection parameter. If the handler is terminated,
## it restarts automatically after a delay specified in seconds.
##
## @doc log.overload_kill.restart_after
## @doc log.file_handlers.<name>.overload_kill.restart_after
## ValueType: Duration
## Default: 5s
overload_kill.restart_after = 5s
@ -544,7 +672,7 @@ log {
## Note that there would be no warning if any messages were
## dropped because of burst control.
##
## @doc log.burst_limit.enable
## @doc log.file_handlers.<name>.burst_limit.enable
## ValueType: Boolean
## Default: false
burst_limit.enable = false
@ -553,17 +681,18 @@ log {
## a time frame. After the limit is reached, successive events are
## dropped until the end of the time frame defined by `window_time`.
##
## @doc log.burst_limit.max_count
## @doc log.file_handlers.<name>.burst_limit.max_count
## ValueType: Integer
## Default: 10000
burst_limit.max_count = 10000
## See the previous description of burst_limit_max_count.
##
## @doc log.burst_limit.window_time
## @doc log.file_handlers.<name>.burst_limit.window_time
## ValueType: duration
## Default: 1s
burst_limit.window_time = 1s
}
}
##==================================================================

View File

@ -159,42 +159,25 @@ fields("rpc") ->
];
fields("log") ->
[ {"primary_level", t(log_level(), undefined, warning)}
, {"console_handler", ref("console_handler")}
[ {"console_handler", ref("console_handler")}
, {"file_handlers", ref("file_handlers")}
, {"time_offset", t(string(), undefined, "system")}
, {"chars_limit", #{type => hoconsc:union([unlimited, range(1, inf)]),
default => unlimited
}}
, {"supervisor_reports", t(union([error, progress]), undefined, error)}
, {"max_depth", t(union([unlimited, integer()]),
"kernel.error_logger_format_depth", 80)}
, {"formatter", t(union([text, json]), undefined, text)}
, {"single_line", t(boolean(), undefined, true)}
, {"sync_mode_qlen", t(integer(), undefined, 100)}
, {"drop_mode_qlen", t(integer(), undefined, 3000)}
, {"flush_qlen", t(integer(), undefined, 8000)}
, {"overload_kill", ref("log_overload_kill")}
, {"burst_limit", ref("log_burst_limit")}
, {"error_logger", t(atom(), "kernel.error_logger", silent)}
];
fields("console_handler") ->
[ {"enable", t(boolean(), undefined, false)}
, {"level", t(log_level(), undefined, warning)}
];
] ++ log_handler_common_confs();
fields("file_handlers") ->
[ {"$name", ref("log_file_handler")}
];
fields("log_file_handler") ->
[ {"level", t(log_level(), undefined, warning)}
, {"file", t(file(), undefined, undefined)}
[ {"file", t(file(), undefined, undefined)}
, {"rotation", ref("log_rotation")}
, {"max_size", #{type => union([infinity, emqx_schema:bytesize()]),
default => "10MB"}}
];
] ++ log_handler_common_confs();
fields("log_rotation") ->
[ {"enable", t(boolean(), undefined, true)}
@ -213,6 +196,7 @@ fields("log_burst_limit") ->
, {"max_count", t(integer(), undefined, 10000)}
, {"window_time", t(emqx_schema:duration(), undefined, "1s")}
];
fields(Name) ->
find_field(Name, ?MERGED_CONFIGS).
@ -259,49 +243,31 @@ tr_cluster__discovery(Conf) ->
Strategy = conf_get("cluster.discovery_strategy", Conf),
{Strategy, filter(options(Strategy, Conf))}.
tr_logger_level(Conf) -> conf_get("log.primary_level", Conf).
tr_logger_level(Conf) ->
%% TODO: use the lowest level of all the handlers
io:format(standard_error, "primary level", []),
conf_get("log.console_handler.level", Conf).
tr_logger(Conf) ->
CharsLimit = case conf_get("log.chars_limit", Conf) of
unlimited -> unlimited;
V when V > 0 -> V
end,
SingleLine = conf_get("log.single_line", Conf),
FmtName = conf_get("log.formatter", Conf),
Formatter = formatter(FmtName, CharsLimit, SingleLine),
BasicConf = #{
sync_mode_qlen => conf_get("log.sync_mode_qlen", Conf),
drop_mode_qlen => conf_get("log.drop_mode_qlen", Conf),
flush_qlen => conf_get("log.flush_qlen", Conf),
overload_kill_enable => conf_get("log.overload_kill.enable", Conf),
overload_kill_qlen => conf_get("log.overload_kill.qlen", Conf),
overload_kill_mem_size => conf_get("log.overload_kill.mem_size", Conf),
overload_kill_restart_after => conf_get("log.overload_kill.restart_after", Conf),
burst_limit_enable => conf_get("log.burst_limit.enable", Conf),
burst_limit_max_count => conf_get("log.burst_limit.max_count", Conf),
burst_limit_window_time => conf_get("log.burst_limit.window_time", Conf)
},
Filters = case conf_get("log.supervisor_reports", Conf) of
error -> [{drop_progress_reports, {fun logger_filters:progress/2, stop}}];
progress -> []
end,
%% For the default logger that outputs to console
ConsoleHandler =
case conf_get("log.console_handler.enable", Conf) of
true ->
ConsoleConf = conf_get("log.console_handler", Conf),
[{handler, console, logger_std_h, #{
level => conf_get("log.console_handler.level", Conf),
config => BasicConf#{type => standard_io},
formatter => Formatter,
filters => Filters
config => (log_handler_conf(ConsoleConf)) #{type => standard_io},
formatter => log_formatter(ConsoleConf),
filters => log_filter(ConsoleConf)
}}];
false -> []
end,
%% For the file logger
FileHandlers =
[{handler, binary_to_atom(HandlerName, latin1), logger_disk_log_h, #{
[begin
{handler, binary_to_atom(HandlerName, latin1), logger_disk_log_h, #{
level => conf_get("level", SubConf),
config => BasicConf#{
config => (log_handler_conf(SubConf)) #{
type => case conf_get("rotation.enable", SubConf) of
true -> wrap;
_ -> halt
@ -310,21 +276,74 @@ tr_logger(Conf) ->
max_no_files => conf_get("rotation.count", SubConf),
max_no_bytes => conf_get("max_size", SubConf)
},
formatter => Formatter,
filters => Filters,
formatter => log_formatter(SubConf),
filters => log_filter(SubConf),
filesync_repeat_interval => no_repeat
}}
|| {HandlerName, SubConf} <- maps:to_list(conf_get("log.file_handlers", Conf, #{}))],
end || {HandlerName, SubConf} <- maps:to_list(conf_get("log.file_handlers", Conf, #{}))],
[{handler, default, undefined}] ++ ConsoleHandler ++ FileHandlers.
log_handler_common_confs() ->
[ {"level", t(log_level(), undefined, warning)}
, {"time_offset", t(string(), undefined, "system")}
, {"chars_limit", #{type => hoconsc:union([unlimited, range(1, inf)]),
default => unlimited
}}
, {"formatter", t(union([text, json]), undefined, text)}
, {"single_line", t(boolean(), undefined, true)}
, {"sync_mode_qlen", t(integer(), undefined, 100)}
, {"drop_mode_qlen", t(integer(), undefined, 3000)}
, {"flush_qlen", t(integer(), undefined, 8000)}
, {"overload_kill", ref("log_overload_kill")}
, {"burst_limit", ref("log_burst_limit")}
, {"supervisor_reports", t(union([error, progress]), undefined, error)}
, {"max_depth", t(union([unlimited, integer()]), undefined, 100)}
].
log_handler_conf(Conf) ->
SycModeQlen = conf_get("sync_mode_qlen", Conf),
DropModeQlen = conf_get("drop_mode_qlen", Conf),
FlushQlen = conf_get("flush_qlen", Conf),
Overkill = conf_get("overload_kill", Conf),
BurstLimit = conf_get("burst_limit", Conf),
#{
sync_mode_qlen => SycModeQlen,
drop_mode_qlen => DropModeQlen,
flush_qlen => FlushQlen,
overload_kill_enable => conf_get("enable", Overkill),
overload_kill_qlen => conf_get("qlen", Overkill),
overload_kill_mem_size => conf_get("mem_size", Overkill),
overload_kill_restart_after => conf_get("restart_after", Overkill),
burst_limit_enable => conf_get("enable", BurstLimit),
burst_limit_max_count => conf_get("max_count", BurstLimit),
burst_limit_window_time => conf_get("window_time", BurstLimit)
}.
log_formatter(Conf) ->
io:format(standard_error, "log_formatter: ~p~n", [Conf]),
CharsLimit = case conf_get("chars_limit", Conf) of
unlimited -> unlimited;
V when V > 0 -> V
end,
TimeOffSet = case conf_get("time_offset", Conf) of
"system" -> "";
"utc" -> 0;
OffSetStr -> OffSetStr
end,
SingleLine = conf_get("single_line", Conf),
Depth = conf_get("max_depth", Conf),
do_formatter(conf_get("formatter", Conf), CharsLimit, SingleLine, TimeOffSet, Depth).
%% helpers
formatter(json, CharsLimit, SingleLine) ->
do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth) ->
{emqx_logger_jsonfmt,
#{chars_limit => CharsLimit,
single_line => SingleLine
single_line => SingleLine,
time_offset => TimeOffSet,
depth => Depth
}};
formatter(text, CharsLimit, SingleLine) ->
do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth) ->
{emqx_logger_textfmt,
#{template =>
[time," [",level,"] ",
@ -337,9 +356,17 @@ formatter(text, CharsLimit, SingleLine) ->
[]}]},
msg,"\n"],
chars_limit => CharsLimit,
single_line => SingleLine
single_line => SingleLine,
time_offset => TimeOffSet,
depth => Depth
}}.
log_filter(Conf) ->
case conf_get("supervisor_reports", Conf) of
error -> [{drop_progress_reports, {fun logger_filters:progress/2, stop}}];
progress -> []
end.
%% utils
-spec(conf_get(string() | [string()], hocon:config()) -> term()).
conf_get(Key, Conf) ->