Add log overload protection parameters

etc/emqx.conf

@@ -414,6 +414,9 @@ log.dir = {{ platform_log_dir }}
 
 ## The log filename for logs of level specified in "log.level".
 ##
+## If `log.rotation` is enabled, this is the base name of the
+## files. Each file in a rotated log is named <base_name>.N, where N is an integer.
+##
 ## Value: String
 ## Default: emqx.log
 log.file = emqx.log
@@ -424,6 +427,14 @@ log.file = emqx.log
 ## Default: No Limit
 #log.chars_limit = 8192
 
+## Enables the log rotation.
+## With this enabled, new log files will be created when the current
+## log file is full, max to `log.rotation.size` files will be created.
+##
+## Value: on | off
+## Default: on
+log.rotation = on
+
 ## Maximum size of each log file.
 ##
 ## Value: Number
@@ -449,6 +460,100 @@ log.rotation.count = 5
 #log.info.file  = info.log
 #log.error.file = error.log
 
+## The max allowed queue length before switching to sync mode.
+##
+## Log overload protection parameter. If the message queue grows
+## larger than this value the handler switches from anync to sync mode.
+##
+## Default: 100
+##
+#log.sync_mode_qlen = 100
+
+## The max allowed queue length before switching to drop mode.
+##
+## Log overload protection parameter. When the message queue grows
+## larger than this threshold, the handler switches to a mode in which
+## it drops all new events that senders want to log.
+##
+## Default: 3000
+##
+#log.drop_mode_qlen = 3000
+
+## The max allowed queue length before switching to flush mode.
+##
+## Log overload protection parameter. If the length of the message queue
+## grows larger than this threshold, a flush (delete) operation takes place.
+## To flush events, the handler discards the messages in the message queue
+## by receiving them in a loop without logging.
+##
+## Default: 8000
+##
+#log.flush_qlen = 8000
+
+## Kill the log handler when it gets overloaded.
+##
+## Log overload protection parameter. It is possible that a handler,
+## even if it can successfully manage peaks of high load without crashing,
+## can build up a large message queue, or use a large amount of memory.
+## We could kill the log handler in these cases and restart it after a
+## few seconds.
+##
+## Default: on
+##
+#log.overload_kill = on
+
+## The max allowed queue length before killing the log hanlder.
+##
+## Log overload protection parameter. This is the maximum allowed queue
+## length. If the message queue grows larger than this, the handler
+## process is terminated.
+##
+## Default: 20000
+##
+#log.overload_kill_qlen = 20000
+
+## The max allowed memory size before killing the log hanlder.
+##
+## Log overload protection parameter. This is the maximum memory size
+## that the handler process is allowed to use. If the handler grows
+## larger than this, the process is terminated.
+##
+## Default: 30MB
+##
+#log.overload_kill_mem_size = 30MB
+
+## Restart the log hanlder after some seconds.
+##
+## Log overload protection parameter. If the handler is terminated,
+## it restarts automatically after a delay specified in seconds.
+## The value "infinity" prevents restarts.
+##
+## Default: 5s
+##
+#log.overload_kill_restart_after = 5s
+
+## Max burst count and time window for burst control.
+##
+## Log overload protection parameter. Large bursts of log events - many
+## events received by the handler under a short period of time - can
+## potentially cause problems. By specifying the maximum number of events
+## to be handled within a certain time frame, the handler can avoid
+## choking the log with massive amounts of printouts.
+##
+## This config controls the maximum number of events to handle within
+## a time frame. After the limit is reached, successive events are
+## dropped until the end of the time frame.
+##
+## Note that there would be no warning if any messages were
+## dropped because of burst control.
+##
+## Comment this config out to disable the burst control feature.
+##
+## Value: MaxBurstCount,TimeWindow
+## Default: disabled
+##
+#log.burst_limit = 20000, 1s
+
 ##--------------------------------------------------------------------
 ## Authentication/Access Control
 ##--------------------------------------------------------------------

priv/emqx.schema

@@ -477,11 +477,21 @@ end}.
   {datatype, integer}
 ]}.
 
+{mapping, "log.rotation", "kernel.logger", [
+  {default, on},
+  {datatype, flag}
+]}.
+
 {mapping, "log.rotation.size", "kernel.logger", [
   {default, "10MB"},
   {datatype, bytesize}
 ]}.
 
+{mapping, "log.size", "kernel.logger", [
+  {default, infinity},
+  {datatype, [bytesize, atom]}
+]}.
+
 {mapping, "log.rotation.count", "kernel.logger", [
   {default, 5},
   {datatype, integer}
@@ -491,6 +501,46 @@ end}.
   {datatype, file}
 ]}.
 
+{mapping, "log.sync_mode_qlen", "kernel.logger", [
+  {default, 100},
+  {datatype, integer}
+]}.
+
+{mapping, "log.drop_mode_qlen", "kernel.logger", [
+  {default, 3000},
+  {datatype, integer}
+]}.
+
+{mapping, "log.flush_qlen", "kernel.logger", [
+  {default, 8000},
+  {datatype, integer}
+]}.
+
+{mapping, "log.overload_kill", "kernel.logger", [
+  {default, on},
+  {datatype, flag}
+]}.
+
+{mapping, "log.overload_kill_mem_size", "kernel.logger", [
+  {default, "30MB"},
+  {datatype, bytesize}
+]}.
+
+{mapping, "log.overload_kill_qlen", "kernel.logger", [
+  {default, 20000},
+  {datatype, integer}
+]}.
+
+{mapping, "log.overload_kill_restart_after", "kernel.logger", [
+  {default, "5s"},
+  {datatype, [{duration, ms}, atom]}
+]}.
+
+{mapping, "log.burst_limit", "kernel.logger", [
+  {default, "disabled"},
+  {datatype, string}
+]}.
+
 {mapping, "log.sasl", "sasl.sasl_error_logger", [
   {default, off},
   {datatype, flag},
@@ -521,6 +571,10 @@ end}.
 {translation, "kernel.logger", fun(Conf) ->
     LogTo = cuttlefish:conf_get("log.to", Conf),
     LogLevel = cuttlefish:conf_get("log.level", Conf),
+    LogType = case cuttlefish:conf_get("log.rotation", Conf) of
+                  true -> wrap;
+                  false -> halt
+              end,
     CharsLimit = case cuttlefish:conf_get("log.chars_limit", Conf) of
                      -1 -> unlimited;
                      V -> V
@@ -537,11 +591,37 @@ end}.
                               []}]},
                        msg,"\n"],
                     chars_limit => CharsLimit}},
+    {BustLimitOn, {MaxBurstCount, TimeWindow}} =
+        case string:tokens(cuttlefish:conf_get("log.burst_limit", Conf), ", ") of
+            ["disabled"] -> {false, {20000, 1000}};
+            [Count, Window] ->
+                {true, {list_to_integer(Count),
+                        case cuttlefish_duration:parse(Window, ms) of
+                          Secs when is_integer(Secs) -> Secs;
+                          {error, Reason1} -> error(Reason1)
+                        end}}
+        end,
     FileConf =  fun(Filename) ->
-                  #{type => wrap,
+                  BasicConf =
+                  #{type => LogType,
                     file => filename:join(cuttlefish:conf_get("log.dir", Conf), Filename),
                     max_no_files => cuttlefish:conf_get("log.rotation.count", Conf),
-                    max_no_bytes => cuttlefish:conf_get("log.rotation.size", Conf)}
+                    sync_mode_qlen => cuttlefish:conf_get("log.sync_mode_qlen", Conf),
+                    drop_mode_qlen => cuttlefish:conf_get("log.drop_mode_qlen", Conf),
+                    flush_qlen => cuttlefish:conf_get("log.flush_qlen", Conf),
+                    overload_kill_enable => cuttlefish:conf_get("log.overload_kill", Conf),
+                    overload_kill_qlen => cuttlefish:conf_get("log.overload_kill_qlen", Conf),
+                    overload_kill_mem_size => cuttlefish:conf_get("log.overload_kill_mem_size", Conf),
+                    overload_kill_restart_after => cuttlefish:conf_get("log.overload_kill_restart_after", Conf),
+                    burst_limit_enable => BustLimitOn,
+                    burst_limit_max_count => MaxBurstCount,
+                    burst_limit_window_time => TimeWindow
+                    },
+                  MaxNoBytes = case LogType of
+                    wrap -> cuttlefish:conf_get("log.rotation.size", Conf);
+                    halt -> cuttlefish:conf_get("log.size", Conf)
+                  end,
+                  BasicConf#{max_no_bytes => MaxNoBytes}
                 end,
 
     %% For the default logger that outputs to console