refactor(log): move default values to schema

apps/emqx/src/emqx_schema.erl

@@ -3187,7 +3187,12 @@ resolve_env(Name0) ->
 -ifdef(TEST).
 %% when running tests, we need to mock the env variables
 special_env("EMQX_ETC_DIR") ->
-    {ok, filename:join([code:lib_dir(emqx), etc])}.
+    {ok, filename:join([code:lib_dir(emqx), etc])};
+special_env("EMQX_LOG_DIR") ->
+    {ok, "log"};
+special_env(_Name) ->
+    %% only in tests
+    error.
 -else.
 special_env(_Name) -> error.
 -endif.

apps/emqx/test/emqx_common_test_helpers.erl

@@ -271,8 +271,7 @@ mustache_vars(App, Opts) ->
     ExtraMustacheVars = maps:get(extra_mustache_vars, Opts, #{}),
     Defaults = #{
         platform_data_dir => app_path(App, "data"),
-        platform_etc_dir => app_path(App, "etc"),
-        platform_log_dir => app_path(App, "log")
+        platform_etc_dir => app_path(App, "etc")
     },
     maps:merge(Defaults, ExtraMustacheVars).
 

apps/emqx/test/emqx_listeners_SUITE.erl

@@ -266,8 +266,7 @@ render_config_file() ->
 mustache_vars() ->
     [
         {platform_data_dir, local_path(["data"])},
-        {platform_etc_dir, local_path(["etc"])},
-        {platform_log_dir, local_path(["log"])}
+        {platform_etc_dir, local_path(["etc"])}
     ].
 
 generate_config() ->

apps/emqx_conf/etc/emqx_conf.conf

@@ -15,13 +15,6 @@ node {
   data_dir = "{{ platform_data_dir }}"
 }
 
-log {
-  file_handlers.default {
-    level = warning
-    file = "{{ platform_log_dir }}/emqx.log"
-  }
-}
-
 cluster {
   name = emqxcl
   discovery_strategy = manual

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -93,7 +93,10 @@ roots() ->
             {"log",
                 sc(
                     ?R_REF("log"),
-                    #{translate_to => ["kernel"]}
+                    #{
+                        translate_to => ["kernel"],
+                        importance => ?IMPORTANCE_HIGH
+                    }
                 )},
             {"rpc",
                 sc(
@@ -862,15 +865,25 @@ fields("rpc") ->
     ];
 fields("log") ->
     [
-        {"console_handler", ?R_REF("console_handler")},
+        {"console_handler",
+            sc(
+                ?R_REF("console_handler"),
+                #{importance => ?IMPORTANCE_HIGH}
+            )},
         {"file_handlers",
             sc(
                 map(name, ?R_REF("log_file_handler")),
-                #{desc => ?DESC("log_file_handlers")}
+                #{
+                    desc => ?DESC("log_file_handlers"),
+                    %% because file_handlers is a map
+                    %% so there has to be a default value in order to populate the raw configs
+                    default => #{<<"default">> => #{<<"level">> => <<"warning">>}},
+                    importance => ?IMPORTANCE_HIGH
+                }
             )}
     ];
 fields("console_handler") ->
-    log_handler_common_confs(false);
+    log_handler_common_confs(console);
 fields("log_file_handler") ->
     [
         {"file",
@@ -878,6 +891,8 @@ fields("log_file_handler") ->
                 file(),
                 #{
                     desc => ?DESC("log_file_handler_file"),
+                    default => <<"${EMQX_LOG_DIR}/emqx.log">>,
+                    converter => fun emqx_schema:naive_env_interpolation/1,
                     validator => fun validate_file_location/1
                 }
             )},
@@ -891,10 +906,11 @@ fields("log_file_handler") ->
                 hoconsc:union([infinity, emqx_schema:bytesize()]),
                 #{
                     default => <<"50MB">>,
-                    desc => ?DESC("log_file_handler_max_size")
+                    desc => ?DESC("log_file_handler_max_size"),
+                    importance => ?IMPORTANCE_MEDIUM
                 }
             )}
-    ] ++ log_handler_common_confs(true);
+    ] ++ log_handler_common_confs(file);
 fields("log_rotation") ->
     [
         {"enable",
@@ -1103,14 +1119,33 @@ tr_logger_level(Conf) ->
 tr_logger_handlers(Conf) ->
     emqx_config_logger:tr_handlers(Conf).
 
-log_handler_common_confs(Enable) ->
+log_handler_common_confs(Handler) ->
+    lists:map(
+        fun
+            ({_Name, #{importance := _}} = F) -> F;
+            ({Name, Sc}) -> {Name, Sc#{importance => ?IMPORTANCE_LOW}}
+        end,
+        do_log_handler_common_confs(Handler)
+    ).
+do_log_handler_common_confs(Handler) ->
+    %% we rarely support dynamic defaults like this
+    %% for this one, we have build-time defualut the same as runtime default
+    %% so it's less tricky
+    EnableValues =
+        case Handler of
+            console -> ["console", "both"];
+            file -> ["file", "both", "", false]
+        end,
+    EnvValue = os:getenv("EMQX_DEFAULT_LOG_HANDLER"),
+    Enable = lists:member(EnvValue, EnableValues),
     [
         {"enable",
             sc(
                 boolean(),
                 #{
                     default => Enable,
-                    desc => ?DESC("common_handler_enable")
+                    desc => ?DESC("common_handler_enable"),
+                    importance => ?IMPORTANCE_LOW
                 }
             )},
         {"level",
@@ -1127,7 +1162,8 @@ log_handler_common_confs(Enable) ->
                 #{
                     default => <<"system">>,
                     desc => ?DESC("common_handler_time_offset"),
-                    validator => fun validate_time_offset/1
+                    validator => fun validate_time_offset/1,
+                    importance => ?IMPORTANCE_LOW
                 }
             )},
         {"chars_limit",
@@ -1135,7 +1171,8 @@ log_handler_common_confs(Enable) ->
                 hoconsc:union([unlimited, range(100, inf)]),
                 #{
                     default => unlimited,
-                    desc => ?DESC("common_handler_chars_limit")
+                    desc => ?DESC("common_handler_chars_limit"),
+                    importance => ?IMPORTANCE_LOW
                 }
             )},
         {"formatter",
@@ -1143,7 +1180,8 @@ log_handler_common_confs(Enable) ->
                 hoconsc:enum([text, json]),
                 #{
                     default => text,
-                    desc => ?DESC("common_handler_formatter")
+                    desc => ?DESC("common_handler_formatter"),
+                    importance => ?IMPORTANCE_MEDIUM
                 }
             )},
         {"single_line",
@@ -1151,7 +1189,8 @@ log_handler_common_confs(Enable) ->
                 boolean(),
                 #{
                     default => true,
-                    desc => ?DESC("common_handler_single_line")
+                    desc => ?DESC("common_handler_single_line"),
+                    importance => ?IMPORTANCE_LOW
                 }
             )},
         {"sync_mode_qlen",

bin/emqx

@@ -861,7 +861,13 @@ wait_until_return_val() {
     done
 }
 
-# backward compatible with 4.x
+# First, there is EMQX_DEFAULT_LOG_HANDLER which can control the default values
+# to be used when generating configs.
+# It's set in docker entrypoint and in systemd service file.
+#
+# To be backward compatible with 4.x and v5.0.0 ~ v5.0.24/e5.0.2:
+# if EMQX_LOG__TO is set, we try to enable handlers from environment variables.
+# i.e. it overrides the default value set in EMQX_DEFAULT_LOG_HANDLER
 tr_log_to_env() {
     local log_to=${EMQX_LOG__TO:-undefined}
     # unset because it's unknown to 5.0
@@ -893,13 +899,11 @@ tr_log_to_env() {
 
 maybe_log_to_console() {
     if [ "${EMQX_LOG__TO:-}" = 'default' ]; then
-        # want to use config file defaults, do nothing
+        # want to use defaults, do nothing
         unset EMQX_LOG__TO
     else
         tr_log_to_env
-        # ensure defaults
-        export EMQX_LOG__CONSOLE_HANDLER__ENABLE="${EMQX_LOG__CONSOLE_HANDLER__ENABLE:-true}"
-        export EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE="${EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE:-false}"
+        export EMQX_DEFAULT_LOG_HANDLER=${EMQX_DEFAULT_LOG_HANDLER:-console}
     fi
 }
 

deploy/docker/docker-entrypoint.sh

@@ -1,9 +1,7 @@
 #!/usr/bin/env bash
-## EMQ docker image start script
-# Huang Rui <vowstar@gmail.com>
-# EMQX Team <support@emqx.io>
 
-## Shell setting
+## EMQ docker image start script
+
 if [[ -n "$DEBUG" ]]; then
     set -ex
 else

deploy/packages/emqx.service

@@ -10,8 +10,8 @@ Group=emqx
 Type=simple
 Environment=HOME=/var/lib/emqx
 
-# Enable logging to file
-Environment=EMQX_LOG__TO=default
+# log to file by default (if no log handler config)
+Environment=EMQX_DEFAULT_LOG_HANDLER=file
 
 # Start 'foreground' but not 'start' (daemon) mode.
 # Because systemd monitor/restarts 'simple' services

mix.exs

@@ -665,7 +665,6 @@ defmodule EMQXUmbrella.MixProject do
       emqx_default_erlang_cookie: default_cookie(),
       platform_data_dir: "data",
       platform_etc_dir: "etc",
-      platform_log_dir: "log",
       platform_plugins_dir: "plugins",
       runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
       emqx_etc_dir: "$RUNNER_ROOT_DIR/etc",
@@ -688,7 +687,6 @@ defmodule EMQXUmbrella.MixProject do
       emqx_default_erlang_cookie: default_cookie(),
       platform_data_dir: "/var/lib/emqx",
       platform_etc_dir: "/etc/emqx",
-      platform_log_dir: "/var/log/emqx",
       platform_plugins_dir: "/var/lib/emqx/plugins",
       runner_bin_dir: "/usr/bin",
       emqx_etc_dir: "/etc/emqx",

rebar.config.erl

@@ -335,7 +335,6 @@ overlay_vars_pkg(bin) ->
     [
         {platform_data_dir, "data"},
         {platform_etc_dir, "etc"},
-        {platform_log_dir, "log"},
         {platform_plugins_dir, "plugins"},
         {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"},
         {emqx_etc_dir, "$RUNNER_ROOT_DIR/etc"},
@@ -348,7 +347,6 @@ overlay_vars_pkg(pkg) ->
     [
         {platform_data_dir, "/var/lib/emqx"},
         {platform_etc_dir, "/etc/emqx"},
-        {platform_log_dir, "/var/log/emqx"},
         {platform_plugins_dir, "/var/lib/emqx/plugins"},
         {runner_bin_dir, "/usr/bin"},
         {emqx_etc_dir, "/etc/emqx"},

rel/i18n/emqx_conf_schema.hocon

@@ -1369,9 +1369,12 @@ this is where to look."""
 
   desc_log {
     desc {
-      en: """EMQX logging supports multiple sinks for the log events.
-Each sink is represented by a _log handler_, which can be configured independently."""
-      zh: """EMQX 日志记录支持日志事件的多个接收器。 每个接收器由一个_log handler_表示，可以独立配置。"""
+      en: """EMQX supports multiple log handlers, one console handler and multiple file handlers.
+EMQX by default logs to console when running in docker or in console/foreground mode,
+otherwise it logs to file $EMQX_LOG_DIR/emqx.log.
+For advanced configuration, you can find more parameters in this section."""
+      zh: """EMQX 支持同时多个日志输出，一个控制台输出，和多个文件输出。
+默认情况下，EMQX 运行在容器中，或者在 'console' 或 'foreground' 模式下运行时，会输出到 控制台，否则输出到文件。"""
     }
     label {
       en: "Log"