Merge pull request #10444 from zmstone/0418-cherry-pick-fixes-for-e503

0418 cherry pick fixes for e503

apps/emqx/src/emqx_schema.erl

@@ -1633,7 +1633,9 @@ fields("sysmon") ->
         {"top",
             sc(
                 ref("sysmon_top"),
-                #{}
+                %% Userful monitoring solution when benchmarking,
+                %% but hardly common enough for regular users.
+                #{importance => ?IMPORTANCE_HIDDEN}
             )}
     ];
 fields("sysmon_vm") ->
@@ -2235,6 +2237,7 @@ common_ssl_opts_schema(Defaults) ->
                 #{
                     default => <<"emqx_tls_psk:lookup">>,
                     converter => fun ?MODULE:user_lookup_fun_tr/2,
+                    importance => ?IMPORTANCE_HIDDEN,
                     desc => ?DESC(common_ssl_opts_schema_user_lookup_fun)
                 }
             )},
@@ -2762,10 +2765,16 @@ str(S) when is_list(S) ->
     S.
 
 authentication(Which) ->
-    Desc =
+    {Importance, Desc} =
         case Which of
-            global -> ?DESC(global_authentication);
+            global ->
-            listener -> ?DESC(listener_authentication)
+                %% For root level authentication, it is recommended to configure
+                %% from the dashboard or API.
+                %% Hence it's considered a low-importance when it comes to
+                %% configuration importance.
+                {?IMPORTANCE_LOW, ?DESC(global_authentication)};
+            listener ->
+                {?IMPORTANCE_HIDDEN, ?DESC(listener_authentication)}
         end,
     %% poor man's dependency injection
     %% this is due to the fact that authn is implemented outside of 'emqx' app.
@@ -2781,7 +2790,7 @@ authentication(Which) ->
     hoconsc:mk(Type, #{
         desc => Desc,
         converter => fun ensure_array/2,
-        importance => ?IMPORTANCE_HIDDEN
+        importance => Importance
     }).
 
 %% the older version schema allows individual element (instead of a chain) in config

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -494,7 +494,10 @@ authz_fields() ->
                     default => [],
                     desc => ?DESC(sources),
                     %% doc_lift is force a root level reference instead of nesting sub-structs
-                    extra => #{doc_lift => true}
+                    extra => #{doc_lift => true},
+                    %% it is recommended to configure authz sources from dashboard
+                    %% hance the importance level for config is low
+                    importance => ?IMPORTANCE_LOW
                 }
             )}
     ].

apps/emqx_bridge/src/schema/emqx_bridge_schema.erl

@@ -137,7 +137,7 @@ namespace() -> "bridge".
 tags() ->
     [<<"Bridge">>].
 
-roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_HIDDEN})}].
+roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_LOW})}].
 
 fields(bridges) ->
     [

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -100,7 +100,7 @@ roots() ->
                     ?R_REF("rpc"),
                     #{
                         translate_to => ["gen_rpc"],
-                        importance => ?IMPORTANCE_HIDDEN
+                        importance => ?IMPORTANCE_LOW
                     }
                 )}
         ] ++
@@ -1288,7 +1288,7 @@ emqx_schema_high_prio_roots() ->
                 ?R_REF("authorization"),
                 #{
                     desc => ?DESC(authorization),
-                    importance => ?IMPORTANCE_HIDDEN
+                    importance => ?IMPORTANCE_HIGH
                 }
             )},
     lists:keyreplace("authorization", 1, Roots, Authz).

apps/emqx_exhook/src/emqx_exhook_schema.erl

@@ -32,7 +32,7 @@
 namespace() -> exhook.
 
 roots() ->
-    [{exhook, ?HOCON(?R_REF(exhook), #{importance => ?IMPORTANCE_HIDDEN})}].
+    [{exhook, ?HOCON(?R_REF(exhook), #{importance => ?IMPORTANCE_LOW})}].
 
 fields(exhook) ->
     [

apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl

@@ -38,7 +38,7 @@ namespace() -> rule_engine.
 tags() ->
     [<<"Rule Engine">>].
 
-roots() -> [{"rule_engine", ?HOCON(?R_REF("rule_engine"), #{importance => ?IMPORTANCE_HIDDEN})}].
+roots() -> [{"rule_engine", ?HOCON(?R_REF("rule_engine"), #{importance => ?IMPORTANCE_LOW})}].
 
 fields("rule_engine") ->
     rule_engine_settings() ++

changes/ce/feat-10391.en.md

@@ -1 +1,7 @@
-hide exhook/rewrite/topic_metric/persistent_session_store/overload_protection from the docs and configuration file.
+Hide a large number of advanced options to simplify the configuration file.
+
+That includes `rewrite`, `topic_metric`, `persistent_session_store`, `overload_protection`,
+`flapping_detect`, `conn_congestion`, `stats,auto_subscribe`, `broker_perf`,
+`shared_subscription_group`, `slow_subs`, `ssl_options.user_lookup_fun` and some advance items
+in `node` and `dashboard` section, [#10358](https://github.com/emqx/emqx/pull/10358),
+[#10381](https://github.com/emqx/emqx/pull/10381), [#10385](https://github.com/emqx/emqx/pull/10385).

rel/emqx_conf.template.en.md

@@ -7,21 +7,27 @@ and a superset of JSON.
 EMQX configuration consists of two layers.
 From bottom up:
 
-1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.<br/>
+1. Cluster-synced configs: `$EMQX_NODE__DATA_DIR/configs/cluster.hocon`.
-   Changes in this layer require a full node restart to take effect.
+2. Local node configs: `emqx.conf` + `EMQX_` prefixed environment variables.
-1. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster-override.conf`
+
+:::tip Tip
+Prior to v5.0.23 and e5.0.3, the cluster-synced configs are stored in
+`cluster-override.conf` which is applied on top of the local configs.
+
+If upgraded from an earlier version, as long as `cluster-override.conf` exists,
+`cluster.hocon` will not be created, and `cluster-override.conf` will stay on
+top of the overriding layers.
+:::
 
 When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir`
 is used.
 
-The `cluster-override.conf` file is overwritten at runtime when changes
+The `cluster.hocon` file is overwritten at runtime when changes
-are made from dashboard UI, management HTTP API, or CLI. When clustered,
+are made from Dashboard, management HTTP API, or CLI. When clustered,
 after EMQX restarts, it copies the file from the node which has the greatest `uptime`.
 
 :::tip Tip
-Some of the configs (such as `node.name`) are boot-only configs and not overridable.
+To avoid confusion, don't add the same keys in both `cluster.hocon` and `emqx.conf`.
-Config values from `*-override.conf` are **not** mapped to boot configs for
-the config fields attributed with `mapping: path.to.boot.config.key`
 :::
 
 For detailed override rules, see [Config Overlay Rules](#config-overlay-rules).

rel/emqx_conf.template.zh.md

@@ -5,9 +5,15 @@ HOCON（Human-Optimized Config Object Notation）是一个JSON的超集，非常
 
 EMQX的配置文件可分为二层，自底向上依次是：
 
-1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。<br/>
+1. 集群同步配置：`$EMQX_NODE__DATA_DIR/configs/cluster.hocon`。
-   修改这一层的配置之后，需要重启节点来使之生效。
+2. 本地节点配置：`emqx.conf` 加上 `EMQX_` 前缀的环境变量。
-1. 集群范围重载层：`$EMQX_NODE__DATA_DIR/configs/cluster-override.conf`
+
+:::tip Tip
+在 v5.0.23 或 e5.0.3 之前，集群同步配置保存在文件 `cluster-override.conf` 中，并且它覆盖在配置的最上层。
+
+如果从之前的版本升级上来，只要 `cluster-override.conf` 文件存在，
+EMQX 就不会创建 `cluster.hocon`，并且 `cluster-override.conf` 会继续覆盖在配置的最上层。
+:::
 
 如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置，那么该目录会从 `emqx.conf` 的 `node.data_dir` 配置中读取。
 
@@ -16,9 +22,7 @@ EMQX的配置文件可分为二层，自底向上依次是：
 当EMQX运行在集群中时，一个EMQX节点重启之后，会从集群中其他节点复制该文件内容到本地。
 
 :::tip Tip
-有些配置项是不能被重载的（例如 `node.name`）。
+为避免歧义，应尽量避免让 `cluster.hocon` 和 `emqx.conf` 出现配置交集。
-配置项如果有 `mapping: path.to.boot.config.key` 这个属性，
-则不能被添加到重载文件 `*-override.conf` 中。
 :::
 
 更多的重载规则，请参考下文 [配置重载规则](#配置重载规则)。