Merge pull request #11180 from zhongwencool/conf-load-api

feat: add conf load api

apps/emqx/src/emqx_config_handler.erl

@@ -532,7 +532,8 @@ schema(SchemaModule, [RootKey | _]) ->
     {Field, Translations} =
         case lists:keyfind(bin(RootKey), 1, Roots) of
             {_, {Ref, ?REF(Ref)}} -> {Ref, ?R_REF(SchemaModule, Ref)};
-            {_, {Name, Field0}} -> parse_translations(Field0, Name, SchemaModule)
+            {_, {Name, Field0}} -> parse_translations(Field0, Name, SchemaModule);
+            false -> throw({root_key_not_found, RootKey})
         end,
     #{
         roots => [Field],

apps/emqx_conf/src/emqx_conf.erl

@@ -30,7 +30,6 @@
 -export([reset/2, reset/3]).
 -export([dump_schema/2]).
 -export([schema_module/0]).
--export([check_config/2]).
 
 %% TODO: move to emqx_dashboard when we stop building api schema at build time
 -export([
@@ -208,15 +207,6 @@ schema_module() ->
         Value -> list_to_existing_atom(Value)
     end.
 
-check_config(Mod, Raw) ->
-    try
-        {_AppEnvs, CheckedConf} = emqx_config:check_config(Mod, Raw),
-        {ok, CheckedConf}
-    catch
-        throw:Error ->
-            {error, Error}
-    end.
-
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------

apps/emqx_conf/src/emqx_conf_cli.erl

@@ -26,6 +26,8 @@
     unload/0
 ]).
 
+-export([keys/0, get_config/0, get_config/1, load_config/2]).
+
 -include_lib("hocon/include/hoconsc.hrl").
 
 %% kept cluster_call for compatibility
@@ -42,7 +44,7 @@ unload() ->
     emqx_ctl:unregister_command(?CONF).
 
 conf(["show_keys" | _]) ->
-    print_keys(get_config());
+    print_keys(keys());
 conf(["show"]) ->
     print_hocon(get_config());
 conf(["show", Key]) ->
@@ -150,9 +152,9 @@ status() ->
     ),
     emqx_ctl:print("-----------------------------------------------\n").
 
-print_keys(Config) ->
+print_keys(Keys) ->
-    Keys = lists:sort(maps:keys(Config)),
+    SortKeys = lists:sort(Keys),
-    emqx_ctl:print("~1p~n", [[binary_to_existing_atom(K) || K <- Keys]]).
+    emqx_ctl:print("~1p~n", [[binary_to_existing_atom(K) || K <- SortKeys]]).
 
 print(Json) ->
     emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Json)]).
@@ -166,6 +168,9 @@ get_config() ->
     AllConf = fill_defaults(emqx:get_raw_config([])),
     drop_hidden_roots(AllConf).
 
+keys() ->
+    emqx_config:get_root_names() -- hidden_roots().
+
 drop_hidden_roots(Conf) ->
     lists:foldl(fun(K, Acc) -> maps:remove(K, Acc) end, Conf, hidden_roots()).
 
@@ -186,37 +191,47 @@ get_config(Key) ->
     end.
 
 -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}).
-load_config(Path, ReplaceOrMerge) ->
+load_config(Path, ReplaceOrMerge) when is_list(Path) ->
     case hocon:files([Path]) of
         {ok, RawConf} when RawConf =:= #{} ->
             emqx_ctl:warning("load ~ts is empty~n", [Path]),
             {error, empty_hocon_file};
         {ok, RawConf} ->
-            case check_config(RawConf) of
+            load_config_from_raw(RawConf, ReplaceOrMerge);
-                ok ->
-                    lists:foreach(
-                        fun({K, V}) -> update_config_cluster(K, V, ReplaceOrMerge) end,
-                        to_sorted_list(RawConf)
-                    );
-                {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} ->
-                    emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]),
-                    emqx_ctl:warning(
-                        "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n"
-                    ),
-                    {error, Reason};
-                {error, Errors} ->
-                    emqx_ctl:warning("load ~ts schema check failed~n", [Path]),
-                    lists:foreach(
-                        fun({Key, Error}) ->
-                            emqx_ctl:warning("~ts: ~p~n", [Key, Error])
-                        end,
-                        Errors
-                    ),
-                    {error, Errors}
-            end;
         {error, Reason} ->
             emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]),
             {error, bad_hocon_file}
+    end;
+load_config(Bin, ReplaceOrMerge) when is_binary(Bin) ->
+    case hocon:binary(Bin) of
+        {ok, RawConf} ->
+            load_config_from_raw(RawConf, ReplaceOrMerge);
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+load_config_from_raw(RawConf, ReplaceOrMerge) ->
+    case check_config(RawConf) of
+        ok ->
+            lists:foreach(
+                fun({K, V}) -> update_config_cluster(K, V, ReplaceOrMerge) end,
+                to_sorted_list(RawConf)
+            );
+        {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} ->
+            emqx_ctl:warning("load config failed~n~ts~n", [Reason]),
+            emqx_ctl:warning(
+                "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n"
+            ),
+            {error, Reason};
+        {error, Errors} ->
+            emqx_ctl:warning("load schema check failed~n"),
+            lists:foreach(
+                fun({Key, Error}) ->
+                    emqx_ctl:warning("~ts: ~p~n", [Key, Error])
+                end,
+                Errors
+            ),
+            {error, Errors}
     end.
 
 update_config_cluster(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge) ->
@@ -265,8 +280,7 @@ check_keys_is_not_readonly(Conf) ->
 check_config_schema(Conf) ->
     SchemaMod = emqx_conf:schema_module(),
     Fold = fun({Key, Value}, Acc) ->
-        Schema = emqx_config_handler:schema(SchemaMod, [Key]),
+        case check_config(SchemaMod, Key, Value) of
-        case emqx_conf:check_config(Schema, #{Key => Value}) of
             {ok, _} -> Acc;
             {error, Reason} -> [{Key, Reason} | Acc]
         end
@@ -319,11 +333,12 @@ load_etc_config_file() ->
 filter_readonly_config(Raw) ->
     SchemaMod = emqx_conf:schema_module(),
     RawDefault = fill_defaults(Raw),
-    case emqx_conf:check_config(SchemaMod, RawDefault) of
+    try
-        {ok, _CheckedConf} ->
+        _ = emqx_config:check_config(SchemaMod, RawDefault),
-            ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
+        ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
-            {ok, maps:without(ReadOnlyKeys, Raw)};
+        {ok, maps:without(ReadOnlyKeys, Raw)}
-        {error, Error} ->
+    catch
+        throw:Error ->
             ?SLOG(error, #{
                 msg => "bad_etc_config_schema_found",
                 error => Error
@@ -377,3 +392,13 @@ filter_cluster_conf(#{<<"cluster">> := #{<<"discovery_strategy">> := Strategy} =
     Conf#{<<"cluster">> => Cluster1};
 filter_cluster_conf(Conf) ->
     Conf.
+
+check_config(SchemaMod, Key, Value) ->
+    try
+        Schema = emqx_config_handler:schema(SchemaMod, [Key]),
+        {_AppEnvs, CheckedConf} = emqx_config:check_config(Schema, #{Key => Value}),
+        {ok, CheckedConf}
+    catch
+        throw:Error ->
+            {error, Error}
+    end.

apps/emqx_management/src/emqx_mgmt_api_configs.erl

@@ -44,9 +44,34 @@
     <<"sys_topics">>,
     <<"sysmon">>,
     <<"log">>
-    %% <<"zones">>
 ]).
 
+%% erlfmt-ignore
+-define(SYSMON_EXAMPLE,
+    <<"""
+    sysmon {
+      os {
+        cpu_check_interval = 60s
+        cpu_high_watermark = 80%
+        cpu_low_watermark = 60%
+        mem_check_interval = 60s
+        procmem_high_watermark = 5%
+        sysmem_high_watermark = 70%
+        }
+        vm {
+        busy_dist_port = true
+        busy_port = true
+        large_heap = 32MB
+        long_gc = disabled
+        long_schedule = 240ms
+        process_check_interval = 30s
+        process_high_watermark = 80%
+        process_low_watermark = 60%
+        }
+    }
+    """>>
+).
+
 api_spec() ->
     emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
 
@@ -66,24 +91,62 @@ schema("/configs") ->
         'operationId' => configs,
         get => #{
             tags => ?TAGS,
-            description => ?DESC(get_conf_node),
+            description => ?DESC(get_configs),
             parameters => [
+                {key,
+                    hoconsc:mk(
+                        hoconsc:enum([binary_to_atom(K) || K <- emqx_conf_cli:keys()]),
+                        #{in => query, example => <<"sysmon">>, required => false}
+                    )},
                 {node,
                     hoconsc:mk(
                         typerefl:atom(),
                         #{
                             in => query,
                             required => false,
-                            example => <<"emqx@127.0.0.1">>,
+                            description => ?DESC(node_name),
-                            description => ?DESC(node_name)
+                            hidden => true
                         }
                     )}
             ],
             responses => #{
-                200 => lists:map(fun({_, Schema}) -> Schema end, config_list()),
+                200 => #{
+                    content =>
+                        %% use proplists( not map) to make user text/plain is default in swagger
+                        [
+                            {'text/plain', #{
+                                schema => #{type => string, example => ?SYSMON_EXAMPLE}
+                            }},
+                            {'application/json', #{
+                                schema => #{type => object, example => #{<<"deprecated">> => true}}
+                            }}
+                        ]
+                },
                 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND']),
                 500 => emqx_dashboard_swagger:error_codes(['BAD_NODE'])
             }
+        },
+        put => #{
+            tags => ?TAGS,
+            description => ?DESC(update_configs),
+            parameters => [
+                {mode,
+                    hoconsc:mk(
+                        hoconsc:enum([replace, merge]),
+                        #{in => query, default => merge, required => false}
+                    )}
+            ],
+            'requestBody' => #{
+                content =>
+                    #{
+                        'text/plain' =>
+                            #{schema => #{type => string, example => ?SYSMON_EXAMPLE}}
+                    }
+            },
+            responses => #{
+                200 => <<"Configurations updated">>,
+                400 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED'])
+            }
         }
     };
 schema("/configs_reset/:rootname") ->
@@ -272,9 +335,21 @@ config_reset(post, _Params, Req) ->
             {400, #{code => 'REST_FAILED', message => ?ERR_MSG(Reason)}}
     end.
 
-configs(get, Params, _Req) ->
+configs(get, #{query_string := QueryStr, headers := Headers}, _Req) ->
-    QS = maps:get(query_string, Params, #{}),
+    %% Should deprecated json v1 since 5.2.0
-    Node = maps:get(<<"node">>, QS, node()),
+    case maps:get(<<"accept">>, Headers, <<"text/plain">>) of
+        <<"application/json">> -> get_configs_v1(QueryStr);
+        <<"text/plain">> -> get_configs_v2(QueryStr)
+    end;
+configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) ->
+    case emqx_conf_cli:load_config(Conf, Mode) of
+        ok -> {200};
+        {error, [{_, Reason}]} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Reason)}};
+        {error, Errors} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Errors)}}
+    end.
+
+get_configs_v1(QueryStr) ->
+    Node = maps:get(<<"node">>, QueryStr, node()),
     case
         lists:member(Node, emqx:running_nodes()) andalso
             emqx_management_proto_v2:get_full_config(Node)
@@ -289,6 +364,18 @@ configs(get, Params, _Req) ->
             {200, Res}
     end.
 
+get_configs_v2(QueryStr) ->
+    Conf =
+        case maps:find(<<"key">>, QueryStr) of
+            error -> emqx_conf_cli:get_config();
+            {ok, Key} -> emqx_conf_cli:get_config(atom_to_binary(Key))
+        end,
+    {
+        200,
+        #{<<"content-type">> => <<"text/plain">>},
+        iolist_to_binary(hocon_pp:do(Conf, #{}))
+    }.
+
 limiter(get, _Params, _Req) ->
     {200, format_limiter_config(get_raw_config(limiter))};
 limiter(put, #{body := NewConf}, _Req) ->

apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl

@@ -40,8 +40,8 @@ end_per_testcase(TestCase = t_configs_node, Config) ->
 end_per_testcase(_TestCase, Config) ->
     Config.
 
-t_get(_Config) ->
+t_get_with_json(_Config) ->
-    {ok, Configs} = get_configs(),
+    {ok, Configs} = get_configs_with_json(),
     maps:map(
         fun(Name, Value) ->
             {ok, Config} = get_config(Name),
@@ -268,6 +268,7 @@ t_dashboard(_Config) ->
     timer:sleep(1500),
     ok.
 
+%% v1 version json
 t_configs_node({'init', Config}) ->
     Node = node(),
     meck:expect(emqx, running_nodes, fun() -> [Node, bad_node, other_node] end),
@@ -286,16 +287,41 @@ t_configs_node({'end', _}) ->
 t_configs_node(_) ->
     Node = atom_to_list(node()),
 
-    ?assertEqual({ok, <<"self">>}, get_configs(Node, #{return_all => true})),
+    ?assertEqual({ok, <<"self">>}, get_configs_with_json(Node, #{return_all => true})),
-    ?assertEqual({ok, <<"other">>}, get_configs("other_node", #{return_all => true})),
+    ?assertEqual({ok, <<"other">>}, get_configs_with_json("other_node", #{return_all => true})),
 
-    {ExpType, ExpRes} = get_configs("unknown_node", #{return_all => true}),
+    {ExpType, ExpRes} = get_configs_with_json("unknown_node", #{return_all => true}),
     ?assertEqual(error, ExpType),
     ?assertMatch({{_, 404, _}, _, _}, ExpRes),
     {_, _, Body} = ExpRes,
     ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_utils_json:decode(Body, [return_maps])),
 
-    ?assertMatch({error, {_, 500, _}}, get_configs("bad_node")).
+    ?assertMatch({error, {_, 500, _}}, get_configs_with_json("bad_node")).
+
+%% v2 version binary
+t_configs_key(_Config) ->
+    Keys = lists:sort(emqx_conf_cli:keys()),
+    {ok, Hocon} = get_configs_with_binary(undefined),
+    ?assertEqual(Keys, lists:sort(maps:keys(Hocon))),
+    {ok, Log} = get_configs_with_binary("log"),
+    ?assertMatch(
+        #{
+            <<"log">> := #{
+                <<"console">> := #{
+                    <<"enable">> := _,
+                    <<"formatter">> := <<"text">>,
+                    <<"level">> := <<"warning">>,
+                    <<"time_offset">> := <<"system">>
+                },
+                <<"file">> := _
+            }
+        },
+        Log
+    ),
+    Log1 = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"error">>),
+    ?assertEqual([], update_configs_with_binary(iolist_to_binary(hocon_pp:do(Log1, #{})))),
+    ?assertEqual(<<"error">>, read_conf([<<"log">>, <<"console">>, <<"level">>])),
+    ok.
 
 %% Helpers
 
@@ -308,25 +334,52 @@ get_config(Name) ->
             Error
     end.
 
-get_configs() ->
+get_configs_with_json() ->
-    get_configs([], #{}).
+    get_configs_with_json([], #{}).
 
-get_configs(Node) ->
+get_configs_with_json(Node) ->
-    get_configs(Node, #{}).
+    get_configs_with_json(Node, #{}).
 
-get_configs(Node, Opts) ->
+get_configs_with_json(Node, Opts) ->
     Path =
         case Node of
             [] -> ["configs"];
             _ -> ["configs?node=" ++ Node]
         end,
     URI = emqx_mgmt_api_test_util:api_path(Path),
-    case emqx_mgmt_api_test_util:request_api(get, URI, [], [], [], Opts) of
+    Auth = emqx_mgmt_api_test_util:auth_header_(),
+    Headers = [{"accept", "application/json"}, Auth],
+    case emqx_mgmt_api_test_util:request_api(get, URI, [], Headers, [], Opts) of
         {ok, {_, _, Res}} -> {ok, emqx_utils_json:decode(Res, [return_maps])};
         {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])};
         Error -> Error
     end.
 
+get_configs_with_binary(Key) ->
+    Path =
+        case Key of
+            undefined -> ["configs"];
+            _ -> ["configs?key=" ++ Key]
+        end,
+    URI = emqx_mgmt_api_test_util:api_path(Path),
+    Auth = emqx_mgmt_api_test_util:auth_header_(),
+    Headers = [{"accept", "text/plain"}, Auth],
+    case emqx_mgmt_api_test_util:request_api(get, URI, [], Headers, [], #{return_all => true}) of
+        {ok, {_, _, Res}} -> hocon:binary(Res);
+        {ok, Res} -> hocon:binary(Res);
+        Error -> Error
+    end.
+
+update_configs_with_binary(Bin) ->
+    Path = emqx_mgmt_api_test_util:api_path(["configs"]),
+    Auth = emqx_mgmt_api_test_util:auth_header_(),
+    Headers = [{"accept", "text/plain"}, Auth],
+    case httpc:request(put, {Path, Headers, "text/plain", Bin}, [], []) of
+        {ok, {_, _, Res}} -> Res;
+        {ok, Res} -> Res;
+        Error -> Error
+    end.
+
 update_config(Name, Change) ->
     AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
     UpdatePath = emqx_mgmt_api_test_util:api_path(["configs", Name]),

changes/ce/feat-11180.en.md

@@ -0,0 +1 @@
+Adding a new configuration API `/configs`(GET/PUT) that supports to reload the hocon format configuration file.

rel/i18n/emqx_mgmt_api_configs.hocon

@@ -1,14 +1,19 @@
 emqx_mgmt_api_configs {
 
-get_conf_node.desc:
+get_configs.desc:
-"""Get all the configurations of the specified node, including hot and non-hot updatable items."""
+"""Get all the configurations of the specified keys, including hot and non-hot updatable items."""
-get_conf_node.label:
+get_configs.label:
-"""Get all the configurations for node."""
+"""Get all the configurations."""
+
+update_configs.desc:
+"""Update the configurations of the specified keys."""
+update_configs.label:
+"""Update Configurations."""
 
 node_name.desc:
-"""Node's name. If not specified, the configs on the node which receives the HTTP request will be returned."""
+"""Node's name. Will deprecated in 5.2.0."""
 node_name.label:
-"""Node's name"""
+"""Node's name (deprecated)."""
 
 rest_conf_query.desc:
 """Reset the config entry specified by the query string parameter `conf_path`.<br/>