diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 3752d67ab..96690c26e 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -532,7 +532,8 @@ schema(SchemaModule, [RootKey | _]) -> {Field, Translations} = case lists:keyfind(bin(RootKey), 1, Roots) of {_, {Ref, ?REF(Ref)}} -> {Ref, ?R_REF(SchemaModule, Ref)}; - {_, {Name, Field0}} -> parse_translations(Field0, Name, SchemaModule) + {_, {Name, Field0}} -> parse_translations(Field0, Name, SchemaModule); + false -> throw({root_key_not_found, RootKey}) end, #{ roots => [Field], diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 51c353edf..d5f39bcb0 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -30,7 +30,6 @@ -export([reset/2, reset/3]). -export([dump_schema/2]). -export([schema_module/0]). --export([check_config/2]). %% TODO: move to emqx_dashboard when we stop building api schema at build time -export([ @@ -208,15 +207,6 @@ schema_module() -> Value -> list_to_existing_atom(Value) end. -check_config(Mod, Raw) -> - try - {_AppEnvs, CheckedConf} = emqx_config:check_config(Mod, Raw), - {ok, CheckedConf} - catch - throw:Error -> - {error, Error} - end. - %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 44dda04f7..05eb60531 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -26,6 +26,8 @@ unload/0 ]). +-export([keys/0, get_config/0, get_config/1, load_config/2]). + -include_lib("hocon/include/hoconsc.hrl"). %% kept cluster_call for compatibility @@ -42,7 +44,7 @@ unload() -> emqx_ctl:unregister_command(?CONF). conf(["show_keys" | _]) -> - print_keys(get_config()); + print_keys(keys()); conf(["show"]) -> print_hocon(get_config()); conf(["show", Key]) -> @@ -150,9 +152,9 @@ status() -> ), emqx_ctl:print("-----------------------------------------------\n"). -print_keys(Config) -> - Keys = lists:sort(maps:keys(Config)), - emqx_ctl:print("~1p~n", [[binary_to_existing_atom(K) || K <- Keys]]). +print_keys(Keys) -> + SortKeys = lists:sort(Keys), + emqx_ctl:print("~1p~n", [[binary_to_existing_atom(K) || K <- SortKeys]]). print(Json) -> emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Json)]). @@ -166,6 +168,9 @@ get_config() -> AllConf = fill_defaults(emqx:get_raw_config([])), drop_hidden_roots(AllConf). +keys() -> + emqx_config:get_root_names() -- hidden_roots(). + drop_hidden_roots(Conf) -> lists:foldl(fun(K, Acc) -> maps:remove(K, Acc) end, Conf, hidden_roots()). @@ -186,37 +191,47 @@ get_config(Key) -> end. -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). -load_config(Path, ReplaceOrMerge) -> +load_config(Path, ReplaceOrMerge) when is_list(Path) -> case hocon:files([Path]) of {ok, RawConf} when RawConf =:= #{} -> emqx_ctl:warning("load ~ts is empty~n", [Path]), {error, empty_hocon_file}; {ok, RawConf} -> - case check_config(RawConf) of - ok -> - lists:foreach( - fun({K, V}) -> update_config_cluster(K, V, ReplaceOrMerge) end, - to_sorted_list(RawConf) - ); - {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} -> - emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]), - emqx_ctl:warning( - "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n" - ), - {error, Reason}; - {error, Errors} -> - emqx_ctl:warning("load ~ts schema check failed~n", [Path]), - lists:foreach( - fun({Key, Error}) -> - emqx_ctl:warning("~ts: ~p~n", [Key, Error]) - end, - Errors - ), - {error, Errors} - end; + load_config_from_raw(RawConf, ReplaceOrMerge); {error, Reason} -> emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]), {error, bad_hocon_file} + end; +load_config(Bin, ReplaceOrMerge) when is_binary(Bin) -> + case hocon:binary(Bin) of + {ok, RawConf} -> + load_config_from_raw(RawConf, ReplaceOrMerge); + {error, Reason} -> + {error, Reason} + end. + +load_config_from_raw(RawConf, ReplaceOrMerge) -> + case check_config(RawConf) of + ok -> + lists:foreach( + fun({K, V}) -> update_config_cluster(K, V, ReplaceOrMerge) end, + to_sorted_list(RawConf) + ); + {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} -> + emqx_ctl:warning("load config failed~n~ts~n", [Reason]), + emqx_ctl:warning( + "Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n" + ), + {error, Reason}; + {error, Errors} -> + emqx_ctl:warning("load schema check failed~n"), + lists:foreach( + fun({Key, Error}) -> + emqx_ctl:warning("~ts: ~p~n", [Key, Error]) + end, + Errors + ), + {error, Errors} end. update_config_cluster(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge) -> @@ -265,8 +280,7 @@ check_keys_is_not_readonly(Conf) -> check_config_schema(Conf) -> SchemaMod = emqx_conf:schema_module(), Fold = fun({Key, Value}, Acc) -> - Schema = emqx_config_handler:schema(SchemaMod, [Key]), - case emqx_conf:check_config(Schema, #{Key => Value}) of + case check_config(SchemaMod, Key, Value) of {ok, _} -> Acc; {error, Reason} -> [{Key, Reason} | Acc] end @@ -319,11 +333,12 @@ load_etc_config_file() -> filter_readonly_config(Raw) -> SchemaMod = emqx_conf:schema_module(), RawDefault = fill_defaults(Raw), - case emqx_conf:check_config(SchemaMod, RawDefault) of - {ok, _CheckedConf} -> - ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], - {ok, maps:without(ReadOnlyKeys, Raw)}; - {error, Error} -> + try + _ = emqx_config:check_config(SchemaMod, RawDefault), + ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], + {ok, maps:without(ReadOnlyKeys, Raw)} + catch + throw:Error -> ?SLOG(error, #{ msg => "bad_etc_config_schema_found", error => Error @@ -377,3 +392,13 @@ filter_cluster_conf(#{<<"cluster">> := #{<<"discovery_strategy">> := Strategy} = Conf#{<<"cluster">> => Cluster1}; filter_cluster_conf(Conf) -> Conf. + +check_config(SchemaMod, Key, Value) -> + try + Schema = emqx_config_handler:schema(SchemaMod, [Key]), + {_AppEnvs, CheckedConf} = emqx_config:check_config(Schema, #{Key => Value}), + {ok, CheckedConf} + catch + throw:Error -> + {error, Error} + end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 3a93bd0be..6991bb11c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -44,9 +44,34 @@ <<"sys_topics">>, <<"sysmon">>, <<"log">> - %% <<"zones">> ]). +%% erlfmt-ignore +-define(SYSMON_EXAMPLE, + <<""" + sysmon { + os { + cpu_check_interval = 60s + cpu_high_watermark = 80% + cpu_low_watermark = 60% + mem_check_interval = 60s + procmem_high_watermark = 5% + sysmem_high_watermark = 70% + } + vm { + busy_dist_port = true + busy_port = true + large_heap = 32MB + long_gc = disabled + long_schedule = 240ms + process_check_interval = 30s + process_high_watermark = 80% + process_low_watermark = 60% + } + } + """>> +). + api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -66,24 +91,62 @@ schema("/configs") -> 'operationId' => configs, get => #{ tags => ?TAGS, - description => ?DESC(get_conf_node), + description => ?DESC(get_configs), parameters => [ + {key, + hoconsc:mk( + hoconsc:enum([binary_to_atom(K) || K <- emqx_conf_cli:keys()]), + #{in => query, example => <<"sysmon">>, required => false} + )}, {node, hoconsc:mk( typerefl:atom(), #{ in => query, required => false, - example => <<"emqx@127.0.0.1">>, - description => ?DESC(node_name) + description => ?DESC(node_name), + hidden => true } )} ], responses => #{ - 200 => lists:map(fun({_, Schema}) -> Schema end, config_list()), + 200 => #{ + content => + %% use proplists( not map) to make user text/plain is default in swagger + [ + {'text/plain', #{ + schema => #{type => string, example => ?SYSMON_EXAMPLE} + }}, + {'application/json', #{ + schema => #{type => object, example => #{<<"deprecated">> => true}} + }} + ] + }, 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND']), 500 => emqx_dashboard_swagger:error_codes(['BAD_NODE']) } + }, + put => #{ + tags => ?TAGS, + description => ?DESC(update_configs), + parameters => [ + {mode, + hoconsc:mk( + hoconsc:enum([replace, merge]), + #{in => query, default => merge, required => false} + )} + ], + 'requestBody' => #{ + content => + #{ + 'text/plain' => + #{schema => #{type => string, example => ?SYSMON_EXAMPLE}} + } + }, + responses => #{ + 200 => <<"Configurations updated">>, + 400 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED']) + } } }; schema("/configs_reset/:rootname") -> @@ -272,9 +335,21 @@ config_reset(post, _Params, Req) -> {400, #{code => 'REST_FAILED', message => ?ERR_MSG(Reason)}} end. -configs(get, Params, _Req) -> - QS = maps:get(query_string, Params, #{}), - Node = maps:get(<<"node">>, QS, node()), +configs(get, #{query_string := QueryStr, headers := Headers}, _Req) -> + %% Should deprecated json v1 since 5.2.0 + case maps:get(<<"accept">>, Headers, <<"text/plain">>) of + <<"application/json">> -> get_configs_v1(QueryStr); + <<"text/plain">> -> get_configs_v2(QueryStr) + end; +configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> + case emqx_conf_cli:load_config(Conf, Mode) of + ok -> {200}; + {error, [{_, Reason}]} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Reason)}}; + {error, Errors} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Errors)}} + end. + +get_configs_v1(QueryStr) -> + Node = maps:get(<<"node">>, QueryStr, node()), case lists:member(Node, emqx:running_nodes()) andalso emqx_management_proto_v2:get_full_config(Node) @@ -289,6 +364,18 @@ configs(get, Params, _Req) -> {200, Res} end. +get_configs_v2(QueryStr) -> + Conf = + case maps:find(<<"key">>, QueryStr) of + error -> emqx_conf_cli:get_config(); + {ok, Key} -> emqx_conf_cli:get_config(atom_to_binary(Key)) + end, + { + 200, + #{<<"content-type">> => <<"text/plain">>}, + iolist_to_binary(hocon_pp:do(Conf, #{})) + }. + limiter(get, _Params, _Req) -> {200, format_limiter_config(get_raw_config(limiter))}; limiter(put, #{body := NewConf}, _Req) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 0d7d57864..0e54a3e22 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -40,8 +40,8 @@ end_per_testcase(TestCase = t_configs_node, Config) -> end_per_testcase(_TestCase, Config) -> Config. -t_get(_Config) -> - {ok, Configs} = get_configs(), +t_get_with_json(_Config) -> + {ok, Configs} = get_configs_with_json(), maps:map( fun(Name, Value) -> {ok, Config} = get_config(Name), @@ -268,6 +268,7 @@ t_dashboard(_Config) -> timer:sleep(1500), ok. +%% v1 version json t_configs_node({'init', Config}) -> Node = node(), meck:expect(emqx, running_nodes, fun() -> [Node, bad_node, other_node] end), @@ -286,16 +287,41 @@ t_configs_node({'end', _}) -> t_configs_node(_) -> Node = atom_to_list(node()), - ?assertEqual({ok, <<"self">>}, get_configs(Node, #{return_all => true})), - ?assertEqual({ok, <<"other">>}, get_configs("other_node", #{return_all => true})), + ?assertEqual({ok, <<"self">>}, get_configs_with_json(Node, #{return_all => true})), + ?assertEqual({ok, <<"other">>}, get_configs_with_json("other_node", #{return_all => true})), - {ExpType, ExpRes} = get_configs("unknown_node", #{return_all => true}), + {ExpType, ExpRes} = get_configs_with_json("unknown_node", #{return_all => true}), ?assertEqual(error, ExpType), ?assertMatch({{_, 404, _}, _, _}, ExpRes), {_, _, Body} = ExpRes, ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_utils_json:decode(Body, [return_maps])), - ?assertMatch({error, {_, 500, _}}, get_configs("bad_node")). + ?assertMatch({error, {_, 500, _}}, get_configs_with_json("bad_node")). + +%% v2 version binary +t_configs_key(_Config) -> + Keys = lists:sort(emqx_conf_cli:keys()), + {ok, Hocon} = get_configs_with_binary(undefined), + ?assertEqual(Keys, lists:sort(maps:keys(Hocon))), + {ok, Log} = get_configs_with_binary("log"), + ?assertMatch( + #{ + <<"log">> := #{ + <<"console">> := #{ + <<"enable">> := _, + <<"formatter">> := <<"text">>, + <<"level">> := <<"warning">>, + <<"time_offset">> := <<"system">> + }, + <<"file">> := _ + } + }, + Log + ), + Log1 = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"error">>), + ?assertEqual([], update_configs_with_binary(iolist_to_binary(hocon_pp:do(Log1, #{})))), + ?assertEqual(<<"error">>, read_conf([<<"log">>, <<"console">>, <<"level">>])), + ok. %% Helpers @@ -308,25 +334,52 @@ get_config(Name) -> Error end. -get_configs() -> - get_configs([], #{}). +get_configs_with_json() -> + get_configs_with_json([], #{}). -get_configs(Node) -> - get_configs(Node, #{}). +get_configs_with_json(Node) -> + get_configs_with_json(Node, #{}). -get_configs(Node, Opts) -> +get_configs_with_json(Node, Opts) -> Path = case Node of [] -> ["configs"]; _ -> ["configs?node=" ++ Node] end, URI = emqx_mgmt_api_test_util:api_path(Path), - case emqx_mgmt_api_test_util:request_api(get, URI, [], [], [], Opts) of + Auth = emqx_mgmt_api_test_util:auth_header_(), + Headers = [{"accept", "application/json"}, Auth], + case emqx_mgmt_api_test_util:request_api(get, URI, [], Headers, [], Opts) of {ok, {_, _, Res}} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. +get_configs_with_binary(Key) -> + Path = + case Key of + undefined -> ["configs"]; + _ -> ["configs?key=" ++ Key] + end, + URI = emqx_mgmt_api_test_util:api_path(Path), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Headers = [{"accept", "text/plain"}, Auth], + case emqx_mgmt_api_test_util:request_api(get, URI, [], Headers, [], #{return_all => true}) of + {ok, {_, _, Res}} -> hocon:binary(Res); + {ok, Res} -> hocon:binary(Res); + Error -> Error + end. + +update_configs_with_binary(Bin) -> + Path = emqx_mgmt_api_test_util:api_path(["configs"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Headers = [{"accept", "text/plain"}, Auth], + case httpc:request(put, {Path, Headers, "text/plain", Bin}, [], []) of + {ok, {_, _, Res}} -> Res; + {ok, Res} -> Res; + Error -> Error + end. + update_config(Name, Change) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), UpdatePath = emqx_mgmt_api_test_util:api_path(["configs", Name]), diff --git a/changes/ce/feat-11180.en.md b/changes/ce/feat-11180.en.md new file mode 100644 index 000000000..0f3d45e23 --- /dev/null +++ b/changes/ce/feat-11180.en.md @@ -0,0 +1 @@ +Adding a new configuration API `/configs`(GET/PUT) that supports to reload the hocon format configuration file. diff --git a/rel/i18n/emqx_mgmt_api_configs.hocon b/rel/i18n/emqx_mgmt_api_configs.hocon index 292658900..47852349e 100644 --- a/rel/i18n/emqx_mgmt_api_configs.hocon +++ b/rel/i18n/emqx_mgmt_api_configs.hocon @@ -1,14 +1,19 @@ emqx_mgmt_api_configs { -get_conf_node.desc: -"""Get all the configurations of the specified node, including hot and non-hot updatable items.""" -get_conf_node.label: -"""Get all the configurations for node.""" +get_configs.desc: +"""Get all the configurations of the specified keys, including hot and non-hot updatable items.""" +get_configs.label: +"""Get all the configurations.""" + +update_configs.desc: +"""Update the configurations of the specified keys.""" +update_configs.label: +"""Update Configurations.""" node_name.desc: -"""Node's name. If not specified, the configs on the node which receives the HTTP request will be returned.""" +"""Node's name. Will deprecated in 5.2.0.""" node_name.label: -"""Node's name""" +"""Node's name (deprecated).""" rest_conf_query.desc: """Reset the config entry specified by the query string parameter `conf_path`.