Merge pull request #11180 from zhongwencool/conf-load-api
feat: add conf load api
This commit is contained in:
zhongwencool
GitHub
commit
a45a16351c
|
|
@ -532,7 +532,8 @@ schema(SchemaModule, [RootKey | _]) ->
|
|||
{Field, Translations} =
|
||||
case lists:keyfind(bin(RootKey), 1, Roots) of
|
||||
{_, {Ref, ?REF(Ref)}} -> {Ref, ?R_REF(SchemaModule, Ref)};
|
||||
{_, {Name, Field0}} -> parse_translations(Field0, Name, SchemaModule)
|
||||
{_, {Name, Field0}} -> parse_translations(Field0, Name, SchemaModule);
|
||||
false -> throw({root_key_not_found, RootKey})
|
||||
end,
|
||||
#{
|
||||
roots => [Field],
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@
|
|||
-export([reset/2, reset/3]).
|
||||
-export([dump_schema/2]).
|
||||
-export([schema_module/0]).
|
||||
-export([check_config/2]).
|
||||
|
||||
%% TODO: move to emqx_dashboard when we stop building api schema at build time
|
||||
-export([
|
||||
|
|
@ -208,15 +207,6 @@ schema_module() ->
|
|||
Value -> list_to_existing_atom(Value)
|
||||
end.
|
||||
|
||||
check_config(Mod, Raw) ->
|
||||
try
|
||||
{_AppEnvs, CheckedConf} = emqx_config:check_config(Mod, Raw),
|
||||
{ok, CheckedConf}
|
||||
catch
|
||||
throw:Error ->
|
||||
{error, Error}
|
||||
end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% Internal functions
|
||||
%%--------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@
|
|||
unload/0
|
||||
]).
|
||||
|
||||
-export([keys/0, get_config/0, get_config/1, load_config/2]).
|
||||
|
||||
-include_lib("hocon/include/hoconsc.hrl").
|
||||
|
||||
%% kept cluster_call for compatibility
|
||||
|
|
@ -42,7 +44,7 @@ unload() ->
|
|||
emqx_ctl:unregister_command(?CONF).
|
||||
|
||||
conf(["show_keys" | _]) ->
|
||||
print_keys(get_config());
|
||||
print_keys(keys());
|
||||
conf(["show"]) ->
|
||||
print_hocon(get_config());
|
||||
conf(["show", Key]) ->
|
||||
|
|
@ -150,9 +152,9 @@ status() ->
|
|||
),
|
||||
emqx_ctl:print("-----------------------------------------------\n").
|
||||
|
||||
print_keys(Config) ->
|
||||
Keys = lists:sort(maps:keys(Config)),
|
||||
emqx_ctl:print("~1p~n", [[binary_to_existing_atom(K) || K <- Keys]]).
|
||||
print_keys(Keys) ->
|
||||
SortKeys = lists:sort(Keys),
|
||||
emqx_ctl:print("~1p~n", [[binary_to_existing_atom(K) || K <- SortKeys]]).
|
||||
|
||||
print(Json) ->
|
||||
emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Json)]).
|
||||
|
|
@ -166,6 +168,9 @@ get_config() ->
|
|||
AllConf = fill_defaults(emqx:get_raw_config([])),
|
||||
drop_hidden_roots(AllConf).
|
||||
|
||||
keys() ->
|
||||
emqx_config:get_root_names() -- hidden_roots().
|
||||
|
||||
drop_hidden_roots(Conf) ->
|
||||
lists:foldl(fun(K, Acc) -> maps:remove(K, Acc) end, Conf, hidden_roots()).
|
||||
|
||||
|
|
@ -186,12 +191,26 @@ get_config(Key) ->
|
|||
end.
|
||||
|
||||
-define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}).
|
||||
load_config(Path, ReplaceOrMerge) ->
|
||||
load_config(Path, ReplaceOrMerge) when is_list(Path) ->
|
||||
case hocon:files([Path]) of
|
||||
{ok, RawConf} when RawConf =:= #{} ->
|
||||
emqx_ctl:warning("load ~ts is empty~n", [Path]),
|
||||
{error, empty_hocon_file};
|
||||
{ok, RawConf} ->
|
||||
load_config_from_raw(RawConf, ReplaceOrMerge);
|
||||
{error, Reason} ->
|
||||
emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]),
|
||||
{error, bad_hocon_file}
|
||||
end;
|
||||
load_config(Bin, ReplaceOrMerge) when is_binary(Bin) ->
|
||||
case hocon:binary(Bin) of
|
||||
{ok, RawConf} ->
|
||||
load_config_from_raw(RawConf, ReplaceOrMerge);
|
||||
{error, Reason} ->
|
||||
{error, Reason}
|
||||
end.
|
||||
|
||||
load_config_from_raw(RawConf, ReplaceOrMerge) ->
|
||||
case check_config(RawConf) of
|
||||
ok ->
|
||||
lists:foreach(
|
||||
|
|
@ -199,13 +218,13 @@ load_config(Path, ReplaceOrMerge) ->
|
|||
to_sorted_list(RawConf)
|
||||
);
|
||||
{error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} ->
|
||||
emqx_ctl:warning("load ~ts failed~n~ts~n", [Path, Reason]),
|
||||
emqx_ctl:warning("load config failed~n~ts~n", [Reason]),
|
||||
emqx_ctl:warning(
|
||||
"Maybe try `emqx_ctl conf reload` to reload etc/emqx.conf on local node~n"
|
||||
),
|
||||
{error, Reason};
|
||||
{error, Errors} ->
|
||||
emqx_ctl:warning("load ~ts schema check failed~n", [Path]),
|
||||
emqx_ctl:warning("load schema check failed~n"),
|
||||
lists:foreach(
|
||||
fun({Key, Error}) ->
|
||||
emqx_ctl:warning("~ts: ~p~n", [Key, Error])
|
||||
|
|
@ -213,10 +232,6 @@ load_config(Path, ReplaceOrMerge) ->
|
|||
Errors
|
||||
),
|
||||
{error, Errors}
|
||||
end;
|
||||
{error, Reason} ->
|
||||
emqx_ctl:warning("load ~ts failed~n~p~n", [Path, Reason]),
|
||||
{error, bad_hocon_file}
|
||||
end.
|
||||
|
||||
update_config_cluster(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge) ->
|
||||
|
|
@ -265,8 +280,7 @@ check_keys_is_not_readonly(Conf) ->
|
|||
check_config_schema(Conf) ->
|
||||
SchemaMod = emqx_conf:schema_module(),
|
||||
Fold = fun({Key, Value}, Acc) ->
|
||||
Schema = emqx_config_handler:schema(SchemaMod, [Key]),
|
||||
case emqx_conf:check_config(Schema, #{Key => Value}) of
|
||||
case check_config(SchemaMod, Key, Value) of
|
||||
{ok, _} -> Acc;
|
||||
{error, Reason} -> [{Key, Reason} | Acc]
|
||||
end
|
||||
|
|
@ -319,11 +333,12 @@ load_etc_config_file() ->
|
|||
filter_readonly_config(Raw) ->
|
||||
SchemaMod = emqx_conf:schema_module(),
|
||||
RawDefault = fill_defaults(Raw),
|
||||
case emqx_conf:check_config(SchemaMod, RawDefault) of
|
||||
{ok, _CheckedConf} ->
|
||||
try
|
||||
_ = emqx_config:check_config(SchemaMod, RawDefault),
|
||||
ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
|
||||
{ok, maps:without(ReadOnlyKeys, Raw)};
|
||||
{error, Error} ->
|
||||
{ok, maps:without(ReadOnlyKeys, Raw)}
|
||||
catch
|
||||
throw:Error ->
|
||||
?SLOG(error, #{
|
||||
msg => "bad_etc_config_schema_found",
|
||||
error => Error
|
||||
|
|
@ -377,3 +392,13 @@ filter_cluster_conf(#{<<"cluster">> := #{<<"discovery_strategy">> := Strategy} =
|
|||
Conf#{<<"cluster">> => Cluster1};
|
||||
filter_cluster_conf(Conf) ->
|
||||
Conf.
|
||||
|
||||
check_config(SchemaMod, Key, Value) ->
|
||||
try
|
||||
Schema = emqx_config_handler:schema(SchemaMod, [Key]),
|
||||
{_AppEnvs, CheckedConf} = emqx_config:check_config(Schema, #{Key => Value}),
|
||||
{ok, CheckedConf}
|
||||
catch
|
||||
throw:Error ->
|
||||
{error, Error}
|
||||
end.
|
||||
|
|
|
|||
|
|
@ -44,9 +44,34 @@
|
|||
<<"sys_topics">>,
|
||||
<<"sysmon">>,
|
||||
<<"log">>
|
||||
%% <<"zones">>
|
||||
]).
|
||||
|
||||
%% erlfmt-ignore
|
||||
-define(SYSMON_EXAMPLE,
|
||||
<<"""
|
||||
sysmon {
|
||||
os {
|
||||
cpu_check_interval = 60s
|
||||
cpu_high_watermark = 80%
|
||||
cpu_low_watermark = 60%
|
||||
mem_check_interval = 60s
|
||||
procmem_high_watermark = 5%
|
||||
sysmem_high_watermark = 70%
|
||||
}
|
||||
vm {
|
||||
busy_dist_port = true
|
||||
busy_port = true
|
||||
large_heap = 32MB
|
||||
long_gc = disabled
|
||||
long_schedule = 240ms
|
||||
process_check_interval = 30s
|
||||
process_high_watermark = 80%
|
||||
process_low_watermark = 60%
|
||||
}
|
||||
}
|
||||
""">>
|
||||
).
|
||||
|
||||
api_spec() ->
|
||||
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
|
||||
|
||||
|
|
@ -66,24 +91,62 @@ schema("/configs") ->
|
|||
'operationId' => configs,
|
||||
get => #{
|
||||
tags => ?TAGS,
|
||||
description => ?DESC(get_conf_node),
|
||||
description => ?DESC(get_configs),
|
||||
parameters => [
|
||||
{key,
|
||||
hoconsc:mk(
|
||||
hoconsc:enum([binary_to_atom(K) || K <- emqx_conf_cli:keys()]),
|
||||
#{in => query, example => <<"sysmon">>, required => false}
|
||||
)},
|
||||
{node,
|
||||
hoconsc:mk(
|
||||
typerefl:atom(),
|
||||
#{
|
||||
in => query,
|
||||
required => false,
|
||||
example => <<"emqx@127.0.0.1">>,
|
||||
description => ?DESC(node_name)
|
||||
description => ?DESC(node_name),
|
||||
hidden => true
|
||||
}
|
||||
)}
|
||||
],
|
||||
responses => #{
|
||||
200 => lists:map(fun({_, Schema}) -> Schema end, config_list()),
|
||||
200 => #{
|
||||
content =>
|
||||
%% use proplists( not map) to make user text/plain is default in swagger
|
||||
[
|
||||
{'text/plain', #{
|
||||
schema => #{type => string, example => ?SYSMON_EXAMPLE}
|
||||
}},
|
||||
{'application/json', #{
|
||||
schema => #{type => object, example => #{<<"deprecated">> => true}}
|
||||
}}
|
||||
]
|
||||
},
|
||||
404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND']),
|
||||
500 => emqx_dashboard_swagger:error_codes(['BAD_NODE'])
|
||||
}
|
||||
},
|
||||
put => #{
|
||||
tags => ?TAGS,
|
||||
description => ?DESC(update_configs),
|
||||
parameters => [
|
||||
{mode,
|
||||
hoconsc:mk(
|
||||
hoconsc:enum([replace, merge]),
|
||||
#{in => query, default => merge, required => false}
|
||||
)}
|
||||
],
|
||||
'requestBody' => #{
|
||||
content =>
|
||||
#{
|
||||
'text/plain' =>
|
||||
#{schema => #{type => string, example => ?SYSMON_EXAMPLE}}
|
||||
}
|
||||
},
|
||||
responses => #{
|
||||
200 => <<"Configurations updated">>,
|
||||
400 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED'])
|
||||
}
|
||||
}
|
||||
};
|
||||
schema("/configs_reset/:rootname") ->
|
||||
|
|
@ -272,9 +335,21 @@ config_reset(post, _Params, Req) ->
|
|||
{400, #{code => 'REST_FAILED', message => ?ERR_MSG(Reason)}}
|
||||
end.
|
||||
|
||||
configs(get, Params, _Req) ->
|
||||
QS = maps:get(query_string, Params, #{}),
|
||||
Node = maps:get(<<"node">>, QS, node()),
|
||||
configs(get, #{query_string := QueryStr, headers := Headers}, _Req) ->
|
||||
%% Should deprecated json v1 since 5.2.0
|
||||
case maps:get(<<"accept">>, Headers, <<"text/plain">>) of
|
||||
<<"application/json">> -> get_configs_v1(QueryStr);
|
||||
<<"text/plain">> -> get_configs_v2(QueryStr)
|
||||
end;
|
||||
configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) ->
|
||||
case emqx_conf_cli:load_config(Conf, Mode) of
|
||||
ok -> {200};
|
||||
{error, [{_, Reason}]} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Reason)}};
|
||||
{error, Errors} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Errors)}}
|
||||
end.
|
||||
|
||||
get_configs_v1(QueryStr) ->
|
||||
Node = maps:get(<<"node">>, QueryStr, node()),
|
||||
case
|
||||
lists:member(Node, emqx:running_nodes()) andalso
|
||||
emqx_management_proto_v2:get_full_config(Node)
|
||||
|
|
@ -289,6 +364,18 @@ configs(get, Params, _Req) ->
|
|||
{200, Res}
|
||||
end.
|
||||
|
||||
get_configs_v2(QueryStr) ->
|
||||
Conf =
|
||||
case maps:find(<<"key">>, QueryStr) of
|
||||
error -> emqx_conf_cli:get_config();
|
||||
{ok, Key} -> emqx_conf_cli:get_config(atom_to_binary(Key))
|
||||
end,
|
||||
{
|
||||
200,
|
||||
#{<<"content-type">> => <<"text/plain">>},
|
||||
iolist_to_binary(hocon_pp:do(Conf, #{}))
|
||||
}.
|
||||
|
||||
limiter(get, _Params, _Req) ->
|
||||
{200, format_limiter_config(get_raw_config(limiter))};
|
||||
limiter(put, #{body := NewConf}, _Req) ->
|
||||
|
|
|
|||
|
|
@ -40,8 +40,8 @@ end_per_testcase(TestCase = t_configs_node, Config) ->
|
|||
end_per_testcase(_TestCase, Config) ->
|
||||
Config.
|
||||
|
||||
t_get(_Config) ->
|
||||
{ok, Configs} = get_configs(),
|
||||
t_get_with_json(_Config) ->
|
||||
{ok, Configs} = get_configs_with_json(),
|
||||
maps:map(
|
||||
fun(Name, Value) ->
|
||||
{ok, Config} = get_config(Name),
|
||||
|
|
@ -268,6 +268,7 @@ t_dashboard(_Config) ->
|
|||
timer:sleep(1500),
|
||||
ok.
|
||||
|
||||
%% v1 version json
|
||||
t_configs_node({'init', Config}) ->
|
||||
Node = node(),
|
||||
meck:expect(emqx, running_nodes, fun() -> [Node, bad_node, other_node] end),
|
||||
|
|
@ -286,16 +287,41 @@ t_configs_node({'end', _}) ->
|
|||
t_configs_node(_) ->
|
||||
Node = atom_to_list(node()),
|
||||
|
||||
?assertEqual({ok, <<"self">>}, get_configs(Node, #{return_all => true})),
|
||||
?assertEqual({ok, <<"other">>}, get_configs("other_node", #{return_all => true})),
|
||||
?assertEqual({ok, <<"self">>}, get_configs_with_json(Node, #{return_all => true})),
|
||||
?assertEqual({ok, <<"other">>}, get_configs_with_json("other_node", #{return_all => true})),
|
||||
|
||||
{ExpType, ExpRes} = get_configs("unknown_node", #{return_all => true}),
|
||||
{ExpType, ExpRes} = get_configs_with_json("unknown_node", #{return_all => true}),
|
||||
?assertEqual(error, ExpType),
|
||||
?assertMatch({{_, 404, _}, _, _}, ExpRes),
|
||||
{_, _, Body} = ExpRes,
|
||||
?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_utils_json:decode(Body, [return_maps])),
|
||||
|
||||
?assertMatch({error, {_, 500, _}}, get_configs("bad_node")).
|
||||
?assertMatch({error, {_, 500, _}}, get_configs_with_json("bad_node")).
|
||||
|
||||
%% v2 version binary
|
||||
t_configs_key(_Config) ->
|
||||
Keys = lists:sort(emqx_conf_cli:keys()),
|
||||
{ok, Hocon} = get_configs_with_binary(undefined),
|
||||
?assertEqual(Keys, lists:sort(maps:keys(Hocon))),
|
||||
{ok, Log} = get_configs_with_binary("log"),
|
||||
?assertMatch(
|
||||
#{
|
||||
<<"log">> := #{
|
||||
<<"console">> := #{
|
||||
<<"enable">> := _,
|
||||
<<"formatter">> := <<"text">>,
|
||||
<<"level">> := <<"warning">>,
|
||||
<<"time_offset">> := <<"system">>
|
||||
},
|
||||
<<"file">> := _
|
||||
}
|
||||
},
|
||||
Log
|
||||
),
|
||||
Log1 = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"error">>),
|
||||
?assertEqual([], update_configs_with_binary(iolist_to_binary(hocon_pp:do(Log1, #{})))),
|
||||
?assertEqual(<<"error">>, read_conf([<<"log">>, <<"console">>, <<"level">>])),
|
||||
ok.
|
||||
|
||||
%% Helpers
|
||||
|
||||
|
|
@ -308,25 +334,52 @@ get_config(Name) ->
|
|||
Error
|
||||
end.
|
||||
|
||||
get_configs() ->
|
||||
get_configs([], #{}).
|
||||
get_configs_with_json() ->
|
||||
get_configs_with_json([], #{}).
|
||||
|
||||
get_configs(Node) ->
|
||||
get_configs(Node, #{}).
|
||||
get_configs_with_json(Node) ->
|
||||
get_configs_with_json(Node, #{}).
|
||||
|
||||
get_configs(Node, Opts) ->
|
||||
get_configs_with_json(Node, Opts) ->
|
||||
Path =
|
||||
case Node of
|
||||
[] -> ["configs"];
|
||||
_ -> ["configs?node=" ++ Node]
|
||||
end,
|
||||
URI = emqx_mgmt_api_test_util:api_path(Path),
|
||||
case emqx_mgmt_api_test_util:request_api(get, URI, [], [], [], Opts) of
|
||||
Auth = emqx_mgmt_api_test_util:auth_header_(),
|
||||
Headers = [{"accept", "application/json"}, Auth],
|
||||
case emqx_mgmt_api_test_util:request_api(get, URI, [], Headers, [], Opts) of
|
||||
{ok, {_, _, Res}} -> {ok, emqx_utils_json:decode(Res, [return_maps])};
|
||||
{ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])};
|
||||
Error -> Error
|
||||
end.
|
||||
|
||||
get_configs_with_binary(Key) ->
|
||||
Path =
|
||||
case Key of
|
||||
undefined -> ["configs"];
|
||||
_ -> ["configs?key=" ++ Key]
|
||||
end,
|
||||
URI = emqx_mgmt_api_test_util:api_path(Path),
|
||||
Auth = emqx_mgmt_api_test_util:auth_header_(),
|
||||
Headers = [{"accept", "text/plain"}, Auth],
|
||||
case emqx_mgmt_api_test_util:request_api(get, URI, [], Headers, [], #{return_all => true}) of
|
||||
{ok, {_, _, Res}} -> hocon:binary(Res);
|
||||
{ok, Res} -> hocon:binary(Res);
|
||||
Error -> Error
|
||||
end.
|
||||
|
||||
update_configs_with_binary(Bin) ->
|
||||
Path = emqx_mgmt_api_test_util:api_path(["configs"]),
|
||||
Auth = emqx_mgmt_api_test_util:auth_header_(),
|
||||
Headers = [{"accept", "text/plain"}, Auth],
|
||||
case httpc:request(put, {Path, Headers, "text/plain", Bin}, [], []) of
|
||||
{ok, {_, _, Res}} -> Res;
|
||||
{ok, Res} -> Res;
|
||||
Error -> Error
|
||||
end.
|
||||
|
||||
update_config(Name, Change) ->
|
||||
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
|
||||
UpdatePath = emqx_mgmt_api_test_util:api_path(["configs", Name]),
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Adding a new configuration API `/configs`(GET/PUT) that supports to reload the hocon format configuration file.
|
||||
|
|
@ -1,14 +1,19 @@
|
|||
emqx_mgmt_api_configs {
|
||||
|
||||
get_conf_node.desc:
|
||||
"""Get all the configurations of the specified node, including hot and non-hot updatable items."""
|
||||
get_conf_node.label:
|
||||
"""Get all the configurations for node."""
|
||||
get_configs.desc:
|
||||
"""Get all the configurations of the specified keys, including hot and non-hot updatable items."""
|
||||
get_configs.label:
|
||||
"""Get all the configurations."""
|
||||
|
||||
update_configs.desc:
|
||||
"""Update the configurations of the specified keys."""
|
||||
update_configs.label:
|
||||
"""Update Configurations."""
|
||||
|
||||
node_name.desc:
|
||||
"""Node's name. If not specified, the configs on the node which receives the HTTP request will be returned."""
|
||||
"""Node's name. Will deprecated in 5.2.0."""
|
||||
node_name.label:
|
||||
"""Node's name"""
|
||||
"""Node's name (deprecated)."""
|
||||
|
||||
rest_conf_query.desc:
|
||||
"""Reset the config entry specified by the query string parameter `conf_path`.<br/>
|
||||
|
|
|
|||
Loading…
Reference in New Issue