From d80f20aca3b1b66db2a5e359a11363c9438506e6 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 13 Oct 2021 14:04:43 +0800 Subject: [PATCH] feat(swagger): define mgmt config_api by hocon schema (#5814) * feat(swagger): define mgmt config_api by hocon schema * fix: enum can't defined by integer, use union. * fix: hocon schema union to enum --- .../src/emqx_auto_subscribe_schema.erl | 10 +- .../src/emqx_connector_http.erl | 6 +- .../src/emqx_dashboard_swagger.erl | 136 +++++++++--- .../test/emqx_swagger_parameter_SUITE.erl | 6 +- .../test/emqx_swagger_requestBody_SUITE.erl | 2 +- .../test/emqx_swagger_response_SUITE.erl | 73 ++++++- apps/emqx_exhook/src/emqx_exhook_schema.erl | 4 +- apps/emqx_gateway/src/coap/emqx_coap_api.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 2 +- .../src/emqx_mgmt_api_configs.erl | 200 ++++++++++-------- apps/emqx_modules/src/emqx_modules_schema.erl | 4 +- 11 files changed, 288 insertions(+), 157 deletions(-) diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl index 5b781455d..ba4b058ed 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl @@ -34,10 +34,12 @@ fields("auto_subscribe") -> fields("topic") -> [ {topic, sc(binary(), #{})} - , {qos, sc(typerefl:union([0, 1, 2]), #{default => 0})} - , {rh, sc(typerefl:union([0, 1, 2]), #{default => 0})} - , {rap, sc(typerefl:union([0, 1]), #{default => 0})} - , {nl, sc(typerefl:union([0, 1]), #{default => 0})} + , {qos, sc(hoconsc:union([typerefl:integer(0), typerefl:integer(1), typerefl:integer(2)]), + #{default => 0})} + , {rh, sc(hoconsc:union([typerefl:integer(0), typerefl:integer(1), typerefl:integer(2)]), + #{default => 0})} + , {rap, sc(hoconsc:union([typerefl:integer(0), typerefl:integer(1)]), #{default => 0})} + , {nl, sc(hoconsc:union([typerefl:integer(0), typerefl:integer(1)]), #{default => 0})} ]. %%-------------------------------------------------------------------- diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 92c7c6d64..7c71e09b3 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -79,11 +79,7 @@ fields(config) -> ] ++ emqx_connector_schema_lib:ssl_fields(). method() -> - hoconsc:union([ typerefl:atom(post) - , typerefl:atom(put) - , typerefl:atom(get) - , typerefl:atom(delete) - ]). + hoconsc:enum([post, put, get, delete]). validations() -> [ {check_ssl_opts, fun check_ssl_opts/1} ]. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 91354c05a..47453d246 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -103,7 +103,7 @@ error_codes(Codes = [_ | _], MsgExample) -> ]. support_check_schema(#{check_schema := true}) -> ?DEFAULT_FILTER; -support_check_schema(#{check_schema := Func})when is_function(Func, 2) -> #{filter => Func}; +support_check_schema(#{check_schema := Func}) when is_function(Func, 2) -> #{filter => Func}; support_check_schema(_) -> #{filter => undefined}. parse_spec_ref(Module, Path) -> @@ -191,11 +191,11 @@ parameters(Params, Module) -> lists:foldl(fun(Param, {Acc, RefsAcc}) -> case Param of ?REF(StructName) -> - {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(Module, StructName)} |Acc], - [{Module, StructName, parameter}|RefsAcc]}; + {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(Module, StructName)} | Acc], + [{Module, StructName, parameter} | RefsAcc]}; ?R_REF(RModule, StructName) -> - {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(RModule, StructName)} |Acc], - [{RModule, StructName, parameter}|RefsAcc]}; + {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(RModule, StructName)} | Acc], + [{RModule, StructName, parameter} | RefsAcc]}; {Name, Type} -> In = hocon_schema:field_schema(Type, in), In =:= undefined andalso throw({error, <<"missing in:path/query field in parameters">>}), @@ -300,10 +300,13 @@ components([{Module, Field, parameter} | Refs], SpecAcc, SubRefsAcc) -> NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Param}, components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc). +%% Semantic error at components.schemas.xxx:xx:xx +%% Component names can only contain the characters A-Z a-z 0-9 - . _ +%% So replace ':' by '-'. namespace(Module) -> case hocon_schema:namespace(Module) of undefined -> Module; - NameSpace -> NameSpace + NameSpace -> re:replace(to_bin(NameSpace), ":","-",[global]) end. hocon_schema_to_spec(?R_REF(Module, StructName), _LocalModule) -> @@ -312,13 +315,20 @@ hocon_schema_to_spec(?R_REF(Module, StructName), _LocalModule) -> hocon_schema_to_spec(?REF(StructName), LocalModule) -> {#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(LocalModule, StructName)}, [{LocalModule, StructName}]}; -hocon_schema_to_spec(Type, _LocalModule) when ?IS_TYPEREFL(Type) -> - {typename_to_spec(typerefl:name(Type)), []}; +hocon_schema_to_spec(Type, LocalModule) when ?IS_TYPEREFL(Type) -> + {typename_to_spec(typerefl:name(Type), LocalModule), []}; hocon_schema_to_spec(?ARRAY(Item), LocalModule) -> {Schema, Refs} = hocon_schema_to_spec(Item, LocalModule), {#{type => array, items => Schema}, Refs}; +hocon_schema_to_spec(?LAZY(Item), LocalModule) -> + hocon_schema_to_spec(Item, LocalModule); hocon_schema_to_spec(?ENUM(Items), _LocalModule) -> {#{type => string, enum => Items}, []}; +hocon_schema_to_spec(?MAP(Name, Type), LocalModule) -> + {Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule), + {#{<<"type">> => object, + <<"properties">> => #{<<"$", (to_bin(Name))/binary>> => Schema}}, + SubRefs}; hocon_schema_to_spec(?UNION(Types), LocalModule) -> {OneOf, Refs} = lists:foldl(fun(Type, {Acc, RefsAcc}) -> {Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule), @@ -328,38 +338,98 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) -> hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> {#{type => string, enum => [Atom]}, []}. -typename_to_spec("boolean()") -> #{type => boolean, example => true}; -typename_to_spec("binary()") -> #{type => string, example =><<"binary example">>}; -typename_to_spec("float()") -> #{type =>number, example =>3.14159}; -typename_to_spec("integer()") -> #{type =>integer, example =>100}; -typename_to_spec("number()") -> #{type =>number, example =>42}; -typename_to_spec("string()") -> #{type =>string, example =><<"string example">>}; -typename_to_spec("atom()") -> #{type =>string, example =>atom}; -typename_to_spec("duration()") -> #{type =>string, example =><<"12m">>}; -typename_to_spec("duration_s()") -> #{type =>string, example =><<"1h">>}; -typename_to_spec("duration_ms()") -> #{type =>string, example =><<"32s">>}; -typename_to_spec("percent()") -> #{type =>number, example =><<"12%">>}; -typename_to_spec("file()") -> #{type =>string, example =><<"/path/to/file">>}; -typename_to_spec("ip_port()") -> #{type => string, example =><<"127.0.0.1:80">>}; -typename_to_spec(Name) -> +%% todo: Find a way to fetch enum value from user_id_type(). +typename_to_spec("user_id_type()", _Mod) -> #{type => string, enum => [clientid, username]}; +typename_to_spec("term()", _Mod) -> #{type => string, example => "term"}; +typename_to_spec("boolean()", _Mod) -> #{type => boolean, example => true}; +typename_to_spec("binary()", _Mod) -> #{type => string, example => <<"binary-example">>}; +typename_to_spec("float()", _Mod) -> #{type => number, example => 3.14159}; +typename_to_spec("integer()", _Mod) -> #{type => integer, example => 100}; +typename_to_spec("non_neg_integer()", _Mod) -> #{type => integer, minimum => 1, example => 100}; +typename_to_spec("number()", _Mod) -> #{type => number, example => 42}; +typename_to_spec("string()", _Mod) -> #{type => string, example => <<"string-example">>}; +typename_to_spec("atom()", _Mod) -> #{type => string, example => atom}; +typename_to_spec("duration()", _Mod) -> #{type => string, example => <<"12m">>}; +typename_to_spec("duration_s()", _Mod) -> #{type => string, example => <<"1h">>}; +typename_to_spec("duration_ms()", _Mod) -> #{type => string, example => <<"32s">>}; +typename_to_spec("percent()", _Mod) -> #{type => number, example => <<"12%">>}; +typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>}; +typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>}; +typename_to_spec("url()", _Mod) -> #{type => string, example => <<"http://127.0.0.1">>}; +typename_to_spec("server()", Mod) -> typename_to_spec("ip_port()", Mod); +typename_to_spec("connect_timeout()", Mod) -> typename_to_spec("timeout()", Mod); +typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, example => infinity}, + #{type => integer, example => 100}], example => infinity}; +typename_to_spec("bytesize()", _Mod) -> #{type => string, example => <<"32MB">>}; +typename_to_spec("wordsize()", _Mod) -> #{type => string, example => <<"1024KB">>}; +typename_to_spec("map()", _Mod) -> #{type => string, example => <<>>}; +typename_to_spec("comma_separated_list()", _Mod) -> #{type => string, example => <<"item1,item2">>}; +typename_to_spec("comma_separated_atoms()", _Mod) -> #{type => string, example => <<"item1,item2">>}; +typename_to_spec("pool_type()", _Mod) -> #{type => string, enum => [random, hash], example => hash}; +typename_to_spec("log_level()", _Mod) -> + #{type => string, enum => [debug, info, notice, warning, error, critical, alert, emergency, all]}; +typename_to_spec(Name, Mod) -> + Spec = range(Name), + Spec1 = remote_module_type(Spec, Name, Mod), + Spec2 = typerefl_array(Spec1, Name, Mod), + Spec3 = integer(Spec2, Name), + Spec3 =:= nomatch andalso + throw({error, #{msg => <<"Unsupport Type">>, type => Name, module => Mod}}), + Spec3. + +range(Name) -> case string:split(Name, "..") of - [MinStr, MaxStr] -> %% 1..10 - {Min, []} = string:to_integer(MinStr), - {Max, []} = string:to_integer(MaxStr), - #{type => integer, example => Min, minimum => Min, maximum => Max}; - _ -> %% Module:Type(). - case string:split(Name, ":") of - [_Module, Type] -> typename_to_spec(Type); - _ -> throw({error, #{msg => <<"Unsupport Type">>, type => Name}}) - end + [MinStr, MaxStr] -> %% 1..10 1..inf -inf..10 + Schema = #{type => integer}, + Schema1 = add_integer_prop(Schema, minimum, MinStr), + add_integer_prop(Schema1, maximum, MaxStr); + _ -> nomatch end. -to_bin(List) when is_list(List) -> list_to_binary(List); +%% Module:Type +remote_module_type(nomatch, Name, Mod) -> + case string:split(Name, ":") of + [_Module, Type] -> typename_to_spec(Type, Mod); + _ -> nomatch + end; +remote_module_type(Spec, _Name, _Mod) -> Spec. + +%% [string()] or [integer()] or [xxx]. +typerefl_array(nomatch, Name, Mod) -> + case string:trim(Name, leading, "[") of + Name -> nomatch; + Name1 -> + case string:trim(Name1, trailing, "]") of + Name1 -> notmatch; + Name2 -> + Schema = typename_to_spec(Name2, Mod), + #{type => array, items => Schema} + end + end; +typerefl_array(Spec, _Name, _Mod) -> Spec. + +%% integer(1) +integer(nomatch, Name) -> + case string:to_integer(Name) of + {Int, []} -> #{type => integer, enum => [Int], example => Int, default => Int}; + _ -> nomatch + end; +integer(Spec, _Name) -> Spec. + +add_integer_prop(Schema, Key, Value) -> + case string:to_integer(Value) of + {error, no_integer} -> Schema; + {Int, []}when Key =:= minimum -> Schema#{Key => Int, example => Int}; + {Int, []} -> Schema#{Key => Int} + end. + +to_bin([Atom | _] = List) when is_atom(Atom) -> iolist_to_binary(io_lib:format("~p", [List])); +to_bin(List) when is_list(List) -> unicode:characters_to_binary(List); to_bin(B) when is_boolean(B) -> B; to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); to_bin(X) -> X. -parse_object(PropList = [_|_], Module) when is_list(PropList) -> +parse_object(PropList = [_ | _], Module) when is_list(PropList) -> {Props, Required, Refs} = lists:foldl(fun({Name, Hocon}, {Acc, RequiredAcc, RefsAcc}) -> NameBin = to_bin(Name), diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index cea0a915d..9c9958880 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -112,15 +112,15 @@ t_without_in(_Config) -> t_require(_Config) -> ExpectSpec = [#{ in => query,name => userid, required => false, - schema => #{example => <<"binary example">>, type => string}}], + schema => #{example => <<"binary-example">>, type => string}}], validate("/required/false", ExpectSpec), ok. t_nullable(_Config) -> NullableFalse = [#{in => query,name => userid, required => true, - schema => #{example => <<"binary example">>, type => string}}], + schema => #{example => <<"binary-example">>, type => string}}], NullableTrue = [#{in => query,name => userid, - schema => #{example => <<"binary example">>, type => string, + schema => #{example => <<"binary-example">>, type => string, nullable => true}}], validate("/nullable/false", NullableFalse), validate("/nullable/true", NullableTrue), diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 84cbfe5fb..7aa986d1d 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -121,7 +121,7 @@ t_nest_ref(_Config) -> #{<<"emqx_swagger_requestBody_SUITE.good_ref">> => #{<<"properties">> => [ {<<"webhook-host">>, #{default => <<"127.0.0.1:80">>, example => <<"127.0.0.1:80">>,type => string}}, {<<"log_dir">>, #{example => <<"var/log/emqx">>,type => string}}, - {<<"tag">>, #{description => <<"tag">>, example => <<"binary example">>,type => string}}], + {<<"tag">>, #{description => <<"tag">>, example => <<"binary-example">>,type => string}}], <<"type">> => object}}]), {_, Components} = validate("/ref/nest/ref", Spec, Refs), ?assertEqual(ExpectComponents, Components), diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index fd6920549..a9969ba4b 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -13,7 +13,7 @@ -export([all/0, suite/0, groups/0]). -export([paths/0, api_spec/0, schema/1, fields/1]). -export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1, t_error/1, - t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1, + t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1, t_complicated_type/1, t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, t_ref_array_with_key/1, t_ref_array_without_key/1, t_api_spec/1]). @@ -21,7 +21,7 @@ all() -> [{group, spec}]. suite() -> [{timetrap, {minutes, 1}}]. groups() -> [ {spec, [parallel], [ - t_api_spec, t_simple_binary, t_object, t_nest_object, t_error, + t_api_spec, t_simple_binary, t_object, t_nest_object, t_error, t_complicated_type, t_raw_local_ref, t_raw_remote_ref, t_empty, t_hocon_schema_function, t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]} @@ -54,7 +54,7 @@ t_error(_Config) -> #{<<"application/json">> => #{<<"schema">> => #{<<"type">> => object, <<"properties">> => [ - {<<"code">>, #{enum => ['Bad1','Bad2'], type => string}}, + {<<"code">>, #{enum => ['Bad1', 'Bad2'], type => string}}, {<<"message">>, #{description => <<"Details description of the error.">>, example => <<"Bad request desc">>, type => string}}] }}}}, @@ -156,6 +156,38 @@ t_nest_ref(_Config) -> validate(Path, Object, ExpectRefs), ok. +t_complicated_type(_Config) -> + Path = "/ref/complicated_type", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{<<"properties">> => + [ + {<<"no_neg_integer">>, #{example => 100, minimum => 1, type => integer}}, + {<<"url">>, #{example => <<"http://127.0.0.1">>, type => string}}, + {<<"server">>, #{example => <<"127.0.0.1:80">>, type => string}}, + {<<"connect_timeout">>, #{example => infinity, <<"oneOf">> => [ + #{example => infinity, type => string}, + #{example => 100, type => integer}]}}, + {<<"pool_type">>, #{enum => [random, hash], example => hash, type => string}}, + {<<"timeout">>, #{example => infinity, + <<"oneOf">> => + [#{example => infinity, type => string}, #{example => 100, type => integer}]}}, + {<<"bytesize">>, #{example => <<"32MB">>, type => string}}, + {<<"wordsize">>, #{example => <<"1024KB">>, type => string}}, + {<<"maps">>, #{example => <<>>, type => string}}, + {<<"comma_separated_list">>, #{example => <<"item1,item2">>, type => string}}, + {<<"comma_separated_atoms">>, #{example => <<"item1,item2">>, type => string}}, + {<<"log_level">>, + #{enum => [debug, info, notice, warning, error, critical, alert, emergency, all], type => string}}, + {<<"fix_integer">>, #{default => 100, enum => [100], example => 100,type => integer}} + ], + <<"type">> => object}}}}, + {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), + ?assertEqual(test, OperationId), + Response = maps:get(responses, maps:get(post, Spec)), + ?assertEqual(Object, maps:get(<<"200">>, Response)), + ?assertEqual([], Refs), + ok. + + t_ref_array_with_key(_Config) -> Path = "/ref/array/with/key", Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ @@ -163,10 +195,10 @@ t_ref_array_with_key(_Config) -> {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, - {<<"assert">>, #{description => <<"money">>, example => 3.14159,type => number}}, - {<<"number_ex">>, #{description => <<"number example">>, example => 42,type => number}}, - {<<"percent_ex">>, #{description => <<"percent example">>, example => <<"12%">>,type => number}}, - {<<"duration_ms_ex">>, #{description => <<"duration ms example">>, example => <<"32s">>,type => string}}, + {<<"assert">>, #{description => <<"money">>, example => 3.14159, type => number}}, + {<<"number_ex">>, #{description => <<"number example">>, example => 42, type => number}}, + {<<"percent_ex">>, #{description => <<"percent example">>, example => <<"12%">>, type => number}}, + {<<"duration_ms_ex">>, #{description => <<"duration ms example">>, example => <<"32s">>, type => string}}, {<<"atom_ex">>, #{description => <<"atom ex">>, example => atom, type => string}}, {<<"array_refs">>, #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}, type => array}} ]} @@ -201,7 +233,7 @@ t_hocon_schema_function(_Config) -> }}, #{<<"emqx_swagger_remote_schema.ref3">> => #{<<"type">> => object, <<"properties">> => [ - {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}}, + {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>, type => string}}, {<<"version">>, #{description => <<"a good version">>, example => <<"1.0.0">>, type => string}}] }}, #{<<"emqx_swagger_remote_schema.root">> => #{required => [<<"default_password">>, <<"default_username">>], @@ -210,8 +242,8 @@ t_hocon_schema_function(_Config) -> [#{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref2">>}, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref1">>}]}, type => array}}, {<<"default_username">>, - #{default => <<"admin">>, example => <<"string example">>, type => string}}, - {<<"default_password">>, #{default => <<"public">>, example => <<"string example">>, type => string}}, + #{default => <<"admin">>, example => <<"string-example">>, type => string}}, + {<<"default_password">>, #{default => <<"public">>, example => <<"string-example">>, type => string}}, {<<"sample_interval">>, #{default => <<"10s">>, example => <<"1h">>, type => string}}, {<<"token_expired_time">>, #{default => <<"30m">>, example => <<"12m">>, type => string}}], <<"type">> => object}}], @@ -290,6 +322,27 @@ schema("/error") -> 400 => emqx_dashboard_swagger:error_codes(['Bad1', 'Bad2'], <<"Bad request desc">>), 404 => emqx_dashboard_swagger:error_codes(['Not-Found']) }} + }; +schema("/ref/complicated_type") -> + #{ + operationId => test, + post => #{responses => #{ + 200 => [ + {no_neg_integer, hoconsc:mk(non_neg_integer(), #{})}, + {url, hoconsc:mk(emqx_connector_http:url(), #{})}, + {server, hoconsc:mk(emqx_connector_redis:server(), #{})}, + {connect_timeout, hoconsc:mk(emqx_connector_http:connect_timeout(), #{})}, + {pool_type, hoconsc:mk(emqx_connector_http:pool_type(), #{})}, + {timeout, hoconsc:mk(timeout(), #{})}, + {bytesize, hoconsc:mk(emqx_schema:bytesize(), #{})}, + {wordsize, hoconsc:mk(emqx_schema:wordsize(), #{})}, + {maps, hoconsc:mk(map(), #{})}, + {comma_separated_list, hoconsc:mk(emqx_schema:comma_separated_list(), #{})}, + {comma_separated_atoms, hoconsc:mk(emqx_schema:comma_separated_atoms(), #{})}, + {log_level, hoconsc:mk(emqx_machine_schema:log_level(), #{})}, + {fix_integer, hoconsc:mk(typerefl:integer(100), #{})} + ] + }} }. validate(Path, ExpectObject, ExpectRefs) -> diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 64d39eb52..9e988c6d8 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -40,13 +40,13 @@ roots() -> [exhook]. fields(exhook) -> [ {request_failed_action, - sc(union([deny, ignore]), + sc(hoconsc:enum([deny, ignore]), #{default => deny})} , {request_timeout, sc(duration(), #{default => "5s"})} , {auto_reconnect, - sc(union([false, duration()]), + sc(hoconsc:union([false, duration()]), #{ default => "60s" })} , {servers, diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index 3eed9b802..aa448a1a2 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -83,7 +83,7 @@ request_parameters() -> request_properties() -> properties([ {token, string, "message token, can be empty"} , {method, string, "request method type", ["get", "put", "post", "delete"]} - , {timeout, string, "timespan for response", "10s"} + , {timeout, string, "timespan for response"} , {content_type, string, "payload type", [<<"text/plain">>, <<"application/json">>, <<"application/octet-stream">>]} , {payload, string, "payload"}]). diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index bb0bf9dbe..32f24b2dd 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -232,7 +232,7 @@ gateway_common_options() -> common_listener_opts() -> [ {enable, sc(boolean(), true)} - , {bind, sc(union(ip_port(), integer()))} + , {bind, sc(hoconsc:union([ip_port(), integer()]))} , {max_connections, sc(integer(), 1024)} , {max_conn_rate, sc(integer())} , {authentication, authentication()} diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index a859f2002..de2774cae 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -16,97 +16,106 @@ -module(emqx_mgmt_api_configs). +-include_lib("hocon/include/hoconsc.hrl"). -behaviour(minirest_api). --import(emqx_mgmt_util, [ schema/1 - , schema/2 - , error_schema/2 - ]). - -export([api_spec/0]). +-export([paths/0, schema/1, fields/1]). --export([ config/3 - , config_reset/3 - ]). +-export([config/3, config_reset/3, configs/3, get_full_config/0]). -export([get_conf_schema/2, gen_schema/1]). --define(PARAM_CONF_PATH, [#{ - name => conf_path, - in => query, - description => <<"The config path separated by '.' character">>, - required => false, - schema => #{type => string, default => <<".">>} -}]). - --define(PREFIX, "/configs"). --define(PREFIX_RESET, "/configs_reset"). - --define(MAX_DEPTH, 1). - +-define(PREFIX, "/configs/"). +-define(PREFIX_RESET, "/configs_reset/"). -define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))). - --define(CORE_CONFS, [ - %% from emqx_machine_schema - log, rpc, - %% from emqx_schema - zones, mqtt, flapping_detect, force_shutdown, force_gc, conn_congestion, rate_limit, quota, - broker, alarm, sysmon, - %% from other apps - emqx_dashboard, emqx_management]). +-define(EXCLUDES, [listeners, node, cluster, gateway, rule_engine]). api_spec() -> - {config_apis() ++ [config_reset_api()], []}. + emqx_dashboard_swagger:spec(?MODULE). -config_apis() -> - [config_api(ConfPath, Schema) || {ConfPath, Schema} <- - get_conf_schema(emqx:get_config([]), ?MAX_DEPTH), is_core_conf(ConfPath)]. +paths() -> + ["/configs", "/configs_reset/:rootname"] ++ + lists:map(fun({Name, _Type}) -> ?PREFIX ++ to_list(Name) end, config_list(?EXCLUDES)). -config_api(ConfPath, Schema) -> - Path = path_join(ConfPath), - Descr = fun(Str) -> - list_to_binary([Str, " ", path_join(ConfPath, ".")]) - end, - Metadata = #{ +schema("/configs") -> + #{ + operationId => configs, get => #{ - description => Descr("Get configs for"), + tags => [conf], + description => <<"Get all the configurations of the specified node, including hot and non-hot updatable items.">>, + parameters => [ + {node, hoconsc:mk(typerefl:atom(), + #{in => query, required => false, example => <<"emqx@127.0.0.1">>, + desc => <<"Node's name: If you do not fill in the fields, this node will be used by default.">>})}], responses => #{ - <<"200">> => schema(Schema, <<"Get configs successfully">>), - <<"404">> => emqx_mgmt_util:error_schema(<<"Config not found">>, ['NOT_FOUND']) - } - }, - put => #{ - description => Descr("Update configs for"), - 'requestBody' => schema(Schema), - responses => #{ - <<"200">> => schema(Schema, <<"Update configs successfully">>), - <<"400">> => error_schema(<<"Update configs failed">>, ['UPDATE_FAILED']) + 200 => config_list([]) } } - }, - {?PREFIX ++ "/" ++ Path, Metadata, config}. - -config_reset_api() -> - Metadata = #{ + }; +schema("/configs_reset/:rootname") -> + Paths = lists:map(fun({Path, _}) -> Path end, config_list(?EXCLUDES)), + #{ + operationId => config_reset, post => #{ - tags => [configs], + tags => [conf], description => <<"Reset the config entry specified by the query string parameter `conf_path`.
- For a config entry that has default value, this resets it to the default value; - For a config entry that has no default value, an error 400 will be returned">>, - parameters => ?PARAM_CONF_PATH, + %% We only return "200" rather than the new configs that has been changed, as + %% the schema of the changed configs is depends on the request parameter + %% `conf_path`, it cannot be defined here. + parameters => [ + {rootname, hoconsc:mk(hoconsc:enum(Paths), #{in => path, example => <<"authorization">>})}, + {conf_path, hoconsc:mk(typerefl:binary(), + #{in => query, required => false, example => <<"cache.enable">>, + desc => <<"The config path separated by '.' character">>})}], responses => #{ - %% We only return "200" rather than the new configs that has been changed, as - %% the schema of the changed configs is depends on the request parameter - %% `conf_path`, it cannot be defined here. - <<"200">> => schema(<<"Reset configs successfully">>), - <<"400">> => error_schema(<<"It's not able to reset the config">>, ['INVALID_OPERATION']) + 200 => <<"Rest config successfully">>, + 400 => emqx_dashboard_swagger:error_codes(['NO_DEFAULT_VALUE', 'REST_FAILED']) } } - }, - {?PREFIX_RESET, Metadata, config_reset}. + }; +schema(Path) -> + {Root, Schema} = find_schema(Path), + #{ + operationId => config, + get => #{ + tags => [conf], + description => iolist_to_binary([<<"Get the sub-configurations under *">>, Root, <<"*">>]), + responses => #{ + 200 => Schema, + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"config not found">>) + } + }, + put => #{ + tags => [conf], + description => iolist_to_binary([<<"Update the sub-configurations under *">>, Root, <<"*">>]), + requestBody => Schema, + responses => #{ + 200 => Schema, + 400 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED']) + } + } + }. + +find_schema(Path) -> + [_, _Prefix, Root | _] = string:split(Path, "/", all), + Configs = config_list(?EXCLUDES), + case lists:keyfind(Root, 1, Configs) of + {Root, Schema} -> {Root, Schema}; + false -> + RootAtom = list_to_existing_atom(Root), + {Root, element(2, lists:keyfind(RootAtom, 1, Configs))} + end. + +%% we load all configs from emqx_machine_schema, some of them are defined as local ref +%% we need redirect to emqx_machine_schema. +%% such as hoconsc:ref("node") to hoconsc:ref(emqx_machine_schema, "node") +fields(Field) -> emqx_machine_schema:fields(Field). %%%============================================================================================== -%% parameters trans +%% HTTP API Callbacks config(get, _Params, Req) -> Path = conf_path(Req), case emqx_map_lib:deep_find(Path, get_full_config()) of @@ -118,19 +127,33 @@ config(get, _Params, Req) -> config(put, #{body := Body}, Req) -> Path = conf_path(Req), - {ok, #{raw_config := RawConf}} = emqx:update_config(Path, Body, - #{rawconf_with_defaults => true}), - {200, emqx_map_lib:jsonable_map(RawConf)}. + case emqx:update_config(Path, Body, #{rawconf_with_defaults => true}) of + {ok, #{raw_config := RawConf}} -> + {200, emqx_map_lib:jsonable_map(RawConf)}; + {error, Reason} -> + {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Reason)}} + end. config_reset(post, _Params, Req) -> %% reset the config specified by the query string param 'conf_path' Path = conf_path_reset(Req) ++ conf_path_from_querystr(Req), case emqx:reset_config(Path, #{}) of {ok, _} -> {200}; + {error, no_default_value} -> + {400, #{code => 'NO_DEFAULT_VALUE', message => <<"No Default Value.">>}}; {error, Reason} -> - {400, ?ERR_MSG(Reason)} + {400, #{code => 'REST_FAILED', message => ?ERR_MSG(Reason)}} end. +configs(get, Params, _Req) -> + Node = maps:get(node, Params, node()), + Res = rpc:call(Node, ?MODULE, get_full_config, [[]]), + {200, Res}. + +conf_path_reset(Req) -> + <<"/api/v5", ?PREFIX_RESET, Path/binary>> = cowboy_req:path(Req), + string:lexemes(Path, "/ "). + get_full_config() -> emqx_map_lib:jsonable_map( emqx_config:fill_defaults(emqx:get_raw_config([]))). @@ -141,14 +164,17 @@ conf_path_from_querystr(Req) -> Path -> string:lexemes(Path, ". ") end. +config_list(Exclude) -> + Roots = emqx_machine_schema:roots(), + lists:foldl(fun(Key, Acc) -> lists:delete(Key, Acc) end, Roots, Exclude). + +to_list(L) when is_list(L) -> L; +to_list(Atom) when is_atom(Atom) -> atom_to_list(Atom). + conf_path(Req) -> <<"/api/v5", ?PREFIX, Path/binary>> = cowboy_req:path(Req), string:lexemes(Path, "/ "). -conf_path_reset(Req) -> - <<"/api/v5", ?PREFIX_RESET, Path/binary>> = cowboy_req:path(Req), - string:lexemes(Path, "/ "). - get_conf_schema(Conf, MaxDepth) -> get_conf_schema([], maps:to_list(Conf), [], MaxDepth). @@ -158,11 +184,11 @@ get_conf_schema(BasePath, [{Key, Conf} | Confs], Result, MaxDepth) -> Path = BasePath ++ [Key], Depth = length(Path), Result1 = case is_map(Conf) of - true when Depth < MaxDepth -> - get_conf_schema(Path, maps:to_list(Conf), Result, MaxDepth); - true when Depth >= MaxDepth -> Result; - false -> Result - end, + true when Depth < MaxDepth -> + get_conf_schema(Path, maps:to_list(Conf), Result, MaxDepth); + true when Depth >= MaxDepth -> Result; + false -> Result + end, get_conf_schema(BasePath, Confs, [{Path, gen_schema(Conf)} | Result1], MaxDepth). %% TODO: generate from hocon schema @@ -181,7 +207,7 @@ gen_schema(Conf) when is_list(Conf) -> end; gen_schema(Conf) when is_map(Conf) -> #{type => object, properties => - maps:map(fun(_K, V) -> gen_schema(V) end, Conf)}; + maps:map(fun(_K, V) -> gen_schema(V) end, Conf)}; gen_schema(_Conf) -> %% the conf is not of JSON supported type, it may have been converted %% by the hocon schema @@ -189,17 +215,3 @@ gen_schema(_Conf) -> with_default_value(Type, Value) -> Type#{example => emqx_map_lib:binary_string(Value)}. - -path_join(Path) -> - path_join(Path, "/"). - -path_join([P], _Sp) -> str(P); -path_join([P | Path], Sp) -> - str(P) ++ Sp ++ path_join(Path, Sp). - -is_core_conf(Path) -> - lists:member(hd(Path), ?CORE_CONFS). - -str(S) when is_list(S) -> S; -str(S) when is_binary(S) -> binary_to_list(S); -str(S) when is_atom(S) -> atom_to_list(S). diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index c989ecbed..cec75c9b5 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -28,14 +28,12 @@ namespace() -> modules. roots() -> ["delayed", - "recon", "telemetry", "event_message", array("rewrite"), array("topic_metrics")]. -fields(Name) when Name =:= "recon"; - Name =:= "telemetry" -> +fields("telemetry") -> [ {enable, hoconsc:mk(boolean(), #{default => false})} ];