yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Mgmt http api banned (#5998) 

* fix(swagger): don't transform [log,publish] to a list.pwd

* chore: replace banned-api by hocon schema

* fix(api): code style warning
This commit is contained in:
zhongwencool 2021-11-11 16:35:29 +08:00 committed by GitHub
parent a65597e302
commit 7a5da76197
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 156 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/emqx/src/emqx_banned.erl
View File

@ -37,6 +37,7 @@
, info/1
, format/1
, parse/1
, to_timestamp/1
]).
%% gen_server callbacks
@ -108,8 +109,8 @@ parse(Params) ->
Who = pares_who(Params),
By = maps:get(<<"by">>, Params, <<"mgmt_api">>),
Reason = maps:get(<<"reason">>, Params, <<"">>),
At = pares_time(maps:get(<<"at">>, Params, undefined), erlang:system_time(second)),
Until = pares_time(maps:get(<<"until">>, Params, undefined), At + 5 * 60),
At = parse_time(maps:get(<<"at">>, Params, undefined), erlang:system_time(second)),
Until = parse_time(maps:get(<<"until">>, Params, undefined), At + 5 * 60),
#banned{
who = Who,
by = By,
@ -120,15 +121,15 @@ parse(Params) ->
pares_who(#{as := As, who := Who}) ->
pares_who(#{<<"as">> => As, <<"who">> => Who});
pares_who(#{<<"as">> := <<"peerhost">>, <<"who">> := Peerhost0}) ->
pares_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) ->
{ok, Peerhost} = inet:parse_address(binary_to_list(Peerhost0)),
{peerhost, Peerhost};
pares_who(#{<<"as">> := As, <<"who">> := Who}) ->
{binary_to_atom(As, utf8), Who}.
{As, Who}.
pares_time(undefined, Default) ->
parse_time(undefined, Default) ->
Default;
pares_time(Rfc3339, _Default) ->
parse_time(Rfc3339, _Default) ->
to_timestamp(Rfc3339).
maybe_format_host({peerhost, Host}) ->

97
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -22,11 +22,14 @@
-define(DEFAULT_FIELDS, [example, allowReserved, style,
explode, maxLength, allowEmptyValue, deprecated, minimum, maximum]).
-define(INIT_SCHEMA, #{fields => #{}, translations => #{}, validations => [], namespace => undefined}).
-define(INIT_SCHEMA, #{fields => #{}, translations => #{},
validations => [], namespace => undefined}).
-define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])).
-define(TO_COMPONENTS_SCHEMA(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>, ?TO_REF(namespace(_M_), _F_)])).
-define(TO_COMPONENTS_PARAM(_M_, _F_), iolist_to_binary([<<"#/components/parameters/">>, ?TO_REF(namespace(_M_), _F_)])).
-define(TO_COMPONENTS_SCHEMA(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>,
?TO_REF(namespace(_M_), _F_)])).
-define(TO_COMPONENTS_PARAM(_M_, _F_), iolist_to_binary([<<"#/components/parameters/">>,
?TO_REF(namespace(_M_), _F_)])).
-define(MAX_ROW_LIMIT, 100).
@ -116,9 +119,9 @@ translate_req(Request, #{module := Module, path := Path, method := Method}, Chec
#{Method := Spec} = apply(Module, schema, [Path]),
try
Params = maps:get(parameters, Spec, []),
Body = maps:get(requestBody, Spec, []),
Body = maps:get('requestBody', Spec, []),
{Bindings, QueryStr} = check_parameters(Request, Params, Module),
NewBody = check_requestBody(Request, Body, Module, CheckFun, hoconsc:is_schema(Body)),
NewBody = check_request_body(Request, Body, Module, CheckFun, hoconsc:is_schema(Body)),
{ok, Request#{bindings => Bindings, query_string => QueryStr, body => NewBody}}
catch throw:Error ->
{_, [{validation_error, ValidErr}]} = Error,
@ -155,34 +158,41 @@ parse_spec_ref(Module, Path) ->
{Spec, SubRefs} = meta_to_spec(Meta, Module),
{Acc#{Method => Spec}, SubRefs ++ RefsAcc}
end, {#{}, []},
maps:without([operationId], Schema)),
{maps:get(operationId, Schema), Specs, Refs}.
maps:without(['operationId'], Schema)),
{maps:get('operationId', Schema), Specs, Refs}.
check_parameters(Request, Spec, Module) ->
#{bindings := Bindings, query_string := QueryStr} = Request,
BindingsBin = maps:fold(fun(Key, Value, Acc) -> Acc#{atom_to_binary(Key) => Value} end, #{}, Bindings),
BindingsBin = maps:fold(fun(Key, Value, Acc) ->
Acc#{atom_to_binary(Key) => Value}
end, #{}, Bindings),
check_parameter(Spec, BindingsBin, QueryStr, Module, #{}, #{}).
check_parameter([?REF(Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) ->
check_parameter([?R_REF(LocalMod, Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc);
check_parameter([?R_REF(Module, Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) ->
check_parameter([?R_REF(LocalMod, Fields) | Spec],
Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc);
check_parameter([?R_REF(Module, Fields) | Spec],
Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) ->
Params = apply(Module, fields, [Fields]),
check_parameter(Params ++ Spec, Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc);
check_parameter([], _Bindings, _QueryStr, _Module, NewBindings, NewQueryStr) -> {NewBindings, NewQueryStr};
check_parameter([], _Bindings, _QueryStr, _Module, NewBindings, NewQueryStr) ->
{NewBindings, NewQueryStr};
check_parameter([{Name, Type} | Spec], Bindings, QueryStr, Module, BindingsAcc, QueryStrAcc) ->
Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
case hocon_schema:field_schema(Type, in) of
path ->
NewBindings = hocon_schema:check_plain(Schema, Bindings, #{atom_key => true, override_env => false}),
Option = #{atom_key => true, override_env => false},
NewBindings = hocon_schema:check_plain(Schema, Bindings, Option),
NewBindingsAcc = maps:merge(BindingsAcc, NewBindings),
check_parameter(Spec, Bindings, QueryStr, Module, NewBindingsAcc, QueryStrAcc);
query ->
NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, #{override_env => false}),
Option = #{override_env => false},
NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, Option),
NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr),
check_parameter(Spec, Bindings, QueryStr, Module,BindingsAcc, NewQueryStrAcc)
end.
check_requestBody(#{body := Body}, Schema, Module, CheckFun, true) ->
check_request_body(#{body := Body}, Schema, Module, CheckFun, true) ->
Type0 = hocon_schema:field_schema(Schema, type),
Type =
case Type0 of
@ -190,16 +200,17 @@ check_requestBody(#{body := Body}, Schema, Module, CheckFun, true) ->
_ -> Type0
end,
NewSchema = ?INIT_SCHEMA#{roots => [{root, Type}]},
#{<<"root">> := NewBody} = CheckFun(NewSchema, #{<<"root">> => Body}, #{override_env => false}),
Option = #{override_env => false},
#{<<"root">> := NewBody} = CheckFun(NewSchema, #{<<"root">> => Body}, Option),
NewBody;
%% TODO not support nest object check yet, please use ref!
%% RequestBody = [ {per_page, mk(integer(), #{}},
%% 'requestBody' = [ {per_page, mk(integer(), #{}},
%% {nest_object, [
%% {good_nest_1, mk(integer(), #{})},
%% {good_nest_2, mk(ref(?MODULE, good_ref), #{})}
%% ]}
%% ]
check_requestBody(#{body := Body}, Spec, _Module, CheckFun, false) ->
check_request_body(#{body := Body}, Spec, _Module, CheckFun, false) ->
lists:foldl(fun({Name, Type}, Acc) ->
Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
maps:merge(Acc, CheckFun(Schema, Body, #{}))
@ -208,7 +219,7 @@ check_requestBody(#{body := Body}, Spec, _Module, CheckFun, false) ->
%% tags, description, summary, security, deprecated
meta_to_spec(Meta, Module) ->
{Params, Refs1} = parameters(maps:get(parameters, Meta, []), Module),
{RequestBody, Refs2} = requestBody(maps:get(requestBody, Meta, []), Module),
{RequestBody, Refs2} = request_body(maps:get('requestBody', Meta, []), Module),
{Responses, Refs3} = responses(maps:get(responses, Meta, #{}), Module),
{
to_spec(Meta, Params, RequestBody, Responses),
@ -216,25 +227,22 @@ meta_to_spec(Meta, Module) ->
}.
to_spec(Meta, Params, [], Responses) ->
Spec = maps:without([parameters, requestBody, responses], Meta),
Spec = maps:without([parameters, 'requestBody', responses], Meta),
Spec#{parameters => Params, responses => Responses};
to_spec(Meta, Params, RequestBody, Responses) ->
Spec = to_spec(Meta, Params, [], Responses),
maps:put(requestBody, RequestBody, Spec).
maps:put('requestBody', RequestBody, Spec).
parameters(Params, Module) ->
{SpecList, AllRefs} =
lists:foldl(fun(Param, {Acc, RefsAcc}) ->
case Param of
?REF(StructName) ->
{[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(Module, StructName)} | Acc],
[{Module, StructName, parameter} | RefsAcc]};
?R_REF(RModule, StructName) ->
{[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(RModule, StructName)} | Acc],
[{RModule, StructName, parameter} | RefsAcc]};
?REF(StructName) -> to_ref(Module, StructName, Acc, RefsAcc);
?R_REF(RModule, StructName) -> to_ref(RModule, StructName, Acc, RefsAcc);
{Name, Type} ->
In = hocon_schema:field_schema(Type, in),
In =:= undefined andalso throw({error, <<"missing in:path/query field in parameters">>}),
In =:= undefined andalso
throw({error, <<"missing in:path/query field in parameters">>}),
Nullable = hocon_schema:field_schema(Type, nullable),
Default = hocon_schema:field_schema(Type, default),
HoconType = hocon_schema:field_schema(Type, type),
@ -278,8 +286,8 @@ trans_desc(Spec, Hocon) ->
Desc -> Spec#{description => to_bin(Desc)}
end.
requestBody([], _Module) -> {[], []};
requestBody(Schema, Module) ->
request_body([], _Module) -> {[], []};
request_body(Schema, Module) ->
{{Props, Refs}, Examples} =
case hoconsc:is_schema(Schema) of
true ->
@ -311,7 +319,10 @@ response(Status, Schema, {Acc, RefsAcc, Module}) ->
{Spec, Refs} = hocon_schema_to_spec(Hocon, Module),
Init = trans_desc(#{}, Schema),
Content = content(Spec, Examples),
{Acc#{integer_to_binary(Status) => Init#{<<"content">> => Content}}, Refs ++ RefsAcc, Module};
{
Acc#{integer_to_binary(Status) => Init#{<<"content">> => Content}},
Refs ++ RefsAcc, Module
};
false ->
{Props, Refs} = parse_object(Schema, Module),
Content = #{<<"content">> => content(Props)},
@ -401,11 +412,16 @@ typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, examp
typename_to_spec("bytesize()", _Mod) -> #{type => string, example => <<"32MB">>};
typename_to_spec("wordsize()", _Mod) -> #{type => string, example => <<"1024KB">>};
typename_to_spec("map()", _Mod) -> #{type => object, example => #{}};
typename_to_spec("comma_separated_list()", _Mod) -> #{type => string, example => <<"item1,item2">>};
typename_to_spec("comma_separated_atoms()", _Mod) -> #{type => string, example => <<"item1,item2">>};
typename_to_spec("pool_type()", _Mod) -> #{type => string, enum => [random, hash], example => hash};
typename_to_spec("comma_separated_list()", _Mod) ->
#{type => string, example => <<"item1,item2">>};
typename_to_spec("comma_separated_atoms()", _Mod) ->
#{type => string, example => <<"item1,item2">>};
typename_to_spec("pool_type()", _Mod) ->
#{type => string, enum => [random, hash], example => hash};
typename_to_spec("log_level()", _Mod) ->
#{type => string, enum => [debug, info, notice, warning, error, critical, alert, emergency, all]};
#{ type => string,
enum => [debug, info, notice, warning, error, critical, alert, emergency, all]
};
typename_to_spec("rate()", _Mod) ->
#{type => string, example => <<"10M/s">>};
typename_to_spec("bucket_rate()", _Mod) ->
@ -465,9 +481,12 @@ add_integer_prop(Schema, Key, Value) ->
{Int, []} -> Schema#{Key => Int}
end.
to_bin([Atom | _] = List) when is_atom(Atom) -> iolist_to_binary(io_lib:format("~p", [List]));
to_bin(List) when is_list(List) -> unicode:characters_to_binary(List);
to_bin(B) when is_boolean(B) -> B;
to_bin(List) when is_list(List) ->
case io_lib:printable_list(List) of
true -> unicode:characters_to_binary(List);
false -> List
end;
to_bin(Boolean) when is_boolean(Boolean) -> Boolean;
to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
to_bin(X) -> X.
@ -513,3 +532,7 @@ content(ApiSpec, undefined) ->
#{<<"application/json">> => #{<<"schema">> => ApiSpec}};
content(ApiSpec, Examples) when is_map(Examples) ->
#{<<"application/json">> => Examples#{<<"schema">> => ApiSpec}}.
to_ref(Mod, StructName, Acc, RefsAcc) ->
Ref = #{<<"$ref">> => ?TO_COMPONENTS_PARAM(Mod, StructName)},
{[Ref | Acc], [{Mod, StructName, parameter} | RefsAcc]}.

145
apps/emqx_management/src/emqx_mgmt_api_banned.erl
View File

@ -17,87 +17,119 @@
-module(emqx_mgmt_api_banned).
-include_lib("emqx/include/emqx.hrl").
-include_lib("typerefl/include/types.hrl").
-include("emqx_mgmt.hrl").
-behaviour(minirest_api).
-export([api_spec/0]).
-export([api_spec/0, paths/0, schema/1, fields/1]).
-export([format/1]).
-export([ banned/2
, delete_banned/2
]).
-import(emqx_mgmt_util, [ page_params/0
, schema/1
, object_schema/1
, page_object_schema/1
, properties/1
, error_schema/1
]).
-export([format/1]).
-define(TAB, emqx_banned).
-define(FORMAT_FUN, {?MODULE, format}).
api_spec() ->
{[banned_api(), delete_banned_api()], []}.
-define(BANNED_TYPES, [clientid, username, peerhost]).
properties() ->
properties([
{as, string, <<"Banned type clientid, username, peerhost">>, [clientid, username, peerhost]},
{who, string, <<"Client info as banned type">>},
{by, integer, <<"Commander">>},
{reason, string, <<"Banned reason">>},
{at, integer, <<"Create banned time. Nullable, rfc3339, default is now">>},
{until, string, <<"Cancel banned time. Nullable, rfc3339, default is now + 5 minute">>}
]).
-define(FORMAT_FUN, {?MODULE, format}).
banned_api() ->
Path = "/banned",
MetaData = #{
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).
paths() ->
["/banned", "/banned/:as/:who"].
schema("/banned") ->
#{
'operationId' => banned,
get => #{
description => <<"List banned">>,
parameters => page_params(),
parameters => [
hoconsc:ref(emqx_dashboard_swagger, page),
hoconsc:ref(emqx_dashboard_swagger, limit)
],
responses => #{
<<"200">> =>
page_object_schema(properties())}},
200 =>[
{data, hoconsc:mk(hoconsc:array(hoconsc:ref(ban)), #{})},
{meta, hoconsc:mk(hoconsc:ref(meta), #{})}
]
}
},
post => #{
description => <<"Create banned">>,
'requestBody' => object_schema(properties()),
'requestBody' => hoconsc:mk(hoconsc:ref(ban)),
responses => #{
<<"200">> => schema(<<"Create success">>)}}},
{Path, MetaData, banned}.
delete_banned_api() ->
Path = "/banned/:as/:who",
MetaData = #{
200 => <<"Create success">>
}
}
};
schema("/banned/:as/:who") ->
#{
'operationId' => delete_banned,
delete => #{
description => <<"Delete banned">>,
parameters => [
#{
name => as,
{as, hoconsc:mk(hoconsc:enum(?BANNED_TYPES), #{
desc => <<"Banned type">>,
in => path,
required => true,
description => <<"Banned type">>,
schema => #{type => string, enum => ?BANNED_TYPES}
},
#{
name => who,
example => username})},
{who, hoconsc:mk(binary(), #{
desc => <<"Client info as banned type">>,
in => path,
required => true,
description => <<"Client info as banned type">>,
schema => #{type => string}
}
],
example => <<"Badass">>})}
],
responses => #{
<<"200">> => schema(<<"Delete banned success">>),
<<"404">> => error_schema(<<"Banned not found">>)}}},
{Path, MetaData, delete_banned}.
200 => <<"Delete banned success">>,
404 => emqx_dashboard_swagger:error_codes(['RESOURCE_NOT_FOUND'],
<<"Banned not found">>)
}
}
}.
fields(ban) ->
[
{as, hoconsc:mk(hoconsc:enum(?BANNED_TYPES), #{
desc => <<"Banned type clientid, username, peerhost">>,
nullable => false,
example => username})},
{who, hoconsc:mk(binary(), #{
desc => <<"Client info as banned type">>,
nullable => false,
example => <<"Badass">>})},
{by, hoconsc:mk(binary(), #{
desc => <<"Commander">>,
nullable => true,
example => <<"mgmt_api">>})},
{reason, hoconsc:mk(binary(), #{
desc => <<"Banned reason">>,
nullable => true,
example => <<"Too many requests">>})},
{at, hoconsc:mk(binary(), #{
desc => <<"Create banned time, rfc3339, now if not specified">>,
nullable => true,
validator => fun is_rfc3339/1,
example => <<"2021-10-25T21:48:47+08:00">>})},
{until, hoconsc:mk(binary(), #{
desc => <<"Cancel banned time, rfc3339, now + 5 minute if not specified">>,
nullable => true,
validator => fun is_rfc3339/1,
example => <<"2021-10-25T21:53:47+08:00">>})
}
];
fields(meta) ->
emqx_dashboard_swagger:fields(page) ++
emqx_dashboard_swagger:fields(limit) ++
[{count, hoconsc:mk(integer(), #{example => 1})}].
is_rfc3339(Time) ->
try
emqx_banned:to_timestamp(Time),
ok
catch _:_ -> {error, Time}
end.
banned(get, #{query_string := Params}) ->
Response = emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN),
@ -109,9 +141,8 @@ banned(post, #{body := Body}) ->
delete_banned(delete, #{bindings := Params}) ->
case emqx_banned:look_up(Params) of
[] ->
As0 = maps:get(as, Params),
Who0 = maps:get(who, Params),
Message = list_to_binary(io_lib:format("~p: ~p not found", [As0, Who0])),
#{as := As0, who := Who0} = Params,
Message = list_to_binary(io_lib:format("~p: ~s not found", [As0, Who0])),
{404, #{code => 'RESOURCE_NOT_FOUND', message => Message}};
_ ->
ok = emqx_banned:delete(Params),

2
rebar.config
View File

@ -54,7 +54,7 @@
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.5"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.6"}}}
, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}}
, {replayq, "0.3.3"}
, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}