yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into hot-confs-sys-topics-limiter

This commit is contained in:
JianBo He 2022-04-19 17:38:49 +08:00 committed by GitHub
parent 86d8dd2fb6 2cc8df24f7
commit ba24f0309d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
143 changed files with 7278 additions and 4308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
.eunit
*.conf.all
test-data/
deps
!deps/.placeholder

5
Makefile
View File

@ -8,7 +8,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-a
export EMQX_DEFAULT_RUNNER = alpine:3.15.1
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v0.28.0
export EMQX_DASHBOARD_VERSION ?= v0.29.0
export DOCKERFILE := deploy/docker/Dockerfile
export EMQX_REL_FORM ?= tgz
ifeq ($(OS),Windows_NT)
@ -61,7 +61,7 @@ get-dashboard:
@$(SCRIPTS)/get-dashboard.sh
.PHONY: eunit
eunit: $(REBAR)
eunit: $(REBAR) conf-segs
@ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c
.PHONY: proper
@ -218,6 +218,7 @@ $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt))))
.PHONY:
conf-segs:
@scripts/merge-config.escript
@scripts/merge-i18n.escript
## elixir target is to create release packages using Elixir's Mix
.PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir)

3
apps/emqx/rebar.config
View File

@ -24,13 +24,12 @@
{deps, [
{lc, {git, "https://github.com/emqx/lc.git", {tag, "0.2.1"}}},
{gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}},
{typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.8.6"}}},
{jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
{cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.1"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.3"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.26.6"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.26.7"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
{recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
{snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.18.0"}}}

11
apps/emqx/src/emqx_frame.erl
View File

@ -310,7 +310,6 @@ parse_packet(
(PacketId =/= undefined) andalso
StrictMode andalso validate_packet_id(PacketId),
{Properties, Payload} = parse_properties(Rest1, Ver, StrictMode),
ok = ensure_topic_name_valid(StrictMode, TopicName, Properties),
Publish = #mqtt_packet_publish{
topic_name = TopicName,
packet_id = PacketId,
@ -425,7 +424,6 @@ parse_will_message(
{Props, Rest} = parse_properties(Bin, Ver, StrictMode),
{Topic, Rest1} = parse_utf8_string(Rest, StrictMode),
{Payload, Rest2} = parse_binary_data(Rest1),
ok = ensure_topic_name_valid(StrictMode, Topic, Props),
{
Packet#mqtt_packet_connect{
will_props = Props,
@ -623,15 +621,6 @@ parse_binary_data(Bin) when
->
?PARSE_ERR(malformed_binary_data_length).
ensure_topic_name_valid(false, _TopicName, _Properties) ->
ok;
ensure_topic_name_valid(true, TopicName, _Properties) when TopicName =/= <<>> ->
ok;
ensure_topic_name_valid(true, <<>>, #{'Topic-Alias' := _}) ->
ok;
ensure_topic_name_valid(true, <<>>, _) ->
error(empty_topic_name).
%%--------------------------------------------------------------------
%% Serialize MQTT Packet
%%--------------------------------------------------------------------

35
apps/emqx/src/emqx_listeners.erl
View File

@ -87,13 +87,22 @@ do_list_raw() ->
Listeners = maps:to_list(RawWithDefault),
lists:flatmap(fun format_raw_listeners/1, Listeners).
format_raw_listeners({Type, Conf}) ->
format_raw_listeners({Type0, Conf}) ->
Type = binary_to_atom(Type0),
lists:map(
fun({LName, LConf0}) when is_map(LConf0) ->
Running = is_running(binary_to_atom(Type), listener_id(Type, LName), LConf0),
Bind = parse_bind(LConf0),
Running = is_running(Type, listener_id(Type, LName), LConf0#{bind => Bind}),
LConf1 = maps:remove(<<"authentication">>, LConf0),
LConf2 = maps:put(<<"running">>, Running, LConf1),
{Type, LName, LConf2}
LConf2 = maps:remove(<<"limiter">>, LConf1),
LConf3 = maps:put(<<"running">>, Running, LConf2),
CurrConn =
case Running of
true -> current_conns(Type, LName, Bind);
false -> 0
end,
LConf4 = maps:put(<<"current_connections">>, CurrConn, LConf3),
{Type0, LName, LConf4}
end,
maps:to_list(Conf)
).
@ -112,16 +121,7 @@ is_running(ListenerId) ->
end.
is_running(Type, ListenerId, Conf) when Type =:= tcp; Type =:= ssl ->
ListenOn =
case Conf of
#{bind := Bind} ->
Bind;
#{<<"bind">> := Bind} ->
case emqx_schema:to_ip_port(binary_to_list(Bind)) of
{ok, L} -> L;
{error, _} -> binary_to_integer(Bind)
end
end,
#{bind := ListenOn} = Conf,
try esockd:listener({ListenerId, ListenOn}) of
Pid when is_pid(Pid) ->
true
@ -545,3 +545,10 @@ str(B) when is_binary(B) ->
binary_to_list(B);
str(S) when is_list(S) ->
S.
parse_bind(#{<<"bind">> := Bind}) when is_integer(Bind) -> Bind;
parse_bind(#{<<"bind">> := Bind}) ->
case emqx_schema:to_ip_port(binary_to_list(Bind)) of
{ok, L} -> L;
{error, _} -> binary_to_integer(Bind)
end.

11
apps/emqx/test/emqx_frame_SUITE.erl
View File

@ -157,17 +157,6 @@ t_parse_malformed_utf8_string(_) ->
ParseState = emqx_frame:initial_parse_state(#{strict_mode => true}),
?ASSERT_FRAME_THROW(utf8_string_invalid, emqx_frame:parse(MalformedPacket, ParseState)).
t_parse_empty_topic_name(_) ->
Packet = ?PUBLISH_PACKET(?QOS_1, <<>>, 1, #{}, <<>>),
?assertEqual(Packet, parse_serialize(Packet, #{strict_mode => false})),
?ASSERT_FRAME_THROW(empty_topic_name, parse_serialize(Packet, #{strict_mode => true})).
t_parse_empty_topic_name_with_alias(_) ->
Props = #{'Topic-Alias' => 16#AB},
Packet = ?PUBLISH_PACKET(?QOS_1, <<>>, 1, Props, <<>>),
?assertEqual(Packet, parse_serialize(Packet, #{strict_mode => false})),
?assertEqual(Packet, parse_serialize(Packet, #{strict_mode => true})).
t_serialize_parse_v3_connect(_) ->
Bin =
<<16, 37, 0, 6, 77, 81, 73, 115, 100, 112, 3, 2, 0, 60, 0, 23, 109, 111, 115, 113, 112, 117,

29
apps/emqx_authn/src/emqx_authn_api.erl
View File

@ -149,8 +149,8 @@ fields(response_users) ->
paginated_list_type(ref(response_user));
fields(pagination_meta) ->
[
{page, non_neg_integer()},
{limit, non_neg_integer()},
{page, pos_integer()},
{limit, pos_integer()},
{count, non_neg_integer()}
].
@ -431,8 +431,10 @@ schema("/authentication/:id/users") ->
description => <<"List users in authenticator in global authentication chain">>,
parameters => [
param_auth_id(),
{page, mk(integer(), #{in => query, desc => <<"Page Index">>, required => false})},
{limit, mk(integer(), #{in => query, desc => <<"Page Limit">>, required => false})},
{page,
mk(pos_integer(), #{in => query, desc => <<"Page Index">>, required => false})},
{limit,
mk(pos_integer(), #{in => query, desc => <<"Page Limit">>, required => false})},
{like_username,
mk(binary(), #{
in => query,
@ -481,8 +483,10 @@ schema("/listeners/:listener_id/authentication/:id/users") ->
parameters => [
param_listener_id(),
param_auth_id(),
{page, mk(integer(), #{in => query, desc => <<"Page Index">>, required => false})},
{limit, mk(integer(), #{in => query, desc => <<"Page Limit">>, required => false})}
{page,
mk(pos_integer(), #{in => query, desc => <<"Page Index">>, required => false})},
{limit,
mk(pos_integer(), #{in => query, desc => <<"Page Limit">>, required => false})}
],
responses => #{
200 => emqx_dashboard_swagger:schema_with_example(
@ -1158,8 +1162,17 @@ delete_user(ChainName, AuthenticatorID, UserID) ->
end.
list_users(ChainName, AuthenticatorID, QueryString) ->
Response = emqx_authentication:list_users(ChainName, AuthenticatorID, QueryString),
emqx_mgmt_util:generate_response(Response).
case emqx_authentication:list_users(ChainName, AuthenticatorID, QueryString) of
{error, page_limit_invalid} ->
{400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
{error, Reason} ->
{400, #{
code => <<"INVALID_PARAMETER">>,
message => list_to_binary(io_lib:format("Reason ~p", [Reason]))
}};
Result ->
{200, Result}
end.
update_config(Path, ConfigRequest) ->
emqx_conf:update(Path, ConfigRequest, #{

74
apps/emqx_authz/src/emqx_authz_api_cache.erl Normal file
View File

@ -0,0 +1,74 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authz_api_cache).
-behaviour(minirest_api).
-export([
api_spec/0,
paths/0,
schema/1
]).
-export([
clean_cache/2
]).
-define(BAD_REQUEST, 'BAD_REQUEST').
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
paths() ->
[
"/authorization/cache"
].
%%--------------------------------------------------------------------
%% Schema for each URI
%%--------------------------------------------------------------------
schema("/authorization/cache") ->
#{
'operationId' => clean_cache,
delete =>
#{
description => <<"Clean all authorization cache in the cluster.">>,
responses =>
#{
204 => <<"No Content">>,
400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>)
}
}
}.
clean_cache(delete, _) ->
case emqx_mgmt:clean_authz_cache_all() of
ok ->
{204};
{error, Reason} ->
{400, #{
code => <<"BAD_REQUEST">>,
message => bin(Reason)
}}
end.
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
bin(Term) -> erlang:iolist_to_binary(io_lib:format("~p", [Term])).

50
apps/emqx_authz/src/emqx_authz_api_mnesia.erl
View File

@ -405,14 +405,23 @@ fields(meta) ->
%%--------------------------------------------------------------------
users(get, #{query_string := QueryString}) ->
Response = emqx_mgmt_api:node_query(
node(),
QueryString,
?ACL_TABLE,
?ACL_USERNAME_QSCHEMA,
?QUERY_USERNAME_FUN
),
emqx_mgmt_util:generate_response(Response);
case
emqx_mgmt_api:node_query(
node(),
QueryString,
?ACL_TABLE,
?ACL_USERNAME_QSCHEMA,
?QUERY_USERNAME_FUN
)
of
{error, page_limit_invalid} ->
{400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
{error, Node, {badrpc, R}} ->
Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])),
{500, #{code => <<"NODE_DOWN">>, message => Message}};
Result ->
{200, Result}
end;
users(post, #{body := Body}) when is_list(Body) ->
lists:foreach(
fun(#{<<"username">> := Username, <<"rules">> := Rules}) ->
@ -423,14 +432,23 @@ users(post, #{body := Body}) when is_list(Body) ->
{204}.
clients(get, #{query_string := QueryString}) ->
Response = emqx_mgmt_api:node_query(
node(),
QueryString,
?ACL_TABLE,
?ACL_CLIENTID_QSCHEMA,
?QUERY_CLIENTID_FUN
),
emqx_mgmt_util:generate_response(Response);
case
emqx_mgmt_api:node_query(
node(),
QueryString,
?ACL_TABLE,
?ACL_CLIENTID_QSCHEMA,
?QUERY_CLIENTID_FUN
)
of
{error, page_limit_invalid} ->
{400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
{error, Node, {badrpc, R}} ->
Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])),
{500, #{code => <<"NODE_DOWN">>, message => Message}};
Result ->
{200, Result}
end;
clients(post, #{body := Body}) when is_list(Body) ->
lists:foreach(
fun(#{<<"clientid">> := ClientID, <<"rules">> := Rules}) ->

79
apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl Normal file
View File

@ -0,0 +1,79 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authz_api_cache_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-import(emqx_dashboard_api_test_helpers, [request/2, uri/1]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
groups() ->
[].
init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_authz, emqx_dashboard, emqx_management],
fun set_special_configs/1
),
Config.
end_per_suite(_Config) ->
{ok, _} = emqx:update_config(
[authorization],
#{
<<"no_match">> => <<"allow">>,
<<"cache">> => #{<<"enable">> => <<"true">>},
<<"sources">> => []
}
),
ok = stop_apps([emqx_resource, emqx_connector]),
emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf, emqx_management]),
ok.
set_special_configs(emqx_dashboard) ->
emqx_dashboard_api_test_helpers:set_default_config();
set_special_configs(emqx_authz) ->
{ok, _} = emqx:update_config([authorization, cache, enable], true),
{ok, _} = emqx:update_config([authorization, no_match], deny),
{ok, _} = emqx:update_config([authorization, sources], []),
ok;
set_special_configs(_App) ->
ok.
t_clean_cahce(_) ->
{ok, C} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]),
{ok, _} = emqtt:connect(C),
{ok, _, _} = emqtt:subscribe(C, <<"a/b/c">>, 0),
ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0),
{ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])),
?assertEqual(2, length(jsx:decode(Result3))),
request(delete, uri(["authorization", "cache"])),
{ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])),
?assertEqual(0, length(jsx:decode(Result4))),
ok.
stop_apps(Apps) ->
lists:foreach(fun application:stop/1, Apps).

23
apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_api_i18n.conf Normal file
View File

@ -0,0 +1,23 @@
emqx_auto_subscribe_api {
list_auto_subscribe_api {
desc {
en: """Get auto subscribe topic list"""
zh: """获取自动订阅主题列表"""
}
}
update_auto_subscribe_api {
desc {
en: """Update auto subscribe topic list"""
zh: """更新自动订阅主题列表"""
}
}
update_auto_subscribe_api_response409 {
desc {
en: """Auto Subscribe topics max limit"""
zh: """超出自定订阅主题列表长度限制"""
}
}
}

85
apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf Normal file
View File

@ -0,0 +1,85 @@
emqx_auto_subscribe_schema {
auto_subscribe {
desc {
en: """After the device logs in successfully, the subscription is automatically completed for the device through the pre-defined subscription representation. Supports the use of placeholders."""
zh: """设备登陆成功之后，通过预设的订阅表示符，为设备自动完成订阅。支持使用占位符。"""
}
lable {
en: """Auto Subscribe"""
zh: """自动订阅"""
}
}
topic {
desc {
en: """Topic name, placeholders are supported. For example: client/${clientid}/username/${username}/host/${host}/port/${port}
Required field, and cannot be empty string"""
zh: """订阅标识符，支持使用占位符，例如 client/${clientid}/username/${username}/host/${host}/port/${port}
必填，且不可为空字符串"""
}
label {
en: """Topic"""
zh: """订阅标识符"""
}
}
qos {
desc {
en: """Default value 0. Quality of service.
At most once (0)
At least once (1)
Exactly once (2)"""
zh: """缺省值为 0，服务质量，
QoS 0：消息最多传递一次，如果当时客户端不可用，则会丢失该消息。
QoS 1：消息传递至少 1 次。
QoS 2：消息仅传送一次。"""
}
label {
en: """Quality of Service"""
zh: """服务质量"""
}
}
rh {
desc {
en: """Default value 0. This option is used to specify whether the server forwards the retained message to the client when establishing a subscription.
Retain Handling is equal to 0, as long as the client successfully subscribes, the server will send the retained message.
Retain Handling is equal to 1, if the client successfully subscribes and this subscription does not exist previously, the server sends the retained message. After all, sometimes the client re-initiate the subscription just to change the QoS, but it does not mean that it wants to receive the reserved messages again.
Retain Handling is equal to 2, even if the client successfully subscribes, the server does not send the retained message."""
zh: """指定订阅建立时服务端是否向客户端发送保留消息，
可选值 0：只要客户端订阅成功，服务端就发送保留消息。
可选值 1：客户端订阅成功且该订阅此前不存在，服务端才发送保留消息。毕竟有些时候客户端重新发起订阅可能只是为了改变一下 QoS，并不意味着它想再次接收保留消息。
可选值 2：即便客户订阅成功，服务端也不会发送保留消息。"""
}
label {
en: """Retain Handling"""
zh: """Retain Handling"""
}
}
rap {
desc {
en: """Default value 0. This option is used to specify whether the server retains the RETAIN mark when forwarding messages to the client, and this option does not affect the RETAIN mark in the retained message. Therefore, when the option Retain As Publish is set to 0, the client will directly distinguish whether this is a normal forwarded message or a retained message according to the RETAIN mark in the message, instead of judging whether this message is the first received after subscribing(the forwarded message may be sent before the retained message, which depends on the specific implementation of different brokers)."""
zh: """缺省值为 0，这一选项用来指定服务端向客户端转发消息时是否要保留其中的 RETAIN 标识，注意这一选项不会影响保留消息中的 RETAIN 标识。因此当 Retain As Publish 选项被设置为 0 时，客户端直接依靠消息中的 RETAIN 标识来区分这是一个正常的转发消息还是一个保留消息，而不是去判断消息是否是自己订阅后收到的第一个消息（转发消息甚至可能会先于保留消息被发送，视不同 Broker 的具体实现而定）。"""
}
label {
en: """Retain As Publish"""
zh: """Retain As Publish"""
}
}
nl {
desc {
en: """Default value 0.
MQTT v3.1.1： if you subscribe to the topic published by yourself, you will receive all messages that you published.
MQTT v5: if you set this option as 1 when subscribing, the server will not forward the message you published to you."""
zh: """缺省值为0，
MQTT v3.1.1：如果设备订阅了自己发布消息的主题，那么将收到自己发布的所有消息。
MQTT v5：如果设备在订阅时将此选项设置为 1，那么服务端将不会向设备转发自己发布的消息"""
}
label {
en: """No Local"""
zh: """No Local"""
}
}
}

29
apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src
View File

@ -1,16 +1,17 @@
%% -*- mode: erlang -*-
{application, emqx_auto_subscribe,
[{description, "An OTP application"},
{vsn, "0.1.0"},
{registered, []},
{mod, {emqx_auto_subscribe_app, []}},
{applications,
[kernel,
stdlib
]},
{env,[]},
{modules, []},
{application, emqx_auto_subscribe, [
{description, "An OTP application"},
{vsn, "0.1.0"},
{registered, []},
{mod, {emqx_auto_subscribe_app, []}},
{applications, [
kernel,
stdlib,
emqx
]},
{env, []},
{modules, []},
{licenses, ["Apache 2.0"]},
{links, []}
]}.
{licenses, ["Apache 2.0"]},
{links, []}
]}.

39
apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl
View File

@ -20,19 +20,21 @@
-define(MAX_AUTO_SUBSCRIBE, 20).
-export([load/0, unload/0]). %
%
-export([load/0, unload/0]).
-export([ max_limit/0
, list/0
, update/1
, post_config_update/5
]).
-export([
max_limit/0,
list/0,
update/1,
post_config_update/5
]).
%% hook callback
-export([on_client_connected/3]).
load() ->
emqx_conf:add_handler([auto_subscribe, topics], ?MODULE),
ok = emqx_conf:add_handler([auto_subscribe, topics], ?MODULE),
update_hook().
unload() ->
@ -56,7 +58,8 @@ post_config_update(_KeyPath, _Req, NewTopics, _OldConf, _AppEnvs) ->
on_client_connected(ClientInfo, ConnInfo, {TopicHandler, Options}) ->
case erlang:apply(TopicHandler, handle, [ClientInfo, ConnInfo, Options]) of
[] -> ok;
[] ->
ok;
TopicTables ->
_ = self() ! {subscribe, TopicTables},
ok
@ -71,17 +74,21 @@ format(Rules) when is_list(Rules) ->
[format(Rule) || Rule <- Rules];
format(Rule = #{topic := Topic}) when is_map(Rule) ->
#{
topic => Topic,
qos => maps:get(qos, Rule, 0),
rh => maps:get(rh, Rule, 0),
rap => maps:get(rap, Rule, 0),
nl => maps:get(nl, Rule, 0)
topic => Topic,
qos => maps:get(qos, Rule, 0),
rh => maps:get(rh, Rule, 0),
rap => maps:get(rap, Rule, 0),
nl => maps:get(nl, Rule, 0)
}.
update_(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE ->
case emqx_conf:update([auto_subscribe, topics],
Topics,
#{rawconf_with_defaults => true, override_to => cluster}) of
case
emqx_conf:update(
[auto_subscribe, topics],
Topics,
#{rawconf_with_defaults => true, override_to => cluster}
)
of
{ok, #{raw_config := NewTopics}} ->
{ok, NewTopics};
{error, Reason} ->

14
apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl
View File

@ -29,6 +29,7 @@
-define(EXCEED_LIMIT, 'EXCEED_LIMIT').
-define(BAD_REQUEST, 'BAD_REQUEST').
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
api_spec() ->
@ -41,20 +42,21 @@ schema("/mqtt/auto_subscribe") ->
#{
'operationId' => auto_subscribe,
get => #{
description => <<"Auto subscribe list">>,
description => ?DESC(list_auto_subscribe_api),
responses => #{
200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe")
}
},
put => #{
description => <<"Update auto subscribe topic list">>,
description => ?DESC(update_auto_subscribe_api),
'requestBody' => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"),
responses => #{
200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"),
400 => emqx_mgmt_util:error_schema(
<<"Request body required">>, [?BAD_REQUEST]),
409 => emqx_mgmt_util:error_schema(
<<"Auto Subscribe topics max limit">>, [?EXCEED_LIMIT])}}
409 => emqx_dashboard_swagger:error_codes(
[?EXCEED_LIMIT],
?DESC(update_auto_subscribe_api_response409))
}
}
}.
%%%==============================================================================================

24
apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl
View File

@ -17,6 +17,7 @@
-behaviour(hocon_schema).
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
@ -32,34 +33,31 @@ roots() ->
fields("auto_subscribe") ->
[ {topics, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "topic")),
#{desc => "List of auto-subscribe topics."})}
#{desc => ?DESC(auto_subscribe)})}
];
fields("topic") ->
[ {topic, sc(binary(), #{
required => true,
example => topic_example(),
desc => "Topic name, placeholders is supported. For example: "
++ binary_to_list(topic_example())})}
desc => ?DESC("topic")})}
, {qos, sc(emqx_schema:qos(), #{
default => 0,
desc => "Quality of service. MQTT definition."})}
desc => ?DESC("qos")})}
, {rh, sc(range(0,2), #{
default => 0,
desc => "Retain handling. MQTT 5.0 definition."})}
desc => ?DESC("rh")})}
, {rap, sc(range(0, 1), #{
default => 0,
desc => "Retain as Published. MQTT 5.0 definition."})}
desc => ?DESC("rap")})}
, {nl, sc(range(0, 1), #{
default => 0,
desc => "Not local. MQTT 5.0 definition."})}
desc => ?DESC(nl)})}
].
desc("auto_subscribe") ->
"Configuration for `auto_subscribe` feature.";
desc("topic") ->
"";
desc(_) ->
undefined.
desc("auto_subscribe") -> ?DESC("auto_subscribe");
desc("topic") -> ?DESC("topic");
desc(_) -> undefined.
topic_example() ->
<<"/clientid/", ?PH_S_CLIENTID,

103
apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl
View File

@ -31,9 +31,11 @@
-define(TOPICS, [?TOPIC_C, ?TOPIC_U, ?TOPIC_H, ?TOPIC_P, ?TOPIC_A, ?TOPIC_S]).
-define(ENSURE_TOPICS , [<<"/c/auto_sub_c">>
, <<"/u/auto_sub_u">>
, ?TOPIC_S]).
-define(ENSURE_TOPICS, [
<<"/c/auto_sub_c">>,
<<"/u/auto_sub_u">>,
?TOPIC_S
]).
-define(CLIENT_ID, <<"auto_sub_c">>).
-define(CLIENT_USERNAME, <<"auto_sub_u">>).
@ -45,60 +47,58 @@ init_per_suite(Config) ->
mria:start(),
application:stop(?APP),
meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_schema, fields, fun("auto_subscribe") ->
meck:passthrough(["auto_subscribe"]) ++
emqx_auto_subscribe_schema:fields("auto_subscribe");
(F) -> meck:passthrough([F])
end),
meck:expect(emqx_schema, fields, fun
("auto_subscribe") ->
meck:passthrough(["auto_subscribe"]) ++
emqx_auto_subscribe_schema:fields("auto_subscribe");
(F) ->
meck:passthrough([F])
end),
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, remove, fun(_) -> ok end ),
meck:expect(emqx_resource, remove, fun(_) -> ok end),
application:load(emqx_dashboard),
application:load(?APP),
ok = emqx_common_test_helpers:load_config(emqx_auto_subscribe_schema,
<<"auto_subscribe {
topics = [
{
topic = \"/c/${clientid}\"
},
{
topic = \"/u/${username}\"
},
{
topic = \"/h/${host}\"
},
{
topic = \"/p/${port}\"
},
{
topic = \"/client/${clientid}/username/${username}/host/${host}/port/${port}\"
},
{
topic = \"/topic/simple\"
qos = 1
rh = 0
rap = 0
nl = 0
}
]
}">>),
emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard, ?APP],
fun set_special_configs/1),
ok = emqx_common_test_helpers:load_config(
emqx_auto_subscribe_schema,
<<"auto_subscribe {\n"
" topics = [\n"
" {\n"
" topic = \"/c/${clientid}\"\n"
" },\n"
" {\n"
" topic = \"/u/${username}\"\n"
" },\n"
" {\n"
" topic = \"/h/${host}\"\n"
" },\n"
" {\n"
" topic = \"/p/${port}\"\n"
" },\n"
" {\n"
" topic = \"/client/${clientid}/username/${username}/host/${host}/port/${port}\"\n"
" },\n"
" {\n"
" topic = \"/topic/simple\"\n"
" qos = 1\n"
" rh = 0\n"
" rap = 0\n"
" nl = 0\n"
" }\n"
" ]\n"
" }">>
),
emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_dashboard, ?APP],
fun set_special_configs/1
),
Config.
set_special_configs(emqx_dashboard) ->
Config = #{
default_username => <<"admin">>,
default_password => <<"public">>,
listeners => [#{
protocol => http,
port => 18083
}]
},
emqx_config:put([emqx_dashboard], Config),
emqx_dashboard_api_test_helpers:set_default_config(),
ok;
set_special_configs(_) ->
ok.
@ -106,10 +106,10 @@ set_special_configs(_) ->
topic_config(T) ->
#{
topic => T,
qos => 0,
rh => 0,
rap => 0,
nl => 0
qos => 0,
rh => 0,
rap => 0,
nl => 0
}.
end_per_suite(_) ->
@ -148,7 +148,6 @@ t_update(_) ->
?assertEqual(1, erlang:length(GETResponseMap)),
ok.
check_subs(Count) ->
Subs = ets:tab2list(emqx_suboption),
ct:pal("---> ~p ~p ~n", [Subs, Count]),

9
apps/emqx_bridge/src/emqx_bridge.erl
View File

@ -97,11 +97,12 @@ on_message_publish(Message = #message{topic = Topic, flags = Flags}) ->
send_to_matched_egress_bridges(Topic, Msg) ->
lists:foreach(fun (Id) ->
try send_message(Id, Msg) of
ok -> ok;
Error -> ?SLOG(error, #{msg => "send_message_to_bridge_failed",
bridge => Id, error => Error})
{error, Reason} ->
?SLOG(error, #{msg => "send_message_to_bridge_failed",
bridge => Id, error => Reason});
_ -> ok
catch Err:Reason:ST ->
?SLOG(error, #{msg => "send_message_to_bridge_crash",
?SLOG(error, #{msg => "send_message_to_bridge_exception",
bridge => Id, error => Err, reason => Reason,
stacktrace => ST})
end

301
apps/emqx_conf/src/emqx_conf.erl
View File

@ -25,7 +25,7 @@
-export([update/3, update/4]).
-export([remove/2, remove/3]).
-export([reset/2, reset/3]).
-export([dump_schema/1, dump_schema/2]).
-export([dump_schema/1, dump_schema/3]).
-export([schema_module/0]).
%% for rpc
@ -80,15 +80,22 @@ get_node_and_config(KeyPath) ->
{node(), emqx:get_config(KeyPath, config_not_found)}.
%% @doc Update all value of key path in cluster-override.conf or local-override.conf.
-spec update(emqx_map_lib:config_key_path(), emqx_config:update_request(),
emqx_config:update_opts()) ->
-spec update(
emqx_map_lib:config_key_path(),
emqx_config:update_request(),
emqx_config:update_opts()
) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
update(KeyPath, UpdateReq, Opts) ->
check_cluster_rpc_result(emqx_conf_proto_v1:update(KeyPath, UpdateReq, Opts)).
%% @doc Update the specified node's key path in local-override.conf.
-spec update(node(), emqx_map_lib:config_key_path(), emqx_config:update_request(),
emqx_config:update_opts()) ->
-spec update(
node(),
emqx_map_lib:config_key_path(),
emqx_config:update_request(),
emqx_config:update_opts()
) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()} | emqx_rpc:badrpc().
update(Node, KeyPath, UpdateReq, Opts0) when Node =:= node() ->
emqx:update_config(KeyPath, UpdateReq, Opts0#{override_to => local});
@ -126,25 +133,41 @@ reset(Node, KeyPath, Opts) ->
%% @doc Called from build script.
-spec dump_schema(file:name_all()) -> ok.
dump_schema(Dir) ->
dump_schema(Dir, emqx_conf_schema).
I18nFile = emqx:etc_file("i18n.conf"),
dump_schema(Dir, emqx_conf_schema, I18nFile).
dump_schema(Dir, SchemaModule) ->
SchemaMdFile = filename:join([Dir, "config.md"]),
io:format(user, "===< Generating: ~s~n", [SchemaMdFile ]),
ok = gen_doc(SchemaMdFile, SchemaModule),
dump_schema(Dir, SchemaModule, I18nFile) ->
lists:foreach(
fun(Lang) ->
gen_config_md(Dir, I18nFile, SchemaModule, Lang),
gen_hot_conf_schema_json(Dir, I18nFile, Lang)
end,
[en, zh]
),
gen_schema_json(Dir, I18nFile, SchemaModule).
%% for scripts/spellcheck.
%% for scripts/spellcheck.
gen_schema_json(Dir, I18nFile, SchemaModule) ->
SchemaJsonFile = filename:join([Dir, "schema.json"]),
io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]),
JsonMap = hocon_schema_json:gen(SchemaModule),
Opts = #{desc_file => I18nFile, lang => "en"},
JsonMap = hocon_schema_json:gen(SchemaModule, Opts),
IoData = jsx:encode(JsonMap, [space, {indent, 4}]),
ok = file:write_file(SchemaJsonFile, IoData),
ok = file:write_file(SchemaJsonFile, IoData).
%% hot-update configuration schema
HotConfigSchemaFile = filename:join([Dir, "hot-config-schema.json"]),
gen_hot_conf_schema_json(Dir, I18nFile, Lang) ->
emqx_dashboard:init_i18n(I18nFile, Lang),
JsonFile = "hot-config-schema-" ++ atom_to_list(Lang) ++ ".json",
HotConfigSchemaFile = filename:join([Dir, JsonFile]),
io:format(user, "===< Generating: ~s~n", [HotConfigSchemaFile]),
ok = gen_hot_conf_schema(HotConfigSchemaFile),
ok.
emqx_dashboard:clear_i18n().
gen_config_md(Dir, I18nFile, SchemaModule, Lang0) ->
Lang = atom_to_list(Lang0),
SchemaMdFile = filename:join([Dir, "config-" ++ Lang ++ ".md"]),
io:format(user, "===< Generating: ~s~n", [SchemaMdFile]),
ok = gen_doc(SchemaMdFile, SchemaModule, I18nFile, Lang).
%% @doc return the root schema module.
-spec schema_module() -> module().
@ -158,63 +181,96 @@ schema_module() ->
%% Internal functions
%%--------------------------------------------------------------------
-spec gen_doc(file:name_all(), module()) -> ok.
gen_doc(File, SchemaModule) ->
-spec gen_doc(file:name_all(), module(), file:name_all(), string()) -> ok.
gen_doc(File, SchemaModule, I18nFile, Lang) ->
Version = emqx_release:version(),
Title = "# " ++ emqx_release:description() ++ " " ++ Version ++ " Configuration",
BodyFile = filename:join([code:lib_dir(emqx_conf), "etc", "emqx_conf.md"]),
{ok, Body} = file:read_file(BodyFile),
Doc = hocon_schema_md:gen(SchemaModule, #{title => Title, body => Body}),
Opts = #{title => Title, body => Body, desc_file => I18nFile, lang => Lang},
Doc = hocon_schema_md:gen(SchemaModule, Opts),
file:write_file(File, Doc).
check_cluster_rpc_result(Result) ->
case Result of
{ok, _TnxId, Res} -> Res;
{ok, _TnxId, Res} ->
Res;
{retry, TnxId, Res, Nodes} ->
%% The init MFA return ok, but other nodes failed.
%% We return ok and alert an alarm.
?SLOG(error, #{msg => "failed_to_update_config_in_cluster", nodes => Nodes,
tnx_id => TnxId}),
?SLOG(error, #{
msg => "failed_to_update_config_in_cluster",
nodes => Nodes,
tnx_id => TnxId
}),
Res;
{error, Error} -> %% all MFA return not ok or {ok, term()}.
%% all MFA return not ok or {ok, term()}.
{error, Error} ->
Error
end.
%% Only gen hot_conf schema, not all configuration fields.
gen_hot_conf_schema(File) ->
{ApiSpec0, Components0} = emqx_dashboard_swagger:spec(emqx_mgmt_api_configs,
#{schema_converter => fun hocon_schema_to_spec/2}),
ApiSpec = lists:foldl(fun({Path, Spec, _, _}, Acc) ->
NewSpec = maps:fold(fun(Method, #{responses := Responses}, SubAcc) ->
case Responses of
#{<<"200">> :=
#{<<"content">> := #{<<"application/json">> := #{<<"schema">> := Schema}}}} ->
SubAcc#{Method => Schema};
_ -> SubAcc
end
end, #{}, Spec),
Acc#{list_to_atom(Path) => NewSpec} end, #{}, ApiSpec0),
{ApiSpec0, Components0} = emqx_dashboard_swagger:spec(
emqx_mgmt_api_configs,
#{schema_converter => fun hocon_schema_to_spec/2}
),
ApiSpec = lists:foldl(
fun({Path, Spec, _, _}, Acc) ->
NewSpec = maps:fold(
fun(Method, #{responses := Responses}, SubAcc) ->
case Responses of
#{
<<"200">> :=
#{
<<"content">> := #{
<<"application/json">> := #{<<"schema">> := Schema}
}
}
} ->
SubAcc#{Method => Schema};
_ ->
SubAcc
end
end,
#{},
Spec
),
Acc#{list_to_atom(Path) => NewSpec}
end,
#{},
ApiSpec0
),
Components = lists:foldl(fun(M, Acc) -> maps:merge(M, Acc) end, #{}, Components0),
IoData = jsx:encode(#{
info => #{title => <<"EMQX Hot Conf Schema">>, version => <<"0.1.0">>},
paths => ApiSpec,
components => #{schemas => Components}
}, [space, {indent, 4}]),
IoData = jsx:encode(
#{
info => #{title => <<"EMQX Hot Conf Schema">>, version => <<"0.1.0">>},
paths => ApiSpec,
components => #{schemas => Components}
},
[space, {indent, 4}]
),
file:write_file(File, IoData).
-define(INIT_SCHEMA, #{fields => #{}, translations => #{},
validations => [], namespace => undefined}).
-define(INIT_SCHEMA, #{
fields => #{},
translations => #{},
validations => [],
namespace => undefined
}).
-define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])).
-define(TO_COMPONENTS_SCHEMA(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>,
?TO_REF(emqx_dashboard_swagger:namespace(_M_), _F_)])).
-define(TO_COMPONENTS_SCHEMA(_M_, _F_),
iolist_to_binary([
<<"#/components/schemas/">>,
?TO_REF(emqx_dashboard_swagger:namespace(_M_), _F_)
])
).
hocon_schema_to_spec(?R_REF(Module, StructName), _LocalModule) ->
{#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(Module, StructName)},
[{Module, StructName}]};
{#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(Module, StructName)}, [{Module, StructName}]};
hocon_schema_to_spec(?REF(StructName), LocalModule) ->
{#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(LocalModule, StructName)},
[{LocalModule, StructName}]};
{#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(LocalModule, StructName)}, [{LocalModule, StructName}]};
hocon_schema_to_spec(Type, LocalModule) when ?IS_TYPEREFL(Type) ->
{typename_to_spec(typerefl:name(Type), LocalModule), []};
hocon_schema_to_spec(?ARRAY(Item), LocalModule) ->
@ -226,50 +282,97 @@ hocon_schema_to_spec(?ENUM(Items), _LocalModule) ->
{#{type => enum, symbols => Items}, []};
hocon_schema_to_spec(?MAP(Name, Type), LocalModule) ->
{Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule),
{#{<<"type">> => object,
<<"properties">> => #{<<"$", (to_bin(Name))/binary>> => Schema}},
SubRefs};
{
#{
<<"type">> => object,
<<"properties">> => #{<<"$", (to_bin(Name))/binary>> => Schema}
},
SubRefs
};
hocon_schema_to_spec(?UNION(Types), LocalModule) ->
{OneOf, Refs} = lists:foldl(fun(Type, {Acc, RefsAcc}) ->
{Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule),
{[Schema | Acc], SubRefs ++ RefsAcc}
end, {[], []}, Types),
{OneOf, Refs} = lists:foldl(
fun(Type, {Acc, RefsAcc}) ->
{Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule),
{[Schema | Acc], SubRefs ++ RefsAcc}
end,
{[], []},
Types
),
{#{<<"oneOf">> => OneOf}, Refs};
hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->
{#{type => enum, symbols => [Atom]}, []}.
typename_to_spec("user_id_type()", _Mod) -> #{type => enum, symbols => [clientid, username]};
typename_to_spec("term()", _Mod) -> #{type => string};
typename_to_spec("boolean()", _Mod) -> #{type => boolean};
typename_to_spec("binary()", _Mod) -> #{type => string};
typename_to_spec("float()", _Mod) -> #{type => number};
typename_to_spec("integer()", _Mod) -> #{type => number};
typename_to_spec("non_neg_integer()", _Mod) -> #{type => number, minimum => 1};
typename_to_spec("number()", _Mod) -> #{type => number};
typename_to_spec("string()", _Mod) -> #{type => string};
typename_to_spec("atom()", _Mod) -> #{type => string};
typename_to_spec("duration()", _Mod) -> #{type => duration};
typename_to_spec("duration_s()", _Mod) -> #{type => duration};
typename_to_spec("duration_ms()", _Mod) -> #{type => duration};
typename_to_spec("percent()", _Mod) -> #{type => percent};
typename_to_spec("file()", _Mod) -> #{type => string};
typename_to_spec("ip_port()", _Mod) -> #{type => ip_port};
typename_to_spec("url()", _Mod) -> #{type => url};
typename_to_spec("bytesize()", _Mod) -> #{type => 'byteSize'};
typename_to_spec("wordsize()", _Mod) -> #{type => 'byteSize'};
typename_to_spec("qos()", _Mod) -> #{type => enum, symbols => [0, 1, 2]};
typename_to_spec("comma_separated_list()", _Mod) -> #{type => comma_separated_string};
typename_to_spec("comma_separated_atoms()", _Mod) -> #{type => comma_separated_string};
typename_to_spec("pool_type()", _Mod) -> #{type => enum, symbols => [random, hash]};
typename_to_spec("user_id_type()", _Mod) ->
#{type => enum, symbols => [clientid, username]};
typename_to_spec("term()", _Mod) ->
#{type => string};
typename_to_spec("boolean()", _Mod) ->
#{type => boolean};
typename_to_spec("binary()", _Mod) ->
#{type => string};
typename_to_spec("float()", _Mod) ->
#{type => number};
typename_to_spec("integer()", _Mod) ->
#{type => number};
typename_to_spec("non_neg_integer()", _Mod) ->
#{type => number, minimum => 1};
typename_to_spec("number()", _Mod) ->
#{type => number};
typename_to_spec("string()", _Mod) ->
#{type => string};
typename_to_spec("atom()", _Mod) ->
#{type => string};
typename_to_spec("duration()", _Mod) ->
#{type => duration};
typename_to_spec("duration_s()", _Mod) ->
#{type => duration};
typename_to_spec("duration_ms()", _Mod) ->
#{type => duration};
typename_to_spec("percent()", _Mod) ->
#{type => percent};
typename_to_spec("file()", _Mod) ->
#{type => string};
typename_to_spec("ip_port()", _Mod) ->
#{type => ip_port};
typename_to_spec("url()", _Mod) ->
#{type => url};
typename_to_spec("bytesize()", _Mod) ->
#{type => 'byteSize'};
typename_to_spec("wordsize()", _Mod) ->
#{type => 'byteSize'};
typename_to_spec("qos()", _Mod) ->
#{type => enum, symbols => [0, 1, 2]};
typename_to_spec("comma_separated_list()", _Mod) ->
#{type => comma_separated_string};
typename_to_spec("comma_separated_atoms()", _Mod) ->
#{type => comma_separated_string};
typename_to_spec("pool_type()", _Mod) ->
#{type => enum, symbols => [random, hash]};
typename_to_spec("log_level()", _Mod) ->
#{type => enum, symbols => [debug, info, notice, warning, error,
critical, alert, emergency, all]};
typename_to_spec("rate()", _Mod) -> #{type => string};
typename_to_spec("capacity()", _Mod) -> #{type => string};
typename_to_spec("burst_rate()", _Mod) -> #{type => string};
typename_to_spec("failure_strategy()", _Mod) -> #{type => enum, symbols => [force, drop, throw]};
typename_to_spec("initial()", _Mod) -> #{type => string};
#{
type => enum,
symbols => [
debug,
info,
notice,
warning,
error,
critical,
alert,
emergency,
all
]
};
typename_to_spec("rate()", _Mod) ->
#{type => string};
typename_to_spec("capacity()", _Mod) ->
#{type => string};
typename_to_spec("burst_rate()", _Mod) ->
#{type => string};
typename_to_spec("failure_strategy()", _Mod) ->
#{type => enum, symbols => [force, drop, throw]};
typename_to_spec("initial()", _Mod) ->
#{type => string};
typename_to_spec(Name, Mod) ->
Spec = range(Name),
Spec1 = remote_module_type(Spec, Name, Mod),
@ -282,11 +385,13 @@ default_type(Type) -> Type.
range(Name) ->
case string:split(Name, "..") of
[MinStr, MaxStr] -> %% 1..10 1..inf -inf..10
%% 1..10 1..inf -inf..10
[MinStr, MaxStr] ->
Schema = #{type => number},
Schema1 = add_integer_prop(Schema, minimum, MinStr),
add_integer_prop(Schema1, maximum, MaxStr);
_ -> nomatch
_ ->
nomatch
end.
%% Module:Type
@ -295,21 +400,25 @@ remote_module_type(nomatch, Name, Mod) ->
[_Module, Type] -> typename_to_spec(Type, Mod);
_ -> nomatch
end;
remote_module_type(Spec, _Name, _Mod) -> Spec.
remote_module_type(Spec, _Name, _Mod) ->
Spec.
%% [string()] or [integer()] or [xxx].
typerefl_array(nomatch, Name, Mod) ->
case string:trim(Name, leading, "[") of
Name -> nomatch;
Name ->
nomatch;
Name1 ->
case string:trim(Name1, trailing, "]") of
Name1 -> notmatch;
Name1 ->
notmatch;
Name2 ->
Schema = typename_to_spec(Name2, Mod),
#{type => array, items => Schema}
end
end;
typerefl_array(Spec, _Name, _Mod) -> Spec.
typerefl_array(Spec, _Name, _Mod) ->
Spec.
%% integer(1)
integer(nomatch, Name) ->
@ -317,12 +426,13 @@ integer(nomatch, Name) ->
{Int, []} -> #{type => enum, symbols => [Int], default => Int};
_ -> nomatch
end;
integer(Spec, _Name) -> Spec.
integer(Spec, _Name) ->
Spec.
add_integer_prop(Schema, Key, Value) ->
case string:to_integer(Value) of
{error, no_integer} -> Schema;
{Int, []}when Key =:= minimum -> Schema#{Key => Int};
{Int, []} when Key =:= minimum -> Schema#{Key => Int};
{Int, []} -> Schema#{Key => Int}
end.
@ -333,4 +443,5 @@ to_bin(List) when is_list(List) ->
end;
to_bin(Boolean) when is_boolean(Boolean) -> Boolean;
to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
to_bin(X) -> X.
to_bin(X) ->
X.

3
apps/emqx_conf/test/emqx_conf_schema_tests.erl
View File

@ -9,5 +9,6 @@
doc_gen_test() ->
Dir = "tmp",
ok = filelib:ensure_dir(filename:join("tmp", foo)),
_ = emqx_conf:dump_schema(Dir),
I18nFile = filename:join(["_build", "test", "lib", "emqx_dashboard", "etc", "i18n.conf.all"]),
_ = emqx_conf:dump_schema(Dir, emqx_conf_schema, I18nFile),
ok.

2
apps/emqx_connector/src/emqx_connector_http.erl
View File

@ -95,7 +95,7 @@ For example: `http://localhost:9901/`
, desc => "The type of the pool. Can be one of `random`, `hash`."
})}
, {pool_size,
sc(non_neg_integer(),
sc(pos_integer(),
#{ default => 8
, desc => "The pool size."
})}

15
apps/emqx_connector/src/emqx_connector_mongo.erl
View File

@ -76,8 +76,8 @@ fields(sharded) ->
, {w_mode, fun w_mode/1}
] ++ mongo_fields();
fields(topology) ->
[ {pool_size, fun internal_pool_size/1}
, {max_overflow, fun emqx_connector_schema_lib:pool_size/1}
[ {pool_size, fun emqx_connector_schema_lib:pool_size/1}
, {max_overflow, fun max_overflow/1}
, {overflow_ttl, fun duration/1}
, {overflow_check_period, fun duration/1}
, {local_threshold_ms, fun duration/1}
@ -114,12 +114,6 @@ mongo_fields() ->
] ++
emqx_connector_schema_lib:ssl_fields().
internal_pool_size(type) -> integer();
internal_pool_size(desc) -> "Pool size on start.";
internal_pool_size(default) -> 1;
internal_pool_size(validator) -> [?MIN(1)];
internal_pool_size(_) -> undefined.
%% ===================================================================
on_start(InstId, Config = #{mongo_type := Type,
@ -334,6 +328,11 @@ duration(desc) -> "Time interval, such as timeout or TTL.";
duration(required) -> false;
duration(_) -> undefined.
max_overflow(type) -> non_neg_integer();
max_overflow(desc) -> "Max Overflow.";
max_overflow(default) -> 0;
max_overflow(_) -> undefined.
replica_set_name(type) -> binary();
replica_set_name(desc) -> "Name of the replica set.";
replica_set_name(required) -> false;

4
apps/emqx_connector/src/emqx_connector_schema_lib.erl
View File

@ -34,7 +34,7 @@
]).
-type database() :: binary().
-type pool_size() :: integer().
-type pool_size() :: pos_integer().
-type username() :: binary().
-type password() :: binary().
@ -72,7 +72,7 @@ database(required) -> true;
database(validator) -> [?NOT_EMPTY("the value of the field 'database' cannot be empty")];
database(_) -> undefined.
pool_size(type) -> integer();
pool_size(type) -> pos_integer();
pool_size(desc) -> "Size of the connection pool.";
pool_size(default) -> 8;
pool_size(validator) -> [?MIN(1)];

2
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
View File

@ -93,7 +93,7 @@ topic filters for 'remote_topic' of ingress connections.
"messages in case of ACK not received.",
#{default => "15s"})}
, {max_inflight,
sc(integer(),
sc(non_neg_integer(),
#{ default => 32
, desc => "Max inflight (sent, but un-acked) messages of the MQTT protocol"
})}

51
apps/emqx_dashboard/etc/emqx_dashboard.conf
View File

@ -10,34 +10,29 @@ dashboard {
sample_interval = 10s
## JWT token expiration time.
token_expired_time = 60m
listeners = [
{
protocol = http
num_acceptors = 4
max_connections = 512
bind = 18083
backlog = 512
send_timeout = 5s
inet6 = false
ipv6_v6only = false
}
# ,
# {
# protocol = https
# bind = "127.0.0.1:18084"
# num_acceptors = 2
# backlog = 512
# send_timeout = 5s
# inet6 = false
# ipv6_v6only = false
# certfile = "etc/certs/cert.pem"
# keyfile = "etc/certs/key.pem"
# cacertfile = "etc/certs/cacert.pem"
# verify = verify_peer
# versions = ["tlsv1.3","tlsv1.2","tlsv1.1","tlsv1"]
# ciphers = ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256","TLS_CHACHA20_POLY1305_SHA256","TLS_AES_128_CCM_SHA256","TLS_AES_128_CCM_8_SHA256","ECDHE-ECDSA-AES256-GCM-SHA384","ECDHE-RSA-AES256-GCM-SHA384","ECDHE-ECDSA-AES256-SHA384","ECDHE-RSA-AES256-SHA384","ECDHE-ECDSA-DES-CBC3-SHA","ECDH-ECDSA-AES256-GCM-SHA384","ECDH-RSA-AES256-GCM-SHA384","ECDH-ECDSA-AES256-SHA384","ECDH-RSA-AES256-SHA384","DHE-DSS-AES256-GCM-SHA384","DHE-DSS-AES256-SHA256","AES256-GCM-SHA384","AES256-SHA256","ECDHE-ECDSA-AES128-GCM-SHA256","ECDHE-RSA-AES128-GCM-SHA256","ECDHE-ECDSA-AES128-SHA256","ECDHE-RSA-AES128-SHA256","ECDH-ECDSA-AES128-GCM-SHA256","ECDH-RSA-AES128-GCM-SHA256","ECDH-ECDSA-AES128-SHA256","ECDH-RSA-AES128-SHA256","DHE-DSS-AES128-GCM-SHA256","DHE-DSS-AES128-SHA256","AES128-GCM-SHA256","AES128-SHA256","ECDHE-ECDSA-AES256-SHA","ECDHE-RSA-AES256-SHA","DHE-DSS-AES256-SHA","ECDH-ECDSA-AES256-SHA","ECDH-RSA-AES256-SHA","AES256-SHA","ECDHE-ECDSA-AES128-SHA","ECDHE-RSA-AES128-SHA","DHE-DSS-AES128-SHA","ECDH-ECDSA-AES128-SHA","ECDH-RSA-AES128-SHA","AES128-SHA"]
# }
]
listeners.http {
num_acceptors = 4
max_connections = 512
bind = 18083
backlog = 512
send_timeout = 5s
inet6 = false
ipv6_v6only = false
}
#listeners.https {
# bind = "127.0.0.1:18084"
# num_acceptors = 4
# backlog = 512
# send_timeout = 5s
# inet6 = false
# ipv6_v6only = false
# certfile = "etc/certs/cert.pem"
# keyfile = "etc/certs/key.pem"
# cacertfile = "etc/certs/cacert.pem"
# verify = verify_peer
# versions = ["tlsv1.3","tlsv1.2","tlsv1.1","tlsv1"]
# ciphers = ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256","TLS_CHACHA20_POLY1305_SHA256","TLS_AES_128_CCM_SHA256","TLS_AES_128_CCM_8_SHA256","ECDHE-ECDSA-AES256-GCM-SHA384","ECDHE-RSA-AES256-GCM-SHA384","ECDHE-ECDSA-AES256-SHA384","ECDHE-RSA-AES256-SHA384","ECDHE-ECDSA-DES-CBC3-SHA","ECDH-ECDSA-AES256-GCM-SHA384","ECDH-RSA-AES256-GCM-SHA384","ECDH-ECDSA-AES256-SHA384","ECDH-RSA-AES256-SHA384","DHE-DSS-AES256-GCM-SHA384","DHE-DSS-AES256-SHA256","AES256-GCM-SHA384","AES256-SHA256","ECDHE-ECDSA-AES128-GCM-SHA256","ECDHE-RSA-AES128-GCM-SHA256","ECDHE-ECDSA-AES128-SHA256","ECDHE-RSA-AES128-SHA256","ECDH-ECDSA-AES128-GCM-SHA256","ECDH-RSA-AES128-GCM-SHA256","ECDH-ECDSA-AES128-SHA256","ECDH-RSA-AES128-SHA256","DHE-DSS-AES128-GCM-SHA256","DHE-DSS-AES128-SHA256","AES128-GCM-SHA256","AES128-SHA256","ECDHE-ECDSA-AES256-SHA","ECDHE-RSA-AES256-SHA","DHE-DSS-AES256-SHA","ECDH-ECDSA-AES256-SHA","ECDH-RSA-AES256-SHA","AES256-SHA","ECDHE-ECDSA-AES128-SHA","ECDHE-RSA-AES128-SHA","DHE-DSS-AES128-SHA","ECDH-ECDSA-AES128-SHA","ECDH-RSA-AES128-SHA","AES128-SHA"]
#}
## CORS Support. don't set cors true if you don't know what it means.
# cors = false

12
apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf Normal file
View File

@ -0,0 +1,12 @@
emqx_dashboard_schema {
protocol {
desc {
en: "Protocol Name"
zh: "协议名"
}
label: {
en: "Protocol"
zh: "协议"
}
}
}

5
apps/emqx_dashboard/rebar.config
View File

@ -1,9 +1,6 @@
%% -*- mode: erlang -*-
{deps,
[ {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.8.6"}}}
, {emqx, {path, "../emqx"}}
]}.
{deps, [{emqx, {path, "../emqx"}}]}.
{edoc_opts, [{preprocess, true}]}.
{erl_opts, [warn_unused_vars,

198
apps/emqx_dashboard/src/emqx_dashboard.erl
View File

@ -18,11 +18,19 @@
-define(APP, ?MODULE).
-export([
start_listeners/0,
start_listeners/1,
stop_listeners/1,
stop_listeners/0
]).
-export([ start_listeners/0
, start_listeners/1
, stop_listeners/1
, stop_listeners/0]).
-export([
init_i18n/2,
init_i18n/0,
get_i18n/0,
clear_i18n/0
]).
%% Authorization
-export([authorize/1]).
@ -48,6 +56,7 @@ stop_listeners() ->
start_listeners(Listeners) ->
{ok, _} = application:ensure_all_started(minirest),
init_i18n(),
Authorization = {?MODULE, authorize},
GlobalSpec = #{
openapi => "3.0.0",
@ -58,12 +67,15 @@ start_listeners(Listeners) ->
'securitySchemes' => #{
'basicAuth' => #{type => http, scheme => basic},
'bearerAuth' => #{type => http, scheme => bearer}
}}},
Dispatch = [ {"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}
, {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}
, {?BASE_PATH ++ "/[...]", emqx_dashboard_bad_api, []}
, {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}
],
}
}
},
Dispatch = [
{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}},
{"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}},
{?BASE_PATH ++ "/[...]", emqx_dashboard_bad_api, []},
{'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}
],
BaseMinirest = #{
base_path => ?BASE_PATH,
modules => minirest_api:find_api_modules(apps()),
@ -74,75 +86,109 @@ start_listeners(Listeners) ->
middlewares => [cowboy_router, ?EMQX_MIDDLE, cowboy_handler]
},
Res =
lists:foldl(fun({Name, Protocol, Bind, RanchOptions}, Acc) ->
Minirest = BaseMinirest#{protocol => Protocol},
case minirest:start(Name, RanchOptions, Minirest) of
{ok, _} ->
?ULOG("Start listener ~ts on ~ts successfully.~n", [Name, emqx_listeners:format_addr(Bind)]),
Acc;
{error, _Reason} ->
%% Don't record the reason because minirest already does(too much logs noise).
[Name | Acc]
end
end, [], listeners(Listeners)),
lists:foldl(
fun({Name, Protocol, Bind, RanchOptions}, Acc) ->
Minirest = BaseMinirest#{protocol => Protocol},
case minirest:start(Name, RanchOptions, Minirest) of
{ok, _} ->
?ULOG("Start listener ~ts on ~ts successfully.~n", [
Name, emqx_listeners:format_addr(Bind)
]),
Acc;
{error, _Reason} ->
%% Don't record the reason because minirest already does(too much logs noise).
[Name | Acc]
end
end,
[],
listeners(Listeners)
),
clear_i18n(),
case Res of
[] -> ok;
_ -> {error, Res}
end.
stop_listeners(Listeners) ->
[begin
case minirest:stop(Name) of
ok ->
?ULOG("Stop listener ~ts on ~ts successfully.~n", [Name, emqx_listeners:format_addr(Port)]);
{error, not_found} ->
?SLOG(warning, #{msg => "stop_listener_failed", name => Name, port => Port})
[
begin
case minirest:stop(Name) of
ok ->
?ULOG("Stop listener ~ts on ~ts successfully.~n", [
Name, emqx_listeners:format_addr(Port)
]);
{error, not_found} ->
?SLOG(warning, #{msg => "stop_listener_failed", name => Name, port => Port})
end
end
end || {Name, _, Port, _} <- listeners(Listeners)],
|| {Name, _, Port, _} <- listeners(Listeners)
],
ok.
get_i18n() ->
application:get_env(emqx_dashboard, i18n).
init_i18n(File, Lang) ->
Cache = hocon_schema:new_cache(File),
application:set_env(emqx_dashboard, i18n, #{lang => atom_to_binary(Lang), cache => Cache}).
clear_i18n() ->
case application:get_env(emqx_dashboard, i18n) of
{ok, #{cache := Cache}} ->
hocon_schema:delete_cache(Cache),
application:unset_env(emqx_dashboard, i18n);
undefined ->
ok
end.
%%--------------------------------------------------------------------
%% internal
apps() ->
[App || {App, _, _} <- application:loaded_applications(),
[
App
|| {App, _, _} <- application:loaded_applications(),
case re:run(atom_to_list(App), "^emqx") of
{match,[{0,4}]} -> true;
{match, [{0, 4}]} -> true;
_ -> false
end].
end
].
listeners(Listeners) ->
[begin
Protocol = maps:get(protocol, ListenerOption0, http),
{ListenerOption, Bind} = ip_port(ListenerOption0),
Name = listener_name(Protocol, ListenerOption),
RanchOptions = ranch_opts(maps:without([protocol], ListenerOption)),
{Name, Protocol, Bind, RanchOptions}
end || ListenerOption0 <- Listeners].
lists:map(fun({Protocol, Conf}) ->
{Conf1, Bind} = ip_port(Conf),
{listener_name(Protocol, Conf1), Protocol, Bind, ranch_opts(Conf1)}
end, maps:to_list(Listeners)).
ip_port(Opts) -> ip_port(maps:take(bind, Opts), Opts).
ip_port(error, Opts) -> {Opts#{port => 18083}, 18083};
ip_port(error, Opts) -> {Opts#{port => 18083}, 18083};
ip_port({Port, Opts}, _) when is_integer(Port) -> {Opts#{port => Port}, Port};
ip_port({{IP, Port}, Opts}, _) -> {Opts#{port => Port, ip => IP}, {IP, Port}}.
init_i18n() ->
File = i18n_file(),
Lang = emqx_conf:get([dashboard, i18n_lang], en),
init_i18n(File, Lang).
ranch_opts(RanchOptions) ->
Keys = [ {ack_timeout, handshake_timeout}
, connection_type
, max_connections
, num_acceptors
, shutdown
, socket],
Keys = [
{ack_timeout, handshake_timeout},
connection_type,
max_connections,
num_acceptors,
shutdown,
socket
],
{S, R} = lists:foldl(fun key_take/2, {RanchOptions, #{}}, Keys),
R#{socket_opts => maps:fold(fun key_only/3, [], S)}.
key_take(Key, {All, R}) ->
{K, KX} = case Key of
{K1, K2} -> {K1, K2};
_ -> {Key, Key}
end,
key_take(Key, {All, R}) ->
{K, KX} =
case Key of
{K1, K2} -> {K1, K2};
_ -> {Key, Key}
end,
case maps:get(K, All, undefined) of
undefined ->
{All, R};
@ -150,20 +196,22 @@ key_take(Key, {All, R}) ->
{maps:remove(K, All), R#{KX => V}}
end.
key_only(K , true , S) -> [K | S];
key_only(_K, false, S) -> S;
key_only(K , V , S) -> [{K, V} | S].
key_only(K, true, S) -> [K | S];
key_only(_K, false, S) -> S;
key_only(K, V, S) -> [{K, V} | S].
listener_name(Protocol, #{port := Port, ip := IP}) ->
Name = "dashboard:"
++ atom_to_list(Protocol) ++ ":"
++ inet:ntoa(IP) ++ ":"
++ integer_to_list(Port),
Name =
"dashboard:" ++
atom_to_list(Protocol) ++ ":" ++
inet:ntoa(IP) ++ ":" ++
integer_to_list(Port),
list_to_atom(Name);
listener_name(Protocol, #{port := Port}) ->
Name = "dashboard:"
++ atom_to_list(Protocol) ++ ":"
++ integer_to_list(Port),
Name =
"dashboard:" ++
atom_to_list(Protocol) ++ ":" ++
integer_to_list(Port),
list_to_atom(Name).
authorize(Req) ->
@ -180,11 +228,13 @@ authorize(Req) ->
{error, <<"not_allowed">>} ->
return_unauthorized(
?WRONG_USERNAME_OR_PWD,
<<"Check username/password">>);
<<"Check username/password">>
);
{error, _} ->
return_unauthorized(
?WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET,
<<"Check username/password or api_key/api_secret">>)
<<"Check username/password or api_key/api_secret">>
)
end;
{error, _} ->
return_unauthorized(<<"WORNG_USERNAME_OR_PWD">>, <<"Check username/password">>)
@ -199,12 +249,22 @@ authorize(Req) ->
{401, 'BAD_TOKEN', <<"Get a token by POST /login">>}
end;
_ ->
return_unauthorized(<<"AUTHORIZATION_HEADER_ERROR">>,
<<"Support authorization: basic/bearer ">>)
return_unauthorized(
<<"AUTHORIZATION_HEADER_ERROR">>,
<<"Support authorization: basic/bearer ">>
)
end.
return_unauthorized(Code, Message) ->
{401, #{<<"WWW-Authenticate">> =>
<<"Basic Realm=\"minirest-server\"">>},
#{code => Code, message => Message}
}.
{401,
#{
<<"WWW-Authenticate">> =>
<<"Basic Realm=\"minirest-server\"">>
},
#{code => Code, message => Message}}.
i18n_file() ->
case application:get_env(emqx_dashboard, i18n_file) of
undefined -> emqx:etc_file("i18n.conf");
{ok, File} -> File
end.

13
apps/emqx_dashboard/src/emqx_dashboard_monitor.erl
View File

@ -18,6 +18,8 @@
-include("emqx_dashboard.hrl").
-include_lib("emqx/include/logger.hrl").
-behaviour(gen_server).
-boot_mnesia({mnesia, [boot]}).
@ -128,7 +130,16 @@ current_rate() ->
current_rate(all) ->
current_rate();
current_rate(Node) when Node == node() ->
do_call(current_rate);
try
{ok, Rate} = do_call(current_rate),
{ok, Rate}
catch _E:R ->
?SLOG(warning, #{msg => "Dashboard monitor error", reason => R}),
%% Rate map 0, ensure api will not crash.
%% When joining cluster, dashboard monitor restart.
Rate0 = [{Key, 0} || Key <- ?GAUGE_SAMPLER_LIST ++ maps:values(?DELTA_SAMPLER_RATE_MAP)],
{ok, maps:from_list(Rate0)}
end;
current_rate(Node) ->
case emqx_dashboard_proto_v1:current_rate(Node) of
{badrpc, Reason} ->

2
apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
View File

@ -107,7 +107,7 @@ fields(sampler) ->
Samplers =
[{SamplerName, hoconsc:mk(integer(), #{desc => swagger_desc(SamplerName)})}
|| SamplerName <- ?SAMPLER_LIST],
[{time_stamp, hoconsc:mk(integer(), #{desc => <<"Timestamp">>})} | Samplers];
[{time_stamp, hoconsc:mk(non_neg_integer(), #{desc => <<"Timestamp">>})} | Samplers];
fields(sampler_current) ->
[{SamplerName, hoconsc:mk(integer(), #{desc => swagger_desc(SamplerName)})}

243
apps/emqx_dashboard/src/emqx_dashboard_schema.erl
View File

@ -15,90 +15,140 @@
%%--------------------------------------------------------------------
-module(emqx_dashboard_schema).
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-export([ roots/0
, fields/1
, namespace/0
, desc/1
]).
-export([
roots/0,
fields/1,
namespace/0,
desc/1
]).
namespace() -> <<"dashboard">>.
roots() -> ["dashboard"].
fields("dashboard") ->
[ {listeners,
sc(hoconsc:array(hoconsc:union([hoconsc:ref(?MODULE, "http"),
hoconsc:ref(?MODULE, "https")])),
#{ desc =>
"HTTP(s) listeners are identified by their protocol type and are
used to serve dashboard UI and restful HTTP API.<br>
Listeners must have a unique combination of port number and IP address.<br>
For example, an HTTP listener can listen on all configured IP addresses
on a given port for a machine by specifying the IP address 0.0.0.0.<br>
Alternatively, the HTTP listener can specify a unique IP address for each listener,
but use the same port."})}
, {default_username, fun default_username/1}
, {default_password, fun default_password/1}
, {sample_interval, sc(emqx_schema:duration_s(),
#{ default => "10s"
, desc => "How often to update metrics displayed in the dashboard.<br/>"
"Note: `sample_interval` should be a divisor of 60."
})}
, {token_expired_time, sc(emqx_schema:duration(),
#{ default => "30m"
, desc => "JWT token expiration time."
})}
, {cors, fun cors/1}
[
{listeners,
sc(
ref("listeners"),
#{
desc =>
"HTTP(s) listeners are identified by their protocol type and are\n"
"used to serve dashboard UI and restful HTTP API.<br>\n"
"Listeners must have a unique combination of port number and IP address.<br>\n"
"For example, an HTTP listener can listen on all configured IP addresses\n"
"on a given port for a machine by specifying the IP address 0.0.0.0.<br>\n"
"Alternatively, the HTTP listener can specify a unique IP address for each listener,\n"
"but use the same port."
}
)},
{default_username, fun default_username/1},
{default_password, fun default_password/1},
{sample_interval,
sc(
emqx_schema:duration_s(),
#{
default => "10s",
desc =>
"How often to update metrics displayed in the dashboard.<br/>"
"Note: `sample_interval` should be a divisor of 60."
}
)},
{token_expired_time,
sc(
emqx_schema:duration(),
#{
default => "30m",
desc => "JWT token expiration time."
}
)},
{cors, fun cors/1},
{i18n_lang, fun i18n_lang/1}
];
fields("listeners") ->
[
{"http",
sc(
ref("http"),
#{
desc => "TCP listeners",
required => {false, recursively}
}
)},
{"https",
sc(
ref("https"),
#{
desc => "SSL listeners",
required => {false, recursively}
}
)}
];
fields("http") ->
[ {"protocol", sc(
hoconsc:enum([http, https]),
#{ desc => "HTTP/HTTPS protocol."
, required => true
, default => http
})}
, {"bind", fun bind/1}
, {"num_acceptors", sc(
integer(),
#{ default => 4
, desc => "Socket acceptor pool size for TCP protocols."
})}
, {"max_connections",
sc(integer(),
#{ default => 512
, desc => "Maximum number of simultaneous connections."
})}
, {"backlog",
sc(integer(),
#{ default => 1024
, desc => "Defines the maximum length that the queue of pending connections can grow to."
})}
, {"send_timeout",
sc(emqx_schema:duration(),
#{ default => "5s"
, desc => "Send timeout for the socket."
})}
, {"inet6",
sc(boolean(),
#{ default => false
, desc => "Sets up the listener for IPv6."
})}
, {"ipv6_v6only",
sc(boolean(),
#{ default => false
, desc => "Disable IPv4-to-IPv6 mapping for the listener."
})}
[
{"bind", fun bind/1},
{"num_acceptors",
sc(
integer(),
#{
default => 4,
desc => "Socket acceptor pool size for TCP protocols."
}
)},
{"max_connections",
sc(
integer(),
#{
default => 512,
desc => "Maximum number of simultaneous connections."
}
)},
{"backlog",
sc(
integer(),
#{
default => 1024,
desc =>
"Defines the maximum length that the queue of pending connections can grow to."
}
)},
{"send_timeout",
sc(
emqx_schema:duration(),
#{
default => "5s",
desc => "Send timeout for the socket."
}
)},
{"inet6",
sc(
boolean(),
#{
default => false,
desc => "Sets up the listener for IPv6."
}
)},
{"ipv6_v6only",
sc(
boolean(),
#{
default => false,
desc => "Disable IPv4-to-IPv6 mapping for the listener."
}
)}
];
fields("https") ->
fields("http") ++
proplists:delete("fail_if_no_peer_cert",
emqx_schema:server_ssl_opts_schema(#{}, true)).
proplists:delete(
"fail_if_no_peer_cert",
emqx_schema:server_ssl_opts_schema(#{}, true)
).
desc("dashboard") ->
"Configuration for EMQX dashboard.";
desc("listeners") ->
"Configuration for the dashboard listener.";
desc("http") ->
"Configuration for the dashboard listener (plaintext).";
desc("https") ->
@ -119,23 +169,44 @@ default_username(desc) -> "The default username of the automatically created das
default_username('readOnly') -> true;
default_username(_) -> undefined.
default_password(type) -> string();
default_password(default) -> "public";
default_password(required) -> true;
default_password('readOnly') -> true;
default_password(sensitive) -> true;
default_password(desc) -> """
The initial default password for dashboard 'admin' user.
For safety, it should be changed as soon as possible.""";
default_password(_) -> undefined.
default_password(type) ->
string();
default_password(default) ->
"public";
default_password(required) ->
true;
default_password('readOnly') ->
true;
default_password(sensitive) ->
true;
default_password(desc) ->
""
"\n"
"The initial default password for dashboard 'admin' user.\n"
"For safety, it should be changed as soon as possible."
"";
default_password(_) ->
undefined.
cors(type) -> boolean();
cors(default) -> false;
cors(required) -> false;
cors(type) ->
boolean();
cors(default) ->
false;
cors(required) ->
false;
cors(desc) ->
"Support Cross-Origin Resource Sharing (CORS).
Allows a server to indicate any origins (domain, scheme, or port) other than
its own from which a browser should permit loading resources.";
cors(_) -> undefined.
"Support Cross-Origin Resource Sharing (CORS).\n"
"Allows a server to indicate any origins (domain, scheme, or port) other than\n"
"its own from which a browser should permit loading resources.";
cors(_) ->
undefined.
i18n_lang(type) -> ?ENUM([en, zh]);
i18n_lang(default) -> en;
i18n_lang('readOnly') -> true;
i18n_lang(desc) -> "Internationalization language support.";
i18n_lang(_) -> undefined.
sc(Type, Meta) -> hoconsc:mk(Type, Meta).
ref(Field) -> hoconsc:ref(?MODULE, Field).

621
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -28,101 +28,136 @@
-export([filter_check_request/2, filter_check_request_and_translate_body/2]).
-ifdef(TEST).
-export([ parse_spec_ref/3
, components/2
]).
-export([
parse_spec_ref/3,
components/2
]).
-endif.
-define(METHODS, [get, post, put, head, delete, patch, options, trace]).
-define(DEFAULT_FIELDS, [example, allowReserved, style, format, readOnly,
explode, maxLength, allowEmptyValue, deprecated, minimum, maximum]).
-define(DEFAULT_FIELDS, [
example,
allowReserved,
style,
format,
readOnly,
explode,
maxLength,
allowEmptyValue,
deprecated,
minimum,
maximum
]).
-define(INIT_SCHEMA, #{fields => #{}, translations => #{},
validations => [], namespace => undefined}).
-define(INIT_SCHEMA, #{
fields => #{},
translations => #{},
validations => [],
namespace => undefined
}).
-define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])).
-define(TO_COMPONENTS_SCHEMA(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>,
?TO_REF(namespace(_M_), _F_)])).
-define(TO_COMPONENTS_PARAM(_M_, _F_), iolist_to_binary([<<"#/components/parameters/">>,
?TO_REF(namespace(_M_), _F_)])).
-define(TO_COMPONENTS_SCHEMA(_M_, _F_),
iolist_to_binary([
<<"#/components/schemas/">>,
?TO_REF(namespace(_M_), _F_)
])
).
-define(TO_COMPONENTS_PARAM(_M_, _F_),
iolist_to_binary([
<<"#/components/parameters/">>,
?TO_REF(namespace(_M_), _F_)
])
).
-define(MAX_ROW_LIMIT, 1000).
-define(DEFAULT_ROW, 100).
-type(request() :: #{bindings => map(), query_string => map(), body => map()}).
-type(request_meta() :: #{module => module(), path => string(), method => atom()}).
-type request() :: #{bindings => map(), query_string => map(), body => map()}.
-type request_meta() :: #{module => module(), path => string(), method => atom()}.
-type(filter_result() :: {ok, request()} | {400, 'BAD_REQUEST', binary()}).
-type(filter() :: fun((request(), request_meta()) -> filter_result())).
-type filter_result() :: {ok, request()} | {400, 'BAD_REQUEST', binary()}.
-type filter() :: fun((request(), request_meta()) -> filter_result()).
-type(spec_opts() :: #{check_schema => boolean() | filter(),
translate_body => boolean(),
schema_converter => fun((hocon_schema:schema(), Module::atom()) -> map())
}).
-type spec_opts() :: #{
check_schema => boolean() | filter(),
translate_body => boolean(),
schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map())
}.
-type(route_path() :: string() | binary()).
-type(route_methods() :: map()).
-type(route_handler() :: atom()).
-type(route_options() :: #{filter => filter() | undefined}).
-type route_path() :: string() | binary().
-type route_methods() :: map().
-type route_handler() :: atom().
-type route_options() :: #{filter => filter() | undefined}.
-type(api_spec_entry() :: {route_path(), route_methods(), route_handler(), route_options()}).
-type(api_spec_component() :: map()).
-type api_spec_entry() :: {route_path(), route_methods(), route_handler(), route_options()}.
-type api_spec_component() :: map().
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
%% @equiv spec(Module, #{check_schema => false})
-spec(spec(module()) -> {list(api_spec_entry()), list(api_spec_component())}).
-spec spec(module()) -> {list(api_spec_entry()), list(api_spec_component())}.
spec(Module) -> spec(Module, #{check_schema => false}).
-spec(spec(module(), spec_opts()) -> {list(api_spec_entry()), list(api_spec_component())}).
-spec spec(module(), spec_opts()) -> {list(api_spec_entry()), list(api_spec_component())}.
spec(Module, Options) ->
Paths = apply(Module, paths, []),
{ApiSpec, AllRefs} =
lists:foldl(fun(Path, {AllAcc, AllRefsAcc}) ->
{OperationId, Specs, Refs} = parse_spec_ref(Module, Path, Options),
CheckSchema = support_check_schema(Options),
{[{filename:join("/", Path), Specs, OperationId, CheckSchema} | AllAcc],
Refs ++ AllRefsAcc}
end, {[], []}, Paths),
lists:foldl(
fun(Path, {AllAcc, AllRefsAcc}) ->
{OperationId, Specs, Refs} = parse_spec_ref(Module, Path, Options),
CheckSchema = support_check_schema(Options),
{
[{filename:join("/", Path), Specs, OperationId, CheckSchema} | AllAcc],
Refs ++ AllRefsAcc
}
end,
{[], []},
Paths
),
{ApiSpec, components(lists:usort(AllRefs), Options)}.
-spec(namespace() -> hocon_schema:name()).
-spec namespace() -> hocon_schema:name().
namespace() -> "public".
-spec(fields(hocon_schema:name()) -> hocon_schema:fields()).
-spec fields(hocon_schema:name()) -> hocon_schema:fields().
fields(page) ->
Desc = <<"Page number of the results to fetch.">>,
Meta = #{in => query, desc => Desc, default => 1, example => 1},
[{page, hoconsc:mk(integer(), Meta)}];
[{page, hoconsc:mk(pos_integer(), Meta)}];
fields(limit) ->
Desc = iolist_to_binary([<<"Results per page(max ">>,
integer_to_binary(?MAX_ROW_LIMIT), <<")">>]),
Desc = iolist_to_binary([
<<"Results per page(max ">>,
integer_to_binary(?MAX_ROW_LIMIT),
<<")">>
]),
Meta = #{in => query, desc => Desc, default => ?DEFAULT_ROW, example => 50},
[{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}].
-spec(schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema_map()).
-spec schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema_map().
schema_with_example(Type, Example) ->
hoconsc:mk(Type, #{examples => #{<<"example">> => Example}}).
-spec(schema_with_examples(hocon_schema:type(), map()) -> hocon_schema:field_schema_map()).
-spec schema_with_examples(hocon_schema:type(), map()) -> hocon_schema:field_schema_map().
schema_with_examples(Type, Examples) ->
hoconsc:mk(Type, #{examples => #{<<"examples">> => Examples}}).
-spec(error_codes(list(atom())) -> hocon_schema:fields()).
-spec error_codes(list(atom())) -> hocon_schema:fields().
error_codes(Codes) ->
error_codes(Codes, <<"Error code to troubleshoot problems.">>).
-spec(error_codes(nonempty_list(atom()), binary()) -> hocon_schema:fields()).
error_codes(Codes = [_ | _], MsgExample) ->
-spec error_codes(nonempty_list(atom()), binary() | {desc, module(), term()}) ->
hocon_schema:fields().
error_codes(Codes = [_ | _], MsgDesc) ->
[
{code, hoconsc:mk(hoconsc:enum(Codes))},
{message, hoconsc:mk(string(), #{
desc => <<"Details description of the error.">>,
example => MsgExample
})}
{message,
hoconsc:mk(string(), #{
desc => MsgDesc
})}
].
%%------------------------------------------------------------------------------
@ -143,10 +178,13 @@ translate_req(Request, #{module := Module, path := Path, method := Method}, Chec
{Bindings, QueryStr} = check_parameters(Request, Params, Module),
NewBody = check_request_body(Request, Body, Module, CheckFun, hoconsc:is_schema(Body)),
{ok, Request#{bindings => Bindings, query_string => QueryStr, body => NewBody}}
catch throw:{_, ValidErrors} ->
Msg = [io_lib:format("~ts : ~p", [Key, Reason]) ||
{validation_error, #{path := Key, reason := Reason}} <- ValidErrors],
{400, 'BAD_REQUEST', iolist_to_binary(string:join(Msg, ","))}
catch
throw:{_, ValidErrors} ->
Msg = [
io_lib:format("~ts : ~p", [Key, Reason])
|| {validation_error, #{path := Key, reason := Reason}} <- ValidErrors
],
{400, 'BAD_REQUEST', iolist_to_binary(string:join(Msg, ","))}
end.
check_and_translate(Schema, Map, Opts) ->
@ -169,30 +207,51 @@ parse_spec_ref(Module, Path, Options) ->
Schema =
try
erlang:apply(Module, schema, [Path])
catch error: Reason -> %% better error message
throw({error, #{mfa => {Module, schema, [Path]}, reason => Reason}})
%% better error message
catch
error:Reason ->
throw({error, #{mfa => {Module, schema, [Path]}, reason => Reason}})
end,
{Specs, Refs} = maps:fold(fun(Method, Meta, {Acc, RefsAcc}) ->
(not lists:member(Method, ?METHODS))
andalso throw({error, #{module => Module, path => Path, method => Method}}),
{Spec, SubRefs} = meta_to_spec(Meta, Module, Options),
{Acc#{Method => Spec}, SubRefs ++ RefsAcc}
end, {#{}, []},
maps:without(['operationId'], Schema)),
{Specs, Refs} = maps:fold(
fun(Method, Meta, {Acc, RefsAcc}) ->
(not lists:member(Method, ?METHODS)) andalso
throw({error, #{module => Module, path => Path, method => Method}}),
{Spec, SubRefs} = meta_to_spec(Meta, Module, Options),
{Acc#{Method => Spec}, SubRefs ++ RefsAcc}
end,
{#{}, []},
maps:without(['operationId'], Schema)
),
{maps:get('operationId', Schema), Specs, Refs}.
check_parameters(Request, Spec, Module) ->
#{bindings := Bindings, query_string := QueryStr} = Request,
BindingsBin = maps:fold(fun(Key, Value, Acc) ->
Acc#{atom_to_binary(Key) => Value}
end, #{}, Bindings),
BindingsBin = maps:fold(
fun(Key, Value, Acc) ->
Acc#{atom_to_binary(Key) => Value}
end,
#{},
Bindings
),
check_parameter(Spec, BindingsBin, QueryStr, Module, #{}, #{}).
check_parameter([?REF(Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) ->
check_parameter([?R_REF(LocalMod, Fields) | Spec],
Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc);
check_parameter([?R_REF(Module, Fields) | Spec],
Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) ->
check_parameter(
[?R_REF(LocalMod, Fields) | Spec],
Bindings,
QueryStr,
LocalMod,
BindingsAcc,
QueryStrAcc
);
check_parameter(
[?R_REF(Module, Fields) | Spec],
Bindings,
QueryStr,
LocalMod,
BindingsAcc,
QueryStrAcc
) ->
Params = apply(Module, fields, [Fields]),
check_parameter(Params ++ Spec, Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc);
check_parameter([], _Bindings, _QueryStr, _Module, NewBindings, NewQueryStr) ->
@ -209,7 +268,7 @@ check_parameter([{Name, Type} | Spec], Bindings, QueryStr, Module, BindingsAcc,
Option = #{},
NewQueryStr = hocon_tconf:check_plain(Schema, QueryStr, Option),
NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr),
check_parameter(Spec, Bindings, QueryStr, Module,BindingsAcc, NewQueryStrAcc)
check_parameter(Spec, Bindings, QueryStr, Module, BindingsAcc, NewQueryStrAcc)
end.
check_request_body(#{body := Body}, Schema, Module, CheckFun, true) ->
@ -230,21 +289,24 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) ->
%% {good_nest_2, mk(ref(?MODULE, good_ref), #{})}
%% ]}
%% ]
check_request_body(#{body := Body}, Spec, _Module, CheckFun, false)when is_list(Spec) ->
lists:foldl(fun({Name, Type}, Acc) ->
Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
maps:merge(Acc, CheckFun(Schema, Body, #{}))
end, #{}, Spec);
check_request_body(#{body := Body}, Spec, _Module, CheckFun, false) when is_list(Spec) ->
lists:foldl(
fun({Name, Type}, Acc) ->
Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
maps:merge(Acc, CheckFun(Schema, Body, #{}))
end,
#{},
Spec
);
%% requestBody => #{content => #{ 'application/octet-stream' =>
%% #{schema => #{ type => string, format => binary}}}
check_request_body(#{body := Body}, Spec, _Module, _CheckFun, false)when is_map(Spec) ->
check_request_body(#{body := Body}, Spec, _Module, _CheckFun, false) when is_map(Spec) ->
Body.
%% tags, description, summary, security, deprecated
meta_to_spec(Meta, Module, Options) ->
{Params, Refs1} = parameters(maps:get(parameters, Meta, []), Module),
{RequestBody, Refs2} = request_body(maps:get('requestBody', Meta, []), Module),
{RequestBody, Refs2} = request_body(maps:get('requestBody', Meta, []), Module, Options),
{Responses, Refs3} = responses(maps:get(responses, Meta, #{}), Module, Options),
{
generate_method_desc(to_spec(Meta, Params, RequestBody, Responses)),
@ -258,95 +320,132 @@ to_spec(Meta, Params, RequestBody, Responses) ->
Spec = to_spec(Meta, Params, [], Responses),
maps:put('requestBody', RequestBody, Spec).
generate_method_desc(Spec0 = #{desc := Desc}) ->
Spec = maps:remove(desc, Spec0),
Spec#{description => to_bin(Desc)};
generate_method_desc(Spec = #{description := Desc}) ->
Spec#{description => to_bin(Desc)};
generate_method_desc(Spec = #{desc := _Desc}) ->
trans_description(maps:remove(desc, Spec), Spec);
generate_method_desc(Spec = #{description := _Desc}) ->
trans_description(Spec, Spec);
generate_method_desc(Spec) ->
Spec.
parameters(Params, Module) ->
{SpecList, AllRefs} =
lists:foldl(fun(Param, {Acc, RefsAcc}) ->
case Param of
?REF(StructName) -> to_ref(Module, StructName, Acc, RefsAcc);
?R_REF(RModule, StructName) -> to_ref(RModule, StructName, Acc, RefsAcc);
{Name, Type} ->
In = hocon_schema:field_schema(Type, in),
In =:= undefined andalso
throw({error, <<"missing in:path/query field in parameters">>}),
Required = hocon_schema:field_schema(Type, required),
Default = hocon_schema:field_schema(Type, default),
HoconType = hocon_schema:field_schema(Type, type),
Meta = init_meta(Default),
{ParamType, Refs} = hocon_schema_to_spec(HoconType, Module),
Spec0 = init_prop([required | ?DEFAULT_FIELDS],
#{schema => maps:merge(ParamType, Meta), name => Name, in => In}, Type),
Spec1 = trans_required(Spec0, Required, In),
Spec2 = trans_desc(Spec1, Type),
{[Spec2 | Acc], Refs ++ RefsAcc}
end
end, {[], []}, Params),
lists:foldl(
fun(Param, {Acc, RefsAcc}) ->
case Param of
?REF(StructName) ->
to_ref(Module, StructName, Acc, RefsAcc);
?R_REF(RModule, StructName) ->
to_ref(RModule, StructName, Acc, RefsAcc);
{Name, Type} ->
In = hocon_schema:field_schema(Type, in),
In =:= undefined andalso
throw({error, <<"missing in:path/query field in parameters">>}),
Required = hocon_schema:field_schema(Type, required),
Default = hocon_schema:field_schema(Type, default),
HoconType = hocon_schema:field_schema(Type, type),
Meta = init_meta(Default),
{ParamType, Refs} = hocon_schema_to_spec(HoconType, Module),
Spec0 = init_prop(
[required | ?DEFAULT_FIELDS],
#{schema => maps:merge(ParamType, Meta), name => Name, in => In},
Type
),
Spec1 = trans_required(Spec0, Required, In),
Spec2 = trans_description(Spec1, Type),
{[Spec2 | Acc], Refs ++ RefsAcc}
end
end,
{[], []},
Params
),
{lists:reverse(SpecList), AllRefs}.
init_meta(undefined) -> #{};
init_meta(Default) -> #{default => Default}.
init_prop(Keys, Init, Type) ->
lists:foldl(fun(Key, Acc) ->
case hocon_schema:field_schema(Type, Key) of
undefined -> Acc;
Schema -> Acc#{Key => to_bin(Schema)}
end
end, Init, Keys).
lists:foldl(
fun(Key, Acc) ->
case hocon_schema:field_schema(Type, Key) of
undefined -> Acc;
Schema -> Acc#{Key => to_bin(Schema)}
end
end,
Init,
Keys
).
trans_required(Spec, true, _) -> Spec#{required => true};
trans_required(Spec, _, path) -> Spec#{required => true};
trans_required(Spec, _, _) -> Spec.
trans_desc(Init, Hocon, Func, Name) ->
Spec0 = trans_desc(Init, Hocon),
Spec0 = trans_description(Init, Hocon),
case Func =:= fun hocon_schema_to_spec/2 of
true -> Spec0;
true ->
Spec0;
false ->
Spec1 = Spec0#{label => Name},
Spec1 = trans_label(Spec0, Hocon, Name),
case Spec1 of
#{description := _} -> Spec1;
_ -> Spec1#{description => <<Name/binary, " Description">>}
end
end.
trans_desc(Spec, Hocon) ->
case hocon_schema:field_schema(Hocon, desc) of
undefined ->
case hocon_schema:field_schema(Hocon, description) of
undefined ->
Spec;
Desc ->
Spec#{description => to_bin(Desc)}
end;
Desc -> Spec#{description => to_bin(Desc)}
trans_description(Spec, Hocon) ->
case trans_desc(<<"desc">>, Hocon, undefined) of
undefined -> Spec;
Value -> Spec#{description => Value}
end.
request_body(#{content := _} = Content, _Module) -> {Content, []};
request_body([], _Module) -> {[], []};
request_body(Schema, Module) ->
trans_label(Spec, Hocon, Default) ->
Label = trans_desc(<<"label">>, Hocon, Default),
Spec#{label => Label}.
trans_desc(Key, Hocon, Default) ->
case resolve_desc(Key, desc_struct(Hocon)) of
undefined -> Default;
Value -> to_bin(Value)
end.
desc_struct(Hocon) ->
case hocon_schema:field_schema(Hocon, desc) of
undefined -> hocon_schema:field_schema(Hocon, description);
Struct -> Struct
end.
resolve_desc(_Key, Bin) when is_binary(Bin) -> Bin;
resolve_desc(Key, Struct) ->
{ok, #{cache := Cache, lang := Lang}} = emqx_dashboard:get_i18n(),
Desc = hocon_schema:resolve_schema(Struct, Cache),
case is_map(Desc) of
true -> emqx_map_lib:deep_get([Key, Lang], Desc, undefined);
false -> Desc
end.
request_body(#{content := _} = Content, _Module, _Options) ->
{Content, []};
request_body([], _Module, _Options) ->
{[], []};
request_body(Schema, Module, Options) ->
{{Props, Refs}, Examples} =
case hoconsc:is_schema(Schema) of
true ->
HoconSchema = hocon_schema:field_schema(Schema, type),
SchemaExamples = hocon_schema:field_schema(Schema, examples),
{hocon_schema_to_spec(HoconSchema, Module), SchemaExamples};
false -> {parse_object(Schema, Module, #{}), undefined}
false ->
{parse_object(Schema, Module, Options), undefined}
end,
{#{<<"content">> => content(Props, Examples)},
Refs}.
{#{<<"content">> => content(Props, Examples)}, Refs}.
responses(Responses, Module, Options) ->
{Spec, Refs, _, _} = maps:fold(fun response/3, {#{}, [], Module, Options}, Responses),
{Spec, Refs}.
response(Status, ?DESC(_Mod, _Id) = Schema, {Acc, RefsAcc, Module, Options}) ->
Desc = trans_description(#{}, #{desc => Schema}),
{Acc#{integer_to_binary(Status) => Desc}, RefsAcc, Module, Options};
response(Status, Bin, {Acc, RefsAcc, Module, Options}) when is_binary(Bin) ->
{Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module, Options};
%% Support swagger raw object(file download).
@ -359,33 +458,49 @@ response(Status, ?R_REF(_Mod, _Name) = RRef, {Acc, RefsAcc, Module, Options}) ->
SchemaToSpec = schema_converter(Options),
{Spec, Refs} = SchemaToSpec(RRef, Module),
Content = content(Spec),
{Acc#{integer_to_binary(Status) =>
#{<<"content">> => Content}}, Refs ++ RefsAcc, Module, Options};
{
Acc#{
integer_to_binary(Status) =>
#{<<"content">> => Content}
},
Refs ++ RefsAcc,
Module,
Options
};
response(Status, Schema, {Acc, RefsAcc, Module, Options}) ->
case hoconsc:is_schema(Schema) of
true ->
Hocon = hocon_schema:field_schema(Schema, type),
Examples = hocon_schema:field_schema(Schema, examples),
{Spec, Refs} = hocon_schema_to_spec(Hocon, Module),
Init = trans_desc(#{}, Schema),
Init = trans_description(#{}, Schema),
Content = content(Spec, Examples),
{
Acc#{integer_to_binary(Status) => Init#{<<"content">> => Content}},
Refs ++ RefsAcc, Module, Options
Refs ++ RefsAcc,
Module,
Options
};
false ->
{Props, Refs} = parse_object(Schema, Module, Options),
Init = trans_desc(#{}, Schema),
Init = trans_description(#{}, Schema),
Content = Init#{<<"content">> => content(Props)},
{Acc#{integer_to_binary(Status) => Content}, Refs ++ RefsAcc, Module, Options}
end.
components(Refs, Options) ->
lists:sort(maps:fold(fun(K, V, Acc) -> [#{K => V} | Acc] end, [],
components(Options, Refs, #{}, []))).
lists:sort(
maps:fold(
fun(K, V, Acc) -> [#{K => V} | Acc] end,
[],
components(Options, Refs, #{}, [])
)
).
components(_Options, [], SpecAcc, []) -> SpecAcc;
components(Options, [], SpecAcc, SubRefAcc) -> components(Options, SubRefAcc, SpecAcc, []);
components(_Options, [], SpecAcc, []) ->
SpecAcc;
components(Options, [], SpecAcc, SubRefAcc) ->
components(Options, SubRefAcc, SpecAcc, []);
components(Options, [{Module, Field} | Refs], SpecAcc, SubRefsAcc) ->
Props = hocon_schema_fields(Module, Field),
Namespace = namespace(Module),
@ -404,7 +519,9 @@ hocon_schema_fields(Module, StructName) ->
case apply(Module, fields, [StructName]) of
#{fields := Fields, desc := _} ->
%% evil here, as it's match hocon_schema's internal representation
Fields; %% TODO: make use of desc ?
%% TODO: make use of desc ?
Fields;
Other ->
Other
end.
@ -415,15 +532,13 @@ hocon_schema_fields(Module, StructName) ->
namespace(Module) ->
case hocon_schema:namespace(Module) of
undefined -> Module;
NameSpace -> re:replace(to_bin(NameSpace), ":","-",[global])
NameSpace -> re:replace(to_bin(NameSpace), ":", "-", [global])
end.
hocon_schema_to_spec(?R_REF(Module, StructName), _LocalModule) ->
{#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(Module, StructName)},
[{Module, StructName}]};
{#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(Module, StructName)}, [{Module, StructName}]};
hocon_schema_to_spec(?REF(StructName), LocalModule) ->
{#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(LocalModule, StructName)},
[{LocalModule, StructName}]};
{#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(LocalModule, StructName)}, [{LocalModule, StructName}]};
hocon_schema_to_spec(Type, LocalModule) when ?IS_TYPEREFL(Type) ->
{typename_to_spec(typerefl:name(Type), LocalModule), []};
hocon_schema_to_spec(?ARRAY(Item), LocalModule) ->
@ -435,66 +550,111 @@ hocon_schema_to_spec(?ENUM(Items), _LocalModule) ->
{#{type => string, enum => Items}, []};
hocon_schema_to_spec(?MAP(Name, Type), LocalModule) ->
{Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule),
{#{<<"type">> => object,
<<"properties">> => #{<<"$", (to_bin(Name))/binary>> => Schema}},
SubRefs};
{
#{
<<"type">> => object,
<<"properties">> => #{<<"$", (to_bin(Name))/binary>> => Schema}
},
SubRefs
};
hocon_schema_to_spec(?UNION(Types), LocalModule) ->
{OneOf, Refs} = lists:foldl(fun(Type, {Acc, RefsAcc}) ->
{Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule),
{[Schema | Acc], SubRefs ++ RefsAcc}
end, {[], []}, Types),
{OneOf, Refs} = lists:foldl(
fun(Type, {Acc, RefsAcc}) ->
{Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule),
{[Schema | Acc], SubRefs ++ RefsAcc}
end,
{[], []},
Types
),
{#{<<"oneOf">> => OneOf}, Refs};
hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->
{#{type => string, enum => [Atom]}, []}.
%% todo: Find a way to fetch enum value from user_id_type().
typename_to_spec("user_id_type()", _Mod) -> #{type => string, enum => [clientid, username]};
typename_to_spec("term()", _Mod) -> #{type => string, example => "any"};
typename_to_spec("boolean()", _Mod) -> #{type => boolean, example => true};
typename_to_spec("binary()", _Mod) -> #{type => string, example => <<"binary-example">>};
typename_to_spec("float()", _Mod) -> #{type => number, example => 3.14159};
typename_to_spec("integer()", _Mod) -> #{type => integer, example => 100};
typename_to_spec("non_neg_integer()", _Mod) -> #{type => integer, minimum => 1, example => 100};
typename_to_spec("number()", _Mod) -> #{type => number, example => 42};
typename_to_spec("string()", _Mod) -> #{type => string, example => <<"string-example">>};
typename_to_spec("atom()", _Mod) -> #{type => string, example => atom};
typename_to_spec("user_id_type()", _Mod) ->
#{type => string, enum => [clientid, username]};
typename_to_spec("term()", _Mod) ->
#{type => string};
typename_to_spec("boolean()", _Mod) ->
#{type => boolean};
typename_to_spec("binary()", _Mod) ->
#{type => string};
typename_to_spec("float()", _Mod) ->
#{type => number};
typename_to_spec("integer()", _Mod) ->
#{type => integer};
typename_to_spec("non_neg_integer()", _Mod) ->
#{type => integer, minimum => 0};
typename_to_spec("pos_integer()", _Mod) ->
#{type => integer, minimum => 1};
typename_to_spec("number()", _Mod) ->
#{type => number};
typename_to_spec("string()", _Mod) ->
#{type => string};
typename_to_spec("atom()", _Mod) ->
#{type => string};
typename_to_spec("epoch_second()", _Mod) ->
#{<<"oneOf">> => [
#{type => integer, example => 1640995200, description => <<"epoch-second">>},
#{type => string, example => <<"2022-01-01T00:00:00.000Z">>, format => <<"date-time">>}]
};
typename_to_spec("epoch_millisecond()", _Mod) ->
#{<<"oneOf">> => [
#{type => integer, example => 1640995200000, description => <<"epoch-millisecond">>},
#{type => string, example => <<"2022-01-01T00:00:00.000Z">>, format => <<"date-time">>}]
#{
<<"oneOf">> => [
#{type => integer, example => 1640995200, description => <<"epoch-second">>},
#{type => string, example => <<"2022-01-01T00:00:00.000Z">>, format => <<"date-time">>}
]
};
typename_to_spec("duration()", _Mod) -> #{type => string, example => <<"12m">>};
typename_to_spec("duration_s()", _Mod) -> #{type => string, example => <<"1h">>};
typename_to_spec("duration_ms()", _Mod) -> #{type => string, example => <<"32s">>};
typename_to_spec("percent()", _Mod) -> #{type => number, example => <<"12%">>};
typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>};
typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>};
typename_to_spec("epoch_millisecond()", _Mod) ->
#{
<<"oneOf">> => [
#{type => integer, example => 1640995200000, description => <<"epoch-millisecond">>},
#{type => string, example => <<"2022-01-01T00:00:00.000Z">>, format => <<"date-time">>}
]
};
typename_to_spec("duration()", _Mod) ->
#{type => string, example => <<"12m">>};
typename_to_spec("duration_s()", _Mod) ->
#{type => string, example => <<"1h">>};
typename_to_spec("duration_ms()", _Mod) ->
#{type => string, example => <<"32s">>};
typename_to_spec("percent()", _Mod) ->
#{type => number, example => <<"12%">>};
typename_to_spec("file()", _Mod) ->
#{type => string, example => <<"/path/to/file">>};
typename_to_spec("ip_port()", _Mod) ->
#{type => string, example => <<"127.0.0.1:80">>};
typename_to_spec("ip_ports()", _Mod) ->
#{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>};
typename_to_spec("url()", _Mod) -> #{type => string, example => <<"http://127.0.0.1">>};
typename_to_spec("connect_timeout()", Mod) -> typename_to_spec("timeout()", Mod);
typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, example => infinity},
#{type => integer, example => 100}], example => infinity};
typename_to_spec("bytesize()", _Mod) -> #{type => string, example => <<"32MB">>};
typename_to_spec("wordsize()", _Mod) -> #{type => string, example => <<"1024KB">>};
typename_to_spec("map()", _Mod) -> #{type => object, example => #{}};
typename_to_spec("#{" ++ _, Mod) -> typename_to_spec("map()", Mod);
typename_to_spec("qos()", _Mod) -> #{type => string, enum => [0, 1, 2], example => 0};
typename_to_spec("{binary(), binary()}", _Mod) -> #{type => object, example => #{}};
typename_to_spec("url()", _Mod) ->
#{type => string, example => <<"http://127.0.0.1">>};
typename_to_spec("connect_timeout()", Mod) ->
typename_to_spec("timeout()", Mod);
typename_to_spec("timeout()", _Mod) ->
#{
<<"oneOf">> => [
#{type => string, example => infinity},
#{type => integer}
],
example => infinity
};
typename_to_spec("bytesize()", _Mod) ->
#{type => string, example => <<"32MB">>};
typename_to_spec("wordsize()", _Mod) ->
#{type => string, example => <<"1024KB">>};
typename_to_spec("map()", _Mod) ->
#{type => object, example => #{}};
typename_to_spec("#{" ++ _, Mod) ->
typename_to_spec("map()", Mod);
typename_to_spec("qos()", _Mod) ->
#{type => string, enum => [0, 1, 2]};
typename_to_spec("{binary(), binary()}", _Mod) ->
#{type => object, example => #{}};
typename_to_spec("comma_separated_list()", _Mod) ->
#{type => string, example => <<"item1,item2">>};
typename_to_spec("comma_separated_atoms()", _Mod) ->
#{type => string, example => <<"item1,item2">>};
typename_to_spec("pool_type()", _Mod) ->
#{type => string, enum => [random, hash], example => hash};
#{type => string, enum => [random, hash]};
typename_to_spec("log_level()", _Mod) ->
#{ type => string,
enum => [debug, info, notice, warning, error, critical, alert, emergency, all]
#{
type => string,
enum => [debug, info, notice, warning, error, critical, alert, emergency, all]
};
typename_to_spec("rate()", _Mod) ->
#{type => string, example => <<"10M/s">>};
@ -515,16 +675,18 @@ typename_to_spec(Name, Mod) ->
Spec2 = typerefl_array(Spec1, Name, Mod),
Spec3 = integer(Spec2, Name),
Spec3 =:= nomatch andalso
throw({error, #{msg => <<"Unsupported Type">>, type => Name, module => Mod}}),
Spec3.
throw({error, #{msg => <<"Unsupported Type">>, type => Name, module => Mod}}),
Spec3.
range(Name) ->
case string:split(Name, "..") of
[MinStr, MaxStr] -> %% 1..10 1..inf -inf..10
%% 1..10 1..inf -inf..10
[MinStr, MaxStr] ->
Schema = #{type => integer},
Schema1 = add_integer_prop(Schema, minimum, MinStr),
add_integer_prop(Schema1, maximum, MaxStr);
_ -> nomatch
_ ->
nomatch
end.
%% Module:Type
@ -533,34 +695,39 @@ remote_module_type(nomatch, Name, Mod) ->
[_Module, Type] -> typename_to_spec(Type, Mod);
_ -> nomatch
end;
remote_module_type(Spec, _Name, _Mod) -> Spec.
remote_module_type(Spec, _Name, _Mod) ->
Spec.
%% [string()] or [integer()] or [xxx].
typerefl_array(nomatch, Name, Mod) ->
case string:trim(Name, leading, "[") of
Name -> nomatch;
Name ->
nomatch;
Name1 ->
case string:trim(Name1, trailing, "]") of
Name1 -> notmatch;
Name1 ->
notmatch;
Name2 ->
Schema = typename_to_spec(Name2, Mod),
#{type => array, items => Schema}
end
end;
typerefl_array(Spec, _Name, _Mod) -> Spec.
typerefl_array(Spec, _Name, _Mod) ->
Spec.
%% integer(1)
integer(nomatch, Name) ->
case string:to_integer(Name) of
{Int, []} -> #{type => integer, enum => [Int], example => Int, default => Int};
{Int, []} -> #{type => integer, enum => [Int], default => Int};
_ -> nomatch
end;
integer(Spec, _Name) -> Spec.
integer(Spec, _Name) ->
Spec.
add_integer_prop(Schema, Key, Value) ->
case string:to_integer(Value) of
{error, no_integer} -> Schema;
{Int, []}when Key =:= minimum -> Schema#{Key => Int, example => Int};
{Int, []} when Key =:= minimum -> Schema#{Key => Int};
{Int, []} -> Schema#{Key => Int}
end.
@ -571,39 +738,53 @@ to_bin(List) when is_list(List) ->
end;
to_bin(Boolean) when is_boolean(Boolean) -> Boolean;
to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
to_bin(X) -> X.
to_bin(X) ->
X.
parse_object(PropList = [_ | _], Module, Options) when is_list(PropList) ->
{Props, Required, Refs} =
lists:foldl(fun({Name, Hocon}, {Acc, RequiredAcc, RefsAcc}) ->
NameBin = to_bin(Name),
case hoconsc:is_schema(Hocon) of
true ->
HoconType = hocon_schema:field_schema(Hocon, type),
Init0 = init_prop([default | ?DEFAULT_FIELDS], #{}, Hocon),
SchemaToSpec = schema_converter(Options),
Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin),
{Prop, Refs1} = SchemaToSpec(HoconType, Module),
NewRequiredAcc =
case is_required(Hocon) of
true -> [NameBin | RequiredAcc];
false -> RequiredAcc
end,
{[{NameBin, maps:merge(Prop, Init)} | Acc], NewRequiredAcc, Refs1 ++ RefsAcc};
false ->
{SubObject, SubRefs} = parse_object(Hocon, Module, Options),
{[{NameBin, SubObject} | Acc], RequiredAcc, SubRefs ++ RefsAcc}
end
end, {[], [], []}, PropList),
lists:foldl(
fun({Name, Hocon}, {Acc, RequiredAcc, RefsAcc}) ->
NameBin = to_bin(Name),
case hoconsc:is_schema(Hocon) of
true ->
HoconType = hocon_schema:field_schema(Hocon, type),
Init0 = init_prop([default | ?DEFAULT_FIELDS], #{}, Hocon),
SchemaToSpec = schema_converter(Options),
Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin),
{Prop, Refs1} = SchemaToSpec(HoconType, Module),
NewRequiredAcc =
case is_required(Hocon) of
true -> [NameBin | RequiredAcc];
false -> RequiredAcc
end,
{
[{NameBin, maps:merge(Prop, Init)} | Acc],
NewRequiredAcc,
Refs1 ++ RefsAcc
};
false ->
{SubObject, SubRefs} = parse_object(Hocon, Module, Options),
{[{NameBin, SubObject} | Acc], RequiredAcc, SubRefs ++ RefsAcc}
end
end,
{[], [], []},
PropList
),
Object = #{<<"type">> => object, <<"properties">> => lists:reverse(Props)},
case Required of
[] -> {Object, Refs};
_ -> {maps:put(required, Required, Object), Refs}
end;
parse_object(Other, Module, Options) ->
erlang:throw({error,
#{msg => <<"Object only supports not empty proplists">>,
args => Other, module => Module, options => Options}}).
erlang:throw(
{error, #{
msg => <<"Object only supports not empty proplists">>,
args => Other,
module => Module,
options => Options
}}
).
is_required(Hocon) ->
hocon_schema:field_schema(Hocon, required) =:= true.

9
apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
View File

@ -75,18 +75,13 @@ end_per_suite(_Config) ->
mria:stop().
set_special_configs(emqx_management) ->
Listeners = [#{protocol => http, port => 8081}],
Listeners = #{http => #{port => 8081}},
Config = #{listeners => Listeners,
applications => [#{id => "admin", secret => "public"}]},
emqx_config:put([emqx_management], Config),
ok;
set_special_configs(emqx_dashboard) ->
Listeners = [#{protocol => http, port => 18083}],
Config = #{listeners => Listeners,
default_username => <<"admin">>,
default_password => <<"public">>
},
emqx_config:put([dashboard], Config),
emqx_dashboard_api_test_helpers:set_default_config(),
ok;
set_special_configs(_) ->
ok.

16
apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl
View File

@ -32,12 +32,22 @@ set_default_config() ->
set_default_config(<<"admin">>).
set_default_config(DefaultUsername) ->
Config = #{listeners => [#{protocol => http,
port => 18083}],
Config = #{listeners => #{
http => #{
port => 18083
}
},
default_username => DefaultUsername,
default_password => <<"public">>
default_password => <<"public">>,
i18n_lang => en
},
emqx_config:put([dashboard], Config),
I18nFile = filename:join([
filename:dirname(code:priv_dir(emqx_dashboard)),
"etc",
"i18n.conf.all"
]),
application:set_env(emqx_dashboard, i18n_file, I18nFile),
ok.
request(Method, Url) ->

10
apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl
View File

@ -35,15 +35,7 @@ init_per_suite(Config) ->
Config.
set_special_configs(emqx_dashboard) ->
Config = #{
default_username => <<"admin">>,
default_password => <<"public">>,
listeners => [#{
protocol => http,
port => 18083
}]
},
emqx_config:put([emqx_dashboard], Config),
emqx_dashboard_api_test_helpers:set_default_config(),
ok;
set_special_configs(_) ->
ok.

10
apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl
View File

@ -35,15 +35,7 @@ init_per_suite(Config) ->
Config.
set_special_configs(emqx_dashboard) ->
Config = #{
default_username => <<"admin">>,
default_password => <<"public">>,
listeners => [#{
protocol => http,
port => 18083
}]
},
emqx_config:put([emqx_dashboard], Config),
emqx_dashboard_api_test_helpers:set_default_config(),
ok;
set_special_configs(_) ->
ok.

10
apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
View File

@ -40,15 +40,7 @@ end_per_suite(Config) ->
Config.
set_special_configs(emqx_dashboard) ->
Config = #{
default_username => <<"admin">>,
default_password => <<"public">>,
listeners => [#{
protocol => http,
port => 18083
}]
},
emqx_config:put([dashboard], Config),
emqx_dashboard_api_test_helpers:set_default_config(),
ok;
set_special_configs(_) ->
ok.

46
apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl
View File

@ -4,6 +4,7 @@
%% API
-export([paths/0, api_spec/0, schema/1, fields/1]).
-export([init_per_suite/1, end_per_suite/1]).
-export([t_in_path/1, t_in_query/1, t_in_mix/1, t_without_in/1, t_ref/1, t_public_ref/1]).
-export([t_require/1, t_nullable/1, t_method/1, t_api_spec/1]).
-export([t_in_path_trans/1, t_in_query_trans/1, t_in_mix_trans/1, t_ref_trans/1]).
@ -26,6 +27,27 @@ groups() -> [
t_in_path_trans_error, t_in_query_trans_error, t_in_mix_trans_error]}
].
init_per_suite(Config) ->
mria:start(),
application:load(emqx_dashboard),
emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1),
emqx_dashboard:init_i18n(),
Config.
set_special_configs(emqx_dashboard) ->
emqx_dashboard_api_test_helpers:set_default_config(),
ok;
set_special_configs(_) ->
ok.
end_per_suite(Config) ->
end_suite(),
Config.
end_suite() ->
application:unload(emqx_management),
emqx_common_test_helpers:stop_apps([emqx_dashboard]).
t_in_path(_Config) ->
Expect =
[#{description => <<"Indicates which sorts of issues to return">>,
@ -40,9 +62,9 @@ t_in_query(_Config) ->
Expect =
[#{description => <<"results per page (max 100)">>,
example => 1, in => query, name => per_page,
schema => #{example => 1, maximum => 100, minimum => 1, type => integer}},
schema => #{maximum => 100, minimum => 1, type => integer}},
#{description => <<"QOS">>, in => query, name => qos,
schema => #{enum => [0, 1, 2], example => 0, type => string}}],
schema => #{enum => [0, 1, 2], type => string}}],
validate("/test/in/query", Expect),
ok.
@ -74,12 +96,12 @@ t_public_ref(_Config) ->
], Refs),
ExpectRefs = [
#{<<"public.limit">> => #{description => <<"Results per page(max 1000)">>,
example => 50,in => query,name => limit,
schema => #{default => 100,example => 1,maximum => 1000,
in => query,name => limit, example => 50,
schema => #{default => 100,maximum => 1000,
minimum => 1,type => integer}}},
#{<<"public.page">> => #{description => <<"Page number of the results to fetch.">>,
example => 1,in => query,name => page,
schema => #{default => 1,example => 100,type => integer}}}],
in => query,name => page,example => 1,
schema => #{default => 1,minimum => 1,type => integer}}}],
?assertEqual(ExpectRefs, emqx_dashboard_swagger:components(Refs,#{})),
ok.
@ -92,11 +114,11 @@ t_in_mix(_Config) ->
example => <<"12m">>,in => path,name => state,required => true,
schema => #{example => <<"1h">>,type => string}},
#{example => 10,in => query,name => per_page, required => false,
schema => #{default => 5,example => 1,maximum => 50,minimum => 1, type => integer}},
#{in => query,name => is_admin, schema => #{example => true,type => boolean}},
schema => #{default => 5,maximum => 50,minimum => 1, type => integer}},
#{in => query,name => is_admin, schema => #{type => boolean}},
#{in => query,name => timeout,
schema => #{<<"oneOf">> => [#{enum => [infinity],type => string},
#{example => 30,maximum => 60,minimum => 30, type => integer}]}}],
#{maximum => 60,minimum => 30, type => integer}]}}],
ExpectMeta = #{
tags => [tags, good],
description => <<"good description">>,
@ -116,15 +138,15 @@ t_without_in(_Config) ->
t_require(_Config) ->
ExpectSpec = [#{
in => query,name => userid, required => false,
schema => #{example => <<"binary-example">>, type => string}}],
schema => #{type => string}}],
validate("/required/false", ExpectSpec),
ok.
t_nullable(_Config) ->
NullableFalse = [#{in => query,name => userid, required => true,
schema => #{example => <<"binary-example">>, type => string}}],
schema => #{type => string}}],
NullableTrue = [#{in => query,name => userid,
schema => #{example => <<"binary-example">>, type => string}, required => false}],
schema => #{type => string}, required => false}],
validate("/nullable/false", NullableFalse),
validate("/nullable/true", NullableTrue),
ok.

72
apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl
View File

@ -3,41 +3,36 @@
-behaviour(minirest_api).
-behaviour(hocon_schema).
%% API
-export([paths/0, api_spec/0, schema/1, fields/1]).
-export([t_object/1, t_nest_object/1, t_api_spec/1,
t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1,
t_ref_array_with_key/1, t_ref_array_without_key/1, t_sub_fields/1
]).
-export([
t_object_trans/1, t_object_notrans/1, t_nest_object_trans/1, t_local_ref_trans/1,
t_remote_ref_trans/1, t_nest_ref_trans/1,
t_ref_array_with_key_trans/1, t_ref_array_without_key_trans/1,
t_ref_trans_error/1, t_object_trans_error/1
]).
-export([all/0, suite/0, groups/0]).
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-import(hoconsc, [mk/2]).
all() -> [{group, spec}, {group, validation}].
all() -> emqx_common_test_helpers:all(?MODULE).
suite() -> [{timetrap, {minutes, 1}}].
groups() -> [
{spec, [parallel], [
t_api_spec, t_object, t_nest_object,
t_local_ref, t_remote_ref, t_bad_ref, t_none_ref,
t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]},
{validation, [parallel],
[
t_object_trans, t_object_notrans, t_local_ref_trans, t_remote_ref_trans,
t_ref_array_with_key_trans, t_ref_array_without_key_trans, t_nest_ref_trans,
t_ref_trans_error, t_object_trans_error
%% t_nest_object_trans,
]}
].
init_per_suite(Config) ->
mria:start(),
application:load(emqx_dashboard),
emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1),
emqx_dashboard:init_i18n(),
Config.
set_special_configs(emqx_dashboard) ->
emqx_dashboard_api_test_helpers:set_default_config(),
ok;
set_special_configs(_) ->
ok.
end_per_suite(Config) ->
end_suite(),
Config.
end_suite() ->
application:unload(emqx_management),
emqx_common_test_helpers:stop_apps([emqx_dashboard]).
t_object(_Config) ->
Spec = #{
@ -48,7 +43,7 @@ t_object(_Config) ->
#{required => [<<"timeout">>, <<"per_page">>],
<<"properties">> =>[
{<<"per_page">>, #{description => <<"good per page desc">>,
example => 1, maximum => 100, minimum => 1, type => integer}},
maximum => 100, minimum => 1, type => integer}},
{<<"timeout">>, #{default => 5, <<"oneOf">> =>
[#{example => <<"1h">>, type => string},
#{enum => [infinity], type => string}]}},
@ -69,13 +64,13 @@ t_nest_object(_Config) ->
#{required => [<<"timeout">>],
<<"properties">> =>
[{<<"per_page">>, #{description => <<"good per page desc">>,
example => 1, maximum => 100, minimum => 1, type => integer}},
maximum => 100, minimum => 1, type => integer}},
{<<"timeout">>, #{default => 5, <<"oneOf">> =>
[#{example => <<"1h">>, type => string},
#{enum => [infinity], type => string}]}},
{<<"nest_object">>,
#{<<"properties">> =>
[{<<"good_nest_1">>, #{example => 100, type => integer}},
[{<<"good_nest_1">>, #{type => integer}},
{<<"good_nest_2">>, #{<<"$ref">> =>
<<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],
<<"type">> => object}},
@ -110,7 +105,7 @@ t_remote_ref(_Config) ->
{_, Components} = validate("/ref/remote", Spec, Refs),
ExpectComponents = [
#{<<"emqx_swagger_remote_schema.ref2">> => #{<<"properties">> => [
{<<"page">>, #{description => <<"good page">>,example => 1,
{<<"page">>, #{description => <<"good page">>,
maximum => 100,minimum => 1,type => integer}},
{<<"another_ref">>, #{<<"$ref">> =>
<<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}], <<"type">> => object}},
@ -141,8 +136,7 @@ t_nest_ref(_Config) ->
{<<"webhook-host">>, #{default => <<"127.0.0.1:80">>,
example => <<"127.0.0.1:80">>,type => string}},
{<<"log_dir">>, #{example => <<"var/log/emqx">>,type => string}},
{<<"tag">>, #{description => <<"tag">>,
example => <<"binary-example">>,type => string}}],
{<<"tag">>, #{description => <<"tag">>,type => string}}],
<<"type">> => object}}]),
{_, Components} = validate("/ref/nest/ref", Spec, Refs),
?assertEqual(ExpectComponents, Components),
@ -186,7 +180,7 @@ t_ref_array_with_key(_Config) ->
<<"type">> => object, <<"properties">> =>
[
{<<"per_page">>, #{description => <<"good per page desc">>,
example => 1, maximum => 100, minimum => 1, type => integer}},
maximum => 100, minimum => 1, type => integer}},
{<<"timeout">>, #{default => 5, <<"oneOf">> =>
[#{example => <<"1h">>, type => string},
#{enum => [infinity], type => string}]}},
@ -281,7 +275,7 @@ t_object_notrans(_Config) ->
?assertEqual(Body, ActualBody),
ok.
t_nest_object_trans(_Config) ->
todo_t_nest_object_check(_Config) ->
Path = "/nest/object",
Body = #{
<<"timeout">> => "10m",
@ -306,7 +300,7 @@ t_nest_object_trans(_Config) ->
body => #{<<"per_page">> => 10,
<<"timeout">> => 600}
},
{ok, NewRequest} = trans_requestBody(Path, Body),
{ok, NewRequest} = check_requestBody(Path, Body),
?assertEqual(Expect, NewRequest),
ok.
@ -487,6 +481,10 @@ trans_requestBody(Path, Body) ->
trans_requestBody(Path, Body,
fun emqx_dashboard_swagger:filter_check_request_and_translate_body/2).
check_requestBody(Path, Body) ->
trans_requestBody(Path, Body,
fun emqx_dashboard_swagger:filter_check_request/2).
trans_requestBody(Path, Body, Filter) ->
Meta = #{module => ?MODULE, method => post, path => Path},
Request = #{bindings => #{}, query_string => #{}, body => Body},

80
apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl
View File

@ -10,22 +10,31 @@
-include_lib("hocon/include/hoconsc.hrl").
-import(hoconsc, [mk/2]).
-export([all/0, suite/0, groups/0]).
-export([paths/0, api_spec/0, schema/1, fields/1]).
-export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1, t_error/1,
t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1, t_complicated_type/1,
t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, t_sub_fields/1,
t_ref_array_with_key/1, t_ref_array_without_key/1, t_api_spec/1]).
-compile(nowarn_export_all).
-compile(export_all).
all() -> [{group, spec}].
suite() -> [{timetrap, {minutes, 1}}].
groups() -> [
{spec, [parallel], [
t_api_spec, t_simple_binary, t_object, t_nest_object, t_error, t_complicated_type,
t_raw_local_ref, t_raw_remote_ref, t_empty, t_hocon_schema_function,
t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, t_sub_fields,
t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]}
].
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
mria:start(),
application:load(emqx_dashboard),
emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1),
emqx_dashboard:init_i18n(),
Config.
set_special_configs(emqx_dashboard) ->
emqx_dashboard_api_test_helpers:set_default_config(),
ok;
set_special_configs(_) ->
ok.
end_per_suite(Config) ->
end_suite(),
Config.
end_suite() ->
application:unload(emqx_management),
emqx_common_test_helpers:stop_apps([emqx_dashboard]).
t_simple_binary(_config) ->
Path = "/simple/bin",
@ -41,7 +50,7 @@ t_object(_config) ->
#{<<"schema">> => #{required => [<<"timeout">>, <<"per_page">>],
<<"properties">> => [
{<<"per_page">>, #{description => <<"good per page desc">>,
example => 1, maximum => 100, minimum => 1, type => integer}},
maximum => 100, minimum => 1, type => integer}},
{<<"timeout">>, #{default => 5, <<"oneOf">> =>
[#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}},
{<<"inner_ref">>, #{<<"$ref">> =>
@ -58,16 +67,15 @@ t_error(_Config) ->
<<"properties">> =>
[
{<<"code">>, #{enum => ['Bad1', 'Bad2'], type => string}},
{<<"message">>, #{description => <<"Details description of the error.">>,
example => <<"Bad request desc">>, type => string}}]
{<<"message">>, #{description => <<"Bad request desc">>, type => string}}]
}}}},
Error404 = #{<<"content">> =>
#{<<"application/json">> => #{<<"schema">> => #{<<"type">> => object,
<<"properties">> =>
[
{<<"code">>, #{enum => ['Not-Found'], type => string}},
{<<"message">>, #{description => <<"Details description of the error.">>,
example => <<"Error code to troubleshoot problems.">>, type => string}}]
{<<"message">>, #{
description => <<"Error code to troubleshoot problems.">>, type => string}}]
}}}},
{OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}),
?assertEqual(test, OperationId),
@ -83,12 +91,12 @@ t_nest_object(_Config) ->
Object =
#{<<"content">> => #{<<"application/json">> => #{<<"schema">> =>
#{required => [<<"timeout">>], <<"type">> => object, <<"properties">> => [
{<<"per_page">>, #{description => <<"good per page desc">>, example => 1,
{<<"per_page">>, #{description => <<"good per page desc">>,
maximum => 100, minimum => 1, type => integer}},
{<<"timeout">>, #{default => 5, <<"oneOf">> =>
[#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}},
{<<"nest_object">>, #{<<"type">> => object, <<"properties">> => [
{<<"good_nest_1">>, #{example => 100, type => integer}},
{<<"good_nest_1">>, #{type => integer}},
{<<"good_nest_2">>, #{<<"$ref">> =>
<<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}
}]}},
@ -176,16 +184,15 @@ t_complicated_type(_Config) ->
Object = #{<<"content">> => #{<<"application/json">> =>
#{<<"schema">> => #{<<"properties">> =>
[
{<<"no_neg_integer">>, #{example => 100, minimum => 1, type => integer}},
{<<"no_neg_integer">>, #{minimum => 0, type => integer}},
{<<"url">>, #{example => <<"http://127.0.0.1">>, type => string}},
{<<"server">>, #{example => <<"127.0.0.1:80">>, type => string}},
{<<"connect_timeout">>, #{example => infinity, <<"oneOf">> => [
#{example => infinity, type => string},
#{example => 100, type => integer}]}},
{<<"pool_type">>, #{enum => [random, hash], example => hash, type => string}},
#{type => integer}]}},
{<<"pool_type">>, #{enum => [random, hash], type => string}},
{<<"timeout">>, #{example => infinity,
<<"oneOf">> =>
[#{example => infinity, type => string}, #{example => 100, type => integer}]}},
<<"oneOf">> => [#{example => infinity, type => string}, #{type => integer}]}},
{<<"bytesize">>, #{example => <<"32MB">>, type => string}},
{<<"wordsize">>, #{example => <<"1024KB">>, type => string}},
{<<"maps">>, #{example => #{}, type => object}},
@ -194,7 +201,7 @@ t_complicated_type(_Config) ->
{<<"log_level">>,
#{enum => [debug, info, notice, warning, error, critical, alert, emergency, all],
type => string}},
{<<"fix_integer">>, #{default => 100, enum => [100], example => 100,type => integer}}
{<<"fix_integer">>, #{default => 100, enum => [100],type => integer}}
],
<<"type">> => object}}}},
{OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}),
@ -210,17 +217,16 @@ t_ref_array_with_key(_Config) ->
Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{
required => [<<"timeout">>], <<"type">> => object, <<"properties">> => [
{<<"per_page">>, #{description => <<"good per page desc">>,
example => 1, maximum => 100, minimum => 1, type => integer}},
maximum => 100, minimum => 1, type => integer}},
{<<"timeout">>, #{default => 5, <<"oneOf">> =>
[#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}},
{<<"assert">>, #{description => <<"money">>, example => 3.14159, type => number}},
{<<"number_ex">>, #{description => <<"number example">>,
example => 42, type => number}},
{<<"assert">>, #{description => <<"money">>, type => number}},
{<<"number_ex">>, #{description => <<"number example">>, type => number}},
{<<"percent_ex">>, #{description => <<"percent example">>,
example => <<"12%">>, type => number}},
{<<"duration_ms_ex">>, #{description => <<"duration ms example">>,
example => <<"32s">>, type => string}},
{<<"atom_ex">>, #{description => <<"atom ex">>, example => atom, type => string}},
{<<"atom_ex">>, #{description => <<"atom ex">>, type => string}},
{<<"array_refs">>, #{items => #{<<"$ref">> =>
<<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}, type => array}}
]}
@ -245,12 +251,12 @@ t_hocon_schema_function(_Config) ->
#{<<"emqx_swagger_remote_schema.ref1">> => #{<<"type">> => object,
<<"properties">> => [
{<<"protocol">>, #{enum => [http, https], type => string}},
{<<"port">>, #{default => 18083, example => 100, type => integer}}]
{<<"port">>, #{default => 18083, type => integer}}]
}},
#{<<"emqx_swagger_remote_schema.ref2">> => #{<<"type">> => object,
<<"properties">> => [
{<<"page">>, #{description => <<"good page">>,
example => 1, maximum => 100, minimum => 1, type => integer}},
maximum => 100, minimum => 1, type => integer}},
{<<"another_ref">>, #{<<"$ref">> =>
<<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}
]
@ -270,9 +276,9 @@ t_hocon_schema_function(_Config) ->
#{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref1">>}]},
type => array}},
{<<"default_username">>,
#{default => <<"admin">>, example => <<"string-example">>, type => string}},
#{default => <<"admin">>, type => string}},
{<<"default_password">>,
#{default => <<"public">>, example => <<"string-example">>, type => string}},
#{default => <<"public">>, type => string}},
{<<"sample_interval">>,
#{default => <<"10s">>, example => <<"1h">>, type => string}},
{<<"token_expired_time">>,

2
apps/emqx_exhook/src/emqx_exhook_api.erl
View File

@ -170,7 +170,7 @@ fields(node_metrics) ->
fields(node_status) ->
[
{node, mk(string(), #{})},
{status, mk(enum([running, waiting, stopped, error]), #{})}
{status, mk(enum([connected, connecting, unconnected, disable, error]), #{})}
];
fields(hook_info) ->
[

443
apps/emqx_exhook/src/emqx_exhook_mgr.erl
View File

@ -63,21 +63,25 @@
-export([roots/0]).
%% Running servers
-type state() :: #{
running := servers(),
%% Wait to reload servers
waiting := servers(),
%% Marked stopped servers
stopped := servers(),
%% Timer references
trefs := map(),
orders := orders()
}.
-type state() :: #{servers := servers()}.
-type server_name() :: binary().
-type servers() :: #{server_name() => server()}.
-type server() :: server_options().
-type server_options() :: map().
-type server_name() :: binary().
-type status() ::
connected
| connecting
| unconnected
| disable.
-type server() :: #{
status := status(),
timer := reference(),
order := integer(),
%% include the content of server_options
atom() => any()
}.
-type servers() :: #{server_name() => server()}.
-type position() ::
front
@ -85,19 +89,10 @@
| {before, binary()}
| {'after', binary()}.
-type orders() :: #{server_name() => integer()}.
-type server_info() :: #{
name := server_name(),
status := running | waiting | stopped,
atom() => term()
}.
-define(DEFAULT_TIMEOUT, 60000).
-define(REFRESH_INTERVAL, timer:seconds(5)).
-export_type([servers/0, server/0, server_info/0]).
-export_type([servers/0, server/0]).
%%--------------------------------------------------------------------
%% APIs
@ -113,7 +108,7 @@ start_link() ->
list() ->
call(list).
-spec lookup(server_name()) -> not_found | server_info().
-spec lookup(server_name()) -> not_found | server().
lookup(Name) ->
call({lookup, Name}).
@ -195,104 +190,56 @@ init([]) ->
process_flag(trap_exit, true),
emqx_conf:add_handler([exhook, servers], ?MODULE),
ServerL = emqx:get_config([exhook, servers]),
{Waiting, Running, Stopped} = load_all_servers(ServerL),
Orders = reorder(ServerL),
Servers = load_all_servers(ServerL),
Servers2 = reorder(ServerL, Servers),
refresh_tick(),
{ok,
ensure_reload_timer(
#{
waiting => Waiting,
running => Running,
stopped => Stopped,
trefs => #{},
orders => Orders
}
)}.
{ok, #{servers => Servers2}}.
-spec load_all_servers(list(server_options())) -> {servers(), servers(), servers()}.
-spec load_all_servers(list(server_options())) -> servers().
load_all_servers(ServerL) ->
load_all_servers(ServerL, #{}, #{}, #{}).
load_all_servers(ServerL, #{}).
load_all_servers([#{name := Name} = Options | More], Waiting, Running, Stopped) ->
case emqx_exhook_server:load(Name, Options) of
{ok, ServerState} ->
save(Name, ServerState),
load_all_servers(More, Waiting, Running#{Name => Options}, Stopped);
{error, _} ->
load_all_servers(More, Waiting#{Name => Options}, Running, Stopped);
disable ->
load_all_servers(More, Waiting, Running, Stopped#{Name => Options})
end;
load_all_servers([], Waiting, Running, Stopped) ->
{Waiting, Running, Stopped}.
load_all_servers([#{name := Name} = Options | More], Servers) ->
{_, Server} = do_load_server(options_to_server(Options)),
load_all_servers(More, Servers#{Name => Server});
load_all_servers([], Servers) ->
Servers.
handle_call(
list,
_From,
State = #{
running := Running,
waiting := Waiting,
stopped := Stopped,
orders := Orders
}
State = #{servers := Servers}
) ->
R = get_servers_info(running, Running),
W = get_servers_info(waiting, Waiting),
S = get_servers_info(stopped, Stopped),
Servers = R ++ W ++ S,
OrderServers = sort_name_by_order(Servers, Orders),
Infos = get_servers_info(Servers),
OrderServers = sort_name_by_order(Infos, Servers),
{reply, OrderServers, State};
handle_call(
{update_config, {move, _Name, _Position}, NewConfL},
_From,
State
#{servers := Servers} = State
) ->
Orders = reorder(NewConfL),
{reply, ok, State#{orders := Orders}};
Servers2 = reorder(NewConfL, Servers),
{reply, ok, State#{servers := Servers2}};
handle_call({update_config, {delete, ToDelete}, _}, _From, State) ->
{ok,
#{
orders := Orders,
stopped := Stopped
} = State2} = do_unload_server(ToDelete, State),
State3 = State2#{
stopped := maps:remove(ToDelete, Stopped),
orders := maps:remove(ToDelete, Orders)
},
emqx_exhook_metrics:on_server_deleted(ToDelete),
{reply, ok, State3};
#{servers := Servers} = State2 = do_unload_server(ToDelete, State),
Servers2 = maps:remove(ToDelete, Servers),
{reply, ok, update_servers(Servers2, State2)};
handle_call(
{update_config, {add, RawConf}, NewConfL},
_From,
#{running := Running, waiting := Waitting, stopped := Stopped} = State
#{servers := Servers} = State
) ->
{_, #{name := Name} = Conf} = emqx_config:check_config(?MODULE, RawConf),
case emqx_exhook_server:load(Name, Conf) of
{ok, ServerState} ->
save(Name, ServerState),
State2 = State#{running := Running#{Name => Conf}};
{error, _} ->
StateT = State#{waiting := Waitting#{Name => Conf}},
State2 = ensure_reload_timer(StateT);
disable ->
State2 = State#{stopped := Stopped#{Name => Conf}}
end,
Orders = reorder(NewConfL),
{reply, ok, State2#{orders := Orders}};
{Result, Server} = do_load_server(options_to_server(Conf)),
Servers2 = Servers#{Name => Server},
Servers3 = reorder(NewConfL, Servers2),
{reply, Result, State#{servers := Servers3}};
handle_call({lookup, Name}, _From, State) ->
case where_is_server(Name, State) of
not_found ->
Result = not_found;
{Where, #{Name := Conf}} ->
Result = maps:merge(Conf, #{status => Where})
end,
{reply, Result, State};
{reply, where_is_server(Name, State), State};
handle_call({update_config, {update, Name, _Conf}, NewConfL}, _From, State) ->
{Result, State2} = restart_server(Name, NewConfL, State),
{reply, Result, State2};
@ -303,7 +250,7 @@ handle_call({server_info, Name}, _From, State) ->
case where_is_server(Name, State) of
not_found ->
Result = not_found;
{Status, _} ->
#{status := Status} ->
HooksMetrics = emqx_exhook_metrics:server_metrics(Name),
Result = #{
status => Status,
@ -314,25 +261,9 @@ handle_call({server_info, Name}, _From, State) ->
handle_call(
all_servers_info,
_From,
#{
running := Running,
waiting := Waiting,
stopped := Stopped
} = State
#{servers := Servers} = State
) ->
MakeStatus = fun(Status, Servers, Acc) ->
lists:foldl(
fun(Name, IAcc) -> IAcc#{Name => Status} end,
Acc,
maps:keys(Servers)
)
end,
Status = lists:foldl(
fun({Status, Servers}, Acc) -> MakeStatus(Status, Servers, Acc) end,
#{},
[{running, Running}, {waiting, Waiting}, {stopped, Stopped}]
),
Status = maps:map(fun(_Name, #{status := Status}) -> Status end, Servers),
Metrics = emqx_exhook_metrics:servers_metrics(),
Result = #{
@ -352,23 +283,8 @@ handle_cast(_Msg, State) ->
{noreply, State}.
handle_info({timeout, _Ref, {reload, Name}}, State) ->
{Result, NState} = do_load_server(Name, State),
case Result of
ok ->
{noreply, NState};
{error, not_found} ->
{noreply, NState};
{error, Reason} ->
?SLOG(
warning,
#{
msg => "failed_to_reload_exhook_callback_server",
reason => Reason,
name => Name
}
),
{noreply, ensure_reload_timer(NState)}
end;
{_, NState} = do_reload_server(Name, State),
{noreply, NState};
handle_info(refresh_tick, State) ->
refresh_tick(),
emqx_exhook_metrics:update(?REFRESH_INTERVAL),
@ -376,14 +292,13 @@ handle_info(refresh_tick, State) ->
handle_info(_Info, State) ->
{noreply, State}.
terminate(_Reason, State = #{running := Running}) ->
terminate(_Reason, State = #{servers := Servers}) ->
_ = maps:fold(
fun(Name, _, AccIn) ->
{ok, NAccIn} = do_unload_server(Name, AccIn),
NAccIn
do_unload_server(Name, AccIn)
end,
State,
Running
Servers
),
_ = unload_exhooks(),
ok.
@ -401,122 +316,83 @@ unload_exhooks() ->
|| {Name, {M, F, _A}} <- ?ENABLED_HOOKS
].
-spec do_load_server(server_name(), state()) ->
{{error, not_found}, state()}
| {{error, already_started}, state()}
| {ok, state()}.
do_load_server(Name, State = #{orders := Orders}) ->
case where_is_server(Name, State) of
not_found ->
{{error, not_found}, State};
{running, _} ->
{ok, State};
{Where, Map} ->
State2 = clean_reload_timer(Name, State),
{Options, Map2} = maps:take(Name, Map),
State3 = State2#{Where := Map2},
#{
running := Running,
stopped := Stopped
} = State3,
case emqx_exhook_server:load(Name, Options) of
{ok, ServerState} ->
save(Name, ServerState),
update_order(Orders),
?SLOG(info, #{
msg => "load_exhook_callback_server_ok",
name => Name
}),
{ok, State3#{running := maps:put(Name, Options, Running)}};
{error, Reason} ->
{{error, Reason}, State};
disable ->
{ok, State3#{stopped := Stopped#{Name => Options}}}
do_load_server(#{name := Name} = Server) ->
case emqx_exhook_server:load(Name, Server) of
{ok, ServerState} ->
save(Name, ServerState),
{ok, Server#{status => connected}};
disable ->
{ok, set_disable(Server)};
{ErrorType, Reason} = Error ->
?SLOG(
error,
#{
msg => "failed_to_load_exhook_callback_server",
reason => Reason,
name => Name
}
),
case ErrorType of
load_error ->
{ok, ensure_reload_timer(Server)};
_ ->
{Error, Server#{status => unconnected}}
end
end.
-spec do_unload_server(server_name(), state()) -> {ok, state()}.
do_unload_server(Name, #{stopped := Stopped} = State) ->
do_load_server(#{name := Name} = Server, #{servers := Servers} = State) ->
{Result, Server2} = do_load_server(Server),
{Result, update_servers(Servers#{Name => Server2}, State)}.
-spec do_reload_server(server_name(), state()) ->
{{error, term()}, state()}
| {ok, state()}.
do_reload_server(Name, State = #{servers := Servers}) ->
case where_is_server(Name, State) of
{stopped, _} ->
{ok, State};
{waiting, Waiting} ->
{Options, Waiting2} = maps:take(Name, Waiting),
{ok,
clean_reload_timer(
Name,
State#{
waiting := Waiting2,
stopped := maps:put(Name, Options, Stopped)
}
)};
{running, Running} ->
Service = server(Name),
ok = unsave(Name),
ok = emqx_exhook_server:unload(Service),
{Options, Running2} = maps:take(Name, Running),
{ok, State#{
running := Running2,
stopped := maps:put(Name, Options, Stopped)
}};
not_found ->
{ok, State}
{{error, not_found}, State};
#{timer := undefined} ->
{ok, State};
Server ->
clean_reload_timer(Server),
do_load_server(Server, Servers)
end.
-spec ensure_reload_timer(state()) -> state().
ensure_reload_timer(
State = #{
waiting := Waiting,
stopped := Stopped,
trefs := TRefs
}
) ->
Iter = maps:iterator(Waiting),
{Waitting2, Stopped2, TRefs2} =
ensure_reload_timer(maps:next(Iter), Waiting, Stopped, TRefs),
State#{
waiting := Waitting2,
stopped := Stopped2,
trefs := TRefs2
}.
ensure_reload_timer(none, Waiting, Stopped, TimerRef) ->
{Waiting, Stopped, TimerRef};
ensure_reload_timer(
{Name, #{auto_reconnect := Intv}, Iter},
Waiting,
Stopped,
TimerRef
) when is_integer(Intv) ->
Next = maps:next(Iter),
case maps:is_key(Name, TimerRef) of
true ->
ensure_reload_timer(Next, Waiting, Stopped, TimerRef);
_ ->
Ref = erlang:start_timer(Intv, self(), {reload, Name}),
TimerRef2 = maps:put(Name, Ref, TimerRef),
ensure_reload_timer(Next, Waiting, Stopped, TimerRef2)
end;
ensure_reload_timer({Name, Opts, Iter}, Waiting, Stopped, TimerRef) ->
ensure_reload_timer(
maps:next(Iter),
maps:remove(Name, Waiting),
maps:put(Name, Opts, Stopped),
TimerRef
).
-spec clean_reload_timer(server_name(), state()) -> state().
clean_reload_timer(Name, State = #{trefs := TRefs}) ->
case maps:take(Name, TRefs) of
error ->
-spec do_unload_server(server_name(), state()) -> state().
do_unload_server(Name, #{servers := Servers} = State) ->
case where_is_server(Name, State) of
not_found ->
State;
{TRef, NTRefs} ->
_ = erlang:cancel_timer(TRef),
State#{trefs := NTRefs}
#{status := disable} ->
State;
Server ->
clean_reload_timer(Server),
case server(Name) of
undefined ->
State;
Service ->
ok = unsave(Name),
ok = emqx_exhook_server:unload(Service),
Servers2 = Servers#{Name := set_disable(Server)},
State#{servers := Servers2}
end
end.
ensure_reload_timer(#{timer := Timer} = Server) when is_reference(Timer) ->
Server;
ensure_reload_timer(#{name := Name, auto_reconnect := Intv} = Server) when is_integer(Intv) ->
Ref = erlang:start_timer(Intv, self(), {reload, Name}),
Server#{status := connecting, timer := Ref};
ensure_reload_timer(Server) ->
Server#{status := unconnected}.
-spec clean_reload_timer(server()) -> ok.
clean_reload_timer(#{timer := undefined}) ->
ok;
clean_reload_timer(#{timer := Timer}) ->
_ = erlang:cancel_timer(Timer),
ok.
-spec do_move(binary(), position(), list(server_options())) ->
not_found | list(server_options()).
do_move(Name, Position, ConfL) ->
@ -545,37 +421,32 @@ move_to([H | T], Position, Server, HeadL) ->
move_to([], _Position, _Server, _HeadL) ->
not_found.
-spec reorder(list(server_options())) -> orders().
reorder(ServerL) ->
Orders = reorder(ServerL, 1, #{}),
-spec reorder(list(server_options()), servers()) -> servers().
reorder(ServerL, Servers) ->
Orders = reorder(ServerL, 1, Servers),
update_order(Orders),
Orders.
reorder([#{name := Name} | T], Order, Orders) ->
reorder(T, Order + 1, Orders#{Name => Order});
reorder([], _Order, Orders) ->
Orders.
reorder([#{name := Name} | T], Order, Servers) ->
reorder(T, Order + 1, update_order(Name, Servers, Order));
reorder([], _Order, Servers) ->
Servers.
get_servers_info(Status, Map) ->
update_order(Name, Servers, Order) ->
Server = maps:get(Name, Servers),
Servers#{Name := Server#{order := Order}}.
get_servers_info(Svrs) ->
Fold = fun(Name, Conf, Acc) ->
[
maps:merge(Conf, #{
status => Status,
hooks => hooks(Name)
})
maps:merge(Conf, #{hooks => hooks(Name)})
| Acc
]
end,
maps:fold(Fold, [], Map).
maps:fold(Fold, [], Svrs).
where_is_server(Name, #{running := Running}) when is_map_key(Name, Running) ->
{running, Running};
where_is_server(Name, #{waiting := Waiting}) when is_map_key(Name, Waiting) ->
{waiting, Waiting};
where_is_server(Name, #{stopped := Stopped}) when is_map_key(Name, Stopped) ->
{stopped, Stopped};
where_is_server(_, _) ->
not_found.
where_is_server(Name, #{servers := Servers}) ->
maps:get(Name, Servers, not_found).
-type replace_fun() :: fun((server_options()) -> server_options()).
@ -604,15 +475,10 @@ restart_server(Name, ConfL, State) ->
case where_is_server(Name, State) of
not_found ->
{{error, not_found}, State};
{Where, Map} ->
State2 = State#{Where := Map#{Name := Conf}},
{ok, State3} = do_unload_server(Name, State2),
case do_load_server(Name, State3) of
{ok, State4} ->
{ok, State4};
{Error, State4} ->
{Error, State4}
end
Server ->
Server2 = maps:merge(Server, Conf),
State2 = do_unload_server(Name, State),
do_load_server(Server2, State2)
end
end.
@ -620,9 +486,11 @@ sort_name_by_order(Names, Orders) ->
lists:sort(
fun
(A, B) when is_binary(A) ->
maps:get(A, Orders) < maps:get(B, Orders);
emqx_map_lib:deep_get([A, order], Orders) <
emqx_map_lib:deep_get([B, order], Orders);
(#{name := A}, #{name := B}) ->
maps:get(A, Orders) < maps:get(B, Orders)
emqx_map_lib:deep_get([A, order], Orders) <
emqx_map_lib:deep_get([B, order], Orders)
end,
Names
).
@ -630,6 +498,16 @@ sort_name_by_order(Names, Orders) ->
refresh_tick() ->
erlang:send_after(?REFRESH_INTERVAL, self(), ?FUNCTION_NAME).
options_to_server(Options) ->
maps:merge(Options, #{status => unconnected, timer => undefined, order => 0}).
update_servers(Servers, State) ->
update_order(Servers),
State#{servers := Servers}.
set_disable(Server) ->
Server#{status := disable, timer := undefined}.
%%--------------------------------------------------------------------
%% Server state persistent
save(Name, ServerState) ->
@ -661,8 +539,17 @@ server(Name) ->
Service -> Service
end.
update_order(Orders) ->
update_order(Servers) ->
Running = running(),
Orders = maps:filter(
fun
(Name, #{status := connected}) ->
lists:member(Name, Running);
(_, _) ->
false
end,
Servers
),
Running2 = sort_name_by_order(Running, Orders),
persistent_term:put(?APP, Running2).

2
apps/emqx_exhook/src/emqx_exhook_schema.erl
View File

@ -92,7 +92,7 @@ fields(server) ->
)},
{pool_size,
sc(
integer(),
pos_integer(),
#{
default => 8,
example => 8,

64
apps/emqx_exhook/src/emqx_exhook_server.erl
View File

@ -86,40 +86,44 @@
%% Load/Unload APIs
%%--------------------------------------------------------------------
-spec load(binary(), map()) -> {ok, server()} | {error, term()} | disable.
-spec load(binary(), map()) -> {ok, server()} | {error, term()} | {load_error, term()} | disable.
load(_Name, #{enable := false}) ->
disable;
load(Name, #{request_timeout := Timeout, failed_action := FailedAction} = Opts) ->
ReqOpts = #{timeout => Timeout, failed_action => FailedAction},
{SvrAddr, ClientOpts} = channel_opts(Opts),
case
emqx_exhook_sup:start_grpc_client_channel(
Name,
SvrAddr,
ClientOpts
)
of
{ok, _ChannPoolPid} ->
case do_init(Name, ReqOpts) of
{ok, HookSpecs} ->
%% Register metrics
Prefix = lists:flatten(io_lib:format("exhook.~ts.", [Name])),
ensure_metrics(Prefix, HookSpecs),
%% Ensure hooks
ensure_hooks(HookSpecs),
{ok, #{
name => Name,
options => ReqOpts,
channel => _ChannPoolPid,
hookspec => HookSpecs,
prefix => Prefix
}};
case channel_opts(Opts) of
{ok, {SvrAddr, ClientOpts}} ->
case
emqx_exhook_sup:start_grpc_client_channel(
Name,
SvrAddr,
ClientOpts
)
of
{ok, _ChannPoolPid} ->
case do_init(Name, ReqOpts) of
{ok, HookSpecs} ->
%% Register metrics
Prefix = lists:flatten(io_lib:format("exhook.~ts.", [Name])),
ensure_metrics(Prefix, HookSpecs),
%% Ensure hooks
ensure_hooks(HookSpecs),
{ok, #{
name => Name,
options => ReqOpts,
channel => _ChannPoolPid,
hookspec => HookSpecs,
prefix => Prefix
}};
{error, Reason} ->
emqx_exhook_sup:stop_grpc_client_channel(Name),
{load_error, Reason}
end;
{error, _} = E ->
emqx_exhook_sup:stop_grpc_client_channel(Name),
E
end;
{error, _} = E ->
E
Error ->
Error
end.
%% @private
@ -130,7 +134,7 @@ channel_opts(Opts = #{url := URL}) ->
),
case uri_string:parse(URL) of
#{scheme := <<"http">>, host := Host, port := Port} ->
{format_http_uri("http", Host, Port), ClientOpts};
{ok, {format_http_uri("http", Host, Port), ClientOpts}};
#{scheme := <<"https">>, host := Host, port := Port} ->
SslOpts =
case maps:get(ssl, Opts, undefined) of
@ -154,9 +158,9 @@ channel_opts(Opts = #{url := URL}) ->
transport_opts => SslOpts
}
},
{format_http_uri("https", Host, Port), NClientOpts};
{ok, {format_http_uri("https", Host, Port), NClientOpts}};
Error ->
error({bad_server_url, URL, Error})
{error, {bad_server_url, URL, Error}}
end.
format_http_uri(Scheme, Host, Port) ->

8
apps/emqx_gateway/src/emqx_gateway_api.erl
View File

@ -284,12 +284,12 @@ fields(gateway_overview) ->
)},
{max_connections,
mk(
integer(),
pos_integer(),
#{desc => <<"The Gateway allowed maximum connections/clients">>}
)},
{current_connections,
mk(
integer(),
non_neg_integer(),
#{desc => <<"The Gateway current connected connections/clients">>}
)},
{listeners,
@ -410,11 +410,11 @@ convert_listener_struct(Schema) ->
),
lists:keystore(listeners, 1, Schema1, {listeners, ListenerSchema}).
remove_listener_and_authn(Schmea) ->
remove_listener_and_authn(Schema) ->
lists:keydelete(
authentication,
1,
lists:keydelete(listeners, 1, Schmea)
lists:keydelete(listeners, 1, Schema)
).
listeners_schema(?R_REF(_Mod, tcp_listeners)) ->

4
apps/emqx_gateway/src/emqx_gateway_api_authn.erl
View File

@ -384,7 +384,7 @@ params_paging_in_qs() ->
[
{page,
mk(
integer(),
pos_integer(),
#{
in => query,
required => false,
@ -394,7 +394,7 @@ params_paging_in_qs() ->
)},
{limit,
mk(
integer(),
pos_integer(),
#{
in => query,
required => false,

77
apps/emqx_gateway/src/emqx_gateway_api_clients.erl
View File

@ -101,30 +101,39 @@ clients(get, #{
bindings := #{name := Name0},
query_string := QString
}) ->
with_gateway(Name0, fun(GwName, _) ->
Fun = fun(GwName, _) ->
TabName = emqx_gateway_cm:tabname(info, GwName),
case maps:get(<<"node">>, QString, undefined) of
undefined ->
Response = emqx_mgmt_api:cluster_query(
QString,
TabName,
?CLIENT_QSCHEMA,
?QUERY_FUN
),
emqx_mgmt_util:generate_response(Response);
Node1 ->
Node = binary_to_atom(Node1, utf8),
QStringWithoutNode = maps:without([<<"node">>], QString),
Response = emqx_mgmt_api:node_query(
Node,
QStringWithoutNode,
TabName,
?CLIENT_QSCHEMA,
?QUERY_FUN
),
emqx_mgmt_util:generate_response(Response)
Result =
case maps:get(<<"node">>, QString, undefined) of
undefined ->
emqx_mgmt_api:cluster_query(
QString,
TabName,
?CLIENT_QSCHEMA,
?QUERY_FUN
);
Node0 ->
Node1 = binary_to_atom(Node0, utf8),
QStringWithoutNode = maps:without([<<"node">>], QString),
emqx_mgmt_api:node_query(
Node1,
QStringWithoutNode,
TabName,
?CLIENT_QSCHEMA,
?QUERY_FUN
)
end,
case Result of
{error, page_limit_invalid} ->
{400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
{error, Node, {badrpc, R}} ->
Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])),
{500, #{code => <<"NODE_DOWN">>, message => Message}};
Response ->
{200, Response}
end
end).
end,
with_gateway(Name0, Fun).
clients_insta(get, #{
bindings := #{
@ -392,21 +401,21 @@ format_channel_info({_, Infos, Stats} = R) ->
{heap_size, Stats, 0},
{reductions, Stats, 0}
],
eval(FetchX ++ extra_feilds(R)).
eval(FetchX ++ extra_fields(R)).
extra_feilds({_, Infos, _Stats} = R) ->
extra_feilds(
extra_fields({_, Infos, _Stats} = R) ->
extra_fields(
maps:get(protocol, maps:get(clientinfo, Infos)),
R
).
extra_feilds(lwm2m, {_, Infos, _Stats}) ->
extra_fields(lwm2m, {_, Infos, _Stats}) ->
ClientInfo = maps:get(clientinfo, Infos, #{}),
[
{endpoint_name, ClientInfo},
{lifetime, ClientInfo}
];
extra_feilds(_, _) ->
extra_fields(_, _) ->
[].
eval(Ls) ->
@ -495,7 +504,7 @@ schema("/gateway/:name/clients/:clientid/subscriptions") ->
#{
200 => emqx_dashboard_swagger:schema_with_examples(
hoconsc:array(ref(subscription)),
examples_subsctiption_list()
examples_subscription_list()
)
}
)
@ -506,14 +515,14 @@ schema("/gateway/:name/clients/:clientid/subscriptions") ->
parameters => params_client_insta(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
ref(subscription),
examples_subsctiption()
examples_subscription()
),
responses =>
?STANDARD_RESP(
#{
201 => emqx_dashboard_swagger:schema_with_examples(
ref(subscription),
examples_subsctiption()
examples_subscription()
)
}
)
@ -664,7 +673,7 @@ params_paging() ->
[
{page,
mk(
integer(),
pos_integer(),
#{
in => query,
required => false,
@ -674,7 +683,7 @@ params_paging() ->
)},
{limit,
mk(
integer(),
pos_integer(),
#{
in => query,
desc => <<"Page Limit">>,
@ -1089,7 +1098,7 @@ examples_client() ->
}
}.
examples_subsctiption_list() ->
examples_subscription_list() ->
#{
general_subscription_list =>
#{
@ -1103,7 +1112,7 @@ examples_subsctiption_list() ->
}
}.
examples_subsctiption() ->
examples_subscription() ->
#{
general_subscription =>
#{

14
apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
View File

@ -191,7 +191,7 @@ users(get, #{bindings := #{name := Name0, id := Id}, query_string := Qs}) ->
Name0,
Id,
fun(_GwName, #{id := AuthId, chain_name := ChainName}) ->
emqx_authn_api:list_users(ChainName, AuthId, page_pramas(Qs))
emqx_authn_api:list_users(ChainName, AuthId, page_params(Qs))
end
);
users(post, #{
@ -261,7 +261,7 @@ import_users(post, #{
%%--------------------------------------------------------------------
%% Utils
page_pramas(Qs) ->
page_params(Qs) ->
maps:with([<<"page">>, <<"limit">>], Qs).
%%--------------------------------------------------------------------
@ -555,7 +555,7 @@ params_paging_in_qs() ->
[
{page,
mk(
integer(),
pos_integer(),
#{
in => query,
required => false,
@ -565,7 +565,7 @@ params_paging_in_qs() ->
)},
{limit,
mk(
integer(),
pos_integer(),
#{
in => query,
required => false,
@ -627,10 +627,12 @@ fields(tcp_listener_opts) ->
{high_watermark, mk(binary(), #{})},
{nodelay, mk(boolean(), #{})},
{reuseaddr, boolean()},
%% TODO: duri
{send_timeout, binary()},
{send_timeout_close, boolean()}
];
fields(ssl_listener_opts) ->
%% TODO: maybe use better ssl options schema from emqx_ssl_lib or somewhere
[
{cacertfile, binary()},
{certfile, binary()},
@ -762,7 +764,7 @@ common_listener_opts() ->
required => false,
desc =>
<<
"The Mounpoint for clients of the listener. "
"The Mountpoint for clients of the listener. "
"The gateway-level mountpoint configuration can be overloaded "
"when it is not null or empty string"
>>
@ -774,7 +776,7 @@ common_listener_opts() ->
emqx_authn_schema:authenticator_type(),
#{
required => {false, recursively},
desc => <<"The authenticatior for this listener">>
desc => <<"The authenticator for this listener">>
}
)}
] ++ emqx_gateway_schema:proxy_protocol_opts().

10
apps/emqx_gateway/src/emqx_gateway_schema.erl
View File

@ -28,9 +28,9 @@
-include_lib("typerefl/include/types.hrl").
-type ip_port() :: tuple().
-type duration() :: integer().
-type duration_s() :: integer().
-type bytesize() :: integer().
-type duration() :: non_neg_integer().
-type duration_s() :: non_neg_integer().
-type bytesize() :: pos_integer().
-type comma_separated_list() :: list().
-typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}).
@ -117,7 +117,7 @@ fields(stomp_frame) ->
[
{max_headers,
sc(
integer(),
non_neg_integer(),
#{
default => 10,
desc => "The maximum number of Header"
@ -125,7 +125,7 @@ fields(stomp_frame) ->
)},
{max_headers_length,
sc(
integer(),
non_neg_integer(),
#{
default => 1024,
desc => "The maximum string length of the Header Value"

12
apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl
View File

@ -307,12 +307,12 @@ t_case_stomp_subscribe(_) ->
)
),
timer:sleep(100),
timer:sleep(200),
Msg = emqx_message:make(Topic, Payload),
emqx:publish(Msg),
timer:sleep(100),
{ok, Data1} = gen_tcp:recv(Sock, 0, 2000),
timer:sleep(200),
{ok, Data1} = gen_tcp:recv(Sock, 0, 10000),
{ok, Frame1, _, _} = Mod:parse(Data1),
Checker(Frame1)
end,
@ -406,7 +406,11 @@ t_case_exproto_subscribe(_) ->
%% Helpers
%%------------------------------------------------------------------------------
try_publish_recv(Topic, Publish, Checker) ->
try_publish_recv(Topic, Publish, Checker, 500).
try_publish_recv(Topic, Publish, Checker, Timeout) ->
emqx:subscribe(Topic),
timer:sleep(200),
Clear = fun(Msg) ->
emqx:unsubscribe(Topic),
Checker(Msg)
@ -416,7 +420,7 @@ try_publish_recv(Topic, Publish, Checker) ->
receive
{deliver, Topic, Msg} ->
Clear(Msg)
after 500 ->
after Timeout ->
Clear(timeout)
end.

80
apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf Normal file
View File

@ -0,0 +1,80 @@
emqx_mgmt_api_alarms {
list_alarms_api {
desc {
en: """List alarms"""
zh: """列出告警，获取告警列表"""
}
}
delete_alarms_api {
desc {
en: """Remove all deactivated alarms"""
zh: """删除所有历史告警（非活跃告警）"""
}
}
delete_alarms_api_response204 {
desc {
en: """Remove all deactivated alarms ok"""
zh: """删除所有历史告警（非活跃告警）成功"""
}
}
get_alarms_qs_activated {
desc {
en: """Activate alarms, or deactivate alarms. Default is false"""
zh: """活跃中的告警，或历史告警（非活跃告警），默认为 false"""
}
}
node {
desc {
en: """Alarm in node"""
zh: """告警节点名称"""
}
}
name {
desc {
en: """Alarm name"""
zh: """告警名称"""
}
}
message {
desc {
en: """Alarm readable information"""
zh: """告警信息"""
}
}
details {
desc {
en: """Alarm details information"""
zh: """告警详细信息"""
}
}
duration {
desc {
en: """Alarms duration time; UNIX time stamp, millisecond"""
zh: """告警持续时间，单位：毫秒"""
}
}
activate_at {
desc {
en: """Alarms activate time, RFC 3339"""
zh: """告警开始时间，使用 rfc3339 标准时间格式"""
}
}
deactivate_at {
desc {
en: """Alarms deactivate time, RFC 3339"""
zh: """告警结束时间，使用 rfc3339 标准时间格式"""
}
}
}

113
apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf Normal file
View File

@ -0,0 +1,113 @@
emqx_mgmt_api_banned {
list_banned_api {
desc {
en: """List banned."""
zh: """列出黑名单"""
}
label {
en: """List Banned"""
zh: """列出黑名单"""
}
}
create_banned_api {
desc {
en: """Create banned."""
zh: """创建黑名单"""
}
}
create_banned_api_response400 {
desc {
en: """Banned already existed, or bad args."""
zh: """黑名单已存在，或参数格式有错误"""
}
}
delete_banned_api {
desc {
en: """Delete banned"""
zh: """删除黑名单"""
}
}
delete_banned_api_response404 {
desc {
en: """Banned not found. May be the banned time has been exceeded"""
zh: """黑名单未找到，可能为已经超期失效"""
}
}
create_banned {
desc {
en: """List banned."""
zh: """列出黑名单"""
}
label {
en: """List Banned"""
zh: """列出黑名单"""
}
}
as {
desc {
en: """Banned type clientid, username, peerhost"""
zh: """黑名单类型，可选 clientid、username、peerhost"""
}
label {
en: """Banned Type"""
zh: """黑名单类型"""
}
}
who {
desc {
en: """Client info as banned type"""
zh: """设备信息"""
}
label {
en: """Banned Info"""
zh: """黑名单信息"""
}
}
by {
desc {
en: """Commander"""
zh: """黑名单创建者"""
}
label {
en: """Commander"""
zh: """黑名单创建者"""
}
}
reason {
desc {
en: """Banned reason"""
zh: """黑名单创建原因"""
}
label {
en: """Reason"""
zh: """原因"""
}
}
at {
desc {
en: """Create banned time, rfc3339, now if not specified"""
zh: """黑名单创建时间，默认为当前"""
}
label {
en: """Create banned time"""
zh: """黑名单创建时间"""
}
}
until {
desc {
en: """Cancel banned time, rfc3339, now + 5 minute if not specified"""
zh: """黑名单结束时间，默认为创建时间 + 5 分钟"""
}
label {
en: """Cancel banned time"""
zh: """黑名单结束时间"""
}
}
}

32
apps/emqx_management/rebar.config
View File

@ -1,20 +1,28 @@
%% -*- mode: erlang -*-
{deps, [ {emqx, {path, "../emqx"}}
]}.
{deps, [{emqx, {path, "../emqx"}}]}.
{edoc_opts, [{preprocess, true}]}.
{erl_opts, [warn_unused_vars,
warn_shadow_vars,
warn_unused_import,
warn_obsolete_guard,
warnings_as_errors,
debug_info,
{parse_transform}]}.
{erl_opts, [
warn_unused_vars,
warn_shadow_vars,
warn_unused_import,
warn_obsolete_guard,
warnings_as_errors,
debug_info,
{parse_transform}
]}.
{xref_checks, [undefined_function_calls, undefined_functions,
locals_not_used, deprecated_function_calls,
warnings_as_errors, deprecated_functions]}.
{xref_checks, [
undefined_function_calls,
undefined_functions,
locals_not_used,
deprecated_function_calls,
warnings_as_errors,
deprecated_functions
]}.
{cover_enabled, true}.
{cover_opts, [verbose]}.
{cover_export_enabled, true}.
{project_plugins, [erlfmt]}.

30
apps/emqx_management/src/emqx_management.app.src
View File

@ -1,15 +1,17 @@
%% -*- mode: erlang -*-
{application, emqx_management,
[{description, "EMQX Management API and CLI"},
{vsn, "5.0.0"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_management_sup]},
{applications, [kernel,stdlib,emqx_plugins,minirest,emqx]},
{mod, {emqx_mgmt_app,[]}},
{env, []},
{licenses, ["Apache-2.0"]},
{maintainers, ["EMQX Team <contact@emqx.io>"]},
{links, [{"Homepage", "https://emqx.io/"},
{"Github", "https://github.com/emqx/emqx-management"}
]}
]}.
{application, emqx_management, [
{description, "EMQX Management API and CLI"},
% strict semver, bump manually!
{vsn, "5.0.0"},
{modules, []},
{registered, [emqx_management_sup]},
{applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},
{mod, {emqx_mgmt_app, []}},
{env, []},
{licenses, ["Apache-2.0"]},
{maintainers, ["EMQX Team <contact@emqx.io>"]},
{links, [
{"Homepage", "https://emqx.io/"},
{"Github", "https://github.com/emqx/emqx-management"}
]}
]}.

8
apps/emqx_management/src/emqx_management_schema.erl
View File

@ -19,9 +19,11 @@
-behaviour(hocon_schema).
-export([ namespace/0
, roots/0
, fields/1]).
-export([
namespace/0,
roots/0,
fields/1
]).
namespace() -> management.

263
apps/emqx_management/src/emqx_mgmt.erl
View File

@ -25,76 +25,82 @@
-include_lib("emqx/include/emqx_mqtt.hrl").
%% Nodes and Brokers API
-export([ list_nodes/0
, lookup_node/1
, list_brokers/0
, lookup_broker/1
, node_info/0
, node_info/1
, broker_info/0
, broker_info/1
]).
-export([
list_nodes/0,
lookup_node/1,
list_brokers/0,
lookup_broker/1,
node_info/0,
node_info/1,
broker_info/0,
broker_info/1
]).
%% Metrics and Stats
-export([ get_metrics/0
, get_metrics/1
, get_stats/0
, get_stats/1
]).
-export([
get_metrics/0,
get_metrics/1,
get_stats/0,
get_stats/1
]).
%% Clients, Sessions
-export([ lookup_client/2
, lookup_client/3
, kickout_client/1
, list_authz_cache/1
, list_client_subscriptions/1
, client_subscriptions/2
, clean_authz_cache/1
, clean_authz_cache/2
, clean_authz_cache_all/0
, clean_authz_cache_all/1
, set_ratelimit_policy/2
, set_quota_policy/2
, set_keepalive/2
]).
-export([
lookup_client/2,
lookup_client/3,
kickout_client/1,
list_authz_cache/1,
list_client_subscriptions/1,
client_subscriptions/2,
clean_authz_cache/1,
clean_authz_cache/2,
clean_authz_cache_all/0,
clean_authz_cache_all/1,
set_ratelimit_policy/2,
set_quota_policy/2,
set_keepalive/2
]).
%% Internal funcs
-export([do_call_client/2]).
%% Subscriptions
-export([ list_subscriptions/1
, list_subscriptions_via_topic/2
, list_subscriptions_via_topic/3
, lookup_subscriptions/1
, lookup_subscriptions/2
-export([
list_subscriptions/1,
list_subscriptions_via_topic/2,
list_subscriptions_via_topic/3,
lookup_subscriptions/1,
lookup_subscriptions/2,
, do_list_subscriptions/0
]).
do_list_subscriptions/0
]).
%% PubSub
-export([ subscribe/2
, do_subscribe/2
, publish/1
, unsubscribe/2
, do_unsubscribe/2
]).
-export([
subscribe/2,
do_subscribe/2,
publish/1,
unsubscribe/2,
do_unsubscribe/2
]).
%% Alarms
-export([ get_alarms/1
, get_alarms/2
, deactivate/2
, delete_all_deactivated_alarms/0
, delete_all_deactivated_alarms/1
]).
-export([
get_alarms/1,
get_alarms/2,
deactivate/2,
delete_all_deactivated_alarms/0,
delete_all_deactivated_alarms/1
]).
%% Banned
-export([ create_banned/1
, delete_banned/1
]).
-export([
create_banned/1,
delete_banned/1
]).
%% Common Table API
-export([ max_row_limit/0
]).
-export([max_row_limit/0]).
-define(APP, emqx_management).
@ -113,24 +119,26 @@ list_nodes() ->
lookup_node(Node) -> node_info(Node).
node_info() ->
Memory = emqx_vm:get_memory(),
Memory = emqx_vm:get_memory(),
Info = maps:from_list([{K, list_to_binary(V)} || {K, V} <- emqx_vm:loads()]),
BrokerInfo = emqx_sys:info(),
Info#{node => node(),
otp_release => otp_rel(),
memory_total => proplists:get_value(allocated, Memory),
memory_used => proplists:get_value(used, Memory),
process_available => erlang:system_info(process_limit),
process_used => erlang:system_info(process_count),
Info#{
node => node(),
otp_release => otp_rel(),
memory_total => proplists:get_value(allocated, Memory),
memory_used => proplists:get_value(used, Memory),
process_available => erlang:system_info(process_limit),
process_used => erlang:system_info(process_count),
max_fds => proplists:get_value(
max_fds, lists:usort(lists:flatten(erlang:system_info(check_io)))),
connections => ets:info(emqx_channel, size),
node_status => 'Running',
uptime => proplists:get_value(uptime, BrokerInfo),
version => iolist_to_binary(proplists:get_value(version, BrokerInfo)),
role => mria_rlog:role()
}.
max_fds => proplists:get_value(
max_fds, lists:usort(lists:flatten(erlang:system_info(check_io)))
),
connections => ets:info(emqx_channel, size),
node_status => 'Running',
uptime => proplists:get_value(uptime, BrokerInfo),
version => iolist_to_binary(proplists:get_value(version, BrokerInfo)),
role => mria_rlog:role()
}.
node_info(Node) ->
wrap_rpc(emqx_management_proto_v1:node_info(Node)).
@ -167,18 +175,21 @@ get_metrics(Node) ->
get_stats() ->
GlobalStatsKeys =
[ 'retained.count'
, 'retained.max'
, 'topics.count'
, 'topics.max'
, 'subscriptions.shared.count'
, 'subscriptions.shared.max'
[
'retained.count',
'retained.max',
'topics.count',
'topics.max',
'subscriptions.shared.count',
'subscriptions.shared.max'
],
CountStats = nodes_info_count([
begin
Stats = get_stats(Node),
delete_keys(Stats, GlobalStatsKeys)
end || Node <- mria_mnesia:running_nodes()]),
end
|| Node <- mria_mnesia:running_nodes()
]),
GlobalStats = maps:with(GlobalStatsKeys, maps:from_list(get_stats(node()))),
maps:merge(CountStats, GlobalStats).
@ -207,21 +218,28 @@ nodes_info_count(PropList) ->
%%--------------------------------------------------------------------
lookup_client({clientid, ClientId}, FormatFun) ->
lists:append([lookup_client(Node, {clientid, ClientId}, FormatFun)
|| Node <- mria_mnesia:running_nodes()]);
lists:append([
lookup_client(Node, {clientid, ClientId}, FormatFun)
|| Node <- mria_mnesia:running_nodes()
]);
lookup_client({username, Username}, FormatFun) ->
lists:append([lookup_client(Node, {username, Username}, FormatFun)
|| Node <- mria_mnesia:running_nodes()]).
lists:append([
lookup_client(Node, {username, Username}, FormatFun)
|| Node <- mria_mnesia:running_nodes()
]).
lookup_client(Node, Key, {M, F}) ->
case wrap_rpc(emqx_cm_proto_v1:lookup_client(Node, Key)) of
{error, Err} -> {error, Err};
L -> lists:map(fun({Chan, Info0, Stats}) ->
Info = Info0#{node => Node},
M:F({Chan, Info, Stats})
end,
L)
{error, Err} ->
{error, Err};
L ->
lists:map(
fun({Chan, Info0, Stats}) ->
Info = Info0#{node => Node},
M:F({Chan, Info, Stats})
end,
L
)
end.
kickout_client({ClientID, FormatFun}) ->
@ -266,7 +284,7 @@ clean_authz_cache(Node, ClientId) ->
clean_authz_cache_all() ->
Results = [{Node, clean_authz_cache_all(Node)} || Node <- mria_mnesia:running_nodes()],
case lists:filter(fun({_Node, Item}) -> Item =/= ok end, Results) of
[] -> ok;
[] -> ok;
BadNodes -> {error, BadNodes}
end.
@ -287,9 +305,13 @@ set_keepalive(_ClientId, _Interval) ->
%% @private
call_client(ClientId, Req) ->
Results = [call_client(Node, ClientId, Req) || Node <- mria_mnesia:running_nodes()],
Expected = lists:filter(fun({error, _}) -> false;
(_) -> true
end, Results),
Expected = lists:filter(
fun
({error, _}) -> false;
(_) -> true
end,
Results
),
case Expected of
[] -> {error, not_found};
[Result | _] -> Result
@ -299,13 +321,15 @@ call_client(ClientId, Req) ->
-spec do_call_client(emqx_types:clientid(), term()) -> term().
do_call_client(ClientId, Req) ->
case emqx_cm:lookup_channels(ClientId) of
[] -> {error, not_found};
[] ->
{error, not_found};
Pids when is_list(Pids) ->
Pid = lists:last(Pids),
case emqx_cm:get_chan_info(ClientId, Pid) of
#{conninfo := #{conn_mod := ConnMod}} ->
erlang:apply(ConnMod, call, [Pid, Req]);
undefined -> {error, not_found}
undefined ->
{error, not_found}
end
end.
@ -320,22 +344,28 @@ call_client(Node, ClientId, Req) ->
-spec do_list_subscriptions() -> [map()].
do_list_subscriptions() ->
case check_row_limit([mqtt_subproperty]) of
false -> throw(max_row_limit);
ok -> [#{topic => Topic, clientid => ClientId, options => Options}
|| {{Topic, ClientId}, Options} <- ets:tab2list(mqtt_subproperty)]
false ->
throw(max_row_limit);
ok ->
[
#{topic => Topic, clientid => ClientId, options => Options}
|| {{Topic, ClientId}, Options} <- ets:tab2list(mqtt_subproperty)
]
end.
list_subscriptions(Node) ->
wrap_rpc(emqx_management_proto_v1:list_subscriptions(Node)).
list_subscriptions_via_topic(Topic, FormatFun) ->
lists:append([list_subscriptions_via_topic(Node, Topic, FormatFun)
|| Node <- mria_mnesia:running_nodes()]).
lists:append([
list_subscriptions_via_topic(Node, Topic, FormatFun)
|| Node <- mria_mnesia:running_nodes()
]).
list_subscriptions_via_topic(Node, Topic, _FormatFun = {M, F}) ->
case wrap_rpc(emqx_broker_proto_v1:list_subscriptions_via_topic(Node, Topic)) of
{error, Reason} -> {error, Reason};
Result -> M:F(Result)
Result -> M:F(Result)
end.
lookup_subscriptions(ClientId) ->
@ -354,20 +384,17 @@ subscribe(ClientId, TopicTables) ->
subscribe([Node | Nodes], ClientId, TopicTables) ->
case wrap_rpc(emqx_management_proto_v1:subscribe(Node, ClientId, TopicTables)) of
{error, _} -> subscribe(Nodes, ClientId, TopicTables);
{subscribe, Res} ->
{subscribe, Res, Node}
{subscribe, Res} -> {subscribe, Res, Node}
end;
subscribe([], _ClientId, _TopicTables) ->
{error, channel_not_found}.
-spec do_subscribe(emqx_types:clientid(), emqx_types:topic_filters()) ->
{subscribe, _} | {error, atom()}.
{subscribe, _} | {error, atom()}.
do_subscribe(ClientId, TopicTables) ->
case ets:lookup(emqx_channel, ClientId) of
[] -> {error, channel_not_found};
[{_, Pid}] ->
Pid ! {subscribe, TopicTables}
[{_, Pid}] -> Pid ! {subscribe, TopicTables}
end.
%%TODO: ???
@ -376,12 +403,12 @@ publish(Msg) ->
emqx:publish(Msg).
-spec unsubscribe(emqx_types:clientid(), emqx_types:topic()) ->
{unsubscribe, _} | {error, channel_not_found}.
{unsubscribe, _} | {error, channel_not_found}.
unsubscribe(ClientId, Topic) ->
unsubscribe(mria_mnesia:running_nodes(), ClientId, Topic).
-spec unsubscribe([node()], emqx_types:clientid(), emqx_types:topic()) ->
{unsubscribe, _} | {error, channel_not_found}.
{unsubscribe, _} | {error, channel_not_found}.
unsubscribe([Node | Nodes], ClientId, Topic) ->
case wrap_rpc(emqx_management_proto_v1:unsubscribe(Node, ClientId, Topic)) of
{error, _} -> unsubscribe(Nodes, ClientId, Topic);
@ -391,12 +418,11 @@ unsubscribe([], _ClientId, _Topic) ->
{error, channel_not_found}.
-spec do_unsubscribe(emqx_types:clientid(), emqx_types:topic()) ->
{unsubscribe, _} | {error, _}.
{unsubscribe, _} | {error, _}.
do_unsubscribe(ClientId, Topic) ->
case ets:lookup(emqx_channel, ClientId) of
[] -> {error, channel_not_found};
[{_, Pid}] ->
Pid ! {unsubscribe, [emqx_topic:parse(Topic)]}
[{_, Pid}] -> Pid ! {unsubscribe, [emqx_topic:parse(Topic)]}
end.
%%--------------------------------------------------------------------
@ -426,11 +452,18 @@ add_duration_field([], _Now, Acc) ->
Acc;
add_duration_field([Alarm = #{activated := true, activate_at := ActivateAt} | Rest], Now, Acc) ->
add_duration_field(Rest, Now, [Alarm#{duration => Now - ActivateAt} | Acc]);
add_duration_field( [Alarm = #{ activated := false
, activate_at := ActivateAt
, deactivate_at := DeactivateAt} | Rest]
, Now, Acc) ->
add_duration_field(
[
Alarm = #{
activated := false,
activate_at := ActivateAt,
deactivate_at := DeactivateAt
}
| Rest
],
Now,
Acc
) ->
add_duration_field(Rest, Now, [Alarm#{duration => DeactivateAt - ActivateAt} | Acc]).
%%--------------------------------------------------------------------
@ -462,13 +495,13 @@ check_row_limit([], _Limit) ->
ok;
check_row_limit([Tab | Tables], Limit) ->
case table_size(Tab) > Limit of
true -> false;
true -> false;
false -> check_row_limit(Tables, Limit)
end.
check_results(Results) ->
case lists:any(fun(Item) -> Item =:= ok end, Results) of
true -> ok;
true -> ok;
false -> wrap_rpc(lists:last(Results))
end.

241
apps/emqx_management/src/emqx_mgmt_api.erl
View File

@ -22,16 +22,18 @@
-define(FRESH_SELECT, fresh_select).
-export([ paginate/3
, paginate/4
]).
-export([
paginate/3,
paginate/4
]).
%% first_next query APIs
-export([ node_query/5
, cluster_query/4
, select_table_with_count/5
, b2i/1
]).
-export([
node_query/5,
cluster_query/4,
select_table_with_count/5,
b2i/1
]).
-export([do_query/6]).
@ -50,30 +52,30 @@ do_paginate(Qh, Count, Params, {Module, FormatFun}) ->
Limit = b2i(limit(Params)),
Cursor = qlc:cursor(Qh),
case Page > 1 of
true ->
true ->
_ = qlc:next_answers(Cursor, (Page - 1) * Limit),
ok;
false -> ok
false ->
ok
end,
Rows = qlc:next_answers(Cursor, Limit),
qlc:delete_cursor(Cursor),
#{meta => #{page => Page, limit => Limit, count => Count},
data => [erlang:apply(Module, FormatFun, [Row]) || Row <- Rows]}.
#{
meta => #{page => Page, limit => Limit, count => Count},
data => [erlang:apply(Module, FormatFun, [Row]) || Row <- Rows]
}.
query_handle(Table) when is_atom(Table) ->
qlc:q([R || R <- ets:table(Table)]);
query_handle({Table, Opts}) when is_atom(Table) ->
qlc:q([R || R <- ets:table(Table, Opts)]);
query_handle([Table]) when is_atom(Table) ->
qlc:q([R || R <- ets:table(Table)]);
query_handle([{Table, Opts}]) when is_atom(Table) ->
qlc:q([R || R <- ets:table(Table, Opts)]);
query_handle(Tables) ->
qlc:append([query_handle(T) || T <- Tables]). %
%
qlc:append([query_handle(T) || T <- Tables]).
query_handle(Table, MatchSpec) when is_atom(Table) ->
Options = {traverse, {select, MatchSpec}},
@ -87,16 +89,12 @@ query_handle(Tables, MatchSpec) ->
count(Table) when is_atom(Table) ->
ets:info(Table, size);
count({Table, _}) when is_atom(Table) ->
ets:info(Table, size);
count([Table]) when is_atom(Table) ->
ets:info(Table, size);
count([{Table, _}]) when is_atom(Table) ->
ets:info(Table, size);
count(Tables) ->
lists:sum([count(T) || T <- Tables]).
@ -121,7 +119,7 @@ limit(Params) ->
init_meta(Params) ->
Limit = b2i(limit(Params)),
Page = b2i(page(Params)),
Page = b2i(page(Params)),
#{
page => Page,
limit => Limit,
@ -134,17 +132,24 @@ init_meta(Params) ->
node_query(Node, QString, Tab, QSchema, QueryFun) ->
{_CodCnt, NQString} = parse_qstring(QString, QSchema),
page_limit_check_query( init_meta(QString)
, { fun do_node_query/5
, [Node, Tab, NQString, QueryFun, init_meta(QString)]}).
page_limit_check_query(
init_meta(QString),
{fun do_node_query/5, [Node, Tab, NQString, QueryFun, init_meta(QString)]}
).
%% @private
do_node_query(Node, Tab, QString, QueryFun, Meta) ->
do_node_query(Node, Tab, QString, QueryFun, _Continuation = ?FRESH_SELECT, Meta, _Results = []).
do_node_query( Node, Tab, QString, QueryFun, Continuation
, Meta = #{limit := Limit}
, Results) ->
do_node_query(
Node,
Tab,
QString,
QueryFun,
Continuation,
Meta = #{limit := Limit},
Results
) ->
case do_query(Node, Tab, QString, QueryFun, Continuation, Limit) of
{error, {badrpc, R}} ->
{error, Node, {badrpc, R}};
@ -164,18 +169,33 @@ cluster_query(QString, Tab, QSchema, QueryFun) ->
{_CodCnt, NQString} = parse_qstring(QString, QSchema),
Nodes = mria_mnesia:running_nodes(),
page_limit_check_query(
init_meta(QString)
, {fun do_cluster_query/5, [Nodes, Tab, NQString, QueryFun, init_meta(QString)]}).
init_meta(QString),
{fun do_cluster_query/5, [Nodes, Tab, NQString, QueryFun, init_meta(QString)]}
).
%% @private
do_cluster_query(Nodes, Tab, QString, QueryFun, Meta) ->
do_cluster_query( Nodes, Tab, QString, QueryFun
, _Continuation = ?FRESH_SELECT, Meta, _Results = []).
do_cluster_query(
Nodes,
Tab,
QString,
QueryFun,
_Continuation = ?FRESH_SELECT,
Meta,
_Results = []
).
do_cluster_query([], _Tab, _QString, _QueryFun, _Continuation, Meta, Results) ->
#{meta => Meta, data => Results};
do_cluster_query([Node | Tail] = Nodes, Tab, QString, QueryFun, Continuation,
Meta = #{limit := Limit}, Results) ->
do_cluster_query(
[Node | Tail] = Nodes,
Tab,
QString,
QueryFun,
Continuation,
Meta = #{limit := Limit},
Results
) ->
case do_query(Node, Tab, QString, QueryFun, Continuation, Limit) of
{error, {badrpc, R}} ->
{error, Node, {bar_rpc, R}};
@ -192,11 +212,18 @@ do_cluster_query([Node | Tail] = Nodes, Tab, QString, QueryFun, Continuation,
%%--------------------------------------------------------------------
%% @private This function is exempt from BPAPI
do_query(Node, Tab, QString, {M,F}, Continuation, Limit) when Node =:= node() ->
do_query(Node, Tab, QString, {M, F}, Continuation, Limit) when Node =:= node() ->
erlang:apply(M, F, [Tab, QString, Continuation, Limit]);
do_query(Node, Tab, QString, QueryFun, Continuation, Limit) ->
case rpc:call(Node, ?MODULE, do_query,
[Node, Tab, QString, QueryFun, Continuation, Limit], 50000) of
case
rpc:call(
Node,
?MODULE,
do_query,
[Node, Tab, QString, QueryFun, Continuation, Limit],
50000
)
of
{badrpc, _} = R -> {error, R};
Ret -> Ret
end.
@ -220,8 +247,9 @@ sub_query_result(Len, Rows, Limit, Results, Meta) ->
%% Table Select
%%--------------------------------------------------------------------
select_table_with_count(Tab, {Ms, FuzzyFilterFun}, ?FRESH_SELECT, Limit, FmtFun)
when is_function(FuzzyFilterFun) andalso Limit > 0 ->
select_table_with_count(Tab, {Ms, FuzzyFilterFun}, ?FRESH_SELECT, Limit, FmtFun) when
is_function(FuzzyFilterFun) andalso Limit > 0
->
case ets:select(Tab, Ms, Limit) of
'$end_of_table' ->
{0, [], ?FRESH_SELECT};
@ -229,8 +257,9 @@ select_table_with_count(Tab, {Ms, FuzzyFilterFun}, ?FRESH_SELECT, Limit, FmtFun)
Rows = FuzzyFilterFun(RawResult),
{length(Rows), lists:map(FmtFun, Rows), NContinuation}
end;
select_table_with_count(_Tab, {Ms, FuzzyFilterFun}, Continuation, _Limit, FmtFun)
when is_function(FuzzyFilterFun) ->
select_table_with_count(_Tab, {Ms, FuzzyFilterFun}, Continuation, _Limit, FmtFun) when
is_function(FuzzyFilterFun)
->
case ets:select(ets:repair_continuation(Continuation, Ms)) of
'$end_of_table' ->
{0, [], ?FRESH_SELECT};
@ -238,8 +267,9 @@ select_table_with_count(_Tab, {Ms, FuzzyFilterFun}, Continuation, _Limit, FmtFun
Rows = FuzzyFilterFun(RawResult),
{length(Rows), lists:map(FmtFun, Rows), NContinuation}
end;
select_table_with_count(Tab, Ms, ?FRESH_SELECT, Limit, FmtFun)
when Limit > 0 ->
select_table_with_count(Tab, Ms, ?FRESH_SELECT, Limit, FmtFun) when
Limit > 0
->
case ets:select(Tab, Ms, Limit) of
'$end_of_table' ->
{0, [], ?FRESH_SELECT};
@ -267,36 +297,53 @@ parse_qstring(QString, QSchema) ->
do_parse_qstring([], _, Acc1, Acc2) ->
NAcc2 = [E || E <- Acc2, not lists:keymember(element(1, E), 1, Acc1)],
{lists:reverse(Acc1), lists:reverse(NAcc2)};
do_parse_qstring([{Key, Value} | RestQString], QSchema, Acc1, Acc2) ->
case proplists:get_value(Key, QSchema) of
undefined -> do_parse_qstring(RestQString, QSchema, Acc1, Acc2);
undefined ->
do_parse_qstring(RestQString, QSchema, Acc1, Acc2);
Type ->
case Key of
<<Prefix:4/binary, NKey/binary>>
when Prefix =:= <<"gte_">>;
Prefix =:= <<"lte_">> ->
OpposeKey = case Prefix of
<<"gte_">> -> <<"lte_", NKey/binary>>;
<<"lte_">> -> <<"gte_", NKey/binary>>
end,
<<Prefix:4/binary, NKey/binary>> when
Prefix =:= <<"gte_">>;
Prefix =:= <<"lte_">>
->
OpposeKey =
case Prefix of
<<"gte_">> -> <<"lte_", NKey/binary>>;
<<"lte_">> -> <<"gte_", NKey/binary>>
end,
case lists:keytake(OpposeKey, 1, RestQString) of
false ->
do_parse_qstring( RestQString, QSchema
, [qs(Key, Value, Type) | Acc1], Acc2);
do_parse_qstring(
RestQString,
QSchema,
[qs(Key, Value, Type) | Acc1],
Acc2
);
{value, {K2, V2}, NParams} ->
do_parse_qstring( NParams, QSchema
, [qs(Key, Value, K2, V2, Type) | Acc1], Acc2)
do_parse_qstring(
NParams,
QSchema,
[qs(Key, Value, K2, V2, Type) | Acc1],
Acc2
)
end;
_ ->
case is_fuzzy_key(Key) of
true ->
do_parse_qstring( RestQString, QSchema
, Acc1, [qs(Key, Value, Type) | Acc2]);
do_parse_qstring(
RestQString,
QSchema,
Acc1,
[qs(Key, Value, Type) | Acc2]
);
_ ->
do_parse_qstring( RestQString, QSchema
, [qs(Key, Value, Type) | Acc1], Acc2)
do_parse_qstring(
RestQString,
QSchema,
[qs(Key, Value, Type) | Acc1],
Acc2
)
end
end
end.
@ -310,7 +357,7 @@ qs(K, Value0, Type) ->
try
qs(K, to_type(Value0, Type))
catch
throw : bad_value_type ->
throw:bad_value_type ->
throw({bad_value_type, {K, Type, Value0}})
end.
@ -333,12 +380,11 @@ is_fuzzy_key(_) ->
false.
page_start(1, _) -> 1;
page_start(Page, Limit) -> (Page-1) * Limit + 1.
page_start(Page, Limit) -> (Page - 1) * Limit + 1.
judge_page_with_counting(Len, Meta = #{page := Page, limit := Limit, count := Count}) ->
PageStart = page_start(Page, Limit),
PageEnd = Page * Limit,
PageEnd = Page * Limit,
case Count + Len of
NCount when NCount < PageStart ->
{more, Meta#{count => NCount}};
@ -353,7 +399,7 @@ rows_sub_params(Len, _Meta = #{page := Page, limit := Limit, count := Count}) ->
case (Count - Len) < PageStart of
true ->
NeedNowNum = Count - PageStart + 1,
SubStart = Len - NeedNowNum + 1,
SubStart = Len - NeedNowNum + 1,
{SubStart, NeedNowNum};
false ->
{_SubStart = 1, _NeedNowNum = Len}
@ -361,8 +407,9 @@ rows_sub_params(Len, _Meta = #{page := Page, limit := Limit, count := Count}) ->
page_limit_check_query(Meta, {F, A}) ->
case Meta of
#{page := Page, limit := Limit}
when Page < 1; Limit < 1 ->
#{page := Page, limit := Limit} when
Page < 1; Limit < 1
->
{error, page_limit_invalid};
_ ->
erlang:apply(F, A)
@ -376,7 +423,7 @@ to_type(V, TargetType) ->
try
to_type_(V, TargetType)
catch
_ : _ ->
_:_ ->
throw(bad_value_type)
end.
@ -419,37 +466,43 @@ to_ip_port(IPAddress) ->
-include_lib("eunit/include/eunit.hrl").
params2qs_test() ->
QSchema = [{<<"str">>, binary},
{<<"int">>, integer},
{<<"atom">>, atom},
{<<"ts">>, timestamp},
{<<"gte_range">>, integer},
{<<"lte_range">>, integer},
{<<"like_fuzzy">>, binary},
{<<"match_topic">>, binary}],
QString = [{<<"str">>, <<"abc">>},
{<<"int">>, <<"123">>},
{<<"atom">>, <<"connected">>},
{<<"ts">>, <<"156000">>},
{<<"gte_range">>, <<"1">>},
{<<"lte_range">>, <<"5">>},
{<<"like_fuzzy">>, <<"user">>},
{<<"match_topic">>, <<"t/#">>}],
ExpectedQs = [{str, '=:=', <<"abc">>},
{int, '=:=', 123},
{atom, '=:=', connected},
{ts, '=:=', 156000},
{range, '>=', 1, '=<', 5}
],
FuzzyNQString = [{fuzzy, like, <<"user">>},
{topic, match, <<"t/#">>}],
QSchema = [
{<<"str">>, binary},
{<<"int">>, integer},
{<<"atom">>, atom},
{<<"ts">>, timestamp},
{<<"gte_range">>, integer},
{<<"lte_range">>, integer},
{<<"like_fuzzy">>, binary},
{<<"match_topic">>, binary}
],
QString = [
{<<"str">>, <<"abc">>},
{<<"int">>, <<"123">>},
{<<"atom">>, <<"connected">>},
{<<"ts">>, <<"156000">>},
{<<"gte_range">>, <<"1">>},
{<<"lte_range">>, <<"5">>},
{<<"like_fuzzy">>, <<"user">>},
{<<"match_topic">>, <<"t/#">>}
],
ExpectedQs = [
{str, '=:=', <<"abc">>},
{int, '=:=', 123},
{atom, '=:=', connected},
{ts, '=:=', 156000},
{range, '>=', 1, '=<', 5}
],
FuzzyNQString = [
{fuzzy, like, <<"user">>},
{topic, match, <<"t/#">>}
],
?assertEqual({7, {ExpectedQs, FuzzyNQString}}, parse_qstring(QString, QSchema)),
{0, {[], []}} = parse_qstring([{not_a_predefined_params, val}], QSchema).
-endif.
b2i(Bin) when is_binary(Bin) ->
binary_to_integer(Bin);
b2i(Any) ->

81
apps/emqx_management/src/emqx_mgmt_api_alarms.erl
View File

@ -18,6 +18,7 @@
-behaviour(minirest_api).
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("typerefl/include/types.hrl").
@ -38,13 +39,16 @@ schema("/alarms") ->
#{
'operationId' => alarms,
get => #{
description => <<"EMQX alarms">>,
description => ?DESC(list_alarms_api),
parameters => [
hoconsc:ref(emqx_dashboard_swagger, page),
hoconsc:ref(emqx_dashboard_swagger, limit),
{activated, hoconsc:mk(boolean(), #{in => query,
desc => <<"All alarms, if not specified">>,
required => false})}
{activated,
hoconsc:mk(boolean(), #{
in => query,
desc => ?DESC(get_alarms_qs_activated),
required => false
})}
],
responses => #{
200 => [
@ -53,34 +57,48 @@ schema("/alarms") ->
]
}
},
delete => #{
description => <<"Remove all deactivated alarms">>,
delete => #{
description => ?DESC(delete_alarms_api),
responses => #{
204 => <<"Remove all deactivated alarms ok">>
204 => ?DESC(delete_alarms_api_response204)
}
}
}.
fields(alarm) ->
[
{node, hoconsc:mk(binary(),
#{desc => <<"Alarm in node">>, example => atom_to_list(node())})},
{name, hoconsc:mk(binary(),
#{desc => <<"Alarm name">>, example => <<"high_system_memory_usage">>})},
{message, hoconsc:mk(binary(), #{desc => <<"Alarm readable information">>,
example => <<"System memory usage is higher than 70%">>})},
{details, hoconsc:mk(map(), #{desc => <<"Alarm details information">>,
example => #{<<"high_watermark">> => 70}})},
{duration, hoconsc:mk(integer(),
#{desc => <<"Alarms duration time; UNIX time stamp, millisecond">>,
example => 297056})},
{activate_at, hoconsc:mk(binary(), #{desc => <<"Alarms activate time, RFC 3339">>,
example => <<"2021-10-25T11:52:52.548+08:00">>})},
{deactivate_at, hoconsc:mk(binary(),
#{desc => <<"Nullable, alarms deactivate time, RFC 3339">>,
example => <<"2021-10-31T10:52:52.548+08:00">>})}
{node,
hoconsc:mk(
binary(),
#{desc => ?DESC(node), example => atom_to_list(node())}
)},
{name,
hoconsc:mk(
binary(),
#{desc => ?DESC(node), example => <<"high_system_memory_usage">>}
)},
{message,
hoconsc:mk(binary(), #{
desc => ?DESC(message),
example => <<"System memory usage is higher than 70%">>
})},
{details,
hoconsc:mk(map(), #{
desc => ?DESC(details),
example => #{<<"high_watermark">> => 70}
})},
{duration, hoconsc:mk(integer(), #{desc => ?DESC(duration), example => 297056})},
{activate_at,
hoconsc:mk(binary(), #{
desc => ?DESC(activate_at),
example => <<"2021-10-25T11:52:52.548+08:00">>
})},
{deactivate_at,
hoconsc:mk(binary(), #{
desc => ?DESC(deactivate_at),
example => <<"2021-10-31T10:52:52.548+08:00">>
})}
];
fields(meta) ->
emqx_dashboard_swagger:fields(page) ++
emqx_dashboard_swagger:fields(limit) ++
@ -93,9 +111,15 @@ alarms(get, #{query_string := QString}) ->
true -> ?ACTIVATED_ALARM;
false -> ?DEACTIVATED_ALARM
end,
Response = emqx_mgmt_api:cluster_query(QString, Table, [], {?MODULE, query}),
emqx_mgmt_util:generate_response(Response);
case emqx_mgmt_api:cluster_query(QString, Table, [], {?MODULE, query}) of
{error, page_limit_invalid} ->
{400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
{error, Node, {badrpc, R}} ->
Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])),
{500, #{code => <<"NODE_DOWN">>, message => Message}};
Response ->
{200, Response}
end;
alarms(delete, _Params) ->
_ = emqx_mgmt:delete_all_deactivated_alarms(),
{204}.
@ -104,11 +128,10 @@ alarms(delete, _Params) ->
%% internal
query(Table, _QsSpec, Continuation, Limit) ->
Ms = [{'$1',[],['$1']}],
Ms = [{'$1', [], ['$1']}],
emqx_mgmt_api:select_table_with_count(Table, Ms, Continuation, Limit, fun format_alarm/1).
format_alarm(Alarms) when is_list(Alarms) ->
[emqx_alarm:format(Alarm) || Alarm <- Alarms];
format_alarm(Alarm) ->
emqx_alarm:format(Alarm).

118
apps/emqx_management/src/emqx_mgmt_api_app.erl
View File

@ -31,7 +31,6 @@ api_spec() ->
paths() ->
["/api_key", "/api_key/:name"].
schema("/api_key") ->
#{
'operationId' => api_key,
@ -82,41 +81,80 @@ schema("/api_key/:name") ->
fields(app) ->
[
{name, hoconsc:mk(binary(),
#{desc => "Unique and format by [a-zA-Z0-9-_]",
validator => fun ?MODULE:validate_name/1,
example => <<"EMQX-API-KEY-1">>})},
{api_key, hoconsc:mk(binary(),
#{desc => """TODO:uses HMAC-SHA256 for signing.""",
example => <<"a4697a5c75a769f6">>})},
{api_secret, hoconsc:mk(binary(),
#{desc => """An API secret is a simple encrypted string that identifies"""
"""an application without any principal."""
"""They are useful for accessing public data anonymously,"""
"""and are used to associate API requests.""",
example => <<"MzAyMjk3ODMwMDk0NjIzOTUxNjcwNzQ0NzQ3MTE2NDYyMDI">>})},
{expired_at, hoconsc:mk(hoconsc:union([undefined, emqx_datetime:epoch_second()]),
#{desc => "No longer valid datetime",
example => <<"2021-12-05T02:01:34.186Z">>,
required => false,
default => undefined
})},
{created_at, hoconsc:mk(emqx_datetime:epoch_second(),
#{desc => "ApiKey create datetime",
example => <<"2021-12-01T00:00:00.000Z">>
})},
{desc, hoconsc:mk(binary(),
#{example => <<"Note">>, required => false})},
{name,
hoconsc:mk(
binary(),
#{
desc => "Unique and format by [a-zA-Z0-9-_]",
validator => fun ?MODULE:validate_name/1,
example => <<"EMQX-API-KEY-1">>
}
)},
{api_key,
hoconsc:mk(
binary(),
#{
desc => "" "TODO:uses HMAC-SHA256 for signing." "",
example => <<"a4697a5c75a769f6">>
}
)},
{api_secret,
hoconsc:mk(
binary(),
#{
desc =>
""
"An API secret is a simple encrypted string that identifies"
""
""
"an application without any principal."
""
""
"They are useful for accessing public data anonymously,"
""
""
"and are used to associate API requests."
"",
example => <<"MzAyMjk3ODMwMDk0NjIzOTUxNjcwNzQ0NzQ3MTE2NDYyMDI">>
}
)},
{expired_at,
hoconsc:mk(
hoconsc:union([undefined, emqx_datetime:epoch_second()]),
#{
desc => "No longer valid datetime",
example => <<"2021-12-05T02:01:34.186Z">>,
required => false,
default => undefined
}
)},
{created_at,
hoconsc:mk(
emqx_datetime:epoch_second(),
#{
desc => "ApiKey create datetime",
example => <<"2021-12-01T00:00:00.000Z">>
}
)},
{desc,
hoconsc:mk(
binary(),
#{example => <<"Note">>, required => false}
)},
{enable, hoconsc:mk(boolean(), #{desc => "Enable/Disable", required => false})}
];
fields(name) ->
[{name, hoconsc:mk(binary(),
#{
desc => <<"^[A-Za-z]+[A-Za-z0-9-_]*$">>,
example => <<"EMQX-API-KEY-1">>,
in => path,
validator => fun ?MODULE:validate_name/1
})}
[
{name,
hoconsc:mk(
binary(),
#{
desc => <<"^[A-Za-z]+[A-Za-z0-9-_]*$">>,
example => <<"EMQX-API-KEY-1">>,
in => path,
validator => fun ?MODULE:validate_name/1
}
)}
].
-define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$").
@ -129,7 +167,8 @@ validate_name(Name) ->
nomatch -> {error, "Name should be " ?NAME_RE};
_ -> ok
end;
false -> {error, "Name Length must =< 256"}
false ->
{error, "Name Length must =< 256"}
end.
delete(Keys, Fields) ->
@ -146,10 +185,13 @@ api_key(post, #{body := App}) ->
ExpiredAt = ensure_expired_at(App),
Desc = unicode:characters_to_binary(Desc0, unicode),
case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of
{ok, NewApp} -> {200, format(NewApp)};
{ok, NewApp} ->
{200, format(NewApp)};
{error, Reason} ->
{400, #{code => 'BAD_REQUEST',
message => iolist_to_binary(io_lib:format("~p", [Reason]))}}
{400, #{
code => 'BAD_REQUEST',
message => iolist_to_binary(io_lib:format("~p", [Reason]))
}}
end.
-define(NOT_FOUND_RESPONSE, #{code => 'NOT_FOUND', message => <<"Name NOT FOUND">>}).
@ -184,5 +226,5 @@ format(App = #{expired_at := ExpiredAt0, created_at := CreateAt}) ->
created_at => list_to_binary(calendar:system_time_to_rfc3339(CreateAt))
}.
ensure_expired_at(#{<<"expired_at">> := ExpiredAt})when is_integer(ExpiredAt) -> ExpiredAt;
ensure_expired_at(#{<<"expired_at">> := ExpiredAt}) when is_integer(ExpiredAt) -> ExpiredAt;
ensure_expired_at(_) -> undefined.

126
apps/emqx_management/src/emqx_mgmt_api_banned.erl
View File

@ -16,6 +16,7 @@
-module(emqx_mgmt_api_banned).
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("typerefl/include/types.hrl").
@ -23,16 +24,19 @@
-behaviour(minirest_api).
-export([ api_spec/0
, paths/0
, schema/1
, fields/1]).
-export([
api_spec/0,
paths/0,
schema/1,
fields/1
]).
-export([format/1]).
-export([ banned/2
, delete_banned/2
]).
-export([
banned/2,
delete_banned/2
]).
-define(TAB, emqx_banned).
@ -48,28 +52,29 @@ paths() ->
schema("/banned") ->
#{
'operationId' => banned,
'operationId' => banned,
get => #{
description => <<"List banned">>,
description => ?DESC(list_banned_api),
parameters => [
hoconsc:ref(emqx_dashboard_swagger, page),
hoconsc:ref(emqx_dashboard_swagger, limit)
],
responses => #{
200 =>[
200 => [
{data, hoconsc:mk(hoconsc:array(hoconsc:ref(ban)), #{})},
{meta, hoconsc:mk(hoconsc:ref(meta), #{})}
]
}
},
post => #{
description => <<"Create banned">>,
description => ?DESC(create_banned_api),
'requestBody' => hoconsc:mk(hoconsc:ref(ban)),
responses => #{
200 => [{data, hoconsc:mk(hoconsc:array(hoconsc:ref(ban)), #{})}],
400 => emqx_dashboard_swagger:error_codes(
['ALREADY_EXISTS', 'BAD_REQUEST'],
<<"Banned already existed, or bad args">>)
['ALREADY_EXISTS', 'BAD_REQUEST'],
?DESC(create_banned_api_response400)
)
}
}
};
@ -77,53 +82,71 @@ schema("/banned/:as/:who") ->
#{
'operationId' => delete_banned,
delete => #{
description => <<"Delete banned">>,
description => ?DESC(delete_banned_api),
parameters => [
{as, hoconsc:mk(hoconsc:enum(?BANNED_TYPES), #{
desc => <<"Banned type">>,
in => path,
example => username})},
{who, hoconsc:mk(binary(), #{
desc => <<"Client info as banned type">>,
in => path,
example => <<"Badass">>})}
],
{as,
hoconsc:mk(hoconsc:enum(?BANNED_TYPES), #{
desc => ?DESC(as),
required => true,
in => path,
example => username
})},
{who,
hoconsc:mk(binary(), #{
desc => ?DESC(who),
required => true,
in => path,
example => <<"Badass">>
})}
],
responses => #{
204 => <<"Delete banned success">>,
404 => emqx_dashboard_swagger:error_codes(
['NOT_FOUND'],
<<"Banned not found. May be the banned time has been exceeded">>)
['NOT_FOUND'],
?DESC(delete_banned_api_response404)
)
}
}
}.
fields(ban) ->
[
{as, hoconsc:mk(hoconsc:enum(?BANNED_TYPES), #{
desc => <<"Banned type clientid, username, peerhost">>,
required => true,
example => username})},
{who, hoconsc:mk(binary(), #{
desc => <<"Client info as banned type">>,
required => true,
example => <<"Banned name"/utf8>>})},
{by, hoconsc:mk(binary(), #{
desc => <<"Commander">>,
required => false,
example => <<"mgmt_api">>})},
{reason, hoconsc:mk(binary(), #{
desc => <<"Banned reason">>,
required => false,
example => <<"Too many requests">>})},
{at, hoconsc:mk(emqx_datetime:epoch_second(), #{
desc => <<"Create banned time, rfc3339, now if not specified">>,
required => false,
example => <<"2021-10-25T21:48:47+08:00">>})},
{until, hoconsc:mk(emqx_datetime:epoch_second(), #{
desc => <<"Cancel banned time, rfc3339, now + 5 minute if not specified">>,
required => false,
example => <<"2021-10-25T21:53:47+08:00">>})
}
{as,
hoconsc:mk(hoconsc:enum(?BANNED_TYPES), #{
desc => ?DESC(as),
required => true,
example => username
})},
{who,
hoconsc:mk(binary(), #{
desc => ?DESC(who),
required => true,
example => <<"Banned name"/utf8>>
})},
{by,
hoconsc:mk(binary(), #{
desc => ?DESC(by),
required => false,
example => <<"mgmt_api">>
})},
{reason,
hoconsc:mk(binary(), #{
desc => ?DESC(reason),
required => false,
example => <<"Too many requests">>
})},
{at,
hoconsc:mk(emqx_datetime:epoch_second(), #{
desc => ?DESC(at),
required => false,
example => <<"2021-10-25T21:48:47+08:00">>
})},
{until,
hoconsc:mk(emqx_datetime:epoch_second(), #{
desc => ?DESC(until),
required => false,
example => <<"2021-10-25T21:53:47+08:00">>
})}
];
fields(meta) ->
emqx_dashboard_swagger:fields(page) ++
@ -140,8 +163,7 @@ banned(post, #{body := Body}) ->
Ban ->
case emqx_banned:create(Ban) of
{ok, Banned} -> {200, format(Banned)};
{error, {already_exist, Old}} ->
{400, 'ALREADY_EXISTS', format(Old)}
{error, {already_exist, Old}} -> {400, 'ALREADY_EXISTS', format(Old)}
end
end.

724
apps/emqx_management/src/emqx_mgmt_api_clients.erl
View File

@ -26,63 +26,70 @@
-include("emqx_mgmt.hrl").
%% API
-export([ api_spec/0
, paths/0
, schema/1
, fields/1]).
-export([
api_spec/0,
paths/0,
schema/1,
fields/1
]).
-export([ clients/2
, client/2
, subscriptions/2
, authz_cache/2
, subscribe/2
, unsubscribe/2
, subscribe_batch/2
, set_keepalive/2
]).
-export([
clients/2,
client/2,
subscriptions/2,
authz_cache/2,
subscribe/2,
unsubscribe/2,
subscribe_batch/2,
set_keepalive/2
]).
-export([ query/4
, format_channel_info/1
]).
-export([
query/4,
format_channel_info/1
]).
%% for batch operation
-export([do_subscribe/3]).
-define(CLIENT_QTAB, emqx_channel_info).
-define(CLIENT_QSCHEMA,
[ {<<"node">>, atom}
, {<<"username">>, binary}
, {<<"zone">>, atom}
, {<<"ip_address">>, ip}
, {<<"conn_state">>, atom}
, {<<"clean_start">>, atom}
, {<<"proto_name">>, binary}
, {<<"proto_ver">>, integer}
, {<<"like_clientid">>, binary}
, {<<"like_username">>, binary}
, {<<"gte_created_at">>, timestamp}
, {<<"lte_created_at">>, timestamp}
, {<<"gte_connected_at">>, timestamp}
, {<<"lte_connected_at">>, timestamp}]).
-define(CLIENT_QSCHEMA, [
{<<"node">>, atom},
{<<"username">>, binary},
{<<"zone">>, atom},
{<<"ip_address">>, ip},
{<<"conn_state">>, atom},
{<<"clean_start">>, atom},
{<<"proto_name">>, binary},
{<<"proto_ver">>, integer},
{<<"like_clientid">>, binary},
{<<"like_username">>, binary},
{<<"gte_created_at">>, timestamp},
{<<"lte_created_at">>, timestamp},
{<<"gte_connected_at">>, timestamp},
{<<"lte_connected_at">>, timestamp}
]).
-define(QUERY_FUN, {?MODULE, query}).
-define(FORMAT_FUN, {?MODULE, format_channel_info}).
-define(CLIENT_ID_NOT_FOUND,
<<"{\"code\": \"RESOURCE_NOT_FOUND\", \"reason\": \"Client id not found\"}">>).
<<"{\"code\": \"RESOURCE_NOT_FOUND\", \"reason\": \"Client id not found\"}">>
).
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).
paths() ->
[ "/clients"
, "/clients/:clientid"
, "/clients/:clientid/authz_cache"
, "/clients/:clientid/subscriptions"
, "/clients/:clientid/subscribe"
, "/clients/:clientid/unsubscribe"
, "/clients/:clientid/keepalive"
[
"/clients",
"/clients/:clientid",
"/clients/:clientid/authorization/cache",
"/clients/:clientid/subscriptions",
"/clients/:clientid/subscribe",
"/clients/:clientid/unsubscribe",
"/clients/:clientid/keepalive"
].
schema("/clients") ->
@ -93,69 +100,105 @@ schema("/clients") ->
parameters => [
hoconsc:ref(emqx_dashboard_swagger, page),
hoconsc:ref(emqx_dashboard_swagger, limit),
{node, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Node name">>,
example => atom_to_list(node())})},
{username, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"User name">>})},
{zone, hoconsc:mk(binary(), #{
in => query,
required => false})},
{ip_address, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Client's IP address">>,
example => <<"127.0.0.1">>})},
{conn_state, hoconsc:mk(hoconsc:enum([connected, idle, disconnected]), #{
in => query,
required => false,
desc => <<"The current connection status of the client, ",
"the possible values are connected,idle,disconnected">>})},
{clean_start, hoconsc:mk(boolean(), #{
in => query,
required => false,
description => <<"Whether the client uses a new session">>})},
{proto_name, hoconsc:mk(hoconsc:enum(['MQTT', 'CoAP', 'LwM2M', 'MQTT-SN']), #{
in => query,
required => false,
description => <<"Client protocol name, ",
"the possible values are MQTT,CoAP,LwM2M,MQTT-SN">>})},
{proto_ver, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Client protocol version">>})},
{like_clientid, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Fuzzy search `clientid` as substring">>})},
{like_username, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Fuzzy search `username` as substring">>})},
{gte_created_at, hoconsc:mk(emqx_datetime:epoch_millisecond(), #{
in => query,
required => false,
desc => <<"Search client session creation time by greater",
" than or equal method, rfc3339 or timestamp(millisecond)">>})},
{lte_created_at, hoconsc:mk(emqx_datetime:epoch_millisecond(), #{
in => query,
required => false,
desc => <<"Search client session creation time by less",
" than or equal method, rfc3339 or timestamp(millisecond)">>})},
{gte_connected_at, hoconsc:mk(emqx_datetime:epoch_millisecond(), #{
in => query,
required => false,
desc => <<"Search client connection creation time by greater"
" than or equal method, rfc3339 or timestamp(epoch millisecond)">>})},
{lte_connected_at, hoconsc:mk(emqx_datetime:epoch_millisecond(), #{
in => query,
required => false,
desc => <<"Search client connection creation time by less"
" than or equal method, rfc3339 or timestamp(millisecond)">>})}
{node,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Node name">>,
example => atom_to_list(node())
})},
{username,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"User name">>
})},
{zone,
hoconsc:mk(binary(), #{
in => query,
required => false
})},
{ip_address,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Client's IP address">>,
example => <<"127.0.0.1">>
})},
{conn_state,
hoconsc:mk(hoconsc:enum([connected, idle, disconnected]), #{
in => query,
required => false,
desc =>
<<"The current connection status of the client, ",
"the possible values are connected,idle,disconnected">>
})},
{clean_start,
hoconsc:mk(boolean(), #{
in => query,
required => false,
description => <<"Whether the client uses a new session">>
})},
{proto_name,
hoconsc:mk(hoconsc:enum(['MQTT', 'CoAP', 'LwM2M', 'MQTT-SN']), #{
in => query,
required => false,
description =>
<<"Client protocol name, ",
"the possible values are MQTT,CoAP,LwM2M,MQTT-SN">>
})},
{proto_ver,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Client protocol version">>
})},
{like_clientid,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Fuzzy search `clientid` as substring">>
})},
{like_username,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Fuzzy search `username` as substring">>
})},
{gte_created_at,
hoconsc:mk(emqx_datetime:epoch_millisecond(), #{
in => query,
required => false,
desc =>
<<"Search client session creation time by greater",
" than or equal method, rfc3339 or timestamp(millisecond)">>
})},
{lte_created_at,
hoconsc:mk(emqx_datetime:epoch_millisecond(), #{
in => query,
required => false,
desc =>
<<"Search client session creation time by less",
" than or equal method, rfc3339 or timestamp(millisecond)">>
})},
{gte_connected_at,
hoconsc:mk(emqx_datetime:epoch_millisecond(), #{
in => query,
required => false,
desc => <<
"Search client connection creation time by greater"
" than or equal method, rfc3339 or timestamp(epoch millisecond)"
>>
})},
{lte_connected_at,
hoconsc:mk(emqx_datetime:epoch_millisecond(), #{
in => query,
required => false,
desc => <<
"Search client connection creation time by less"
" than or equal method, rfc3339 or timestamp(millisecond)"
>>
})}
],
responses => #{
200 => [
@ -164,10 +207,11 @@ schema("/clients") ->
],
400 =>
emqx_dashboard_swagger:error_codes(
['INVALID_PARAMETER'], <<"Invalid parameters">>)}
['INVALID_PARAMETER'], <<"Invalid parameters">>
)
}
}
};
schema("/clients/:clientid") ->
#{
'operationId' => client,
@ -177,42 +221,47 @@ schema("/clients/:clientid") ->
responses => #{
200 => hoconsc:mk(hoconsc:ref(?MODULE, client), #{}),
404 => emqx_dashboard_swagger:error_codes(
['CLIENTID_NOT_FOUND'], <<"Client id not found">>)}},
['CLIENTID_NOT_FOUND'], <<"Client id not found">>
)
}
},
delete => #{
description => <<"Kick out client by client ID">>,
parameters => [
{clientid, hoconsc:mk(binary(), #{in => path})}],
{clientid, hoconsc:mk(binary(), #{in => path})}
],
responses => #{
204 => <<"Kick out client successfully">>,
404 => emqx_dashboard_swagger:error_codes(
['CLIENTID_NOT_FOUND'], <<"Client id not found">>)
['CLIENTID_NOT_FOUND'], <<"Client id not found">>
)
}
}
};
schema("/clients/:clientid/authz_cache") ->
schema("/clients/:clientid/authorization/cache") ->
#{
'operationId' => authz_cache,
get => #{
description => <<"Get client authz cache">>,
description => <<"Get client authz cache in the cluster.">>,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
responses => #{
200 => hoconsc:mk(hoconsc:ref(?MODULE, authz_cache), #{}),
404 => emqx_dashboard_swagger:error_codes(
['CLIENTID_NOT_FOUND'], <<"Client id not found">>)
['CLIENTID_NOT_FOUND'], <<"Client id not found">>
)
}
},
delete => #{
description => <<"Clean client authz cache">>,
description => <<"Clean client authz cache in the cluster.">>,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
responses => #{
204 => <<"Kick out client successfully">>,
404 => emqx_dashboard_swagger:error_codes(
['CLIENTID_NOT_FOUND'], <<"Client id not found">>)
['CLIENTID_NOT_FOUND'], <<"Client id not found">>
)
}
}
};
schema("/clients/:clientid/subscriptions") ->
#{
'operationId' => subscriptions,
@ -220,13 +269,15 @@ schema("/clients/:clientid/subscriptions") ->
description => <<"Get client subscriptions">>,
parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
responses => #{
200 => hoconsc:mk(hoconsc:array(hoconsc:ref(emqx_mgmt_api_subscriptions, subscription)), #{}),
200 => hoconsc:mk(
hoconsc:array(hoconsc:ref(emqx_mgmt_api_subscriptions, subscription)), #{}
),
404 => emqx_dashboard_swagger:error_codes(
['CLIENTID_NOT_FOUND'], <<"Client id not found">>)
['CLIENTID_NOT_FOUND'], <<"Client id not found">>
)
}
}
};
schema("/clients/:clientid/subscribe") ->
#{
'operationId' => subscribe,
@ -237,11 +288,11 @@ schema("/clients/:clientid/subscribe") ->
responses => #{
200 => hoconsc:ref(emqx_mgmt_api_subscriptions, subscription),
404 => emqx_dashboard_swagger:error_codes(
['CLIENTID_NOT_FOUND'], <<"Client id not found">>)
['CLIENTID_NOT_FOUND'], <<"Client id not found">>
)
}
}
};
schema("/clients/:clientid/unsubscribe") ->
#{
'operationId' => unsubscribe,
@ -252,11 +303,11 @@ schema("/clients/:clientid/unsubscribe") ->
responses => #{
204 => <<"Unsubscribe OK">>,
404 => emqx_dashboard_swagger:error_codes(
['CLIENTID_NOT_FOUND'], <<"Client id not found">>)
['CLIENTID_NOT_FOUND'], <<"Client id not found">>
)
}
}
};
schema("/clients/:clientid/keepalive") ->
#{
'operationId' => set_keepalive,
@ -267,96 +318,187 @@ schema("/clients/:clientid/keepalive") ->
responses => #{
200 => hoconsc:mk(hoconsc:ref(?MODULE, client), #{}),
404 => emqx_dashboard_swagger:error_codes(
['CLIENTID_NOT_FOUND'], <<"Client id not found">>)
['CLIENTID_NOT_FOUND'], <<"Client id not found">>
)
}
}
}.
fields(client) ->
[
{awaiting_rel_cnt, hoconsc:mk(integer(), #{desc =>
<<"v4 api name [awaiting_rel] Number of awaiting PUBREC packet">>})},
{awaiting_rel_max, hoconsc:mk(integer(), #{desc =>
<<"v4 api name [max_awaiting_rel]. "
"Maximum allowed number of awaiting PUBREC packet">>})},
{clean_start, hoconsc:mk(boolean(), #{desc =>
<<"Indicate whether the client is using a brand new session">>})},
{awaiting_rel_cnt,
hoconsc:mk(integer(), #{
desc =>
<<"v4 api name [awaiting_rel] Number of awaiting PUBREC packet">>
})},
{awaiting_rel_max,
hoconsc:mk(integer(), #{
desc =>
<<
"v4 api name [max_awaiting_rel]. "
"Maximum allowed number of awaiting PUBREC packet"
>>
})},
{clean_start,
hoconsc:mk(boolean(), #{
desc =>
<<"Indicate whether the client is using a brand new session">>
})},
{clientid, hoconsc:mk(binary(), #{desc => <<"Client identifier">>})},
{connected, hoconsc:mk(boolean(), #{desc => <<"Whether the client is connected">>})},
{connected_at, hoconsc:mk(emqx_datetime:epoch_millisecond(),
#{desc => <<"Client connection time, rfc3339 or timestamp(millisecond)">>})},
{created_at, hoconsc:mk(emqx_datetime:epoch_millisecond(),
#{desc => <<"Session creation time, rfc3339 or timestamp(millisecond)">>})},
{disconnected_at, hoconsc:mk(emqx_datetime:epoch_millisecond(), #{desc =>
<<"Client offline time."
" It's Only valid and returned when connected is false, rfc3339 or timestamp(millisecond)">>})},
{expiry_interval, hoconsc:mk(integer(), #{desc =>
<<"Session expiration interval, with the unit of second">>})},
{heap_size, hoconsc:mk(integer(), #{desc =>
<<"Process heap size with the unit of byte">>})},
{connected_at,
hoconsc:mk(
emqx_datetime:epoch_millisecond(),
#{desc => <<"Client connection time, rfc3339 or timestamp(millisecond)">>}
)},
{created_at,
hoconsc:mk(
emqx_datetime:epoch_millisecond(),
#{desc => <<"Session creation time, rfc3339 or timestamp(millisecond)">>}
)},
{disconnected_at,
hoconsc:mk(emqx_datetime:epoch_millisecond(), #{
desc =>
<<
"Client offline time."
" It's Only valid and returned when connected is false, rfc3339 or timestamp(millisecond)"
>>
})},
{expiry_interval,
hoconsc:mk(integer(), #{
desc =>
<<"Session expiration interval, with the unit of second">>
})},
{heap_size,
hoconsc:mk(integer(), #{
desc =>
<<"Process heap size with the unit of byte">>
})},
{inflight_cnt, hoconsc:mk(integer(), #{desc => <<"Current length of inflight">>})},
{inflight_max, hoconsc:mk(integer(), #{desc =>
<<"v4 api name [max_inflight]. Maximum length of inflight">>})},
{inflight_max,
hoconsc:mk(integer(), #{
desc =>
<<"v4 api name [max_inflight]. Maximum length of inflight">>
})},
{ip_address, hoconsc:mk(binary(), #{desc => <<"Client's IP address">>})},
{is_bridge, hoconsc:mk(boolean(), #{desc =>
<<"Indicates whether the client is connectedvia bridge">>})},
{keepalive, hoconsc:mk(integer(), #{desc =>
<<"keepalive time, with the unit of second">>})},
{is_bridge,
hoconsc:mk(boolean(), #{
desc =>
<<"Indicates whether the client is connectedvia bridge">>
})},
{keepalive,
hoconsc:mk(integer(), #{
desc =>
<<"keepalive time, with the unit of second">>
})},
{mailbox_len, hoconsc:mk(integer(), #{desc => <<"Process mailbox size">>})},
{mqueue_dropped, hoconsc:mk(integer(), #{desc =>
<<"Number of messages dropped by the message queue due to exceeding the length">>})},
{mqueue_dropped,
hoconsc:mk(integer(), #{
desc =>
<<"Number of messages dropped by the message queue due to exceeding the length">>
})},
{mqueue_len, hoconsc:mk(integer(), #{desc => <<"Current length of message queue">>})},
{mqueue_max, hoconsc:mk(integer(), #{desc =>
<<"v4 api name [max_mqueue]. Maximum length of message queue">>})},
{node, hoconsc:mk(binary(), #{desc =>
<<"Name of the node to which the client is connected">>})},
{mqueue_max,
hoconsc:mk(integer(), #{
desc =>
<<"v4 api name [max_mqueue]. Maximum length of message queue">>
})},
{node,
hoconsc:mk(binary(), #{
desc =>
<<"Name of the node to which the client is connected">>
})},
{port, hoconsc:mk(integer(), #{desc => <<"Client's port">>})},
{proto_name, hoconsc:mk(binary(), #{desc => <<"Client protocol name">>})},
{proto_ver, hoconsc:mk(integer(), #{desc => <<"Protocol version used by the client">>})},
{recv_cnt, hoconsc:mk(integer(), #{desc => <<"Number of TCP packets received">>})},
{recv_msg, hoconsc:mk(integer(), #{desc => <<"Number of PUBLISH packets received">>})},
{'recv_msg.dropped', hoconsc:mk(integer(), #{desc =>
<<"Number of dropped PUBLISH packets">>})},
{'recv_msg.dropped.await_pubrel_timeout', hoconsc:mk(integer(), #{desc =>
<<"Number of dropped PUBLISH packets due to expired">>})},
{'recv_msg.qos0', hoconsc:mk(integer(), #{desc =>
<<"Number of PUBLISH QoS0 packets received">>})},
{'recv_msg.qos1', hoconsc:mk(integer(), #{desc =>
<<"Number of PUBLISH QoS1 packets received">>})},
{'recv_msg.qos2', hoconsc:mk(integer(), #{desc =>
<<"Number of PUBLISH QoS2 packets received">>})},
{'recv_msg.dropped',
hoconsc:mk(integer(), #{
desc =>
<<"Number of dropped PUBLISH packets">>
})},
{'recv_msg.dropped.await_pubrel_timeout',
hoconsc:mk(integer(), #{
desc =>
<<"Number of dropped PUBLISH packets due to expired">>
})},
{'recv_msg.qos0',
hoconsc:mk(integer(), #{
desc =>
<<"Number of PUBLISH QoS0 packets received">>
})},
{'recv_msg.qos1',
hoconsc:mk(integer(), #{
desc =>
<<"Number of PUBLISH QoS1 packets received">>
})},
{'recv_msg.qos2',
hoconsc:mk(integer(), #{
desc =>
<<"Number of PUBLISH QoS2 packets received">>
})},
{recv_oct, hoconsc:mk(integer(), #{desc => <<"Number of bytes received">>})},
{recv_pkt, hoconsc:mk(integer(), #{desc => <<"Number of MQTT packets received">>})},
{reductions, hoconsc:mk(integer(), #{desc => <<"Erlang reduction">>})},
{send_cnt, hoconsc:mk(integer(), #{desc => <<"Number of TCP packets sent">>})},
{send_msg, hoconsc:mk(integer(), #{desc => <<"Number of PUBLISH packets sent">>})},
{'send_msg.dropped', hoconsc:mk(integer(), #{desc =>
<<"Number of dropped PUBLISH packets">>})},
{'send_msg.dropped.expired', hoconsc:mk(integer(), #{desc =>
<<"Number of dropped PUBLISH packets due to expired">>})},
{'send_msg.dropped.queue_full', hoconsc:mk(integer(), #{desc =>
<<"Number of dropped PUBLISH packets due to queue full">>})},
{'send_msg.dropped.too_large', hoconsc:mk(integer(), #{desc =>
<<"Number of dropped PUBLISH packets due to packet length too large">>})},
{'send_msg.qos0', hoconsc:mk(integer(), #{desc =>
<<"Number of PUBLISH QoS0 packets sent">>})},
{'send_msg.qos1', hoconsc:mk(integer(), #{desc =>
<<"Number of PUBLISH QoS1 packets sent">>})},
{'send_msg.qos2', hoconsc:mk(integer(), #{desc =>
<<"Number of PUBLISH QoS2 packets sent">>})},
{'send_msg.dropped',
hoconsc:mk(integer(), #{
desc =>
<<"Number of dropped PUBLISH packets">>
})},
{'send_msg.dropped.expired',
hoconsc:mk(integer(), #{
desc =>
<<"Number of dropped PUBLISH packets due to expired">>
})},
{'send_msg.dropped.queue_full',
hoconsc:mk(integer(), #{
desc =>
<<"Number of dropped PUBLISH packets due to queue full">>
})},
{'send_msg.dropped.too_large',
hoconsc:mk(integer(), #{
desc =>
<<"Number of dropped PUBLISH packets due to packet length too large">>
})},
{'send_msg.qos0',
hoconsc:mk(integer(), #{
desc =>
<<"Number of PUBLISH QoS0 packets sent">>
})},
{'send_msg.qos1',
hoconsc:mk(integer(), #{
desc =>
<<"Number of PUBLISH QoS1 packets sent">>
})},
{'send_msg.qos2',
hoconsc:mk(integer(), #{
desc =>
<<"Number of PUBLISH QoS2 packets sent">>
})},
{send_oct, hoconsc:mk(integer(), #{desc => <<"Number of bytes sent">>})},
{send_pkt, hoconsc:mk(integer(), #{desc => <<"Number of MQTT packets sent">>})},
{subscriptions_cnt, hoconsc:mk(integer(), #{desc =>
<<"Number of subscriptions established by this client.">>})},
{subscriptions_max, hoconsc:mk(integer(), #{desc =>
<<"v4 api name [max_subscriptions]",
" Maximum number of subscriptions allowed by this client">>})},
{subscriptions_cnt,
hoconsc:mk(integer(), #{
desc =>
<<"Number of subscriptions established by this client.">>
})},
{subscriptions_max,
hoconsc:mk(integer(), #{
desc =>
<<"v4 api name [max_subscriptions]",
" Maximum number of subscriptions allowed by this client">>
})},
{username, hoconsc:mk(binary(), #{desc => <<"User name of client when connecting">>})},
{will_msg, hoconsc:mk(binary(), #{desc => <<"Client will message">>})},
{zone, hoconsc:mk(binary(), #{desc =>
<<"Indicate the configuration group used by the client">>})}
{zone,
hoconsc:mk(binary(), #{
desc =>
<<"Indicate the configuration group used by the client">>
})}
];
fields(authz_cache) ->
[
{access, hoconsc:mk(binary(), #{desc => <<"Access type">>})},
@ -364,23 +506,19 @@ fields(authz_cache) ->
{topic, hoconsc:mk(binary(), #{desc => <<"Topic name">>})},
{updated_time, hoconsc:mk(integer(), #{desc => <<"Update time">>})}
];
fields(keepalive) ->
[
{interval, hoconsc:mk(integer(), #{desc => <<"Keepalive time, with the unit of second">>})}
];
fields(subscribe) ->
[
{topic, hoconsc:mk(binary(), #{desc => <<"Topic">>})},
{qos, hoconsc:mk(emqx_schema:qos(), #{desc => <<"QoS">>})}
];
fields(unsubscribe) ->
[
{topic, hoconsc:mk(binary(), #{desc => <<"Topic">>})}
];
fields(meta) ->
emqx_dashboard_swagger:fields(page) ++
emqx_dashboard_swagger:fields(limit) ++
@ -393,13 +531,11 @@ clients(get, #{query_string := QString}) ->
client(get, #{bindings := Bindings}) ->
lookup(Bindings);
client(delete, #{bindings := Bindings}) ->
kickout(Bindings).
authz_cache(get, #{bindings := Bindings}) ->
get_authz_cache(Bindings);
authz_cache(delete, #{bindings := Bindings}) ->
clean_authz_cache(Bindings).
@ -415,11 +551,14 @@ unsubscribe(post, #{bindings := #{clientid := ClientID}, body := TopicInfo}) ->
%% TODO: batch
subscribe_batch(post, #{bindings := #{clientid := ClientID}, body := TopicInfos}) ->
Topics =
[begin
Topic = maps:get(<<"topic">>, TopicInfo),
Qos = maps:get(<<"qos">>, TopicInfo, 0),
#{topic => Topic, qos => Qos}
end || TopicInfo <- TopicInfos],
[
begin
Topic = maps:get(<<"topic">>, TopicInfo),
Qos = maps:get(<<"qos">>, TopicInfo, 0),
#{topic => Topic, qos => Qos}
end
|| TopicInfo <- TopicInfos
],
subscribe_batch(#{clientid => ClientID, topics => Topics}).
subscriptions(get, #{bindings := #{clientid := ClientID}}) ->
@ -436,16 +575,17 @@ subscriptions(get, #{bindings := #{clientid := ClientID}}) ->
qos => maps:get(qos, SubOpts)
}
end,
{200, lists:map(Formatter, Subs)}
{200, lists:map(Formatter, Subs)}
end.
set_keepalive(put, #{bindings := #{clientid := ClientID}, body := Body}) ->
case maps:find(<<"interval">>, Body) of
error -> {400, 'BAD_REQUEST',"Interval Not Found"};
error ->
{400, 'BAD_REQUEST', "Interval Not Found"};
{ok, Interval} ->
case emqx_mgmt:set_keepalive(emqx_mgmt_util:urldecode(ClientID), Interval) of
ok -> lookup(#{clientid => ClientID});
{error, not_found} ->{404, ?CLIENT_ID_NOT_FOUND};
{error, not_found} -> {404, ?CLIENT_ID_NOT_FOUND};
{error, Reason} -> {400, #{code => 'PARAMS_ERROR', message => Reason}}
end
end.
@ -454,17 +594,34 @@ set_keepalive(put, #{bindings := #{clientid := ClientID}, body := Body}) ->
%% api apply
list_clients(QString) ->
case maps:get(<<"node">>, QString, undefined) of
undefined ->
Response = emqx_mgmt_api:cluster_query(QString, ?CLIENT_QTAB,
?CLIENT_QSCHEMA, ?QUERY_FUN),
emqx_mgmt_util:generate_response(Response);
Node1 ->
Node = binary_to_atom(Node1, utf8),
QStringWithoutNode = maps:without([<<"node">>], QString),
Response = emqx_mgmt_api:node_query(Node, QStringWithoutNode,
?CLIENT_QTAB, ?CLIENT_QSCHEMA, ?QUERY_FUN),
emqx_mgmt_util:generate_response(Response)
Result =
case maps:get(<<"node">>, QString, undefined) of
undefined ->
emqx_mgmt_api:cluster_query(
QString,
?CLIENT_QTAB,
?CLIENT_QSCHEMA,
?QUERY_FUN
);
Node0 ->
Node1 = binary_to_atom(Node0, utf8),
QStringWithoutNode = maps:without([<<"node">>], QString),
emqx_mgmt_api:node_query(
Node1,
QStringWithoutNode,
?CLIENT_QTAB,
?CLIENT_QSCHEMA,
?QUERY_FUN
)
end,
case Result of
{error, page_limit_invalid} ->
{400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
{error, Node, {badrpc, R}} ->
Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])),
{500, #{code => <<"NODE_DOWN">>, message => Message}};
Response ->
{200, Response}
end.
lookup(#{clientid := ClientID}) ->
@ -483,7 +640,7 @@ kickout(#{clientid := ClientID}) ->
{204}
end.
get_authz_cache(#{clientid := ClientID})->
get_authz_cache(#{clientid := ClientID}) ->
case emqx_mgmt:list_authz_cache(ClientID) of
{error, not_found} ->
{404, ?CLIENT_ID_NOT_FOUND};
@ -517,9 +674,9 @@ subscribe(#{clientid := ClientID, topic := Topic, qos := Qos}) ->
Response =
#{
clientid => ClientID,
topic => Topic,
qos => Qos,
node => Node
topic => Topic,
qos => Qos,
node => Node
},
{200, Response}
end.
@ -555,7 +712,7 @@ do_subscribe(ClientID, Topic0, Qos) ->
end.
-spec do_unsubscribe(emqx_types:clientid(), emqx_types:topic()) ->
{unsubscribe, _} | {error, channel_not_found}.
{unsubscribe, _} | {error, channel_not_found}.
do_unsubscribe(ClientID, Topic) ->
case emqx_mgmt:unsubscribe(ClientID, Topic) of
{error, Reason} ->
@ -569,14 +726,23 @@ do_unsubscribe(ClientID, Topic) ->
query(Tab, {QString, []}, Continuation, Limit) ->
Ms = qs2ms(QString),
emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit,
fun format_channel_info/1);
emqx_mgmt_api:select_table_with_count(
Tab,
Ms,
Continuation,
Limit,
fun format_channel_info/1
);
query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
Ms = qs2ms(QString),
FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
emqx_mgmt_api:select_table_with_count(Tab, {Ms, FuzzyFilterFun}, Continuation, Limit,
fun format_channel_info/1).
emqx_mgmt_api:select_table_with_count(
Tab,
{Ms, FuzzyFilterFun},
Continuation,
Limit,
fun format_channel_info/1
).
%%--------------------------------------------------------------------
%% QueryString to Match Spec
@ -588,7 +754,6 @@ qs2ms(Qs) ->
qs2ms([], _, {MtchHead, Conds}) ->
{MtchHead, lists:reverse(Conds)};
qs2ms([{Key, '=:=', Value} | Rest], N, {MtchHead, Conds}) ->
NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(Key, Value)),
qs2ms(Rest, N, {NMtchHead, Conds});
@ -596,13 +761,16 @@ qs2ms([Qs | Rest], N, {MtchHead, Conds}) ->
Holder = binary_to_atom(iolist_to_binary(["$", integer_to_list(N)]), utf8),
NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(element(1, Qs), Holder)),
NConds = put_conds(Qs, Holder, Conds),
qs2ms(Rest, N+1, {NMtchHead, NConds}).
qs2ms(Rest, N + 1, {NMtchHead, NConds}).
put_conds({_, Op, V}, Holder, Conds) ->
[{Op, Holder, V} | Conds];
put_conds({_, Op1, V1, Op2, V2}, Holder, Conds) ->
[{Op2, Holder, V2},
{Op1, Holder, V1} | Conds].
[
{Op2, Holder, V2},
{Op1, Holder, V1}
| Conds
].
ms(clientid, X) ->
#{clientinfo => #{clientid => X}};
@ -630,68 +798,78 @@ ms(created_at, X) ->
fuzzy_filter_fun(Fuzzy) ->
fun(MsRaws) when is_list(MsRaws) ->
lists:filter( fun(E) -> run_fuzzy_filter(E, Fuzzy) end
, MsRaws)
lists:filter(
fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
MsRaws
)
end.
run_fuzzy_filter(_, []) ->
true;
run_fuzzy_filter(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr} | Fuzzy]) ->
Val = case maps:get(Key, ClientInfo, <<>>) of
undefined -> <<>>;
V -> V
end,
Val =
case maps:get(Key, ClientInfo, <<>>) of
undefined -> <<>>;
V -> V
end,
binary:match(Val, SubStr) /= nomatch andalso run_fuzzy_filter(E, Fuzzy).
%%--------------------------------------------------------------------
%% format funcs
format_channel_info({_, ClientInfo, ClientStats}) ->
Node = case ClientInfo of
#{node := N} -> N;
_ -> node()
end,
StatsMap = maps:without([memory, next_pkt_id, total_heap_size],
maps:from_list(ClientStats)),
Node =
case ClientInfo of
#{node := N} -> N;
_ -> node()
end,
StatsMap = maps:without(
[memory, next_pkt_id, total_heap_size],
maps:from_list(ClientStats)
),
ClientInfoMap0 = maps:fold(fun take_maps_from_inner/3, #{}, ClientInfo),
{IpAddress, Port} = peername_dispart(maps:get(peername, ClientInfoMap0)),
Connected = maps:get(conn_state, ClientInfoMap0) =:= connected,
Connected = maps:get(conn_state, ClientInfoMap0) =:= connected,
ClientInfoMap1 = maps:merge(StatsMap, ClientInfoMap0),
ClientInfoMap2 = maps:put(node, Node, ClientInfoMap1),
ClientInfoMap3 = maps:put(ip_address, IpAddress, ClientInfoMap2),
ClientInfoMap4 = maps:put(port, Port, ClientInfoMap3),
ClientInfoMap = maps:put(connected, Connected, ClientInfoMap4),
ClientInfoMap = maps:put(connected, Connected, ClientInfoMap4),
RemoveList =
[ auth_result
, peername
, sockname
, peerhost
, conn_state
, send_pend
, conn_props
, peercert
, sockstate
, subscriptions
, receive_maximum
, protocol
, is_superuser
, sockport
, anonymous
, mountpoint
, socktype
, active_n
, await_rel_timeout
, conn_mod
, sockname
, retry_interval
, upgrade_qos
, id %% sessionID, defined in emqx_session.erl
],
[
auth_result,
peername,
sockname,
peerhost,
conn_state,
send_pend,
conn_props,
peercert,
sockstate,
subscriptions,
receive_maximum,
protocol,
is_superuser,
sockport,
anonymous,
mountpoint,
socktype,
active_n,
await_rel_timeout,
conn_mod,
sockname,
retry_interval,
upgrade_qos,
%% sessionID, defined in emqx_session.erl
id
],
TimesKeys = [created_at, connected_at, disconnected_at],
%% format timestamp to rfc3339
lists:foldl(fun result_format_time_fun/2
, maps:without(RemoveList, ClientInfoMap)
, TimesKeys).
lists:foldl(
fun result_format_time_fun/2,
maps:without(RemoveList, ClientInfoMap),
TimesKeys
).
%% format func helpers
take_maps_from_inner(_Key, Value, Current) when is_map(Value) ->
@ -703,20 +881,22 @@ result_format_time_fun(Key, NClientInfoMap) ->
case NClientInfoMap of
#{Key := TimeStamp} ->
NClientInfoMap#{
Key => emqx_datetime:epoch_to_rfc3339(TimeStamp)};
Key => emqx_datetime:epoch_to_rfc3339(TimeStamp)
};
#{} ->
NClientInfoMap
end.
-spec(peername_dispart(emqx_types:peername()) -> {binary(), inet:port_number()}).
-spec peername_dispart(emqx_types:peername()) -> {binary(), inet:port_number()}.
peername_dispart({Addr, Port}) ->
AddrBinary = list_to_binary(inet:ntoa(Addr)),
%% PortBinary = integer_to_binary(Port),
{AddrBinary, Port}.
format_authz_cache({{PubSub, Topic}, {AuthzResult, Timestamp}}) ->
#{ access => PubSub,
topic => Topic,
result => AuthzResult,
updated_time => Timestamp
}.
#{
access => PubSub,
topic => Topic,
result => AuthzResult,
updated_time => Timestamp
}.

2
apps/emqx_management/src/emqx_mgmt_api_cluster.erl
View File

@ -125,7 +125,7 @@ force_leave(delete, #{bindings := #{node := Node0}}) ->
{400, #{code => 'BAD_REQUEST', message => error_message(Error)}}
end.
-spec(join(node()) -> ok | ignore | {error, term()}).
-spec join(node()) -> ok | ignore | {error, term()}.
join(Node) ->
ekka:join(Node).

180
apps/emqx_management/src/emqx_mgmt_api_configs.erl
View File

@ -23,11 +23,13 @@
-export([api_spec/0, namespace/0]).
-export([paths/0, schema/1, fields/1]).
-export([ config/3
, config_reset/3
, configs/3
, get_full_config/0
, global_zone_configs/3]).
-export([
config/3,
config_reset/3,
configs/3,
get_full_config/0,
global_zone_configs/3
]).
-export([gen_schema/1]).
@ -36,29 +38,30 @@
-define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))).
-define(OPTS, #{rawconf_with_defaults => true, override_to => cluster}).
-define(EXCLUDES, [
<<"exhook">>,
<<"gateway">>,
<<"plugins">>,
<<"bridges">>,
<<"rule_engine">>,
<<"authorization">>,
<<"authentication">>,
<<"rpc">>,
<<"db">>,
<<"connectors">>,
<<"slow_subs">>,
<<"psk_authentication">>,
<<"topic_metrics">>,
<<"rewrite">>,
<<"auto_subscribe">>,
<<"retainer">>,
<<"statsd">>,
<<"delayed">>,
<<"event_message">>,
<<"prometheus">>,
<<"telemetry">>,
<<"sys_topics">>
-define(EXCLUDES,
[
<<"exhook">>,
<<"gateway">>,
<<"plugins">>,
<<"bridges">>,
<<"rule_engine">>,
<<"authorization">>,
<<"authentication">>,
<<"rpc">>,
<<"db">>,
<<"connectors">>,
<<"slow_subs">>,
<<"psk_authentication">>,
<<"topic_metrics">>,
<<"rewrite">>,
<<"auto_subscribe">>,
<<"retainer">>,
<<"statsd">>,
<<"delayed">>,
<<"event_message">>,
<<"prometheus">>,
<<"telemetry">>,
<<"sys_topics">>
] ++ global_zone_roots()
).
@ -69,7 +72,7 @@ namespace() -> "configuration".
paths() ->
["/configs", "/configs_reset/:rootname", "/configs/global_zone"] ++
lists:map(fun({Name, _Type}) -> ?PREFIX ++ binary_to_list(Name) end, config_list()).
lists:map(fun({Name, _Type}) -> ?PREFIX ++ binary_to_list(Name) end, config_list()).
schema("/configs") ->
#{
@ -77,12 +80,20 @@ schema("/configs") ->
get => #{
tags => [conf],
description =>
<<"Get all the configurations of the specified node, including hot and non-hot updatable items.">>,
<<"Get all the configurations of the specified node, including hot and non-hot updatable items.">>,
parameters => [
{node, hoconsc:mk(typerefl:atom(),
#{in => query, required => false, example => <<"emqx@127.0.0.1">>,
desc =>
<<"Node's name: If you do not fill in the fields, this node will be used by default.">>})}],
{node,
hoconsc:mk(
typerefl:atom(),
#{
in => query,
required => false,
example => <<"emqx@127.0.0.1">>,
desc =>
<<"Node's name: If you do not fill in the fields, this node will be used by default.">>
}
)}
],
responses => #{
200 => lists:map(fun({_, Schema}) -> Schema end, config_list())
}
@ -95,18 +106,29 @@ schema("/configs_reset/:rootname") ->
post => #{
tags => [conf],
description =>
<<"Reset the config entry specified by the query string parameter `conf_path`.<br/>
- For a config entry that has default value, this resets it to the default value;
- For a config entry that has no default value, an error 400 will be returned">>,
<<"Reset the config entry specified by the query string parameter `conf_path`.<br/>\n"
"- For a config entry that has default value, this resets it to the default value;\n"
"- For a config entry that has no default value, an error 400 will be returned">>,
%% We only return "200" rather than the new configs that has been changed, as
%% the schema of the changed configs is depends on the request parameter
%% `conf_path`, it cannot be defined here.
parameters => [
{rootname, hoconsc:mk( hoconsc:enum(Paths)
, #{in => path, example => <<"sysmon">>})},
{conf_path, hoconsc:mk(typerefl:binary(),
#{in => query, required => false, example => <<"os.sysmem_high_watermark">>,
desc => <<"The config path separated by '.' character">>})}],
{rootname,
hoconsc:mk(
hoconsc:enum(Paths),
#{in => path, example => <<"sysmon">>}
)},
{conf_path,
hoconsc:mk(
typerefl:binary(),
#{
in => query,
required => false,
example => <<"os.sysmem_high_watermark">>,
desc => <<"The config path separated by '.' character">>
}
)}
],
responses => #{
200 => <<"Rest config successfully">>,
400 => emqx_dashboard_swagger:error_codes(['NO_DEFAULT_VALUE', 'REST_FAILED'])
@ -138,9 +160,11 @@ schema(Path) ->
'operationId' => config,
get => #{
tags => [conf],
description => iolist_to_binary([ <<"Get the sub-configurations under *">>
, RootKey
, <<"*">>]),
description => iolist_to_binary([
<<"Get the sub-configurations under *">>,
RootKey,
<<"*">>
]),
responses => #{
200 => Schema,
404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"config not found">>)
@ -148,9 +172,11 @@ schema(Path) ->
},
put => #{
tags => [conf],
description => iolist_to_binary([ <<"Update the sub-configurations under *">>
, RootKey
, <<"*">>]),
description => iolist_to_binary([
<<"Update the sub-configurations under *">>,
RootKey,
<<"*">>
]),
'requestBody' => Schema,
responses => #{
200 => Schema,
@ -177,7 +203,6 @@ config(get, _Params, Req) ->
Path = conf_path(Req),
{ok, Conf} = emqx_map_lib:deep_find(Path, get_full_config()),
{200, Conf};
config(put, #{body := Body}, Req) ->
Path = conf_path(Req),
case emqx_conf:update(Path, Body, ?OPTS) of
@ -189,21 +214,32 @@ config(put, #{body := Body}, Req) ->
global_zone_configs(get, _Params, _Req) ->
Paths = global_zone_roots(),
Zones = lists:foldl(fun(Path, Acc) -> Acc#{Path => get_config_with_default([Path])} end,
#{}, Paths),
Zones = lists:foldl(
fun(Path, Acc) -> Acc#{Path => get_config_with_default([Path])} end,
#{},
Paths
),
{200, Zones};
global_zone_configs(put, #{body := Body}, _Req) ->
Res =
maps:fold(fun(Path, Value, Acc) ->
case emqx_conf:update([Path], Value, ?OPTS) of
{ok, #{raw_config := RawConf}} ->
Acc#{Path => RawConf};
{error, Reason} ->
?SLOG(error, #{msg => "update global zone failed", reason => Reason,
path => Path, value => Value}),
Acc
end
end, #{}, Body),
maps:fold(
fun(Path, Value, Acc) ->
case emqx_conf:update([Path], Value, ?OPTS) of
{ok, #{raw_config := RawConf}} ->
Acc#{Path => RawConf};
{error, Reason} ->
?SLOG(error, #{
msg => "update global zone failed",
reason => Reason,
path => Path,
value => Value
}),
Acc
end
end,
#{},
Body
),
case maps:size(Res) =:= maps:size(Body) of
true -> {200, Res};
false -> {400, #{code => 'UPDATE_FAILED'}}
@ -213,7 +249,8 @@ config_reset(post, _Params, Req) ->
%% reset the config specified by the query string param 'conf_path'
Path = conf_path_reset(Req) ++ conf_path_from_querystr(Req),
case emqx:reset_config(Path, #{}) of
{ok, _} -> {200};
{ok, _} ->
{200};
{error, no_default_value} ->
{400, #{code => 'NO_DEFAULT_VALUE', message => <<"No Default Value.">>}};
{error, Reason} ->
@ -223,9 +260,8 @@ config_reset(post, _Params, Req) ->
configs(get, Params, _Req) ->
Node = maps:get(node, Params, node()),
case
lists:member(Node, mria_mnesia:running_nodes())
andalso
emqx_management_proto_v1:get_full_config(Node)
lists:member(Node, mria_mnesia:running_nodes()) andalso
emqx_management_proto_v1:get_full_config(Node)
of
false ->
Message = list_to_binary(io_lib:format("Bad node ~p, reason not found", [Node])),
@ -243,8 +279,11 @@ conf_path_reset(Req) ->
get_full_config() ->
emqx_config:fill_defaults(
maps:without(?EXCLUDES,
emqx:get_raw_config([]))).
maps:without(
?EXCLUDES,
emqx:get_raw_config([])
)
).
get_config_with_default(Path) ->
emqx_config:fill_defaults(emqx:get_raw_config(Path)).
@ -279,8 +318,11 @@ gen_schema(Conf) when is_list(Conf) ->
#{type => array, items => gen_schema(hd(Conf))}
end;
gen_schema(Conf) when is_map(Conf) ->
#{type => object, properties =>
maps:map(fun(_K, V) -> gen_schema(V) end, Conf)};
#{
type => object,
properties =>
maps:map(fun(_K, V) -> gen_schema(V) end, Conf)
};
gen_schema(_Conf) ->
%% the conf is not of JSON supported type, it may have been converted
%% by the hocon schema

450
apps/emqx_management/src/emqx_mgmt_api_listeners.erl
View File

@ -22,12 +22,10 @@
-import(emqx_dashboard_swagger, [error_codes/2, error_codes/1]).
-export([
listener_type_status/2,
list_listeners/2,
crud_listeners_by_id/2,
list_listeners_on_node/2,
crud_listener_by_id_on_node/2,
action_listeners_by_id/2,
action_listeners_by_id_on_node/2
action_listeners_by_id/2
]).
%% for rpc call
@ -55,21 +53,35 @@ api_spec() ->
paths() ->
[
"/listeners_status",
"/listeners",
"/listeners/:id",
"/listeners/:id/:action",
"/nodes/:node/listeners",
"/nodes/:node/listeners/:id",
"/nodes/:node/listeners/:id/:action"
"/listeners/:id/:action"
].
schema("/listeners_status") ->
#{
'operationId' => listener_type_status,
get => #{
tags => [<<"listeners">>],
desc => <<"List all running node's listeners live status. group by listener type">>,
responses => #{200 => ?HOCON(?ARRAY(?R_REF(listener_type_status)))}
}
};
schema("/listeners") ->
#{
'operationId' => list_listeners,
get => #{
tags => [<<"listeners">>],
desc => <<"List all running node's listeners.">>,
responses => #{200 => ?HOCON(?ARRAY(?R_REF(listeners)))}
desc => <<"List all running node's listeners for the specified type.">>,
parameters => [
{type,
?HOCON(
?ENUM(listeners_type()),
#{desc => "Listener type", in => query, required => false}
)}
],
responses => #{200 => ?HOCON(?ARRAY(?R_REF(listener_id_status)))}
}
};
schema("/listeners/:id") ->
@ -80,17 +92,29 @@ schema("/listeners/:id") ->
desc => <<"List all running node's listeners for the specified id.">>,
parameters => [?R_REF(listener_id)],
responses => #{
200 => ?HOCON(?ARRAY(?R_REF(listeners)))
200 => ?HOCON(listener_schema(#{bind => true})),
404 => error_codes(['BAD_LISTENER_ID', 'BAD_REQUEST'], ?LISTENER_NOT_FOUND)
}
},
put => #{
tags => [<<"listeners">>],
desc => <<"Create or update the specified listener on all nodes.">>,
desc => <<"Update the specified listener on all nodes.">>,
parameters => [?R_REF(listener_id)],
'requestBody' => ?HOCON(listener_schema(), #{}),
'requestBody' => ?HOCON(listener_schema(#{bind => false}), #{}),
responses => #{
200 => ?HOCON(listener_schema(), #{}),
400 => error_codes(['BAD_LISTENER_ID', 'BAD_REQUEST'], ?LISTENER_NOT_FOUND)
200 => ?HOCON(listener_schema(#{bind => true}), #{}),
400 => error_codes(['BAD_REQUEST']),
404 => error_codes(['BAD_LISTENER_ID', 'BAD_REQUEST'], ?LISTENER_NOT_FOUND)
}
},
post => #{
tags => [<<"listeners">>],
desc => <<"Create the specified listener on all nodes.">>,
parameters => [?R_REF(listener_id)],
'requestBody' => ?HOCON(listener_schema(#{bind => true}), #{}),
responses => #{
200 => ?HOCON(listener_schema(#{bind => true}), #{}),
400 => error_codes(['BAD_LISTENER_ID', 'BAD_REQUEST'])
}
},
delete => #{
@ -118,90 +142,8 @@ schema("/listeners/:id/:action") ->
400 => error_codes(['BAD_REQUEST'])
}
}
};
schema("/nodes/:node/listeners") ->
#{
'operationId' => list_listeners_on_node,
get => #{
tags => [<<"listeners">>],
desc => <<"List all listeners on the specified node.">>,
parameters => [?R_REF(node)],
responses => #{
200 => ?HOCON(?ARRAY(listener_schema())),
400 => error_codes(['BAD_NODE', 'BAD_REQUEST'], ?NODE_NOT_FOUND_OR_DOWN)
}
}
};
schema("/nodes/:node/listeners/:id") ->
#{
'operationId' => crud_listener_by_id_on_node,
get => #{
tags => [<<"listeners">>],
desc => <<"Get the specified listener on the specified node.">>,
parameters => [
?R_REF(listener_id),
?R_REF(node)
],
responses => #{
200 => ?HOCON(listener_schema()),
400 => error_codes(['BAD_REQUEST']),
404 => error_codes(['BAD_LISTEN_ID'], ?NODE_LISTENER_NOT_FOUND)
}
},
put => #{
tags => [<<"listeners">>],
desc => <<"Create or update the specified listener on the specified node.">>,
parameters => [
?R_REF(listener_id),
?R_REF(node)
],
'requestBody' => ?HOCON(listener_schema()),
responses => #{
200 => ?HOCON(listener_schema()),
400 => error_codes(['BAD_REQUEST'])
}
},
delete => #{
tags => [<<"listeners">>],
desc => <<"Delete the specified listener on the specified node.">>,
parameters => [
?R_REF(listener_id),
?R_REF(node)
],
responses => #{
204 => <<"Listener deleted">>,
400 => error_codes(['BAD_REQUEST'])
}
}
};
schema("/nodes/:node/listeners/:id/:action") ->
#{
'operationId' => action_listeners_by_id_on_node,
post => #{
tags => [<<"listeners">>],
desc => <<"Start/stop/restart listeners on a specified node.">>,
parameters => [
?R_REF(node),
?R_REF(listener_id),
?R_REF(action)
],
responses => #{
200 => <<"Updated">>,
400 => error_codes(['BAD_REQUEST'])
}
}
}.
fields(listeners) ->
[
{"node",
?HOCON(atom(), #{
desc => "Node name",
example => "emqx@127.0.0.1",
required => true
})},
{"listeners", ?ARRAY(listener_schema())}
];
fields(listener_id) ->
[
{id,
@ -230,23 +172,56 @@ fields(node) ->
in => path
})}
];
fields(listener_type_status) ->
[
{type, ?HOCON(?ENUM(listeners_type()), #{desc => "Listener type", required => true})},
{enable, ?HOCON(boolean(), #{desc => "Listener enable", required => true})},
{ids, ?HOCON(?ARRAY(string()), #{desc => "Listener Ids", required => true})},
{status, ?HOCON(?R_REF(status))},
{node_status, ?HOCON(?ARRAY(?R_REF(node_status)))}
];
fields(listener_id_status) ->
fields(listener_id) ++
[
{enable, ?HOCON(boolean(), #{desc => "Listener enable", required => true})},
{number, ?HOCON(typerefl:pos_integer(), #{desc => "ListenerId number"})},
{status, ?HOCON(?R_REF(status))},
{node_status, ?HOCON(?ARRAY(?R_REF(node_status)))}
];
fields(status) ->
[
{max_connections,
?HOCON(hoconsc:union([infinity, integer()]), #{desc => "Max connections"})},
{current_connections, ?HOCON(non_neg_integer(), #{desc => "Current connections"})}
];
fields(node_status) ->
fields(node) ++ fields(status);
fields(Type) ->
Listeners = listeners_info(),
Listeners = listeners_info(#{bind => true}) ++ listeners_info(#{bind => false}),
[Schema] = [S || #{ref := ?R_REF(_, T), schema := S} <- Listeners, T =:= Type],
Schema.
listener_schema() ->
?UNION(lists:map(fun(#{ref := Ref}) -> Ref end, listeners_info())).
listener_schema(Opts) ->
?UNION(lists:map(fun(#{ref := Ref}) -> Ref end, listeners_info(Opts))).
listeners_info() ->
listeners_type() ->
lists:map(
fun({Type, _}) -> list_to_existing_atom(Type) end,
hocon_schema:fields(emqx_schema, "listeners")
).
listeners_info(Opts) ->
Listeners = hocon_schema:fields(emqx_schema, "listeners"),
lists:map(
fun({Type, #{type := ?MAP(_Name, ?R_REF(Mod, Field))}}) ->
Fields0 = hocon_schema:fields(Mod, Field),
Fields1 = lists:keydelete("authentication", 1, Fields0),
Fields2 = lists:keydelete("limiter", 1, Fields1),
Fields3 = required_bind(Fields2, Opts),
Ref = listeners_ref(Type, Opts),
TypeAtom = list_to_existing_atom(Type),
#{
ref => ?R_REF(TypeAtom),
ref => ?R_REF(Ref),
schema => [
{type, ?HOCON(?ENUM([TypeAtom]), #{desc => "Listener type", required => true})},
{running, ?HOCON(boolean(), #{desc => "Listener status", required => false})},
@ -255,14 +230,33 @@ listeners_info() ->
desc => "Listener id",
required => true,
validator => fun validate_id/1
})}
| Fields1
})},
{current_connections,
?HOCON(
non_neg_integer(),
#{desc => "Current connections", required => false}
)}
| Fields3
]
}
end,
Listeners
).
required_bind(Fields, #{bind := true}) ->
Fields;
required_bind(Fields, #{bind := false}) ->
{value, {_, Hocon}, Fields1} = lists:keytake("bind", 1, Fields),
[{"bind", Hocon#{required => false}} | Fields1].
listeners_ref(Type, #{bind := Bind}) ->
Suffix =
case Bind of
true -> "_required_bind";
false -> "_not_required_bind"
end,
Type ++ Suffix.
validate_id(Id) ->
case emqx_listeners:parse_listener_id(Id) of
{error, Reason} -> {error, Reason};
@ -270,19 +264,65 @@ validate_id(Id) ->
end.
%% api
list_listeners(get, _Request) ->
{200, list_listeners()}.
listener_type_status(get, _Request) ->
Listeners = maps:to_list(listener_status_by_type(list_listeners(), #{})),
List = lists:map(fun({Type, L}) -> L#{type => Type} end, Listeners),
{200, List}.
crud_listeners_by_id(get, #{bindings := #{id := Id}}) ->
{200, list_listeners_by_id(Id)};
list_listeners(get, #{query_string := Query}) ->
Listeners = list_listeners(),
NodeL =
case maps:find(<<"type">>, Query) of
{ok, Type} -> listener_type_filter(atom_to_binary(Type), Listeners);
error -> Listeners
end,
{200, listener_status_by_id(NodeL)}.
crud_listeners_by_id(get, #{bindings := #{id := Id0}}) ->
Listeners = [
Conf#{<<"id">> => Id, <<"type">> => Type}
|| {Id, Type, Conf} <- emqx_listeners:list_raw(),
Id =:= Id0
],
case Listeners of
[] -> {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}};
[L] -> {200, L}
end;
crud_listeners_by_id(put, #{bindings := #{id := Id}, body := Body0}) ->
case parse_listener_conf(Body0) of
{Id, Type, Name, Conf} ->
case emqx_conf:update([listeners, Type, Name], Conf, ?OPTS(cluster)) of
{ok, #{raw_config := _RawConf}} ->
crud_listeners_by_id(get, #{bindings => #{id => Id}});
{error, Reason} ->
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}}
Key = [listeners, Type, Name],
case emqx_conf:get_raw(Key, undefined) of
undefined ->
{404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}};
PrevConf ->
MergeConf = emqx_map_lib:deep_merge(PrevConf, Conf),
case emqx_conf:update(Key, MergeConf, ?OPTS(cluster)) of
{ok, #{raw_config := _RawConf}} ->
crud_listeners_by_id(get, #{bindings => #{id => Id}});
{error, Reason} ->
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}}
end
end;
{error, Reason} ->
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}};
_ ->
{400, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_ID_INCONSISTENT}}
end;
crud_listeners_by_id(post, #{bindings := #{id := Id}, body := Body0}) ->
case parse_listener_conf(Body0) of
{Id, Type, Name, Conf} ->
Key = [listeners, Type, Name],
case emqx_conf:get(Key, undefined) of
undefined ->
case emqx_conf:update([listeners, Type, Name], Conf, ?OPTS(cluster)) of
{ok, #{raw_config := _RawConf}} ->
crud_listeners_by_id(get, #{bindings => #{id => Id}});
{error, Reason} ->
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}}
end;
_ ->
{400, #{code => 'BAD_LISTENER_ID', message => <<"Already Exist">>}}
end;
{error, Reason} ->
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}};
@ -298,64 +338,16 @@ crud_listeners_by_id(delete, #{bindings := #{id := Id}}) ->
parse_listener_conf(Conf0) ->
Conf1 = maps:remove(<<"running">>, Conf0),
{IdBin, Conf2} = maps:take(<<"id">>, Conf1),
{TypeBin, Conf3} = maps:take(<<"type">>, Conf2),
Conf2 = maps:remove(<<"current_connections">>, Conf1),
{IdBin, Conf3} = maps:take(<<"id">>, Conf2),
{TypeBin, Conf4} = maps:take(<<"type">>, Conf3),
{ok, #{type := Type, name := Name}} = emqx_listeners:parse_listener_id(IdBin),
TypeAtom = binary_to_existing_atom(TypeBin),
case Type =:= TypeAtom of
true -> {binary_to_existing_atom(IdBin), TypeAtom, Name, Conf3};
true -> {binary_to_existing_atom(IdBin), TypeAtom, Name, Conf4};
false -> {error, listener_type_inconsistent}
end.
list_listeners_on_node(get, #{bindings := #{node := Node}}) ->
case list_listeners(Node) of
{error, nodedown} ->
{400, #{code => 'BAD_NODE', message => ?NODE_NOT_FOUND_OR_DOWN}};
{error, Reason} ->
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}};
#{<<"listeners">> := Listener} ->
{200, Listener}
end.
crud_listener_by_id_on_node(get, #{bindings := #{id := Id, node := Node}}) ->
case get_listener(Node, Id) of
{error, not_found} ->
{404, #{code => 'BAD_LISTEN_ID', message => ?NODE_LISTENER_NOT_FOUND}};
{error, Reason} ->
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}};
Listener ->
{200, Listener}
end;
crud_listener_by_id_on_node(put, #{bindings := #{id := Id, node := Node}, body := Body}) ->
case parse_listener_conf(Body) of
{error, Reason} ->
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}};
{Id, Type, _Name, Conf} ->
case update_listener(Node, Id, Conf) of
{error, nodedown} ->
{400, #{code => 'BAD_REQUEST', message => ?NODE_NOT_FOUND_OR_DOWN}};
%% TODO
{error, {eaddrinuse, _}} ->
{400, #{code => 'BAD_REQUEST', message => ?ADDR_PORT_INUSE}};
{error, Reason} ->
{400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}};
{ok, Listener} ->
{200, Listener#{<<"id">> => Id, <<"type">> => Type, <<"running">> => true}}
end;
_ ->
{400, #{code => 'BAD_REQUEST', message => ?LISTENER_ID_INCONSISTENT}}
end;
crud_listener_by_id_on_node(delete, #{bindings := #{id := Id, node := Node}}) ->
case remove_listener(Node, Id) of
ok -> {204};
{error, Reason} -> {400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}}
end.
action_listeners_by_id_on_node(post,
#{bindings := #{id := Id, action := Action, node := Node}}) ->
{_, Result} = action_listeners(Node, Id, Action),
Result.
action_listeners_by_id(post, #{bindings := #{id := Id, action := Action}}) ->
Results = [action_listeners(Node, Id, Action) || Node <- mria_mnesia:running_nodes()],
case
@ -418,31 +410,49 @@ list_listeners() ->
list_listeners(Node) ->
wrap_rpc(emqx_management_proto_v1:list_listeners(Node)).
list_listeners_by_id(Id) ->
listener_id_filter(Id, list_listeners()).
get_listener(Node, Id) ->
case listener_id_filter(Id, [list_listeners(Node)]) of
[#{<<"listeners">> := []}] -> {error, not_found};
[#{<<"listeners">> := [Listener]}] -> Listener
end.
listener_id_filter(Id, Listeners) ->
listener_status_by_id(NodeL) ->
Listeners = maps:to_list(listener_status_by_id(NodeL, #{})),
lists:map(
fun(Conf = #{<<"listeners">> := Listeners0}) ->
Conf#{
<<"listeners">> =>
[C || C = #{<<"id">> := Id0} <- Listeners0, Id =:= Id0]
}
fun({Id, L}) ->
L1 = maps:remove(ids, L),
#{node_status := Nodes} = L1,
L1#{number => maps:size(Nodes), id => Id}
end,
Listeners
).
update_listener(Node, Id, Config) ->
wrap_rpc(emqx_management_proto_v1:update_listener(Node, Id, Config)).
listener_status_by_type([], Acc) ->
Acc;
listener_status_by_type([NodeL | Rest], Acc) ->
#{<<"node">> := Node, <<"listeners">> := Listeners} = NodeL,
Acc1 = lists:foldl(
fun(L, Acc0) -> format_status(<<"type">>, Node, L, Acc0) end,
Acc,
Listeners
),
listener_status_by_type(Rest, Acc1).
remove_listener(Node, Id) ->
wrap_rpc(emqx_management_proto_v1:remove_listener(Node, Id)).
listener_status_by_id([], Acc) ->
Acc;
listener_status_by_id([NodeL | Rest], Acc) ->
#{<<"node">> := Node, <<"listeners">> := Listeners} = NodeL,
Acc1 = lists:foldl(
fun(L, Acc0) -> format_status(<<"id">>, Node, L, Acc0) end,
Acc,
Listeners
),
listener_status_by_id(Rest, Acc1).
listener_type_filter(Type0, Listeners) ->
lists:map(
fun(Conf = #{<<"listeners">> := Listeners0}) ->
Conf#{
<<"listeners">> =>
[C || C = #{<<"type">> := Type} <- Listeners0, Type =:= Type0]
}
end,
Listeners
).
-spec do_list_listeners() -> map().
do_list_listeners() ->
@ -476,3 +486,75 @@ wrap_rpc({badrpc, Reason}) ->
{error, Reason};
wrap_rpc(Res) ->
Res.
format_status(Key, Node, Listener, Acc) ->
#{
<<"id">> := Id,
<<"running">> := Running,
<<"max_connections">> := MaxConnections,
<<"current_connections">> := CurrentConnections
} = Listener,
GroupKey = maps:get(Key, Listener),
case maps:find(GroupKey, Acc) of
error ->
Acc#{
GroupKey => #{
enable => Running,
ids => [Id],
status => #{
max_connections => MaxConnections,
current_connections => CurrentConnections
},
node_status => #{
Node => #{
max_connections => MaxConnections,
current_connections => CurrentConnections
}
}
}
};
{ok, GroupValue} ->
#{
ids := Ids,
status := #{
max_connections := MaxConnections0,
current_connections := CurrentConnections0
},
node_status := NodeStatus0
} = GroupValue,
NodeStatus =
case maps:find(Node, NodeStatus0) of
error ->
#{
Node => #{
max_connections => MaxConnections,
current_connections => CurrentConnections
}
};
{ok, #{
max_connections := PrevMax,
current_connections := PrevCurr
}} ->
NodeStatus0#{
Node => #{
max_connections => max_conn(MaxConnections, PrevMax),
current_connections => CurrentConnections + PrevCurr
}
}
end,
Acc#{
GroupKey =>
GroupValue#{
ids => lists:usort([Id | Ids]),
status => #{
max_connections => max_conn(MaxConnections0, MaxConnections),
current_connections => CurrentConnections0 + CurrentConnections
},
node_status => NodeStatus
}
}
end.
max_conn(_Int1, infinity) -> infinity;
max_conn(infinity, _Int) -> infinity;
max_conn(Int1, Int2) -> Int1 + Int2.

587
apps/emqx_management/src/emqx_mgmt_api_metrics.erl
View File

@ -23,14 +23,16 @@
-import(hoconsc, [mk/2, ref/2]).
%% minirest/dashbaord_swagger behaviour callbacks
-export([ api_spec/0
, paths/0
, schema/1
]).
-export([
api_spec/0,
paths/0,
schema/1
]).
-export([ roots/0
, fields/1
]).
-export([
roots/0,
fields/1
]).
%% http handlers
-export([metrics/2]).
@ -53,9 +55,12 @@ metrics(get, #{query_string := Qs}) ->
true ->
{200, emqx_mgmt:get_metrics()};
false ->
Data = [maps:from_list(
emqx_mgmt:get_metrics(Node) ++ [{node, Node}])
|| Node <- mria_mnesia:running_nodes()],
Data = [
maps:from_list(
emqx_mgmt:get_metrics(Node) ++ [{node, Node}]
)
|| Node <- mria_mnesia:running_nodes()
],
{200, Data}
end.
@ -64,23 +69,34 @@ metrics(get, #{query_string := Qs}) ->
%%--------------------------------------------------------------------
schema("/metrics") ->
#{ 'operationId' => metrics
, get =>
#{ description => <<"EMQX metrics">>
, parameters =>
[{ aggregate
, mk( boolean()
, #{ in => query
, required => false
, desc => <<"Whether to aggregate all nodes Metrics">>})
}]
, responses =>
#{ 200 => hoconsc:union(
[ref(?MODULE, aggregated_metrics),
hoconsc:array(ref(?MODULE, node_metrics))])
}
#{
'operationId' => metrics,
get =>
#{
description => <<"EMQX metrics">>,
parameters =>
[
{aggregate,
mk(
boolean(),
#{
in => query,
required => false,
desc => <<"Whether to aggregate all nodes Metrics">>
}
)}
],
responses =>
#{
200 => hoconsc:union(
[
ref(?MODULE, aggregated_metrics),
hoconsc:array(ref(?MODULE, node_metrics))
]
)
}
}
}.
}.
roots() ->
[].
@ -91,177 +107,354 @@ fields(node_metrics) ->
[{node, mk(binary(), #{desc => <<"Node name">>})}] ++ properties().
properties() ->
[ m('actions.failure',
<<"Number of failure executions of the rule engine action">>)
, m('actions.success',
<<"Number of successful executions of the rule engine action">>)
, m('bytes.received',
<<"Number of bytes received ">>)
, m('bytes.sent',
<<"Number of bytes sent on this connection">>)
, m('client.auth.anonymous',
<<"Number of clients who log in anonymously">>)
, m('client.authenticate',
<<"Number of client authentications">>)
, m('client.check_authz',
<<"Number of Authorization rule checks">>)
, m('client.connack',
<<"Number of CONNACK packet sent">>)
, m('client.connect',
<<"Number of client connections">>)
, m('client.connected',
<<"Number of successful client connections">>)
, m('client.disconnected',
<<"Number of client disconnects">>)
, m('client.subscribe',
<<"Number of client subscriptions">>)
, m('client.unsubscribe',
<<"Number of client unsubscriptions">>)
, m('delivery.dropped',
<<"Total number of discarded messages when sending">>)
, m('delivery.dropped.expired',
<<"Number of messages dropped due to message expiration on sending">>)
, m('delivery.dropped.no_local',
<<"Number of messages that were dropped due to the No Local subscription "
"option when sending">>)
, m('delivery.dropped.qos0_msg',
<<"Number of messages with QoS 0 that were dropped because the message "
"queue was full when sending">>)
, m('delivery.dropped.queue_full',
<<"Number of messages with a non-zero QoS that were dropped because the "
"message queue was full when sending">>)
, m('delivery.dropped.too_large',
<<"The number of messages that were dropped because the length exceeded "
"the limit when sending">>)
, m('messages.acked',
<<"Number of received PUBACK and PUBREC packet">>)
, m('messages.delayed',
<<"Number of delay-published messages">>)
, m('messages.delivered',
<<"Number of messages forwarded to the subscription process internally">>)
, m('messages.dropped',
<<"Total number of messages dropped before forwarding to the subscription process">>)
, m('messages.dropped.await_pubrel_timeout',
<<"Number of messages dropped due to waiting PUBREL timeout">>)
, m('messages.dropped.no_subscribers',
<<"Number of messages dropped due to no subscribers">>)
, m('messages.forward',
<<"Number of messages forwarded to other nodes">>)
, m('messages.publish',
<<"Number of messages published in addition to system messages">>)
, m('messages.qos0.received',
<<"Number of QoS 0 messages received from clients">>)
, m('messages.qos0.sent',
<<"Number of QoS 0 messages sent to clients">>)
, m('messages.qos1.received',
<<"Number of QoS 1 messages received from clients">>)
, m('messages.qos1.sent',
<<"Number of QoS 1 messages sent to clients">>)
, m('messages.qos2.received',
<<"Number of QoS 2 messages received from clients">>)
, m('messages.qos2.sent',
<<"Number of QoS 2 messages sent to clients">>)
, m('messages.received',
<<"Number of messages received from the client, equal to the sum of "
"messages.qos0.received\fmessages.qos1.received and messages.qos2.received">>)
, m('messages.retained',
<<"Number of retained messages">>)
, m('messages.sent',
<<"Number of messages sent to the client, equal to the sum of "
"messages.qos0.sent\fmessages.qos1.sent and messages.qos2.sent">>)
, m('packets.auth.received',
<<"Number of received AUTH packet">>)
, m('packets.auth.sent',
<<"Number of sent AUTH packet">>)
, m('packets.connack.auth_error',
<<"Number of received CONNECT packet with failed authentication">>)
, m('packets.connack.error',
<<"Number of received CONNECT packet with unsuccessful connections">>)
, m('packets.connack.sent',
<<"Number of sent CONNACK packet">>)
, m('packets.connect.received',
<<"Number of received CONNECT packet">>)
, m('packets.disconnect.received',
<<"Number of received DISCONNECT packet">>)
, m('packets.disconnect.sent',
<<"Number of sent DISCONNECT packet">>)
, m('packets.pingreq.received',
<<"Number of received PINGREQ packet">>)
, m('packets.pingresp.sent',
<<"Number of sent PUBRESP packet">>)
, m('packets.puback.inuse',
<<"Number of received PUBACK packet with occupied identifiers">>)
, m('packets.puback.missed',
<<"Number of received packet with identifiers.">>)
, m('packets.puback.received',
<<"Number of received PUBACK packet">>)
, m('packets.puback.sent',
<<"Number of sent PUBACK packet">>)
, m('packets.pubcomp.inuse',
<<"Number of received PUBCOMP packet with occupied identifiers">>)
, m('packets.pubcomp.missed',
<<"Number of missed PUBCOMP packet">>)
, m('packets.pubcomp.received',
<<"Number of received PUBCOMP packet">>)
, m('packets.pubcomp.sent',
<<"Number of sent PUBCOMP packet">>)
, m('packets.publish.auth_error',
<<"Number of received PUBLISH packets with failed the Authorization check">>)
, m('packets.publish.dropped',
<<"Number of messages discarded due to the receiving limit">>)
, m('packets.publish.error',
<<"Number of received PUBLISH packet that cannot be published">>)
, m('packets.publish.inuse',
<<"Number of received PUBLISH packet with occupied identifiers">>)
, m('packets.publish.received',
<<"Number of received PUBLISH packet">>)
, m('packets.publish.sent',
<<"Number of sent PUBLISH packet">>)
, m('packets.pubrec.inuse',
<<"Number of received PUBREC packet with occupied identifiers">>)
, m('packets.pubrec.missed',
<<"Number of received PUBREC packet with unknown identifiers">>)
, m('packets.pubrec.received',
<<"Number of received PUBREC packet">>)
, m('packets.pubrec.sent',
<<"Number of sent PUBREC packet">>)
, m('packets.pubrel.missed',
<<"Number of received PUBREC packet with unknown identifiers">>)
, m('packets.pubrel.received',
<<"Number of received PUBREL packet">>)
, m('packets.pubrel.sent',
<<"Number of sent PUBREL packet">>)
, m('packets.received',
<<"Number of received packet">>)
, m('packets.sent',
<<"Number of sent packet">>)
, m('packets.suback.sent',
<<"Number of sent SUBACK packet">>)
, m('packets.subscribe.auth_error',
<<"Number of received SUBACK packet with failed Authorization check">>)
, m('packets.subscribe.error',
<<"Number of received SUBSCRIBE packet with failed subscriptions">>)
, m('packets.subscribe.received',
<<"Number of received SUBSCRIBE packet">>)
, m('packets.unsuback.sent',
<<"Number of sent UNSUBACK packet">>)
, m('packets.unsubscribe.error',
<<"Number of received UNSUBSCRIBE packet with failed unsubscriptions">>)
, m('packets.unsubscribe.received',
<<"Number of received UNSUBSCRIBE packet">>)
, m('rules.matched',
<<"Number of rule matched">>)
, m('session.created',
<<"Number of sessions created">>)
, m('session.discarded',
<<"Number of sessions dropped because Clean Session or Clean Start is true">>)
, m('session.resumed',
<<"Number of sessions resumed because Clean Session or Clean Start is false">>)
, m('session.takenover',
<<"Number of sessions takenover because Clean Session or Clean Start is false">>)
, m('session.terminated',
<<"Number of terminated sessions">>)
[
m(
'actions.failure',
<<"Number of failure executions of the rule engine action">>
),
m(
'actions.success',
<<"Number of successful executions of the rule engine action">>
),
m(
'bytes.received',
<<"Number of bytes received ">>
),
m(
'bytes.sent',
<<"Number of bytes sent on this connection">>
),
m(
'client.auth.anonymous',
<<"Number of clients who log in anonymously">>
),
m(
'client.authenticate',
<<"Number of client authentications">>
),
m(
'client.check_authz',
<<"Number of Authorization rule checks">>
),
m(
'client.connack',
<<"Number of CONNACK packet sent">>
),
m(
'client.connect',
<<"Number of client connections">>
),
m(
'client.connected',
<<"Number of successful client connections">>
),
m(
'client.disconnected',
<<"Number of client disconnects">>
),
m(
'client.subscribe',
<<"Number of client subscriptions">>
),
m(
'client.unsubscribe',
<<"Number of client unsubscriptions">>
),
m(
'delivery.dropped',
<<"Total number of discarded messages when sending">>
),
m(
'delivery.dropped.expired',
<<"Number of messages dropped due to message expiration on sending">>
),
m(
'delivery.dropped.no_local',
<<
"Number of messages that were dropped due to the No Local subscription "
"option when sending"
>>
),
m(
'delivery.dropped.qos0_msg',
<<
"Number of messages with QoS 0 that were dropped because the message "
"queue was full when sending"
>>
),
m(
'delivery.dropped.queue_full',
<<
"Number of messages with a non-zero QoS that were dropped because the "
"message queue was full when sending"
>>
),
m(
'delivery.dropped.too_large',
<<
"The number of messages that were dropped because the length exceeded "
"the limit when sending"
>>
),
m(
'messages.acked',
<<"Number of received PUBACK and PUBREC packet">>
),
m(
'messages.delayed',
<<"Number of delay-published messages">>
),
m(
'messages.delivered',
<<"Number of messages forwarded to the subscription process internally">>
),
m(
'messages.dropped',
<<"Total number of messages dropped before forwarding to the subscription process">>
),
m(
'messages.dropped.await_pubrel_timeout',
<<"Number of messages dropped due to waiting PUBREL timeout">>
),
m(
'messages.dropped.no_subscribers',
<<"Number of messages dropped due to no subscribers">>
),
m(
'messages.forward',
<<"Number of messages forwarded to other nodes">>
),
m(
'messages.publish',
<<"Number of messages published in addition to system messages">>
),
m(
'messages.qos0.received',
<<"Number of QoS 0 messages received from clients">>
),
m(
'messages.qos0.sent',
<<"Number of QoS 0 messages sent to clients">>
),
m(
'messages.qos1.received',
<<"Number of QoS 1 messages received from clients">>
),
m(
'messages.qos1.sent',
<<"Number of QoS 1 messages sent to clients">>
),
m(
'messages.qos2.received',
<<"Number of QoS 2 messages received from clients">>
),
m(
'messages.qos2.sent',
<<"Number of QoS 2 messages sent to clients">>
),
m(
'messages.received',
<<
"Number of messages received from the client, equal to the sum of "
"messages.qos0.received\fmessages.qos1.received and messages.qos2.received"
>>
),
m(
'messages.retained',
<<"Number of retained messages">>
),
m(
'messages.sent',
<<
"Number of messages sent to the client, equal to the sum of "
"messages.qos0.sent\fmessages.qos1.sent and messages.qos2.sent"
>>
),
m(
'packets.auth.received',
<<"Number of received AUTH packet">>
),
m(
'packets.auth.sent',
<<"Number of sent AUTH packet">>
),
m(
'packets.connack.auth_error',
<<"Number of received CONNECT packet with failed authentication">>
),
m(
'packets.connack.error',
<<"Number of received CONNECT packet with unsuccessful connections">>
),
m(
'packets.connack.sent',
<<"Number of sent CONNACK packet">>
),
m(
'packets.connect.received',
<<"Number of received CONNECT packet">>
),
m(
'packets.disconnect.received',
<<"Number of received DISCONNECT packet">>
),
m(
'packets.disconnect.sent',
<<"Number of sent DISCONNECT packet">>
),
m(
'packets.pingreq.received',
<<"Number of received PINGREQ packet">>
),
m(
'packets.pingresp.sent',
<<"Number of sent PUBRESP packet">>
),
m(
'packets.puback.inuse',
<<"Number of received PUBACK packet with occupied identifiers">>
),
m(
'packets.puback.missed',
<<"Number of received packet with identifiers.">>
),
m(
'packets.puback.received',
<<"Number of received PUBACK packet">>
),
m(
'packets.puback.sent',
<<"Number of sent PUBACK packet">>
),
m(
'packets.pubcomp.inuse',
<<"Number of received PUBCOMP packet with occupied identifiers">>
),
m(
'packets.pubcomp.missed',
<<"Number of missed PUBCOMP packet">>
),
m(
'packets.pubcomp.received',
<<"Number of received PUBCOMP packet">>
),
m(
'packets.pubcomp.sent',
<<"Number of sent PUBCOMP packet">>
),
m(
'packets.publish.auth_error',
<<"Number of received PUBLISH packets with failed the Authorization check">>
),
m(
'packets.publish.dropped',
<<"Number of messages discarded due to the receiving limit">>
),
m(
'packets.publish.error',
<<"Number of received PUBLISH packet that cannot be published">>
),
m(
'packets.publish.inuse',
<<"Number of received PUBLISH packet with occupied identifiers">>
),
m(
'packets.publish.received',
<<"Number of received PUBLISH packet">>
),
m(
'packets.publish.sent',
<<"Number of sent PUBLISH packet">>
),
m(
'packets.pubrec.inuse',
<<"Number of received PUBREC packet with occupied identifiers">>
),
m(
'packets.pubrec.missed',
<<"Number of received PUBREC packet with unknown identifiers">>
),
m(
'packets.pubrec.received',
<<"Number of received PUBREC packet">>
),
m(
'packets.pubrec.sent',
<<"Number of sent PUBREC packet">>
),
m(
'packets.pubrel.missed',
<<"Number of received PUBREC packet with unknown identifiers">>
),
m(
'packets.pubrel.received',
<<"Number of received PUBREL packet">>
),
m(
'packets.pubrel.sent',
<<"Number of sent PUBREL packet">>
),
m(
'packets.received',
<<"Number of received packet">>
),
m(
'packets.sent',
<<"Number of sent packet">>
),
m(
'packets.suback.sent',
<<"Number of sent SUBACK packet">>
),
m(
'packets.subscribe.auth_error',
<<"Number of received SUBACK packet with failed Authorization check">>
),
m(
'packets.subscribe.error',
<<"Number of received SUBSCRIBE packet with failed subscriptions">>
),
m(
'packets.subscribe.received',
<<"Number of received SUBSCRIBE packet">>
),
m(
'packets.unsuback.sent',
<<"Number of sent UNSUBACK packet">>
),
m(
'packets.unsubscribe.error',
<<"Number of received UNSUBSCRIBE packet with failed unsubscriptions">>
),
m(
'packets.unsubscribe.received',
<<"Number of received UNSUBSCRIBE packet">>
),
m(
'rules.matched',
<<"Number of rule matched">>
),
m(
'session.created',
<<"Number of sessions created">>
),
m(
'session.discarded',
<<"Number of sessions dropped because Clean Session or Clean Start is true">>
),
m(
'session.resumed',
<<"Number of sessions resumed because Clean Session or Clean Start is false">>
),
m(
'session.takenover',
<<"Number of sessions takenover because Clean Session or Clean Start is false">>
),
m(
'session.terminated',
<<"Number of terminated sessions">>
)
].
m(K, Desc) ->
{K, mk(integer(), #{desc => Desc})}.
{K, mk(non_neg_integer(), #{desc => Desc})}.

307
apps/emqx_management/src/emqx_mgmt_api_nodes.erl
View File

@ -28,18 +28,20 @@
-define(SOURCE_ERROR, 'SOURCE_ERROR').
%% Swagger specs from hocon schema
-export([ api_spec/0
, schema/1
, paths/0
, fields/1
]).
-export([
api_spec/0,
schema/1,
paths/0,
fields/1
]).
%% API callbacks
-export([ nodes/2
, node/2
, node_metrics/2
, node_stats/2
]).
-export([
nodes/2,
node/2,
node_metrics/2,
node_stats/2
]).
%%--------------------------------------------------------------------
%% API spec funcs
@ -49,123 +51,183 @@ api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
paths() ->
[ "/nodes"
, "/nodes/:node"
, "/nodes/:node/metrics"
, "/nodes/:node/stats"
[
"/nodes",
"/nodes/:node",
"/nodes/:node/metrics",
"/nodes/:node/stats"
].
schema("/nodes") ->
#{ 'operationId' => nodes
, get =>
#{ description => <<"List EMQX nodes">>
, responses =>
#{200 => mk( array(ref(node_info))
, #{desc => <<"List all EMQX nodes">>})}
#{
'operationId' => nodes,
get =>
#{
description => <<"List EMQX nodes">>,
responses =>
#{
200 => mk(
array(ref(node_info)),
#{desc => <<"List all EMQX nodes">>}
)
}
}
};
};
schema("/nodes/:node") ->
#{ 'operationId' => node
, get =>
#{ description => <<"Get node info">>
, parameters => [ref(node_name)]
, responses =>
#{ 200 => mk( ref(node_info)
, #{desc => <<"Get node info successfully">>})
, 400 => node_error()
}
#{
'operationId' => node,
get =>
#{
description => <<"Get node info">>,
parameters => [ref(node_name)],
responses =>
#{
200 => mk(
ref(node_info),
#{desc => <<"Get node info successfully">>}
),
400 => node_error()
}
}
};
};
schema("/nodes/:node/metrics") ->
#{ 'operationId' => node_metrics
, get =>
#{ description => <<"Get node metrics">>
, parameters => [ref(node_name)]
, responses =>
#{ 200 => mk( ref(?NODE_METRICS_MODULE, node_metrics)
, #{desc => <<"Get node metrics successfully">>})
, 400 => node_error()
}
#{
'operationId' => node_metrics,
get =>
#{
description => <<"Get node metrics">>,
parameters => [ref(node_name)],
responses =>
#{
200 => mk(
ref(?NODE_METRICS_MODULE, node_metrics),
#{desc => <<"Get node metrics successfully">>}
),
400 => node_error()
}
}
};
};
schema("/nodes/:node/stats") ->
#{ 'operationId' => node_stats
, get =>
#{ description => <<"Get node stats">>
, parameters => [ref(node_name)]
, responses =>
#{ 200 => mk( ref(?NODE_STATS_MODULE, node_stats_data)
, #{desc => <<"Get node stats successfully">>})
, 400 => node_error()
}
#{
'operationId' => node_stats,
get =>
#{
description => <<"Get node stats">>,
parameters => [ref(node_name)],
responses =>
#{
200 => mk(
ref(?NODE_STATS_MODULE, node_stats_data),
#{desc => <<"Get node stats successfully">>}
),
400 => node_error()
}
}
}.
}.
%%--------------------------------------------------------------------
%% Fields
fields(node_name) ->
[ { node
, mk(atom()
, #{ in => path
, description => <<"Node name">>
, required => true
, example => <<"emqx@127.0.0.1">>
})
}
[
{node,
mk(
atom(),
#{
in => path,
description => <<"Node name">>,
required => true,
example => <<"emqx@127.0.0.1">>
}
)}
];
fields(node_info) ->
[ { node
, mk( atom()
, #{desc => <<"Node name">>, example => <<"emqx@127.0.0.1">>})}
, { connections
, mk( non_neg_integer()
, #{desc => <<"Number of clients currently connected to this node">>, example => 0})}
, { load1
, mk( string()
, #{desc => <<"CPU average load in 1 minute">>, example => "2.66"})}
, { load5
, mk( string()
, #{desc => <<"CPU average load in 5 minute">>, example => "2.66"})}
, { load15
, mk( string()
, #{desc => <<"CPU average load in 15 minute">>, example => "2.66"})}
, { max_fds
, mk( non_neg_integer()
, #{desc => <<"File descriptors limit">>, example => 1024})}
, { memory_total
, mk( emqx_schema:bytesize()
, #{desc => <<"Allocated memory">>, example => "512.00M"})}
, { memory_used
, mk( emqx_schema:bytesize()
, #{desc => <<"Used memory">>, example => "256.00M"})}
, { node_status
, mk( enum(['Running', 'Stopped'])
, #{desc => <<"Node status">>, example => "Running"})}
, { otp_release
, mk( string()
, #{ desc => <<"Erlang/OTP version">>, example => "24.2/12.2"})}
, { process_available
, mk( non_neg_integer()
, #{desc => <<"Erlang processes limit">>, example => 2097152})}
, { process_used
, mk( non_neg_integer()
, #{desc => <<"Running Erlang processes">>, example => 1024})}
, { uptime
, mk( non_neg_integer()
, #{desc => <<"System uptime, milliseconds">>, example => 5120000})}
, { version
, mk( string()
, #{desc => <<"Release version">>, example => "5.0.0-beat.3-00000000"})}
, { sys_path
, mk( string()
, #{desc => <<"Path to system files">>, example => "path/to/emqx"})}
, { log_path
, mk( string()
, #{desc => <<"Path to log files">>, example => "path/to/log | not found"})}
, { role
, mk( enum([core, replicant])
, #{desc => <<"Node role">>, example => "core"})}
[
{node,
mk(
atom(),
#{desc => <<"Node name">>, example => <<"emqx@127.0.0.1">>}
)},
{connections,
mk(
non_neg_integer(),
#{desc => <<"Number of clients currently connected to this node">>, example => 0}
)},
{load1,
mk(
string(),
#{desc => <<"CPU average load in 1 minute">>, example => "2.66"}
)},
{load5,
mk(
string(),
#{desc => <<"CPU average load in 5 minute">>, example => "2.66"}
)},
{load15,
mk(
string(),
#{desc => <<"CPU average load in 15 minute">>, example => "2.66"}
)},
{max_fds,
mk(
non_neg_integer(),
#{desc => <<"File descriptors limit">>, example => 1024}
)},
{memory_total,
mk(
emqx_schema:bytesize(),
#{desc => <<"Allocated memory">>, example => "512.00M"}
)},
{memory_used,
mk(
emqx_schema:bytesize(),
#{desc => <<"Used memory">>, example => "256.00M"}
)},
{node_status,
mk(
enum(['Running', 'Stopped']),
#{desc => <<"Node status">>, example => "Running"}
)},
{otp_release,
mk(
string(),
#{desc => <<"Erlang/OTP version">>, example => "24.2/12.2"}
)},
{process_available,
mk(
non_neg_integer(),
#{desc => <<"Erlang processes limit">>, example => 2097152}
)},
{process_used,
mk(
non_neg_integer(),
#{desc => <<"Running Erlang processes">>, example => 1024}
)},
{uptime,
mk(
non_neg_integer(),
#{desc => <<"System uptime, milliseconds">>, example => 5120000}
)},
{version,
mk(
string(),
#{desc => <<"Release version">>, example => "5.0.0-beat.3-00000000"}
)},
{sys_path,
mk(
string(),
#{desc => <<"Path to system files">>, example => "path/to/emqx"}
)},
{log_path,
mk(
string(),
#{desc => <<"Path to log files">>, example => "path/to/log | not found"}
)},
{role,
mk(
enum([core, replicant]),
#{desc => <<"Node role">>, example => "core"}
)}
].
%%--------------------------------------------------------------------
@ -221,17 +283,20 @@ get_stats(Node) ->
format(_Node, Info = #{memory_total := Total, memory_used := Used}) ->
{ok, SysPathBinary} = file:get_cwd(),
SysPath = list_to_binary(SysPathBinary),
LogPath = case log_path() of
undefined ->
<<"not found">>;
Path0 ->
Path = list_to_binary(Path0),
<<SysPath/binary, Path/binary>>
end,
Info#{ memory_total := emqx_mgmt_util:kmg(Total)
, memory_used := emqx_mgmt_util:kmg(Used)
, sys_path => SysPath
, log_path => LogPath}.
LogPath =
case log_path() of
undefined ->
<<"not found">>;
Path0 ->
Path = list_to_binary(Path0),
<<SysPath/binary, Path/binary>>
end,
Info#{
memory_total := emqx_mgmt_util:kmg(Total),
memory_used := emqx_mgmt_util:kmg(Used),
sys_path => SysPath,
log_path => LogPath
}.
log_path() ->
Configs = logger:get_handler_config(),

333
apps/emqx_management/src/emqx_mgmt_api_plugins.erl
View File

@ -22,27 +22,30 @@
-include_lib("emqx/include/logger.hrl").
%%-include_lib("emqx_plugins/include/emqx_plugins.hrl").
-export([ api_spec/0
, fields/1
, paths/0
, schema/1
, namespace/0
]).
-export([
api_spec/0,
fields/1,
paths/0,
schema/1,
namespace/0
]).
-export([ list_plugins/2
, upload_install/2
, plugin/2
, update_plugin/2
, update_boot_order/2
]).
-export([
list_plugins/2,
upload_install/2,
plugin/2,
update_plugin/2,
update_boot_order/2
]).
-export([ validate_name/1
, get_plugins/0
, install_package/2
, delete_package/1
, describe_package/1
, ensure_action/2
]).
-export([
validate_name/1,
get_plugins/0,
install_package/2,
delete_package/1,
describe_package/1,
ensure_action/2
]).
-define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_.]*$").
@ -65,9 +68,10 @@ schema("/plugins") ->
#{
'operationId' => list_plugins,
get => #{
description => "List all install plugins.<br>"
"Plugins are launched in top-down order.<br>"
"Using `POST /plugins/{name}/move` to change the boot order.",
description =>
"List all install plugins.<br>"
"Plugins are launched in top-down order.<br>"
"Using `POST /plugins/{name}/move` to change the boot order.",
responses => #{
200 => hoconsc:array(hoconsc:ref(plugin))
}
@ -77,20 +81,26 @@ schema("/plugins/install") ->
#{
'operationId' => upload_install,
post => #{
description => "Install a plugin(plugin-vsn.tar.gz)."
"Follow [emqx-plugin-template](https://github.com/emqx/emqx-plugin-template) "
"to develop plugin.",
description =>
"Install a plugin(plugin-vsn.tar.gz)."
"Follow [emqx-plugin-template](https://github.com/emqx/emqx-plugin-template) "
"to develop plugin.",
'requestBody' => #{
content => #{
'multipart/form-data' => #{
schema => #{
type => object,
properties => #{
plugin => #{type => string, format => binary}}},
encoding => #{plugin => #{'contentType' => 'application/gzip'}}}}},
plugin => #{type => string, format => binary}
}
},
encoding => #{plugin => #{'contentType' => 'application/gzip'}}
}
}
},
responses => #{
200 => <<"OK">>,
400 => emqx_dashboard_swagger:error_codes(['UNEXPECTED_ERROR','ALREADY_INSTALLED'])
400 => emqx_dashboard_swagger:error_codes(['UNEXPECTED_ERROR', 'ALREADY_INSTALLED'])
}
}
};
@ -118,12 +128,14 @@ schema("/plugins/:name/:action") ->
#{
'operationId' => update_plugin,
put => #{
description => "start/stop a installed plugin.<br>"
"- **start**: start the plugin.<br>"
"- **stop**: stop the plugin.<br>",
description =>
"start/stop a installed plugin.<br>"
"- **start**: start the plugin.<br>"
"- **stop**: stop the plugin.<br>",
parameters => [
hoconsc:ref(name),
{action, hoconsc:mk(hoconsc:enum([start, stop]), #{desc => "Action", in => path})}],
{action, hoconsc:mk(hoconsc:enum([start, stop]), #{desc => "Action", in => path})}
],
responses => #{
200 => <<"OK">>,
404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>)
@ -143,57 +155,83 @@ schema("/plugins/:name/move") ->
fields(plugin) ->
[
{name, hoconsc:mk(binary(),
#{
desc => "Name-Vsn: without .tar.gz",
validator => fun ?MODULE:validate_name/1,
required => true,
example => "emqx_plugin_template-5.0-rc.1"})
},
{name,
hoconsc:mk(
binary(),
#{
desc => "Name-Vsn: without .tar.gz",
validator => fun ?MODULE:validate_name/1,
required => true,
example => "emqx_plugin_template-5.0-rc.1"
}
)},
{author, hoconsc:mk(list(string()), #{example => [<<"EMQX Team">>]})},
{builder, hoconsc:ref(?MODULE, builder)},
{built_on_otp_release, hoconsc:mk(string(), #{example => "24"})},
{compatibility, hoconsc:mk(map(), #{example => #{<<"emqx">> => <<"~>5.0">>}})},
{git_commit_or_build_date, hoconsc:mk(string(), #{
example => "2021-12-25",
desc => "Last git commit date by `git log -1 --pretty=format:'%cd' "
"--date=format:'%Y-%m-%d`.\n"
" If the last commit date is not available, the build date will be presented."
})},
{git_commit_or_build_date,
hoconsc:mk(string(), #{
example => "2021-12-25",
desc =>
"Last git commit date by `git log -1 --pretty=format:'%cd' "
"--date=format:'%Y-%m-%d`.\n"
" If the last commit date is not available, the build date will be presented."
})},
{functionality, hoconsc:mk(hoconsc:array(string()), #{example => [<<"Demo">>]})},
{git_ref, hoconsc:mk(string(), #{example => "ddab50fafeed6b1faea70fc9ffd8c700d7e26ec1"})},
{metadata_vsn, hoconsc:mk(string(), #{example => "0.1.0"})},
{rel_vsn, hoconsc:mk(binary(),
#{desc => "Plugins release version",
required => true,
example => <<"5.0-rc.1">>})
},
{rel_apps, hoconsc:mk(hoconsc:array(binary()),
#{desc => "Aplications in plugin.",
required => true,
example => [<<"emqx_plugin_template-5.0.0">>, <<"map_sets-1.1.0">>]})
},
{rel_vsn,
hoconsc:mk(
binary(),
#{
desc => "Plugins release version",
required => true,
example => <<"5.0-rc.1">>
}
)},
{rel_apps,
hoconsc:mk(
hoconsc:array(binary()),
#{
desc => "Aplications in plugin.",
required => true,
example => [<<"emqx_plugin_template-5.0.0">>, <<"map_sets-1.1.0">>]
}
)},
{repo, hoconsc:mk(string(), #{example => "https://github.com/emqx/emqx-plugin-template"})},
{description, hoconsc:mk(binary(),
#{desc => "Plugin description.",
required => true,
example => "This is an demo plugin description"})
},
{running_status, hoconsc:mk(hoconsc:array(hoconsc:ref(running_status)),
#{required => true})},
{readme, hoconsc:mk(binary(), #{
example => "This is an demo plugin.",
desc => "only return when `GET /plugins/{name}`.",
required => false})}
{description,
hoconsc:mk(
binary(),
#{
desc => "Plugin description.",
required => true,
example => "This is an demo plugin description"
}
)},
{running_status,
hoconsc:mk(
hoconsc:array(hoconsc:ref(running_status)),
#{required => true}
)},
{readme,
hoconsc:mk(binary(), #{
example => "This is an demo plugin.",
desc => "only return when `GET /plugins/{name}`.",
required => false
})}
];
fields(name) ->
[{name, hoconsc:mk(binary(),
#{
desc => list_to_binary(?NAME_RE),
example => "emqx_plugin_template-5.0-rc.1",
in => path,
validator => fun ?MODULE:validate_name/1
})}
[
{name,
hoconsc:mk(
binary(),
#{
desc => list_to_binary(?NAME_RE),
example => "emqx_plugin_template-5.0-rc.1",
in => path,
validator => fun ?MODULE:validate_name/1
}
)}
];
fields(builder) ->
[
@ -202,27 +240,38 @@ fields(builder) ->
{website, hoconsc:mk(string(), #{example => "www.emqx.com"})}
];
fields(position) ->
[{position, hoconsc:mk(hoconsc:union([front, rear, binary()]),
#{
desc => """
Enable auto-boot at position in the boot list, where Position could be
'front', 'rear', or 'before:other-vsn', 'after:other-vsn'
to specify a relative position.
""",
required => false
})}];
[
{position,
hoconsc:mk(
hoconsc:union([front, rear, binary()]),
#{
desc =>
""
"\n"
" Enable auto-boot at position in the boot list, where Position could be\n"
" 'front', 'rear', or 'before:other-vsn', 'after:other-vsn'\n"
" to specify a relative position.\n"
" "
"",
required => false
}
)}
];
fields(running_status) ->
[
{node, hoconsc:mk(string(), #{example => "emqx@127.0.0.1"})},
{status, hoconsc:mk(hoconsc:enum([running, stopped]), #{
desc => "Install plugin status at runtime</br>"
"1. running: plugin is running.<br>"
"2. stopped: plugin is stopped.<br>"
})}
{status,
hoconsc:mk(hoconsc:enum([running, stopped]), #{
desc =>
"Install plugin status at runtime</br>"
"1. running: plugin is running.<br>"
"2. stopped: plugin is stopped.<br>"
})}
].
move_request_body() ->
emqx_dashboard_swagger:schema_with_examples(hoconsc:ref(?MODULE, position),
emqx_dashboard_swagger:schema_with_examples(
hoconsc:ref(?MODULE, position),
#{
move_to_front => #{
summary => <<"move plugin on the front">>,
@ -240,7 +289,8 @@ move_request_body() ->
summary => <<"move plugin after other plugins">>,
value => #{position => <<"after:emqx_plugin_demo-5.1-rc.2">>}
}
}).
}
).
validate_name(Name) ->
NameLen = byte_size(Name),
@ -250,7 +300,8 @@ validate_name(Name) ->
nomatch -> {error, "Name should be " ?NAME_RE};
_ -> ok
end;
false -> {error, "Name Length must =< 256"}
false ->
{error, "Name Length must =< 256"}
end.
%% API CallBack Begin
@ -271,29 +322,42 @@ upload_install(post, #{body := #{<<"plugin">> := Plugin}}) when is_map(Plugin) -
{AppName, _Vsn} = emqx_plugins:parse_name_vsn(FileName),
AppDir = filename:join(emqx_plugins:install_dir(), AppName),
case filelib:wildcard(AppDir ++ "*.tar.gz") of
[] -> do_install_package(FileName, Bin);
[] ->
do_install_package(FileName, Bin);
OtherVsn ->
{400, #{code => 'ALREADY_INSTALLED',
message => iolist_to_binary(io_lib:format("~p already installed",
[OtherVsn]))}}
{400, #{
code => 'ALREADY_INSTALLED',
message => iolist_to_binary(
io_lib:format(
"~p already installed",
[OtherVsn]
)
)
}}
end;
{ok, _} ->
{400, #{code => 'ALREADY_INSTALLED',
message => iolist_to_binary(io_lib:format("~p is already installed", [FileName]))}}
{400, #{
code => 'ALREADY_INSTALLED',
message => iolist_to_binary(io_lib:format("~p is already installed", [FileName]))
}}
end;
upload_install(post, #{}) ->
{400, #{code => 'BAD_FORM_DATA',
{400, #{
code => 'BAD_FORM_DATA',
message =>
<<"form-data should be `plugin=@packagename-vsn.tar.gz;type=application/x-gzip`">>}
}.
<<"form-data should be `plugin=@packagename-vsn.tar.gz;type=application/x-gzip`">>
}}.
do_install_package(FileName, Bin) ->
{Res, _} = emqx_mgmt_api_plugins_proto_v1:install_package(FileName, Bin),
case lists:filter(fun(R) -> R =/= ok end, Res) of
[] -> {200};
[] ->
{200};
[{error, Reason} | _] ->
{400, #{code => 'UNEXPECTED_ERROR',
message => iolist_to_binary(io_lib:format("~p", [Reason]))}}
{400, #{
code => 'UNEXPECTED_ERROR',
message => iolist_to_binary(io_lib:format("~p", [Reason]))
}}
end.
plugin(get, #{bindings := #{name := Name}}) ->
@ -302,7 +366,6 @@ plugin(get, #{bindings := #{name := Name}}) ->
[Plugin] -> {200, Plugin};
[] -> {404, #{code => 'NOT_FOUND', message => Name}}
end;
plugin(delete, #{bindings := #{name := Name}}) ->
{ok, _TnxId, Res} = emqx_mgmt_api_plugins_proto_v1:delete_package(Name),
return(204, Res).
@ -313,13 +376,17 @@ update_plugin(put, #{bindings := #{name := Name, action := Action}}) ->
update_boot_order(post, #{bindings := #{name := Name}, body := Body}) ->
case parse_position(Body, Name) of
{error, Reason} -> {400, #{code => 'BAD_POSITION', message => Reason}};
{error, Reason} ->
{400, #{code => 'BAD_POSITION', message => Reason}};
Position ->
case emqx_plugins:ensure_enabled(Name, Position) of
ok -> {200};
ok ->
{200};
{error, Reason} ->
{400, #{code => 'MOVE_FAILED',
message => iolist_to_binary(io_lib:format("~p", [Reason]))}}
{400, #{
code => 'MOVE_FAILED',
message => iolist_to_binary(io_lib:format("~p", [Reason]))
}}
end
end.
@ -347,7 +414,8 @@ delete_package(Name) ->
_ = emqx_plugins:ensure_disabled(Name),
_ = emqx_plugins:purge(Name),
_ = emqx_plugins:delete_package(Name);
Error -> Error
Error ->
Error
end.
%% for RPC plugin update
@ -361,15 +429,19 @@ ensure_action(Name, restart) ->
_ = emqx_plugins:ensure_enabled(Name),
_ = emqx_plugins:restart(Name).
return(Code, ok) -> {Code};
return(Code, {ok, Result}) -> {Code, Result};
return(Code, ok) ->
{Code};
return(Code, {ok, Result}) ->
{Code, Result};
return(_, {error, #{error := "bad_info_file", return := {enoent, _}, path := Path}}) ->
{404, #{code => 'NOT_FOUND', message => Path}};
return(_, {error, Reason}) ->
{400, #{code => 'PARAM_ERROR', message => iolist_to_binary(io_lib:format("~p", [Reason]))}}.
parse_position(#{<<"position">> := <<"front">>}, _) -> front;
parse_position(#{<<"position">> := <<"rear">>}, _) -> rear;
parse_position(#{<<"position">> := <<"front">>}, _) ->
front;
parse_position(#{<<"position">> := <<"rear">>}, _) ->
rear;
parse_position(#{<<"position">> := <<"before:", Name/binary>>}, Name) ->
{error, <<"Invalid parameter. Cannot be placed before itself">>};
parse_position(#{<<"position">> := <<"after:", Name/binary>>}, Name) ->
@ -382,7 +454,8 @@ parse_position(#{<<"position">> := <<"before:", Before/binary>>}, _Name) ->
{before, binary_to_list(Before)};
parse_position(#{<<"position">> := <<"after:", After/binary>>}, _Name) ->
{behind, binary_to_list(After)};
parse_position(Position, _) -> {error, iolist_to_binary(io_lib:format("~p", [Position]))}.
parse_position(Position, _) ->
{error, iolist_to_binary(io_lib:format("~p", [Position]))}.
format_plugins(List) ->
StatusMap = aggregate_status(List),
@ -392,13 +465,18 @@ format_plugins(List) ->
pack_status_in_order(List, StatusMap) ->
{Plugins, _} =
lists:foldl(fun({_Node, PluginList}, {Acc, StatusAcc}) ->
pack_plugin_in_order(PluginList, Acc, StatusAcc)
end, {[], StatusMap}, List),
lists:foldl(
fun({_Node, PluginList}, {Acc, StatusAcc}) ->
pack_plugin_in_order(PluginList, Acc, StatusAcc)
end,
{[], StatusMap},
List
),
lists:reverse(Plugins).
pack_plugin_in_order([], Acc, StatusAcc) -> {Acc, StatusAcc};
pack_plugin_in_order(_, Acc, StatusAcc)when map_size(StatusAcc) =:= 0 -> {Acc, StatusAcc};
pack_plugin_in_order([], Acc, StatusAcc) ->
{Acc, StatusAcc};
pack_plugin_in_order(_, Acc, StatusAcc) when map_size(StatusAcc) =:= 0 -> {Acc, StatusAcc};
pack_plugin_in_order([Plugin0 | Plugins], Acc, StatusAcc) ->
#{<<"name">> := Name, <<"rel_vsn">> := Vsn} = Plugin0,
case maps:find({Name, Vsn}, StatusAcc) of
@ -413,15 +491,20 @@ pack_plugin_in_order([Plugin0 | Plugins], Acc, StatusAcc) ->
aggregate_status(List) -> aggregate_status(List, #{}).
aggregate_status([], Acc) -> Acc;
aggregate_status([], Acc) ->
Acc;
aggregate_status([{Node, Plugins} | List], Acc) ->
NewAcc =
lists:foldl(fun(Plugin, SubAcc) ->
#{<<"name">> := Name, <<"rel_vsn">> := Vsn} = Plugin,
Key = {Name, Vsn},
Value = #{node => Node, status => plugin_status(Plugin)},
SubAcc#{Key => [Value | maps:get(Key, Acc, [])]}
end, Acc, Plugins),
lists:foldl(
fun(Plugin, SubAcc) ->
#{<<"name">> := Name, <<"rel_vsn">> := Vsn} = Plugin,
Key = {Name, Vsn},
Value = #{node => Node, status => plugin_status(Plugin)},
SubAcc#{Key => [Value | maps:get(Key, Acc, [])]}
end,
Acc,
Plugins
),
aggregate_status(List, NewAcc).
% running_status: running loaded, stopped

91
apps/emqx_management/src/emqx_mgmt_api_publish.erl
View File

@ -20,14 +20,17 @@
-behaviour(minirest_api).
-export([ api_spec/0
, paths/0
, schema/1
, fields/1
]).
-export([
api_spec/0,
paths/0,
schema/1,
fields/1
]).
-export([ publish/2
, publish_batch/2]).
-export([
publish/2,
publish_batch/2
]).
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).
@ -46,7 +49,6 @@ schema("/publish") ->
}
}
};
schema("/publish/bulk") ->
#{
'operationId' => publish_batch,
@ -61,32 +63,43 @@ schema("/publish/bulk") ->
fields(publish_message) ->
[
{topic, hoconsc:mk(binary(), #{
desc => <<"Topic Name">>,
required => true,
example => <<"api/example/topic">>})},
{qos, hoconsc:mk(emqx_schema:qos(), #{
desc => <<"MQTT QoS">>,
required => false,
default => 0})},
{from, hoconsc:mk(binary(), #{
desc => <<"From client ID">>,
required => false,
example => <<"api_example_client">>})},
{payload, hoconsc:mk(binary(), #{
desc => <<"MQTT Payload">>,
required => true,
example => <<"hello emqx api">>})},
{retain, hoconsc:mk(boolean(), #{
desc => <<"MQTT Retain Message">>,
required => false,
default => false})}
{topic,
hoconsc:mk(binary(), #{
desc => <<"Topic Name">>,
required => true,
example => <<"api/example/topic">>
})},
{qos,
hoconsc:mk(emqx_schema:qos(), #{
desc => <<"MQTT QoS">>,
required => false,
default => 0
})},
{from,
hoconsc:mk(binary(), #{
desc => <<"From client ID">>,
required => false,
example => <<"api_example_client">>
})},
{payload,
hoconsc:mk(binary(), #{
desc => <<"MQTT Payload">>,
required => true,
example => <<"hello emqx api">>
})},
{retain,
hoconsc:mk(boolean(), #{
desc => <<"MQTT Retain Message">>,
required => false,
default => false
})}
];
fields(publish_message_info) ->
[
{id, hoconsc:mk(binary(), #{
desc => <<"Internal Message ID">>})}
{id,
hoconsc:mk(binary(), #{
desc => <<"Internal Message ID">>
})}
] ++ fields(publish_message).
publish(post, #{body := Body}) ->
@ -100,19 +113,21 @@ publish_batch(post, #{body := Body}) ->
{200, format_message(Messages)}.
message(Map) ->
From = maps:get(<<"from">>, Map, http_api),
QoS = maps:get(<<"qos">>, Map, 0),
Topic = maps:get(<<"topic">>, Map),
From = maps:get(<<"from">>, Map, http_api),
QoS = maps:get(<<"qos">>, Map, 0),
Topic = maps:get(<<"topic">>, Map),
Payload = maps:get(<<"payload">>, Map),
Retain = maps:get(<<"retain">>, Map, false),
Retain = maps:get(<<"retain">>, Map, false),
emqx_message:make(From, QoS, Topic, Payload, #{retain => Retain}, #{}).
messages(List) ->
[message(MessageMap) || MessageMap <- List].
format_message(Messages) when is_list(Messages)->
format_message(Messages) when is_list(Messages) ->
[format_message(Message) || Message <- Messages];
format_message(#message{id = ID, qos = Qos, from = From, topic = Topic, payload = Payload, flags = Flags}) ->
format_message(#message{
id = ID, qos = Qos, from = From, topic = Topic, payload = Payload, flags = Flags
}) ->
#{
id => emqx_guid:to_hexstr(ID),
qos => Qos,
@ -124,5 +139,5 @@ format_message(#message{id = ID, qos = Qos, from = From, topic = Topic, payload
to_binary(Data) when is_binary(Data) ->
Data;
to_binary(Data) ->
to_binary(Data) ->
list_to_binary(io_lib:format("~p", [Data])).

183
apps/emqx_management/src/emqx_mgmt_api_stats.erl
View File

@ -19,17 +19,22 @@
-include_lib("typerefl/include/types.hrl").
-import( hoconsc
, [ mk/2
, ref/1
, ref/2
, array/1]).
-import(
hoconsc,
[
mk/2,
ref/1,
ref/2,
array/1
]
).
-export([ api_spec/0
, paths/0
, schema/1
, fields/1
]).
-export([
api_spec/0,
paths/0,
schema/1,
fields/1
]).
-export([list/2]).
@ -40,102 +45,80 @@ paths() ->
["/stats"].
schema("/stats") ->
#{ 'operationId' => list
, get =>
#{ description => <<"EMQX stats">>
, tags => [<<"stats">>]
, parameters => [ref(aggregate)]
, responses =>
#{ 200 => mk( hoconsc:union([ ref(?MODULE, node_stats_data)
, array(ref(?MODULE, aggergate_data))
])
, #{ desc => <<"List stats ok">> })
}
#{
'operationId' => list,
get =>
#{
description => <<"EMQX stats">>,
tags => [<<"stats">>],
parameters => [ref(aggregate)],
responses =>
#{
200 => mk(
hoconsc:union([
ref(?MODULE, node_stats_data),
array(ref(?MODULE, aggergate_data))
]),
#{desc => <<"List stats ok">>}
)
}
}
}.
}.
fields(aggregate) ->
[ { aggregate
, mk( boolean()
, #{ desc => <<"Calculation aggregate for all nodes">>
, in => query
, required => false
, example => false})}
[
{aggregate,
mk(
boolean(),
#{
desc => <<"Calculation aggregate for all nodes">>,
in => query,
required => false,
example => false
}
)}
];
fields(node_stats_data) ->
[ { 'channels.count'
, mk( integer(), #{ desc => <<"sessions.count">>
, example => 0})}
, { 'channels.max'
, mk( integer(), #{ desc => <<"session.max">>
, example => 0})}
, { 'connections.count'
, mk( integer(), #{ desc => <<"Number of current connections">>
, example => 0})}
, { 'connections.max'
, mk( integer(), #{ desc => <<"Historical maximum number of connections">>
, example => 0})}
, { 'delayed.count'
, mk( integer(), #{ desc => <<"Number of delayed messages">>
, example => 0})}
, { 'delayed.max'
, mk( integer(), #{ desc => <<"Historical maximum number of delayed messages">>
, example => 0})}
, { 'live_connections.count'
, mk( integer(), #{ desc => <<"Number of current live connections">>
, example => 0})}
, { 'live_connections.max'
, mk( integer(), #{ desc => <<"Historical maximum number of live connections">>
, example => 0})}
, { 'retained.count'
, mk( integer(), #{ desc => <<"Number of currently retained messages">>
, example => 0})}
, { 'retained.max'
, mk( integer(), #{ desc => <<"Historical maximum number of retained messages">>
, example => 0})}
, { 'sessions.count'
, mk( integer(), #{ desc => <<"Number of current sessions">>
, example => 0})}
, { 'sessions.max'
, mk( integer(), #{ desc => <<"Historical maximum number of sessions">>
, example => 0})}
, { 'suboptions.count'
, mk( integer(), #{ desc => <<"subscriptions.count">>
, example => 0})}
, { 'suboptions.max'
, mk( integer(), #{ desc => <<"subscriptions.max">>
, example => 0})}
, { 'subscribers.count'
, mk( integer(), #{ desc => <<"Number of current subscribers">>
, example => 0})}
, { 'subscribers.max'
, mk( integer(), #{ desc => <<"Historical maximum number of subscribers">>
, example => 0})}
, { 'subscriptions.count'
, mk( integer(), #{ desc => <<"Number of current subscriptions, including shared subscriptions">>
, example => 0})}
, { 'subscriptions.max'
, mk( integer(), #{ desc => <<"Historical maximum number of subscriptions">>
, example => 0})}
, { 'subscriptions.shared.count'
, mk( integer(), #{ desc => <<"Number of current shared subscriptions">>
, example => 0})}
, { 'subscriptions.shared.max'
, mk( integer(), #{ desc => <<"Historical maximum number of shared subscriptions">>
, example => 0})}
, { 'topics.count'
, mk( integer(), #{ desc => <<"Number of current topics">>
, example => 0})}
, { 'topics.max'
, mk( integer(), #{ desc => <<"Historical maximum number of topics">>
, example => 0})}
[
stats_schema('channels.count', <<"sessions.count">>),
stats_schema('channels.max', <<"session.max">>),
stats_schema('connections.count', <<"Number of current connections">>),
stats_schema('connections.max', <<"Historical maximum number of connections">>),
stats_schema('delayed.count', <<"Number of delayed messages">>),
stats_schema('delayed.max', <<"Historical maximum number of delayed messages">>),
stats_schema('live_connections.count', <<"Number of current live connections">>),
stats_schema('live_connections.max', <<"Historical maximum number of live connections">>),
stats_schema('retained.count', <<"Number of currently retained messages">>),
stats_schema('retained.max', <<"Historical maximum number of retained messages">>),
stats_schema('sessions.count', <<"Number of current sessions">>),
stats_schema('sessions.max', <<"Historical maximum number of sessions">>),
stats_schema('suboptions.count', <<"subscriptions.count">>),
stats_schema('suboptions.max', <<"subscriptions.max">>),
stats_schema('subscribers.count', <<"Number of current subscribers">>),
stats_schema('subscribers.max', <<"Historical maximum number of subscribers">>),
stats_schema(
'subscriptions.count',
<<"Number of current subscriptions, including shared subscriptions">>
),
stats_schema('subscriptions.max', <<"Historical maximum number of subscriptions">>),
stats_schema('subscriptions.shared.count', <<"Number of current shared subscriptions">>),
stats_schema(
'subscriptions.shared.max', <<"Historical maximum number of shared subscriptions">>
),
stats_schema('topics.count', <<"Number of current topics">>),
stats_schema('topics.max', <<"Historical maximum number of topics">>)
];
fields(aggergate_data) ->
[ { node
, mk( string(), #{ desc => <<"Node name">>
, example => <<"emqx@127.0.0.1">>})}
[
{node,
mk(string(), #{
desc => <<"Node name">>,
example => <<"emqx@127.0.0.1">>
})}
] ++ fields(node_stats_data).
stats_schema(Name, Desc) ->
{Name, mk(non_neg_integer(), #{desc => Desc, example => 0})}.
%%%==============================================================================================
%% api apply
@ -144,7 +127,9 @@ list(get, #{query_string := Qs}) ->
true ->
{200, emqx_mgmt:get_stats()};
_ ->
Data = [maps:from_list(emqx_mgmt:get_stats(Node) ++ [{node, Node}]) ||
Node <- mria_mnesia:running_nodes()],
Data = [
maps:from_list(emqx_mgmt:get_stats(Node) ++ [{node, Node}])
|| Node <- mria_mnesia:running_nodes()
],
{200, Data}
end.

49
apps/emqx_management/src/emqx_mgmt_api_status.erl
View File

@ -17,10 +17,11 @@
%% API
-behaviour(minirest_api).
-export([ api_spec/0
, paths/0
, schema/1
]).
-export([
api_spec/0,
paths/0,
schema/1
]).
-export([running_status/2]).
@ -31,22 +32,30 @@ paths() ->
["/status"].
schema("/status") ->
#{ 'operationId' => running_status
, get =>
#{ description => <<"Node running status">>
, security => []
, responses =>
#{200 =>
#{ description => <<"Node is running">>
, content =>
#{ 'text/plain' =>
#{ schema => #{type => string}
, example => <<"Node emqx@127.0.0.1 is started\nemqx is running">>}
}
}
}
#{
'operationId' => running_status,
get =>
#{
description => <<"Node running status">>,
security => [],
responses =>
#{
200 =>
#{
description => <<"Node is running">>,
content =>
#{
'text/plain' =>
#{
schema => #{type => string},
example =>
<<"Node emqx@127.0.0.1 is started\nemqx is running">>
}
}
}
}
}
}.
}.
%%--------------------------------------------------------------------
%% API Handler funcs
@ -62,7 +71,7 @@ running_status(get, _Params) ->
end,
AppStatus =
case lists:keysearch(emqx, 1, application:which_applications()) of
false -> not_running;
false -> not_running;
{value, _Val} -> running
end,
Status = io_lib:format("Node ~ts is ~ts~nemqx is ~ts", [node(), BrokerStatus, AppStatus]),

160
apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
View File

@ -22,26 +22,30 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-export([ api_spec/0
, paths/0
, schema/1
, fields/1]).
-export([
api_spec/0,
paths/0,
schema/1,
fields/1
]).
-export([subscriptions/2]).
-export([ query/4
, format/1
]).
-export([
query/4,
format/1
]).
-define(SUBS_QTABLE, emqx_suboption).
-define(SUBS_QSCHEMA,
[ {<<"clientid">>, binary}
, {<<"topic">>, binary}
, {<<"share">>, binary}
, {<<"share_group">>, binary}
, {<<"qos">>, integer}
, {<<"match_topic">>, binary}]).
-define(SUBS_QSCHEMA, [
{<<"clientid">>, binary},
{<<"topic">>, binary},
{<<"share">>, binary},
{<<"share_group">>, binary},
{<<"qos">>, integer},
{<<"match_topic">>, binary}
]).
-define(QUERY_FUN, {?MODULE, query}).
@ -58,7 +62,9 @@ schema("/subscriptions") ->
description => <<"List subscriptions">>,
parameters => parameters(),
responses => #{
200 => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, subscription)), #{})}}
200 => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, subscription)), #{})
}
}
}.
fields(subscription) ->
@ -74,62 +80,89 @@ parameters() ->
hoconsc:ref(emqx_dashboard_swagger, page),
hoconsc:ref(emqx_dashboard_swagger, limit),
{
node, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Node name">>,
example => atom_to_list(node())})
node,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Node name">>,
example => atom_to_list(node())
})
},
{
clientid, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Client ID">>})
clientid,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Client ID">>
})
},
{
qos, hoconsc:mk(emqx_schema:qos(), #{
in => query,
required => false,
desc => <<"QoS">>})
qos,
hoconsc:mk(emqx_schema:qos(), #{
in => query,
required => false,
desc => <<"QoS">>
})
},
{
topic, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Topic, url encoding">>})
topic,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Topic, url encoding">>
})
},
{
match_topic, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Match topic string, url encoding">>})
match_topic,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Match topic string, url encoding">>
})
},
{
share_group, hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Shared subscription group name">>})
share_group,
hoconsc:mk(binary(), #{
in => query,
required => false,
desc => <<"Shared subscription group name">>
})
}
].
subscriptions(get, #{query_string := QString}) ->
case maps:get(<<"node">>, QString, undefined) of
undefined ->
Response = emqx_mgmt_api:cluster_query(QString, ?SUBS_QTABLE,
?SUBS_QSCHEMA, ?QUERY_FUN),
emqx_mgmt_util:generate_response(Response);
Node ->
Response = emqx_mgmt_api:node_query(binary_to_atom(Node, utf8), QString,
?SUBS_QTABLE, ?SUBS_QSCHEMA, ?QUERY_FUN),
emqx_mgmt_util:generate_response(Response)
Response =
case maps:get(<<"node">>, QString, undefined) of
undefined ->
emqx_mgmt_api:cluster_query(
QString,
?SUBS_QTABLE,
?SUBS_QSCHEMA,
?QUERY_FUN
);
Node0 ->
emqx_mgmt_api:node_query(
binary_to_atom(Node0, utf8),
QString,
?SUBS_QTABLE,
?SUBS_QSCHEMA,
?QUERY_FUN
)
end,
case Response of
{error, page_limit_invalid} ->
{400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
{error, Node, {badrpc, R}} ->
Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])),
{500, #{code => <<"NODE_DOWN">>, message => Message}};
Result ->
{200, Result}
end.
format(Items) when is_list(Items) ->
[format(Item) || Item <- Items];
format({{Subscriber, Topic}, Options}) ->
format({Subscriber, Topic, Options});
format({_Subscriber, Topic, Options = #{share := Group}}) ->
QoS = maps:get(qos, Options),
#{
@ -153,19 +186,30 @@ format({_Subscriber, Topic, Options}) ->
query(Tab, {Qs, []}, Continuation, Limit) ->
Ms = qs2ms(Qs),
emqx_mgmt_api:select_table_with_count( Tab, Ms
, Continuation, Limit, fun format/1);
emqx_mgmt_api:select_table_with_count(
Tab,
Ms,
Continuation,
Limit,
fun format/1
);
query(Tab, {Qs, Fuzzy}, Continuation, Limit) ->
Ms = qs2ms(Qs),
FuzzyFilterFun = fuzzy_filter_fun(Fuzzy),
emqx_mgmt_api:select_table_with_count( Tab, {Ms, FuzzyFilterFun}
, Continuation, Limit, fun format/1).
emqx_mgmt_api:select_table_with_count(
Tab,
{Ms, FuzzyFilterFun},
Continuation,
Limit,
fun format/1
).
fuzzy_filter_fun(Fuzzy) ->
fun(MsRaws) when is_list(MsRaws) ->
lists:filter( fun(E) -> run_fuzzy_filter(E, Fuzzy) end
, MsRaws)
lists:filter(
fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
MsRaws
)
end.
run_fuzzy_filter(_, []) ->

73
apps/emqx_management/src/emqx_mgmt_api_topics.erl
View File

@ -22,23 +22,24 @@
%% API
-behaviour(minirest_api).
-export([ api_spec/0
, paths/0
, schema/1
, fields/1
]).
-export([
api_spec/0,
paths/0,
schema/1,
fields/1
]).
-export([ topics/2
, topic/2
]).
-export([
topics/2,
topic/2
]).
-export([ query/4]).
-export([query/4]).
-define(TOPIC_NOT_FOUND, 'TOPIC_NOT_FOUND').
-define(TOPICS_QUERY_SCHEMA, [{<<"topic">>, binary}, {<<"node">>, atom}]).
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).
@ -73,19 +74,23 @@ schema("/topics/:topic") ->
responses => #{
200 => hoconsc:mk(hoconsc:ref(topic), #{}),
404 =>
emqx_dashboard_swagger:error_codes(['TOPIC_NOT_FOUND'],<<"Topic not found">>)
emqx_dashboard_swagger:error_codes(['TOPIC_NOT_FOUND'], <<"Topic not found">>)
}
}
}.
fields(topic) ->
[
{topic, hoconsc:mk(binary(), #{
desc => <<"Topic Name">>,
required => true})},
{node, hoconsc:mk(binary(), #{
desc => <<"Node">>,
required => true})}
{topic,
hoconsc:mk(binary(), #{
desc => <<"Topic Name">>,
required => true
})},
{node,
hoconsc:mk(binary(), #{
desc => <<"Node">>,
required => true
})}
];
fields(meta) ->
emqx_dashboard_swagger:fields(page) ++
@ -103,9 +108,19 @@ topic(get, #{bindings := Bindings}) ->
%%%==============================================================================================
%% api apply
do_list(Params) ->
Response = emqx_mgmt_api:node_query(
node(), Params, emqx_route, ?TOPICS_QUERY_SCHEMA, {?MODULE, query}),
emqx_mgmt_util:generate_response(Response).
case
emqx_mgmt_api:node_query(
node(), Params, emqx_route, ?TOPICS_QUERY_SCHEMA, {?MODULE, query}
)
of
{error, page_limit_invalid} ->
{400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
{error, Node, {badrpc, R}} ->
Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])),
{500, #{code => <<"NODE_DOWN">>, message => Message}};
Response ->
{200, Response}
end.
lookup(#{topic := Topic}) ->
case emqx_router:lookup_routes(Topic) of
@ -121,16 +136,18 @@ generate_topic(Params = #{<<"topic">> := Topic}) ->
Params#{<<"topic">> => uri_string:percent_decode(Topic)};
generate_topic(Params = #{topic := Topic}) ->
Params#{topic => uri_string:percent_decode(Topic)};
generate_topic(Params) -> Params.
generate_topic(Params) ->
Params.
query(Tab, {Qs, _}, Continuation, Limit) ->
Ms = qs2ms(Qs, [{{route, '_', '_'}, [], ['$_']}]),
emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit, fun format/1).
qs2ms([], Res) -> Res;
qs2ms([{topic,'=:=', T} | Qs], [{{route, _, N}, [], ['$_']}]) ->
qs2ms([], Res) ->
Res;
qs2ms([{topic, '=:=', T} | Qs], [{{route, _, N}, [], ['$_']}]) ->
qs2ms(Qs, [{{route, T, N}, [], ['$_']}]);
qs2ms([{node,'=:=', N} | Qs], [{{route, T, _}, [], ['$_']}]) ->
qs2ms([{node, '=:=', N} | Qs], [{{route, T, _}, [], ['$_']}]) ->
qs2ms(Qs, [{{route, T, N}, [], ['$_']}]).
format(#route{topic = Topic, dest = {_, Node}}) ->
@ -140,7 +157,8 @@ format(#route{topic = Topic, dest = Node}) ->
topic_param(In) ->
{
topic, hoconsc:mk(binary(), #{
topic,
hoconsc:mk(binary(), #{
desc => <<"Topic Name">>,
in => In,
required => (In == path),
@ -148,9 +166,10 @@ topic_param(In) ->
})
}.
node_param()->
node_param() ->
{
node, hoconsc:mk(binary(), #{
node,
hoconsc:mk(binary(), #{
desc => <<"Node Name">>,
in => query,
required => false,

461
apps/emqx_management/src/emqx_mgmt_api_trace.erl
View File

@ -21,26 +21,29 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx/include/logger.hrl").
-export([ api_spec/0
, fields/1
, paths/0
, schema/1
, namespace/0
]).
-export([
api_spec/0,
fields/1,
paths/0,
schema/1,
namespace/0
]).
-export([ trace/2
, delete_trace/2
, update_trace/2
, download_trace_log/2
, stream_log_file/2
]).
-export([
trace/2,
delete_trace/2,
update_trace/2,
download_trace_log/2,
stream_log_file/2
]).
-export([validate_name/1]).
%% for rpc
-export([ read_trace_file/3
, get_trace_size/0
]).
-export([
read_trace_file/3,
get_trace_size/0
]).
-define(TO_BIN(_B_), iolist_to_binary(_B_)).
-define(NOT_FOUND(N), {404, #{code => 'NOT_FOUND', message => ?TO_BIN([N, " NOT FOUND"])}}).
@ -53,7 +56,6 @@ api_spec() ->
paths() ->
["/trace", "/trace/:name/stop", "/trace/:name/download", "/trace/:name/log", "/trace/:name"].
schema("/trace") ->
#{
'operationId' => trace,
@ -68,9 +70,14 @@ schema("/trace") ->
'requestBody' => delete([status, log_size], fields(trace)),
responses => #{
200 => hoconsc:ref(trace),
400 => emqx_dashboard_swagger:error_codes(['ALREADY_EXISTS',
'DUPLICATE_CONDITION', 'INVALID_PARAMS'],
<<"trace name already exists">>)
400 => emqx_dashboard_swagger:error_codes(
[
'ALREADY_EXISTS',
'DUPLICATE_CONDITION',
'INVALID_PARAMS'
],
<<"trace name already exists">>
)
}
},
delete => #{
@ -112,12 +119,13 @@ schema("/trace/:name/download") ->
parameters => [hoconsc:ref(name)],
responses => #{
200 =>
#{description => "A trace zip file",
content => #{
'application/octet-stream' =>
#{schema => #{type => "string", format => "binary"}}
#{
description => "A trace zip file",
content => #{
'application/octet-stream' =>
#{schema => #{type => "string", format => "binary"}}
}
}
}
}
}
};
@ -134,92 +142,151 @@ schema("/trace/:name/log") ->
],
responses => #{
200 =>
[
{items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})}
| fields(bytes) ++ fields(position)
]
[
{items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})}
| fields(bytes) ++ fields(position)
]
}
}
}.
fields(trace) ->
[
{name, hoconsc:mk(binary(),
#{desc => "Unique and format by [a-zA-Z0-9-_]",
validator => fun ?MODULE:validate_name/1,
required => true,
example => <<"EMQX-TRACE-1">>})},
{type, hoconsc:mk(hoconsc:enum([clientid, topic, ip_address]),
#{desc => """Filter type""",
required => true,
example => <<"clientid">>})},
{topic, hoconsc:mk(binary(),
#{desc => """support mqtt wildcard topic.""",
required => false,
example => <<"/dev/#">>})},
{clientid, hoconsc:mk(binary(),
#{desc => """mqtt clientid.""",
required => false,
example => <<"dev-001">>})},
{name,
hoconsc:mk(
binary(),
#{
desc => "Unique and format by [a-zA-Z0-9-_]",
validator => fun ?MODULE:validate_name/1,
required => true,
example => <<"EMQX-TRACE-1">>
}
)},
{type,
hoconsc:mk(
hoconsc:enum([clientid, topic, ip_address]),
#{
desc => "" "Filter type" "",
required => true,
example => <<"clientid">>
}
)},
{topic,
hoconsc:mk(
binary(),
#{
desc => "" "support mqtt wildcard topic." "",
required => false,
example => <<"/dev/#">>
}
)},
{clientid,
hoconsc:mk(
binary(),
#{
desc => "" "mqtt clientid." "",
required => false,
example => <<"dev-001">>
}
)},
%% TODO add ip_address type in emqx_schema.erl
{ip_address, hoconsc:mk(binary(),
#{desc => "client ip address",
required => false,
example => <<"127.0.0.1">>
})},
{status, hoconsc:mk(hoconsc:enum([running, stopped, waiting]),
#{desc => "trace status",
required => false,
example => running
})},
{start_at, hoconsc:mk(emqx_datetime:epoch_second(),
#{desc => "rfc3339 timestamp or epoch second",
required => false,
example => <<"2021-11-04T18:17:38+08:00">>
})},
{end_at, hoconsc:mk(emqx_datetime:epoch_second(),
#{desc => "rfc3339 timestamp or epoch second",
required => false,
example => <<"2021-11-05T18:17:38+08:00">>
})},
{log_size, hoconsc:mk(hoconsc:array(map()),
#{desc => "trace log size",
example => [#{<<"node">> => <<"emqx@127.0.0.1">>, <<"size">> => 1024}],
required => false})}
{ip_address,
hoconsc:mk(
binary(),
#{
desc => "client ip address",
required => false,
example => <<"127.0.0.1">>
}
)},
{status,
hoconsc:mk(
hoconsc:enum([running, stopped, waiting]),
#{
desc => "trace status",
required => false,
example => running
}
)},
{start_at,
hoconsc:mk(
emqx_datetime:epoch_second(),
#{
desc => "rfc3339 timestamp or epoch second",
required => false,
example => <<"2021-11-04T18:17:38+08:00">>
}
)},
{end_at,
hoconsc:mk(
emqx_datetime:epoch_second(),
#{
desc => "rfc3339 timestamp or epoch second",
required => false,
example => <<"2021-11-05T18:17:38+08:00">>
}
)},
{log_size,
hoconsc:mk(
hoconsc:array(map()),
#{
desc => "trace log size",
example => [#{<<"node">> => <<"emqx@127.0.0.1">>, <<"size">> => 1024}],
required => false
}
)}
];
fields(name) ->
[{name, hoconsc:mk(binary(),
#{
desc => <<"[a-zA-Z0-9-_]">>,
example => <<"EMQX-TRACE-1">>,
in => path,
validator => fun ?MODULE:validate_name/1
})}
[
{name,
hoconsc:mk(
binary(),
#{
desc => <<"[a-zA-Z0-9-_]">>,
example => <<"EMQX-TRACE-1">>,
in => path,
validator => fun ?MODULE:validate_name/1
}
)}
];
fields(node) ->
[{node, hoconsc:mk(binary(),
#{
desc => "Node name",
in => query,
required => false
})}];
[
{node,
hoconsc:mk(
binary(),
#{
desc => "Node name",
in => query,
required => false
}
)}
];
fields(bytes) ->
[{bytes, hoconsc:mk(integer(),
#{
desc => "Maximum number of bytes to store in request",
in => query,
required => false,
default => 1000
})}];
[
{bytes,
hoconsc:mk(
integer(),
#{
desc => "Maximum number of bytes to store in request",
in => query,
required => false,
default => 1000
}
)}
];
fields(position) ->
[{position, hoconsc:mk(integer(),
#{
desc => "Offset from the current trace position.",
in => query,
required => false,
default => 0
})}].
[
{position,
hoconsc:mk(
integer(),
#{
desc => "Offset from the current trace position.",
in => query,
required => false,
default => 0
}
)}
].
-define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$").
@ -231,7 +298,8 @@ validate_name(Name) ->
nomatch -> {error, "Name should be " ?NAME_RE};
_ -> ok
end;
false -> {error, "Name Length must =< 256"}
false ->
{error, "Name Length must =< 256"}
end.
delete(Keys, Fields) ->
@ -239,32 +307,48 @@ delete(Keys, Fields) ->
trace(get, _Params) ->
case emqx_trace:list() of
[] -> {200, []};
[] ->
{200, []};
List0 ->
List = lists:sort(fun(#{start_at := A}, #{start_at := B}) -> A > B end,
emqx_trace:format(List0)),
List = lists:sort(
fun(#{start_at := A}, #{start_at := B}) -> A > B end,
emqx_trace:format(List0)
),
Nodes = mria_mnesia:running_nodes(),
TraceSize = wrap_rpc(emqx_mgmt_trace_proto_v1:get_trace_size(Nodes)),
AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize),
Now = erlang:system_time(second),
Traces =
lists:map(fun(Trace = #{name := Name, start_at := Start,
end_at := End, enable := Enable, type := Type, filter := Filter}) ->
FileName = emqx_trace:filename(Name, Start),
LogSize = collect_file_size(Nodes, FileName, AllFileSize),
Trace0 = maps:without([enable, filter], Trace),
Trace0#{log_size => LogSize
, Type => iolist_to_binary(Filter)
, start_at => list_to_binary(calendar:system_time_to_rfc3339(Start))
, end_at => list_to_binary(calendar:system_time_to_rfc3339(End))
, status => status(Enable, Start, End, Now)
}
end, List),
lists:map(
fun(
Trace = #{
name := Name,
start_at := Start,
end_at := End,
enable := Enable,
type := Type,
filter := Filter
}
) ->
FileName = emqx_trace:filename(Name, Start),
LogSize = collect_file_size(Nodes, FileName, AllFileSize),
Trace0 = maps:without([enable, filter], Trace),
Trace0#{
log_size => LogSize,
Type => iolist_to_binary(Filter),
start_at => list_to_binary(calendar:system_time_to_rfc3339(Start)),
end_at => list_to_binary(calendar:system_time_to_rfc3339(End)),
status => status(Enable, Start, End, Now)
}
end,
List
),
{200, Traces}
end;
trace(post, #{body := Param}) ->
case emqx_trace:create(Param) of
{ok, Trace0} -> {200, format_trace(Trace0)};
{ok, Trace0} ->
{200, format_trace(Trace0)};
{error, {already_existed, Name}} ->
{400, #{
code => 'ALREADY_EXISTS',
@ -287,18 +371,27 @@ trace(delete, _Param) ->
format_trace(Trace0) ->
[
#{start_at := Start, end_at := End,
enable := Enable, type := Type, filter := Filter} = Trace1
#{
start_at := Start,
end_at := End,
enable := Enable,
type := Type,
filter := Filter
} = Trace1
] = emqx_trace:format([Trace0]),
Now = erlang:system_time(second),
LogSize = lists:foldl(fun(Node, Acc) -> Acc#{Node => 0} end, #{},
mria_mnesia:running_nodes()),
LogSize = lists:foldl(
fun(Node, Acc) -> Acc#{Node => 0} end,
#{},
mria_mnesia:running_nodes()
),
Trace2 = maps:without([enable, filter], Trace1),
Trace2#{log_size => LogSize
, Type => iolist_to_binary(Filter)
, start_at => list_to_binary(calendar:system_time_to_rfc3339(Start))
, end_at => list_to_binary(calendar:system_time_to_rfc3339(End))
, status => status(Enable, Start, End, Now)
Trace2#{
log_size => LogSize,
Type => iolist_to_binary(Filter),
start_at => list_to_binary(calendar:system_time_to_rfc3339(Start)),
end_at => list_to_binary(calendar:system_time_to_rfc3339(End)),
status => status(Enable, Start, End, Now)
}.
delete_trace(delete, #{bindings := #{name := Name}}) ->
@ -334,25 +427,34 @@ download_trace_log(get, #{bindings := #{name := Name}}) ->
<<"content-disposition">> => iolist_to_binary("attachment; filename=" ++ ZipName)
},
{200, Headers, {file_binary, ZipName, Binary}};
{error, not_found} -> ?NOT_FOUND(Name)
{error, not_found} ->
?NOT_FOUND(Name)
end.
group_trace_file(ZipDir, TraceLog, TraceFiles) ->
lists:foldl(fun(Res, Acc) ->
case Res of
{ok, Node, Bin} ->
FileName = Node ++ "-" ++ TraceLog,
ZipName = filename:join([ZipDir, FileName]),
case file:write_file(ZipName, Bin) of
ok -> [FileName | Acc];
_ -> Acc
end;
{error, Node, Reason} ->
?SLOG(error, #{msg => "download_trace_log_error", node => Node,
log => TraceLog, reason => Reason}),
Acc
end
end, [], TraceFiles).
lists:foldl(
fun(Res, Acc) ->
case Res of
{ok, Node, Bin} ->
FileName = Node ++ "-" ++ TraceLog,
ZipName = filename:join([ZipDir, FileName]),
case file:write_file(ZipName, Bin) of
ok -> [FileName | Acc];
_ -> Acc
end;
{error, Node, Reason} ->
?SLOG(error, #{
msg => "download_trace_log_error",
node => Node,
log => TraceLog,
reason => Reason
}),
Acc
end
end,
[],
TraceFiles
).
collect_trace_file(TraceLog) ->
Nodes = mria_mnesia:running_nodes(),
@ -376,18 +478,25 @@ stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) ->
{eof, Size} ->
Meta = #{<<"position">> => Size, <<"bytes">> => Bytes},
{200, #{meta => Meta, items => <<"">>}};
{error, enoent} -> %% the waiting trace should return "" not error.
%% the waiting trace should return "" not error.
{error, enoent} ->
Meta = #{<<"position">> => Position, <<"bytes">> => Bytes},
{200, #{meta => Meta, items => <<"">>}};
{error, Reason} ->
?SLOG(error, #{msg => "read_file_failed",
node => Node, name => Name, reason => Reason,
position => Position, bytes => Bytes}),
?SLOG(error, #{
msg => "read_file_failed",
node => Node,
name => Name,
reason => Reason,
position => Position,
bytes => Bytes
}),
{400, #{code => 'READ_FILE_ERROR', message => Reason}};
{badrpc, nodedown} ->
{400, #{code => 'RPC_ERROR', message => "BadRpc node down"}}
end;
{error, not_found} -> {400, #{code => 'NODE_ERROR', message => <<"Node not found">>}}
{error, not_found} ->
{400, #{code => 'NODE_ERROR', message => <<"Node not found">>}}
end.
-spec get_trace_size() -> #{{node(), file:name_all()} => non_neg_integer()}.
@ -396,23 +505,31 @@ get_trace_size() ->
Node = node(),
case file:list_dir(TraceDir) of
{ok, AllFiles} ->
lists:foldl(fun(File, Acc) ->
FullFileName = filename:join(TraceDir, File),
Acc#{{Node, File} => filelib:file_size(FullFileName)}
end, #{}, lists:delete("zip", AllFiles));
_ -> #{}
lists:foldl(
fun(File, Acc) ->
FullFileName = filename:join(TraceDir, File),
Acc#{{Node, File} => filelib:file_size(FullFileName)}
end,
#{},
lists:delete("zip", AllFiles)
);
_ ->
#{}
end.
%% this is an rpc call for stream_log_file/2
-spec read_trace_file( binary()
, non_neg_integer()
, non_neg_integer()
) -> {ok, binary()}
| {error, _}
| {eof, non_neg_integer()}.
-spec read_trace_file(
binary(),
non_neg_integer(),
non_neg_integer()
) ->
{ok, binary()}
| {error, _}
| {eof, non_neg_integer()}.
read_trace_file(Name, Position, Limit) ->
case emqx_trace:get_trace_filename(Name) of
{error, _} = Error -> Error;
{error, _} = Error ->
Error;
{ok, TraceFile} ->
TraceDir = emqx_trace:trace_dir(),
TracePath = filename:join([TraceDir, TraceFile]),
@ -423,13 +540,16 @@ read_file(Path, Offset, Bytes) ->
case file:open(Path, [read, raw, binary]) of
{ok, IoDevice} ->
try
_ = case Offset of
_ =
case Offset of
0 -> ok;
_ -> file:position(IoDevice, {bof, Offset})
end,
case file:read(IoDevice, Bytes) of
{ok, Bin} -> {ok, Bin};
{error, Reason} -> {error, Reason};
{ok, Bin} ->
{ok, Bin};
{error, Reason} ->
{error, Reason};
eof ->
{ok, #file_info{size = Size}} = file:read_file_info(IoDevice),
{eof, Size}
@ -437,20 +557,27 @@ read_file(Path, Offset, Bytes) ->
after
file:close(IoDevice)
end;
{error, Reason} -> {error, Reason}
{error, Reason} ->
{error, Reason}
end.
to_node(Node) ->
try {ok, binary_to_existing_atom(Node)}
catch _:_ ->
{error, not_found}
try
{ok, binary_to_existing_atom(Node)}
catch
_:_ ->
{error, not_found}
end.
collect_file_size(Nodes, FileName, AllFiles) ->
lists:foldl(fun(Node, Acc) ->
Size = maps:get({Node, FileName}, AllFiles, 0),
Acc#{Node => Size}
end, #{}, Nodes).
lists:foldl(
fun(Node, Acc) ->
Size = maps:get({Node, FileName}, AllFiles, 0),
Acc#{Node => Size}
end,
#{},
Nodes
).
status(false, _Start, _End, _Now) -> <<"stopped">>;
status(true, Start, _End, Now) when Now < Start -> <<"waiting">>;

7
apps/emqx_management/src/emqx_mgmt_app.erl
View File

@ -20,9 +20,10 @@
-define(APP, emqx_management).
-export([ start/2
, stop/1
]).
-export([
start/2,
stop/1
]).
-include("emqx_mgmt.hrl").

73
apps/emqx_management/src/emqx_mgmt_auth.erl
View File

@ -20,14 +20,15 @@
-export([mnesia/1]).
-boot_mnesia({mnesia, [boot]}).
-export([ create/4
, read/1
, update/4
, delete/1
, list/0
]).
-export([
create/4,
read/1,
update/4,
delete/1,
list/0
]).
-export([ authorize/3 ]).
-export([authorize/3]).
-define(APP, emqx_app).
@ -39,7 +40,7 @@
desc = <<>> :: binary() | '_',
expired_at = 0 :: integer() | undefined | '_',
created_at = 0 :: integer() | '_'
}).
}).
mnesia(boot) ->
ok = mria:create_table(?APP, [
@ -47,7 +48,8 @@ mnesia(boot) ->
{rlog_shard, ?COMMON_SHARD},
{storage, disc_copies},
{record_name, ?APP},
{attributes, record_info(fields, ?APP)}]).
{attributes, record_info(fields, ?APP)}
]).
create(Name, Enable, ExpiredAt, Desc) ->
case mnesia:table_info(?APP, size) < 30 of
@ -61,13 +63,14 @@ read(Name) ->
[] -> mnesia:abort(not_found);
[App] -> to_map(App)
end
end,
end,
trans(Fun).
update(Name, Enable, ExpiredAt, Desc) ->
Fun = fun() ->
case mnesia:read(?APP, Name, write) of
[] -> mnesia:abort(not_found);
[] ->
mnesia:abort(not_found);
[App0 = #?APP{enable = Enable0, desc = Desc0}] ->
App =
App0#?APP{
@ -78,22 +81,25 @@ update(Name, Enable, ExpiredAt, Desc) ->
ok = mnesia:write(App),
to_map(App)
end
end,
end,
trans(Fun).
delete(Name) ->
Fun = fun() ->
case mnesia:read(?APP, Name) of
[] -> mnesia:abort(not_found);
[_App] -> mnesia:delete({?APP, Name}) end
end,
[_App] -> mnesia:delete({?APP, Name})
end
end,
trans(Fun).
list() ->
to_map(ets:match_object(?APP, #?APP{_ = '_'})).
authorize(<<"/api/v5/users", _/binary>>, _ApiKey, _ApiSecret) -> {error, <<"not_allowed">>};
authorize(<<"/api/v5/api_key", _/binary>>, _ApiKey, _ApiSecret) -> {error, <<"not_allowed">>};
authorize(<<"/api/v5/users", _/binary>>, _ApiKey, _ApiSecret) ->
{error, <<"not_allowed">>};
authorize(<<"/api/v5/api_key", _/binary>>, _ApiKey, _ApiSecret) ->
{error, <<"not_allowed">>};
authorize(_Path, ApiKey, ApiSecret) ->
Now = erlang:system_time(second),
case find_by_api_key(ApiKey) of
@ -102,28 +108,35 @@ authorize(_Path, ApiKey, ApiSecret) ->
ok -> ok;
error -> {error, "secret_error"}
end;
{ok, true, _ExpiredAt, _SecretHash} -> {error, "secret_expired"};
{ok, false, _ExpiredAt, _SecretHash} -> {error, "secret_disable"};
{error, Reason} -> {error, Reason}
{ok, true, _ExpiredAt, _SecretHash} ->
{error, "secret_expired"};
{ok, false, _ExpiredAt, _SecretHash} ->
{error, "secret_disable"};
{error, Reason} ->
{error, Reason}
end.
find_by_api_key(ApiKey) ->
Fun = fun() -> mnesia:match_object(#?APP{api_key = ApiKey, _ = '_'}) end,
Fun = fun() -> mnesia:match_object(#?APP{api_key = ApiKey, _ = '_'}) end,
case trans(Fun) of
{ok, [#?APP{api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt}]} ->
{ok, Enable, ExpiredAt, SecretHash};
_ -> {error, "not_found"}
_ ->
{error, "not_found"}
end.
ensure_not_undefined(undefined, Old) -> Old;
ensure_not_undefined(New, _Old) -> New.
to_map(Apps)when is_list(Apps) ->
to_map(Apps) when is_list(Apps) ->
Fields = record_info(fields, ?APP),
lists:map(fun(Trace0 = #?APP{}) ->
[_ | Values] = tuple_to_list(Trace0),
maps:remove(api_secret_hash, maps:from_list(lists:zip(Fields, Values)))
end, Apps);
lists:map(
fun(Trace0 = #?APP{}) ->
[_ | Values] = tuple_to_list(Trace0),
maps:remove(api_secret_hash, maps:from_list(lists:zip(Fields, Values)))
end,
Apps
);
to_map(App0) ->
[App] = to_map([App0]),
App.
@ -149,16 +162,18 @@ create_app(Name, Enable, ExpiredAt, Desc) ->
create_app(App = #?APP{api_key = ApiKey, name = Name}) ->
trans(fun() ->
case mnesia:read(?APP, Name) of
[_] -> mnesia:abort(name_already_existed);
[_] ->
mnesia:abort(name_already_existed);
[] ->
case mnesia:match_object(?APP, #?APP{api_key = ApiKey, _ = '_'}, read) of
[] ->
ok = mnesia:write(App),
to_map(App);
_ -> mnesia:abort(api_key_already_existed)
_ ->
mnesia:abort(api_key_already_existed)
end
end
end).
end).
trans(Fun) ->
case mria:transaction(?COMMON_SHARD, Fun) of

5
apps/emqx_management/src/emqx_mgmt_sup.erl
View File

@ -23,8 +23,7 @@
-export([init/1]).
start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
init([]) ->
{ok, {{one_for_one, 1, 5}, []}}.
{ok, {{one_for_one, 1, 5}, []}}.

239
apps/emqx_management/src/emqx_mgmt_util.erl
View File

@ -16,41 +16,40 @@
-module(emqx_mgmt_util).
-export([ strftime/1
, datetime/1
, kmg/1
, ntoa/1
, merge_maps/2
, batch_operation/3
]).
-export([ bad_request/0
, bad_request/1
, properties/1
, page_params/0
, schema/1
, schema/2
, object_schema/1
, object_schema/2
, array_schema/1
, array_schema/2
, object_array_schema/1
, object_array_schema/2
, page_schema/1
, page_object_schema/1
, error_schema/1
, error_schema/2
, batch_schema/1
]).
-export([generate_response/1]).
-export([
strftime/1,
datetime/1,
kmg/1,
ntoa/1,
merge_maps/2,
batch_operation/3
]).
-export([
bad_request/0,
bad_request/1,
properties/1,
page_params/0,
schema/1,
schema/2,
object_schema/1,
object_schema/2,
array_schema/1,
array_schema/2,
object_array_schema/1,
object_array_schema/2,
page_schema/1,
page_object_schema/1,
error_schema/1,
error_schema/2,
batch_schema/1
]).
-export([urldecode/1]).
-define(KB, 1024).
-define(MB, (1024*1024)).
-define(GB, (1024*1024*1024)).
-define(MB, (1024 * 1024)).
-define(GB, (1024 * 1024 * 1024)).
%%--------------------------------------------------------------------
%% Strftime
@ -58,17 +57,17 @@
strftime({MegaSecs, Secs, _MicroSecs}) ->
strftime(datetime(MegaSecs * 1000000 + Secs));
strftime(Secs) when is_integer(Secs) ->
strftime(datetime(Secs));
strftime({{Y,M,D}, {H,MM,S}}) ->
strftime({{Y, M, D}, {H, MM, S}}) ->
lists:flatten(
io_lib:format(
"~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", [Y, M, D, H, MM, S])).
"~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", [Y, M, D, H, MM, S]
)
).
datetime(Timestamp) when is_integer(Timestamp) ->
Epoch = calendar:datetime_to_gregorian_seconds({{1970,1,1}, {0,0,0}}),
Epoch = calendar:datetime_to_gregorian_seconds({{1970, 1, 1}, {0, 0, 0}}),
Universal = calendar:gregorian_seconds_to_datetime(Timestamp + Epoch),
calendar:universal_time_to_local_time(Universal).
@ -83,19 +82,27 @@ kmg(Byte) ->
kmg(F, S) ->
iolist_to_binary(io_lib:format("~.2f~ts", [F, S])).
ntoa({0,0,0,0,0,16#ffff,AB,CD}) ->
ntoa({0, 0, 0, 0, 0, 16#ffff, AB, CD}) ->
inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256});
ntoa(IP) ->
inet_parse:ntoa(IP).
merge_maps(Default, New) ->
maps:fold(fun(K, V, Acc) ->
case maps:get(K, Acc, undefined) of
OldV when is_map(OldV),
is_map(V) -> Acc#{K => merge_maps(OldV, V)};
_ -> Acc#{K => V}
end
end, Default, New).
maps:fold(
fun(K, V, Acc) ->
case maps:get(K, Acc, undefined) of
OldV when
is_map(OldV),
is_map(V)
->
Acc#{K => merge_maps(OldV, V)};
_ ->
Acc#{K => V}
end
end,
Default,
New
).
urldecode(S) ->
emqx_http_lib:uri_decode(S).
@ -126,8 +133,13 @@ array_schema(Schema, Desc) ->
object_array_schema(Properties) when is_map(Properties) ->
json_content_schema(#{type => array, items => #{type => object, properties => Properties}}).
object_array_schema(Properties, Desc) ->
json_content_schema(#{type => array,
items => #{type => object, properties => Properties}}, Desc).
json_content_schema(
#{
type => array,
items => #{type => object, properties => Properties}
},
Desc
).
page_schema(Ref) when is_atom(Ref) ->
page_schema(minirest:ref(atom_to_binary(Ref, utf8)));
@ -137,9 +149,11 @@ page_schema(Schema) ->
properties => #{
meta => #{
type => object,
properties => properties([{page, integer},
{limit, integer},
{count, integer}])
properties => properties([
{page, integer},
{limit, integer},
{count, integer}
])
},
data => #{
type => array,
@ -158,8 +172,10 @@ error_schema(Description) ->
error_schema(Description, Enum) ->
Schema = #{
type => object,
properties => properties([{code, string, <<>>, Enum},
{message, string}])
properties => properties([
{code, string, <<>>, Enum},
{message, string}
])
},
json_content_schema(Schema, Description).
@ -171,20 +187,28 @@ batch_schema(DefName) when is_binary(DefName) ->
properties => #{
success => #{
type => integer,
description => <<"Success count">>},
description => <<"Success count">>
},
failed => #{
type => integer,
description => <<"Failed count">>},
description => <<"Failed count">>
},
detail => #{
type => array,
description => <<"Failed object & reason">>,
items => #{
type => object,
properties =>
#{
data => minirest:ref(DefName),
reason => #{
type => <<"string">>}}}}}},
#{
data => minirest:ref(DefName),
reason => #{
type => <<"string">>
}
}
}
}
}
},
json_content_schema(Schema).
json_content_schema(Schema) when is_map(Schema) ->
@ -214,7 +238,7 @@ batch_operation(Module, Function, [Args | ArgsList], Failed) ->
case erlang:apply(Module, Function, Args) of
ok ->
batch_operation(Module, Function, ArgsList, Failed);
{error ,Reason} ->
{error, Reason} ->
batch_operation(Module, Function, ArgsList, [{Args, Reason} | Failed])
end.
@ -227,52 +251,77 @@ properties([Key | Props], Acc) when is_atom(Key) ->
properties([{Key, Type} | Props], Acc) ->
properties(Props, maps:put(Key, #{type => Type}, Acc));
properties([{Key, object, Props1} | Props], Acc) ->
properties(Props, maps:put(Key, #{type => object,
properties => properties(Props1)}, Acc));
properties(
Props,
maps:put(
Key,
#{
type => object,
properties => properties(Props1)
},
Acc
)
);
properties([{Key, {array, object}, Props1} | Props], Acc) ->
properties(Props, maps:put(Key, #{type => array,
items => #{type => object,
properties => properties(Props1)
}}, Acc));
properties(
Props,
maps:put(
Key,
#{
type => array,
items => #{
type => object,
properties => properties(Props1)
}
},
Acc
)
);
properties([{Key, {array, Type}, Desc} | Props], Acc) ->
properties(Props, maps:put(Key, #{type => array,
items => #{type => Type},
description => Desc}, Acc));
properties(
Props,
maps:put(
Key,
#{
type => array,
items => #{type => Type},
description => Desc
},
Acc
)
);
properties([{Key, Type, Desc} | Props], Acc) ->
properties(Props, maps:put(Key, #{type => Type, description => Desc}, Acc));
properties([{Key, Type, Desc, Enum} | Props], Acc) ->
properties(Props, maps:put(Key, #{type => Type,
description => Desc,
enum => Enum}, Acc)).
properties(
Props,
maps:put(
Key,
#{
type => Type,
description => Desc,
enum => Enum
},
Acc
)
).
page_params() ->
[#{
name => page,
in => query,
description => <<"Page">>,
schema => #{type => integer, default => 1}
},
#{
name => limit,
in => query,
description => <<"Page size">>,
schema => #{type => integer, default => emqx_mgmt:max_row_limit()}
}].
[
#{
name => page,
in => query,
description => <<"Page">>,
schema => #{type => integer, default => 1}
},
#{
name => limit,
in => query,
description => <<"Page size">>,
schema => #{type => integer, default => emqx_mgmt:max_row_limit()}
}
].
bad_request() ->
bad_request(<<"Bad Request">>).
bad_request(Desc) ->
object_schema(properties([{message, string}, {code, string}]), Desc).
%%%==============================================================================================
%% Response util
generate_response(QueryResult) ->
case QueryResult of
{error, page_limit_invalid} ->
{400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
{error, Node, {badrpc, R}} ->
Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, R])),
{500, #{code => <<"NODE_DOWN">>, message => Message}};
Response ->
{200, Response}
end.

13
apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v1.erl
View File

@ -17,12 +17,13 @@
-behaviour(emqx_bpapi).
-export([ introduced_in/0
, get_plugins/0
, install_package/2
, describe_package/1
, delete_package/1
, ensure_action/2
-export([
introduced_in/0,
get_plugins/0,
install_package/2,
describe_package/1,
delete_package/1,
ensure_action/2
]).
-include_lib("emqx/include/bpapi.hrl").

28
apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v1.erl
View File

@ -18,12 +18,13 @@
-behaviour(emqx_bpapi).
-export([ introduced_in/0
-export([
introduced_in/0,
, trace_file/2
, get_trace_size/1
, read_trace_file/4
]).
trace_file/2,
get_trace_size/1,
read_trace_file/4
]).
-include_lib("emqx/include/bpapi.hrl").
@ -31,21 +32,22 @@ introduced_in() ->
"5.0.0".
-spec get_trace_size([node()]) ->
emqx_rpc:multicall_result(#{{node(), file:name_all()} => non_neg_integer()}).
emqx_rpc:multicall_result(#{{node(), file:name_all()} => non_neg_integer()}).
get_trace_size(Nodes) ->
rpc:multicall(Nodes, emqx_mgmt_api_trace, get_trace_size, [], 30000).
-spec trace_file([node()], file:name_all()) ->
emqx_rpc:multicall_result(
{ok, Node :: list(), Binary :: binary()} |
{error, Node :: list(), Reason :: term()}).
emqx_rpc:multicall_result(
{ok, Node :: list(), Binary :: binary()}
| {error, Node :: list(), Reason :: term()}
).
trace_file(Nodes, File) ->
rpc:multicall(Nodes, emqx_trace, trace_file, [File], 60000).
-spec read_trace_file(node(), binary(), non_neg_integer(), non_neg_integer()) ->
{ok, binary()}
| {error, _}
| {eof, non_neg_integer()}
| {badrpc, _}.
{ok, binary()}
| {error, _}
| {eof, non_neg_integer()}
| {badrpc, _}.
read_trace_file(Node, Name, Position, Limit) ->
rpc:call(Node, emqx_mgmt_api_trace, read_trace_file, [Name, Position, Limit]).

5
apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl
View File

@ -15,7 +15,6 @@
%%--------------------------------------------------------------------
-module(emqx_mgmt_api_alarms_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
@ -55,8 +54,8 @@ get_alarms(AssertCount, Activated) ->
Headers = emqx_mgmt_api_test_util:auth_header_(),
{ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, Qs, Headers),
Data = emqx_json:decode(Response, [return_maps]),
Meta = maps:get(<<"meta">>, Data),
Page = maps:get(<<"page">>, Meta),
Meta = maps:get(<<"meta">>, Data),
Page = maps:get(<<"page">>, Meta),
Limit = maps:get(<<"limit">>, Meta),
Count = maps:get(<<"count">>, Meta),
?assertEqual(Page, 1),

67
apps/emqx_management/test/emqx_mgmt_api_app_SUITE.erl
View File

@ -22,10 +22,11 @@
all() -> [{group, parallel}, {group, sequence}].
suite() -> [{timetrap, {minutes, 1}}].
groups() -> [
{parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]},
{sequence, [], [t_create_failed]}
].
groups() ->
[
{parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]},
{sequence, [], [t_create_failed]}
].
init_per_suite(Config) ->
emqx_mgmt_api_test_util:init_suite(),
@ -37,15 +38,20 @@ end_per_suite(_) ->
t_create(_Config) ->
Name = <<"EMQX-API-KEY-1">>,
{ok, Create} = create_app(Name),
?assertMatch(#{<<"api_key">> := _,
<<"api_secret">> := _,
<<"created_at">> := _,
<<"desc">> := _,
<<"enable">> := true,
<<"expired_at">> := _,
<<"name">> := Name}, Create),
?assertMatch(
#{
<<"api_key">> := _,
<<"api_secret">> := _,
<<"created_at">> := _,
<<"desc">> := _,
<<"enable">> := true,
<<"expired_at">> := _,
<<"name">> := Name
},
Create
),
{ok, List} = list_app(),
[App] = lists:filter(fun(#{<<"name">> := NameA}) -> NameA =:= Name end, List),
[App] = lists:filter(fun(#{<<"name">> := NameA}) -> NameA =:= Name end, List),
?assertEqual(false, maps:is_key(<<"api_secret">>, App)),
{ok, App1} = read_app(Name),
?assertEqual(Name, maps:get(<<"name">>, App1)),
@ -64,9 +70,12 @@ t_create_failed(_Config) ->
{ok, List} = list_app(),
CreateNum = 30 - erlang:length(List),
Names = lists:map(fun(Seq) ->
<<"EMQX-API-FAILED-KEY-", (integer_to_binary(Seq))/binary>>
end, lists:seq(1, CreateNum)),
Names = lists:map(
fun(Seq) ->
<<"EMQX-API-FAILED-KEY-", (integer_to_binary(Seq))/binary>>
end,
lists:seq(1, CreateNum)
),
lists:foreach(fun(N) -> {ok, _} = create_app(N) end, Names),
?assertEqual(BadRequest, create_app(<<"EMQX-API-KEY-MAXIMUM">>)),
@ -93,7 +102,8 @@ t_update(_Config) ->
?assertEqual(Name, maps:get(<<"name">>, Update1)),
?assertEqual(false, maps:get(<<"enable">>, Update1)),
?assertEqual(<<"NoteVersion1"/utf8>>, maps:get(<<"desc">>, Update1)),
?assertEqual(calendar:rfc3339_to_system_time(binary_to_list(ExpiredAt)),
?assertEqual(
calendar:rfc3339_to_system_time(binary_to_list(ExpiredAt)),
calendar:rfc3339_to_system_time(binary_to_list(maps:get(<<"expired_at">>, Update1)))
),
Unexpired1 = maps:without([expired_at], Change),
@ -117,10 +127,14 @@ t_delete(_Config) ->
t_authorize(_Config) ->
Name = <<"EMQX-API-AUTHORIZE-KEY">>,
{ok, #{<<"api_key">> := ApiKey, <<"api_secret">> := ApiSecret}} = create_app(Name),
BasicHeader = emqx_common_test_http:auth_header(binary_to_list(ApiKey),
binary_to_list(ApiSecret)),
SecretError = emqx_common_test_http:auth_header(binary_to_list(ApiKey),
binary_to_list(ApiKey)),
BasicHeader = emqx_common_test_http:auth_header(
binary_to_list(ApiKey),
binary_to_list(ApiSecret)
),
SecretError = emqx_common_test_http:auth_header(
binary_to_list(ApiKey),
binary_to_list(ApiKey)
),
KeyError = emqx_common_test_http:auth_header("not_found_key", binary_to_list(ApiSecret)),
Unauthorized = {error, {"HTTP/1.1", 401, "Unauthorized"}},
@ -134,8 +148,10 @@ t_authorize(_Config) ->
?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, ApiKeyPath, BasicHeader)),
?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, UserPath, BasicHeader)),
?assertMatch({ok, #{<<"api_key">> := _, <<"enable">> := false}},
update_app(Name, #{enable => false})),
?assertMatch(
{ok, #{<<"api_key">> := _, <<"enable">> := false}},
update_app(Name, #{enable => false})
),
?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)),
Expired = #{
@ -145,8 +161,10 @@ t_authorize(_Config) ->
?assertMatch({ok, #{<<"api_key">> := _, <<"enable">> := true}}, update_app(Name, Expired)),
?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)),
UnExpired = #{expired_at => undefined},
?assertMatch({ok, #{<<"api_key">> := _, <<"expired_at">> := <<"undefined">>}},
update_app(Name, UnExpired)),
?assertMatch(
{ok, #{<<"api_key">> := _, <<"expired_at">> := <<"undefined">>}},
update_app(Name, UnExpired)
),
{ok, _Status1} = emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader),
ok.
@ -159,7 +177,6 @@ t_create_unexpired_app(_Config) ->
?assertMatch(#{<<"expired_at">> := <<"undefined">>}, Create2),
ok.
list_app() ->
Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
case emqx_mgmt_api_test_util:request_api(get, Path) of

50
apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl
View File

@ -47,13 +47,17 @@ t_create(_Config) ->
until => Until
},
{ok, ClientIdBannedRes} = create_banned(ClientIdBanned),
?assertEqual(#{<<"as">> => As,
<<"at">> => At,
<<"by">> => By,
<<"reason">> => Reason,
<<"until">> => Until,
<<"who">> => ClientId
}, ClientIdBannedRes),
?assertEqual(
#{
<<"as">> => As,
<<"at">> => At,
<<"by">> => By,
<<"reason">> => Reason,
<<"until">> => Until,
<<"who">> => ClientId
},
ClientIdBannedRes
),
PeerHost = <<"192.168.2.13">>,
PeerHostBanned = #{
as => <<"peerhost">>,
@ -64,15 +68,19 @@ t_create(_Config) ->
until => Until
},
{ok, PeerHostBannedRes} = create_banned(PeerHostBanned),
?assertEqual(#{<<"as">> => <<"peerhost">>,
<<"at">> => At,
<<"by">> => By,
<<"reason">> => Reason,
<<"until">> => Until,
<<"who">> => PeerHost
}, PeerHostBannedRes),
?assertEqual(
#{
<<"as">> => <<"peerhost">>,
<<"at">> => At,
<<"by">> => By,
<<"reason">> => Reason,
<<"until">> => Until,
<<"who">> => PeerHost
},
PeerHostBannedRes
),
{ok, #{<<"data">> := List}} = list_banned(),
Bans = lists:sort(lists:map(fun(#{<<"who">> := W, <<"as">> := A}) -> {A, W} end, List)),
Bans = lists:sort(lists:map(fun(#{<<"who">> := W, <<"as">> := A}) -> {A, W} end, List)),
?assertEqual([{<<"clientid">>, ClientId}, {<<"peerhost">>, PeerHost}], Bans),
ok.
@ -94,8 +102,10 @@ t_create_failed(_Config) ->
},
BadRequest = {error, {"HTTP/1.1", 400, "Bad Request"}},
?assertEqual(BadRequest, create_banned(BadPeerHost)),
Expired = BadPeerHost#{until => emqx_banned:to_rfc3339(Now - 1),
who => <<"127.0.0.1">>},
Expired = BadPeerHost#{
until => emqx_banned:to_rfc3339(Now - 1),
who => <<"127.0.0.1">>
},
?assertEqual(BadRequest, create_banned(Expired)),
ok.
@ -117,8 +127,10 @@ t_delete(_Config) ->
},
{ok, _} = create_banned(Banned),
?assertMatch({ok, _}, delete_banned(binary_to_list(As), binary_to_list(Who))),
?assertMatch({error,{"HTTP/1.1",404,"Not Found"}},
delete_banned(binary_to_list(As), binary_to_list(Who))),
?assertMatch(
{error, {"HTTP/1.1", 404, "Not Found"}},
delete_banned(binary_to_list(As), binary_to_list(Who))
),
ok.
list_banned() ->

125
apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
View File

@ -44,20 +44,20 @@ t_clients(_) ->
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
{ok, C1} = emqtt:start_link(#{username => Username1, clientid => ClientId1}),
{ok, _} = emqtt:connect(C1),
{ok, _} = emqtt:connect(C1),
{ok, C2} = emqtt:start_link(#{username => Username2, clientid => ClientId2}),
{ok, _} = emqtt:connect(C2),
{ok, _} = emqtt:connect(C2),
timer:sleep(300),
%% get /clients
ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]),
{ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath),
ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]),
{ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath),
ClientsResponse = emqx_json:decode(Clients, [return_maps]),
ClientsMeta = maps:get(<<"meta">>, ClientsResponse),
ClientsPage = maps:get(<<"page">>, ClientsMeta),
ClientsLimit = maps:get(<<"limit">>, ClientsMeta),
ClientsCount = maps:get(<<"count">>, ClientsMeta),
ClientsMeta = maps:get(<<"meta">>, ClientsResponse),
ClientsPage = maps:get(<<"page">>, ClientsMeta),
ClientsLimit = maps:get(<<"limit">>, ClientsMeta),
ClientsCount = maps:get(<<"count">>, ClientsMeta),
?assertEqual(ClientsPage, 1),
?assertEqual(ClientsLimit, emqx_mgmt:max_row_limit()),
?assertEqual(ClientsCount, 2),
@ -76,29 +76,49 @@ t_clients(_) ->
AfterKickoutResponse2 = emqx_mgmt_api_test_util:request_api(get, Client2Path),
?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse2),
%% get /clients/:clientid/authz_cache should has no authz cache
Client1AuthzCachePath = emqx_mgmt_api_test_util:api_path(["clients",
binary_to_list(ClientId1), "authz_cache"]),
%% get /clients/:clientid/authorization/cache should has no authz cache
Client1AuthzCachePath = emqx_mgmt_api_test_util:api_path([
"clients",
binary_to_list(ClientId1),
"authorization",
"cache"
]),
{ok, Client1AuthzCache} = emqx_mgmt_api_test_util:request_api(get, Client1AuthzCachePath),
?assertEqual("[]", Client1AuthzCache),
%% post /clients/:clientid/subscribe
SubscribeBody = #{topic => Topic, qos => Qos},
SubscribePath = emqx_mgmt_api_test_util:api_path(["clients",
binary_to_list(ClientId1), "subscribe"]),
{ok, _} = emqx_mgmt_api_test_util:request_api(post, SubscribePath,
"", AuthHeader, SubscribeBody),
SubscribePath = emqx_mgmt_api_test_util:api_path([
"clients",
binary_to_list(ClientId1),
"subscribe"
]),
{ok, _} = emqx_mgmt_api_test_util:request_api(
post,
SubscribePath,
"",
AuthHeader,
SubscribeBody
),
timer:sleep(100),
[{AfterSubTopic, #{qos := AfterSubQos}}] = emqx_mgmt:lookup_subscriptions(ClientId1),
?assertEqual(AfterSubTopic, Topic),
?assertEqual(AfterSubQos, Qos),
%% post /clients/:clientid/unsubscribe
UnSubscribePath = emqx_mgmt_api_test_util:api_path(["clients",
binary_to_list(ClientId1), "unsubscribe"]),
UnSubscribePath = emqx_mgmt_api_test_util:api_path([
"clients",
binary_to_list(ClientId1),
"unsubscribe"
]),
UnSubscribeBody = #{topic => Topic},
{ok, _} = emqx_mgmt_api_test_util:request_api(post, UnSubscribePath,
"", AuthHeader, UnSubscribeBody),
{ok, _} = emqx_mgmt_api_test_util:request_api(
post,
UnSubscribePath,
"",
AuthHeader,
UnSubscribeBody
),
timer:sleep(100),
?assertEqual([], emqx_mgmt:lookup_subscriptions(Client1)),
@ -118,44 +138,58 @@ t_query_clients_with_time(_) ->
ClientId2 = <<"client2">>,
{ok, C1} = emqtt:start_link(#{username => Username1, clientid => ClientId1}),
{ok, _} = emqtt:connect(C1),
{ok, _} = emqtt:connect(C1),
{ok, C2} = emqtt:start_link(#{username => Username2, clientid => ClientId2}),
{ok, _} = emqtt:connect(C2),
{ok, _} = emqtt:connect(C2),
timer:sleep(100),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]),
%% get /clients with time(rfc3339)
NowTimeStampInt = erlang:system_time(millisecond),
%% Do not uri_encode `=` to `%3D`
Rfc3339String = emqx_http_lib:uri_encode(binary:bin_to_list(
emqx_datetime:epoch_to_rfc3339(NowTimeStampInt))),
Rfc3339String = emqx_http_lib:uri_encode(
binary:bin_to_list(
emqx_datetime:epoch_to_rfc3339(NowTimeStampInt)
)
),
TimeStampString = emqx_http_lib:uri_encode(integer_to_list(NowTimeStampInt)),
LteKeys = ["lte_created_at=", "lte_connected_at="],
GteKeys = ["gte_created_at=", "gte_connected_at="],
LteParamRfc3339 = [Param ++ Rfc3339String || Param <- LteKeys],
LteParamStamp = [Param ++ TimeStampString || Param <- LteKeys],
GteParamRfc3339 = [Param ++ Rfc3339String || Param <- GteKeys],
GteParamStamp = [Param ++ TimeStampString || Param <- GteKeys],
LteKeys = ["lte_created_at=", "lte_connected_at="],
GteKeys = ["gte_created_at=", "gte_connected_at="],
LteParamRfc3339 = [Param ++ Rfc3339String || Param <- LteKeys],
LteParamStamp = [Param ++ TimeStampString || Param <- LteKeys],
GteParamRfc3339 = [Param ++ Rfc3339String || Param <- GteKeys],
GteParamStamp = [Param ++ TimeStampString || Param <- GteKeys],
RequestResults =
[emqx_mgmt_api_test_util:request_api(get, ClientsPath, Param, AuthHeader)
|| Param <- LteParamRfc3339 ++ LteParamStamp
++ GteParamRfc3339 ++ GteParamStamp],
DecodedResults = [emqx_json:decode(Response, [return_maps])
|| {ok, Response} <- RequestResults],
RequestResults =
[
emqx_mgmt_api_test_util:request_api(get, ClientsPath, Param, AuthHeader)
|| Param <-
LteParamRfc3339 ++ LteParamStamp ++
GteParamRfc3339 ++ GteParamStamp
],
DecodedResults = [
emqx_json:decode(Response, [return_maps])
|| {ok, Response} <- RequestResults
],
{LteResponseDecodeds, GteResponseDecodeds} = lists:split(4, DecodedResults),
%% EachData :: list()
[?assert(time_string_to_epoch_millisecond(CreatedAt) < NowTimeStampInt)
[
?assert(time_string_to_epoch_millisecond(CreatedAt) < NowTimeStampInt)
|| #{<<"data">> := EachData} <- LteResponseDecodeds,
#{<<"created_at">> := CreatedAt} <- EachData],
[?assert(time_string_to_epoch_millisecond(ConnectedAt) < NowTimeStampInt)
#{<<"created_at">> := CreatedAt} <- EachData
],
[
?assert(time_string_to_epoch_millisecond(ConnectedAt) < NowTimeStampInt)
|| #{<<"data">> := EachData} <- LteResponseDecodeds,
#{<<"connected_at">> := ConnectedAt} <- EachData],
[?assertEqual(EachData, [])
|| #{<<"data">> := EachData} <- GteResponseDecodeds],
#{<<"connected_at">> := ConnectedAt} <- EachData
],
[
?assertEqual(EachData, [])
|| #{<<"data">> := EachData} <- GteResponseDecodeds
],
%% testcase cleanup, kickout client1 and client2
Client1Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1)]),
@ -169,7 +203,7 @@ t_keepalive(_Config) ->
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["clients", ClientId, "keepalive"]),
Body = #{interval => 11},
{error,{"HTTP/1.1",404,"Not Found"}} =
{error, {"HTTP/1.1", 404, "Not Found"}} =
emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body),
{ok, C1} = emqtt:start_link(#{username => Username, clientid => ClientId}),
{ok, _} = emqtt:connect(C1),
@ -190,5 +224,6 @@ time_string_to_epoch(DateTime, Unit) when is_binary(DateTime) ->
catch
error:badarg ->
calendar:rfc3339_to_system_time(
binary_to_list(DateTime), [{unit, Unit}])
binary_to_list(DateTime), [{unit, Unit}]
)
end.

33
apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl
View File

@ -32,10 +32,13 @@ end_per_suite(_) ->
t_get(_Config) ->
{ok, Configs} = get_configs(),
maps:map(fun(Name, Value) ->
{ok, Config} = get_config(Name),
?assertEqual(Value, Config)
end, maps:remove(<<"license">>, Configs)),
maps:map(
fun(Name, Value) ->
{ok, Config} = get_config(Name),
?assertEqual(Value, Config)
end,
maps:remove(<<"license">>, Configs)
),
ok.
t_update(_Config) ->
@ -50,8 +53,10 @@ t_update(_Config) ->
%% update failed
ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"),
?assertMatch({error, {"HTTP/1.1", 400, _}},
update_config(<<"sysmon">>, ErrorSysMon)),
?assertMatch(
{error, {"HTTP/1.1", 400, _}},
update_config(<<"sysmon">>, ErrorSysMon)
),
{ok, SysMon2} = get_config(<<"sysmon">>),
?assertEqual(SysMon1, SysMon2),
@ -101,8 +106,10 @@ t_global_zone(_Config) ->
{ok, Zones} = get_global_zone(),
ZonesKeys = lists:map(fun({K, _}) -> K end, hocon_schema:roots(emqx_zone_schema)),
?assertEqual(lists:usort(ZonesKeys), lists:usort(maps:keys(Zones))),
?assertEqual(emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed]),
emqx_map_lib:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones)),
?assertEqual(
emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed]),
emqx_map_lib:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones)
),
NewZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1),
{ok, #{}} = update_global_zone(NewZones),
?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])),
@ -133,7 +140,8 @@ get_config(Name) ->
case emqx_mgmt_api_test_util:request_api(get, Path) of
{ok, Res} ->
{ok, emqx_json:decode(Res, [return_maps])};
Error -> Error
Error ->
Error
end.
get_configs() ->
@ -153,8 +161,11 @@ update_config(Name, Change) ->
reset_config(Name, Key) ->
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
Path = binary_to_list(iolist_to_binary(
emqx_mgmt_api_test_util:api_path(["configs_reset", Name]))),
Path = binary_to_list(
iolist_to_binary(
emqx_mgmt_api_test_util:api_path(["configs_reset", Name])
)
),
case emqx_mgmt_api_test_util:request_api(post, Path, Key, AuthHeader, []) of
{ok, []} -> ok;
Error -> Error

66
apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl
View File

@ -35,8 +35,8 @@ end_per_suite(_) ->
t_list_listeners(_) ->
Path = emqx_mgmt_api_test_util:api_path(["listeners"]),
Res = request(get, Path, [], []),
Expect = emqx_mgmt_api_listeners:do_list_listeners(),
?assertEqual(emqx_json:encode([Expect]), emqx_json:encode(Res)),
#{<<"listeners">> := Expect} = emqx_mgmt_api_listeners:do_list_listeners(),
?assertEqual(length(Expect), length(Res)),
ok.
t_crud_listeners_by_id(_) ->
@ -44,19 +44,18 @@ t_crud_listeners_by_id(_) ->
NewListenerId = <<"tcp:new">>,
TcpPath = emqx_mgmt_api_test_util:api_path(["listeners", TcpListenerId]),
NewPath = emqx_mgmt_api_test_util:api_path(["listeners", NewListenerId]),
[#{<<"listeners">> := [TcpListener], <<"node">> := Node}] = request(get, TcpPath, [], []),
?assertEqual(atom_to_binary(node()), Node),
TcpListener = request(get, TcpPath, [], []),
%% create
?assertEqual({error, not_found}, is_running(NewListenerId)),
?assertMatch([#{<<"listeners">> := []}], request(get, NewPath, [], [])),
?assertMatch({error, {"HTTP/1.1", 404, _}}, request(get, NewPath, [], [])),
NewConf = TcpListener#{
<<"id">> => NewListenerId,
<<"bind">> => <<"0.0.0.0:2883">>
},
[#{<<"listeners">> := [Create]}] = request(put, NewPath, [], NewConf),
Create = request(post, NewPath, [], NewConf),
?assertEqual(lists:sort(maps:keys(TcpListener)), lists:sort(maps:keys(Create))),
[#{<<"listeners">> := [Get1]}] = request(get, NewPath, [], []),
Get1 = request(get, NewPath, [], []),
?assertMatch(Create, Get1),
?assert(is_running(NewListenerId)),
@ -67,64 +66,21 @@ t_crud_listeners_by_id(_) ->
<<"id">> => BadId,
<<"bind">> => <<"0.0.0.0:2883">>
},
?assertEqual({error, {"HTTP/1.1", 400, "Bad Request"}}, request(put, BadPath, [], BadConf)),
?assertMatch({error, {"HTTP/1.1", 400, _}}, request(post, BadPath, [], BadConf)),
%% update
#{<<"acceptors">> := Acceptors} = Create,
Acceptors1 = Acceptors + 10,
[#{<<"listeners">> := [Update]}] =
Update =
request(put, NewPath, [], Create#{<<"acceptors">> => Acceptors1}),
?assertMatch(#{<<"acceptors">> := Acceptors1}, Update),
[#{<<"listeners">> := [Get2]}] = request(get, NewPath, [], []),
?assertMatch(#{<<"acceptors">> := Acceptors1}, Get2),
%% delete
?assertEqual([], delete(NewPath)),
?assertEqual({error, not_found}, is_running(NewListenerId)),
?assertMatch([#{<<"listeners">> := []}], request(get, NewPath, [], [])),
?assertEqual([], delete(NewPath)),
ok.
t_list_listeners_on_node(_) ->
Node = atom_to_list(node()),
Path = emqx_mgmt_api_test_util:api_path(["nodes", Node, "listeners"]),
Listeners = request(get, Path, [], []),
#{<<"listeners">> := Expect} = emqx_mgmt_api_listeners:do_list_listeners(),
?assertEqual(emqx_json:encode(Expect), emqx_json:encode(Listeners)),
ok.
t_crud_listener_by_id_on_node(_) ->
TcpListenerId = <<"tcp:default">>,
NewListenerId = <<"tcp:new1">>,
Node = atom_to_list(node()),
TcpPath = emqx_mgmt_api_test_util:api_path(["nodes", Node, "listeners", TcpListenerId]),
NewPath = emqx_mgmt_api_test_util:api_path(["nodes", Node, "listeners", NewListenerId]),
TcpListener = request(get, TcpPath, [], []),
%% create
?assertEqual({error, not_found}, is_running(NewListenerId)),
?assertMatch({error,{"HTTP/1.1", 404, "Not Found"}}, request(get, NewPath, [], [])),
Create = request(put, NewPath, [], TcpListener#{
<<"id">> => NewListenerId,
<<"bind">> => <<"0.0.0.0:3883">>
}),
?assertEqual(lists:sort(maps:keys(TcpListener)), lists:sort(maps:keys(Create))),
Get1 = request(get, NewPath, [], []),
?assertMatch(Create, Get1),
?assert(is_running(NewListenerId)),
%% update
#{<<"acceptors">> := Acceptors} = Create,
Acceptors1 = Acceptors + 10,
Update = request(put, NewPath, [], Create#{<<"acceptors">> => Acceptors1}),
?assertMatch(#{<<"acceptors">> := Acceptors1}, Update),
Get2 = request(get, NewPath, [], []),
?assertMatch(#{<<"acceptors">> := Acceptors1}, Get2),
%% delete
?assertEqual([], delete(NewPath)),
?assertEqual({error, not_found}, is_running(NewListenerId)),
?assertMatch({error, {"HTTP/1.1", 404, "Not Found"}}, request(get, NewPath, [], [])),
?assertMatch({error, {"HTTP/1.1", 404, _}}, request(get, NewPath, [], [])),
?assertEqual([], delete(NewPath)),
ok.
@ -139,8 +95,8 @@ action_listener(ID, Action, Running) ->
{ok, _} = emqx_mgmt_api_test_util:request_api(post, Path),
timer:sleep(500),
GetPath = emqx_mgmt_api_test_util:api_path(["listeners", ID]),
[#{<<"listeners">> := Listeners}] = request(get, GetPath, [], []),
[listener_stats(Listener, Running) || Listener <- Listeners].
Listener = request(get, GetPath, [], []),
listener_stats(Listener, Running).
request(Method, Url, QueryParams, Body) ->
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),

11
apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl
View File

@ -40,16 +40,17 @@ t_single_node_metrics_api(_) ->
{ok, MetricsResponse} = request_helper("metrics"),
[MetricsFromAPI] = emqx_json:decode(MetricsResponse, [return_maps]),
LocalNodeMetrics = maps:from_list(
emqx_mgmt:get_metrics(node()) ++ [{node, to_bin(node())}]),
emqx_mgmt:get_metrics(node()) ++ [{node, to_bin(node())}]
),
match_helper(LocalNodeMetrics, MetricsFromAPI).
match_helper(SystemMetrics, MetricsFromAPI) ->
length_equal(SystemMetrics, MetricsFromAPI),
Fun =
fun (Key, {SysMetrics, APIMetrics}) ->
Value = maps:get(Key, SysMetrics),
?assertEqual(Value, maps:get(to_bin(Key), APIMetrics)),
{Value, {SysMetrics, APIMetrics}}
fun(Key, {SysMetrics, APIMetrics}) ->
Value = maps:get(Key, SysMetrics),
?assertEqual(Value, maps:get(to_bin(Key), APIMetrics)),
{Value, {SysMetrics, APIMetrics}}
end,
lists:mapfoldl(Fun, {SystemMetrics, MetricsFromAPI}, maps:keys(SystemMetrics)).

18
apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl
View File

@ -67,19 +67,21 @@ t_nodes_api(_) ->
BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode"]),
?assertMatch(
{error, {_, 400, _}},
emqx_mgmt_api_test_util:request_api(get, BadNodePath)).
emqx_mgmt_api_test_util:request_api(get, BadNodePath)
).
t_log_path(_) ->
NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]),
{ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath),
#{<<"log_path">> := Path} = emqx_json:decode(NodeInfo, [return_maps]),
?assertEqual(
<<"emqx-test.log">>,
filename:basename(Path)).
<<"emqx-test.log">>,
filename:basename(Path)
).
t_node_stats_api(_) ->
StatsPath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "stats"]),
SystemStats= emqx_mgmt:get_stats(),
SystemStats = emqx_mgmt:get_stats(),
{ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath),
Stats = emqx_json:decode(StatsResponse, [return_maps]),
Fun =
@ -91,12 +93,13 @@ t_node_stats_api(_) ->
BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode", "stats"]),
?assertMatch(
{error, {_, 400, _}},
emqx_mgmt_api_test_util:request_api(get, BadNodePath)).
emqx_mgmt_api_test_util:request_api(get, BadNodePath)
).
t_node_metrics_api(_) ->
MetricsPath =
emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "metrics"]),
SystemMetrics= emqx_mgmt:get_metrics(),
SystemMetrics = emqx_mgmt:get_metrics(),
{ok, MetricsResponse} = emqx_mgmt_api_test_util:request_api(get, MetricsPath),
Metrics = emqx_json:decode(MetricsResponse, [return_maps]),
Fun =
@ -108,4 +111,5 @@ t_node_metrics_api(_) ->
BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode", "metrics"]),
?assertMatch(
{error, {_, 400, _}},
emqx_mgmt_api_test_util:request_api(get, BadNodePath)).
emqx_mgmt_api_test_util:request_api(get, BadNodePath)
).

43
apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl
View File

@ -56,17 +56,35 @@ todo_t_plugins(Config) ->
ok = emqx_plugins:delete_package(NameVsn),
ok = install_plugin(PackagePath),
{ok, StopRes} = describe_plugins(NameVsn),
?assertMatch(#{<<"running_status">> := [
#{<<"node">> := <<"test@127.0.0.1">>, <<"status">> := <<"stopped">>}]}, StopRes),
?assertMatch(
#{
<<"running_status">> := [
#{<<"node">> := <<"test@127.0.0.1">>, <<"status">> := <<"stopped">>}
]
},
StopRes
),
{ok, StopRes1} = update_plugin(NameVsn, "start"),
?assertEqual([], StopRes1),
{ok, StartRes} = describe_plugins(NameVsn),
?assertMatch(#{<<"running_status">> := [
#{<<"node">> := <<"test@127.0.0.1">>, <<"status">> := <<"running">>}]}, StartRes),
?assertMatch(
#{
<<"running_status">> := [
#{<<"node">> := <<"test@127.0.0.1">>, <<"status">> := <<"running">>}
]
},
StartRes
),
{ok, []} = update_plugin(NameVsn, "stop"),
{ok, StopRes2} = describe_plugins(NameVsn),
?assertMatch(#{<<"running_status">> := [
#{<<"node">> := <<"test@127.0.0.1">>, <<"status">> := <<"stopped">>}]}, StopRes2),
?assertMatch(
#{
<<"running_status">> := [
#{<<"node">> := <<"test@127.0.0.1">>, <<"status">> := <<"stopped">>}
]
},
StopRes2
),
{ok, []} = uninstall_plugin(NameVsn),
ok.
@ -87,8 +105,16 @@ describe_plugins(Name) ->
install_plugin(FilePath) ->
{ok, Token} = emqx_dashboard_admin:sign_token(<<"admin">>, <<"public">>),
Path = emqx_mgmt_api_test_util:api_path(["plugins", "install"]),
case emqx_mgmt_api_test_util:upload_request(Path, FilePath, "plugin",
<<"application/gzip">>, [], Token) of
case
emqx_mgmt_api_test_util:upload_request(
Path,
FilePath,
"plugin",
<<"application/gzip">>,
[],
Token
)
of
{ok, {{"HTTP/1.1", 200, "OK"}, _Headers, <<>>}} -> ok;
Error -> Error
end.
@ -109,7 +135,6 @@ uninstall_plugin(Name) ->
DeletePath = emqx_mgmt_api_test_util:api_path(["plugins", Name]),
emqx_mgmt_api_test_util:request_api(delete, DeletePath).
build_demo_plugin_package(Dir) ->
#{package := Pkg} = emqx_plugins_SUITE:build_demo_plugin_package(),
FileName = "emqx_plugin_template-" ++ ?EMQX_PLUGIN_TEMPLATE_VSN ++ ?PACKAGE_SUFFIX,

22
apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl
View File

@ -37,7 +37,9 @@ end_per_suite(_) ->
emqx_mgmt_api_test_util:end_suite().
t_publish_api(_) ->
{ok, Client} = emqtt:start_link(#{username => <<"api_username">>, clientid => <<"api_clientid">>}),
{ok, Client} = emqtt:start_link(#{
username => <<"api_username">>, clientid => <<"api_clientid">>
}),
{ok, _} = emqtt:connect(Client),
{ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1),
{ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2),
@ -50,14 +52,16 @@ t_publish_api(_) ->
emqtt:disconnect(Client).
t_publish_bulk_api(_) ->
{ok, Client} = emqtt:start_link(#{username => <<"api_username">>, clientid => <<"api_clientid">>}),
{ok, Client} = emqtt:start_link(#{
username => <<"api_username">>, clientid => <<"api_clientid">>
}),
{ok, _} = emqtt:connect(Client),
{ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1),
{ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2),
Payload = <<"hello">>,
Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]),
Auth = emqx_mgmt_api_test_util:auth_header_(),
Body =[#{topic => ?TOPIC1, payload => Payload}, #{topic => ?TOPIC2, payload => Payload}],
Body = [#{topic => ?TOPIC1, payload => Payload}, #{topic => ?TOPIC2, payload => Payload}],
{ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body),
ResponseMap = emqx_json:decode(Response, [return_maps]),
?assertEqual(2, erlang:length(ResponseMap)),
@ -68,12 +72,12 @@ t_publish_bulk_api(_) ->
receive_assert(Topic, Qos, Payload) ->
receive
{publish, Message} ->
ReceiveTopic = maps:get(topic, Message),
ReceiveQos = maps:get(qos, Message),
ReceivePayload = maps:get(payload, Message),
?assertEqual(ReceiveTopic , Topic),
?assertEqual(ReceiveQos , Qos),
?assertEqual(ReceivePayload , Payload),
ReceiveTopic = maps:get(topic, Message),
ReceiveQos = maps:get(qos, Message),
ReceivePayload = maps:get(payload, Message),
?assertEqual(ReceiveTopic, Topic),
?assertEqual(ReceiveQos, Qos),
?assertEqual(ReceivePayload, Payload),
ok
after 5000 ->
timeout

2
apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl
View File

@ -37,7 +37,7 @@ t_stats_api(_) ->
SystemStats1 = emqx_mgmt:get_stats(),
Fun1 =
fun(Key) ->
?assertEqual(maps:get(Key, SystemStats1), maps:get(atom_to_binary(Key, utf8), Stats1))
?assertEqual(maps:get(Key, SystemStats1), maps:get(atom_to_binary(Key, utf8), Stats1))
end,
lists:foreach(Fun1, maps:keys(SystemStats1)),
StatsPath = emqx_mgmt_api_test_util:api_path(["stats?aggregate=true"]),

14
apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
View File

@ -58,13 +58,13 @@ t_subscription_api(_) ->
fun(#{<<"topic">> := T1}, #{<<"topic">> := T2}) ->
maps:get(T1, ?TOPIC_SORT) =< maps:get(T2, ?TOPIC_SORT)
end,
[Subscriptions1, Subscriptions2] = lists:sort(Sort, Subscriptions),
[Subscriptions1, Subscriptions2] = lists:sort(Sort, Subscriptions),
?assertEqual(maps:get(<<"topic">>, Subscriptions1), ?TOPIC1),
?assertEqual(maps:get(<<"topic">>, Subscriptions2), ?TOPIC2),
?assertEqual(maps:get(<<"clientid">>, Subscriptions1), ?CLIENTID),
?assertEqual(maps:get(<<"clientid">>, Subscriptions2), ?CLIENTID),
QS = uri_string:compose_query([
QS = uri_string:compose_query([
{"clientid", ?CLIENTID},
{"topic", ?TOPIC2_TOPIC_ONLY},
{"node", atom_to_list(node())},
@ -83,11 +83,11 @@ t_subscription_api(_) ->
?assertEqual(length(SubscriptionsList2), 1),
MatchQs = uri_string:compose_query([
{"clientid", ?CLIENTID},
{"node", atom_to_list(node())},
{"qos", "0"},
{"match_topic", "t/#"}
]),
{"clientid", ?CLIENTID},
{"node", atom_to_list(node())},
{"qos", "0"},
{"match_topic", "t/#"}
]),
{ok, MatchRes} = emqx_mgmt_api_test_util:request_api(get, Path, MatchQs, Headers),
MatchData = emqx_json:decode(MatchRes, [return_maps]),

171
apps/emqx_management/test/emqx_mgmt_api_test_util.erl
View File

@ -28,7 +28,6 @@ init_suite(Apps) ->
application:load(emqx_management),
emqx_common_test_helpers:start_apps(Apps ++ [emqx_dashboard], fun set_special_configs/1).
end_suite() ->
end_suite([]).
@ -39,15 +38,7 @@ end_suite(Apps) ->
ok.
set_special_configs(emqx_dashboard) ->
Config = #{
default_username => <<"admin">>,
default_password => <<"public">>,
listeners => [#{
protocol => http,
port => 18083
}]
},
emqx_config:put([dashboard], Config),
emqx_dashboard_api_test_helpers:set_default_config(),
ok;
set_special_configs(_App) ->
ok.
@ -61,36 +52,44 @@ request_api(Method, Url, AuthOrHeaders) ->
request_api(Method, Url, QueryParams, AuthOrHeaders) ->
request_api(Method, Url, QueryParams, AuthOrHeaders, []).
request_api(Method, Url, QueryParams, AuthOrHeaders, [])
when (Method =:= options) orelse
(Method =:= get) orelse
(Method =:= put) orelse
(Method =:= head) orelse
(Method =:= delete) orelse
(Method =:= trace) ->
NewUrl = case QueryParams of
"" -> Url;
_ -> Url ++ "?" ++ QueryParams
end,
request_api(Method, Url, QueryParams, AuthOrHeaders, []) when
(Method =:= options) orelse
(Method =:= get) orelse
(Method =:= put) orelse
(Method =:= head) orelse
(Method =:= delete) orelse
(Method =:= trace)
->
NewUrl =
case QueryParams of
"" -> Url;
_ -> Url ++ "?" ++ QueryParams
end,
do_request_api(Method, {NewUrl, build_http_header(AuthOrHeaders)});
request_api(Method, Url, QueryParams, AuthOrHeaders, Body)
when (Method =:= post) orelse
(Method =:= patch) orelse
(Method =:= put) orelse
(Method =:= delete) ->
NewUrl = case QueryParams of
"" -> Url;
_ -> Url ++ "?" ++ QueryParams
end,
do_request_api(Method, {NewUrl, build_http_header(AuthOrHeaders), "application/json", emqx_json:encode(Body)}).
request_api(Method, Url, QueryParams, AuthOrHeaders, Body) when
(Method =:= post) orelse
(Method =:= patch) orelse
(Method =:= put) orelse
(Method =:= delete)
->
NewUrl =
case QueryParams of
"" -> Url;
_ -> Url ++ "?" ++ QueryParams
end,
do_request_api(
Method,
{NewUrl, build_http_header(AuthOrHeaders), "application/json", emqx_json:encode(Body)}
).
do_request_api(Method, Request)->
do_request_api(Method, Request) ->
ct:pal("Method: ~p, Request: ~p", [Method, Request]),
case httpc:request(Method, Request, [], []) of
{error, socket_closed_remotely} ->
{error, socket_closed_remotely};
{ok, {{"HTTP/1.1", Code, _}, _, Return} }
when Code >= 200 andalso Code =< 299 ->
{ok, {{"HTTP/1.1", Code, _}, _, Return}} when
Code >= 200 andalso Code =< 299
->
{ok, Return};
{ok, {Reason, _, _} = Error} ->
ct:pal("error: ~p~n", [Error]),
@ -105,11 +104,10 @@ auth_header_() ->
build_http_header(X) when is_list(X) ->
X;
build_http_header(X) ->
[X].
api_path(Parts)->
api_path(Parts) ->
?SERVER ++ filename:join([?BASE_PATH | Parts]).
%% Usage:
@ -125,20 +123,27 @@ api_path(Parts)->
%% upload_request(<<"site.com/api/upload">>, <<"path/to/file.png">>,
%% <<"upload">>, <<"image/png">>, RequestData, <<"some-token">>)
-spec upload_request(URL, FilePath, Name, MimeType, RequestData, AuthorizationToken) ->
{ok, binary()} | {error, list()} when
URL:: binary(),
FilePath:: binary(),
Name:: binary(),
MimeType:: binary(),
RequestData:: list(),
AuthorizationToken:: binary().
{ok, binary()} | {error, list()}
when
URL :: binary(),
FilePath :: binary(),
Name :: binary(),
MimeType :: binary(),
RequestData :: list(),
AuthorizationToken :: binary().
upload_request(URL, FilePath, Name, MimeType, RequestData, AuthorizationToken) ->
Method = post,
Filename = filename:basename(FilePath),
{ok, Data} = file:read_file(FilePath),
Boundary = emqx_guid:to_base62(emqx_guid:gen()),
RequestBody = format_multipart_formdata(Data, RequestData, Name,
[Filename], MimeType, Boundary),
RequestBody = format_multipart_formdata(
Data,
RequestData,
Name,
[Filename],
MimeType,
Boundary
),
ContentType = "multipart/form-data; boundary=" ++ binary_to_list(Boundary),
ContentLength = integer_to_list(length(binary_to_list(RequestBody))),
Headers = [
@ -154,34 +159,56 @@ upload_request(URL, FilePath, Name, MimeType, RequestData, AuthorizationToken) -
httpc:request(Method, {URL, Headers, ContentType, RequestBody}, HTTPOptions, Options).
-spec format_multipart_formdata(Data, Params, Name, FileNames, MimeType, Boundary) ->
binary() when
Data:: binary(),
Params:: list(),
Name:: binary(),
FileNames:: list(),
MimeType:: binary(),
Boundary:: binary().
binary()
when
Data :: binary(),
Params :: list(),
Name :: binary(),
FileNames :: list(),
MimeType :: binary(),
Boundary :: binary().
format_multipart_formdata(Data, Params, Name, FileNames, MimeType, Boundary) ->
StartBoundary = erlang:iolist_to_binary([<<"--">>, Boundary]),
LineSeparator = <<"\r\n">>,
WithParams = lists:foldl(fun({Key, Value}, Acc) ->
erlang:iolist_to_binary([
Acc,
StartBoundary, LineSeparator,
<<"Content-Disposition: form-data; name=\"">>, Key, <<"\"">>,
LineSeparator, LineSeparator,
Value, LineSeparator
])
end, <<"">>, Params),
WithPaths = lists:foldl(fun(FileName, Acc) ->
erlang:iolist_to_binary([
Acc,
StartBoundary, LineSeparator,
<<"Content-Disposition: form-data; name=\"">>, Name, <<"\"; filename=\"">>,
FileName, <<"\"">>, LineSeparator,
<<"Content-Type: ">>, MimeType, LineSeparator, LineSeparator,
Data,
LineSeparator
])
end, WithParams, FileNames),
WithParams = lists:foldl(
fun({Key, Value}, Acc) ->
erlang:iolist_to_binary([
Acc,
StartBoundary,
LineSeparator,
<<"Content-Disposition: form-data; name=\"">>,
Key,
<<"\"">>,
LineSeparator,
LineSeparator,
Value,
LineSeparator
])
end,
<<"">>,
Params
),
WithPaths = lists:foldl(
fun(FileName, Acc) ->
erlang:iolist_to_binary([
Acc,
StartBoundary,
LineSeparator,
<<"Content-Disposition: form-data; name=\"">>,
Name,
<<"\"; filename=\"">>,
FileName,
<<"\"">>,
LineSeparator,
<<"Content-Type: ">>,
MimeType,
LineSeparator,
LineSeparator,
Data,
LineSeparator
])
end,
WithParams,
FileNames
),
erlang:iolist_to_binary([WithPaths, StartBoundary, <<"--">>, LineSeparator]).

4
apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
View File

@ -32,7 +32,9 @@ end_per_suite(_) ->
t_nodes_api(_) ->
Topic = <<"test_topic">>,
{ok, Client} = emqtt:start_link(#{username => <<"routes_username">>, clientid => <<"routes_cid">>}),
{ok, Client} = emqtt:start_link(#{
username => <<"routes_username">>, clientid => <<"routes_cid">>
}),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, Topic),

140
apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
View File

@ -59,7 +59,9 @@ t_http_test(_Config) ->
#{
<<"code">> => <<"BAD_REQUEST">>,
<<"message">> => <<"name : mandatory_required_field">>
}, json(Body)),
},
json(Body)
),
Name = <<"test-name">>,
Trace = [
@ -77,32 +79,47 @@ t_http_test(_Config) ->
%% update
{ok, Update} = request_api(put, api_path("trace/test-name/stop"), Header, #{}),
?assertEqual(#{<<"enable">> => false,
<<"name">> => <<"test-name">>}, json(Update)),
?assertEqual(
#{
<<"enable">> => false,
<<"name">> => <<"test-name">>
},
json(Update)
),
?assertMatch({error, {"HTTP/1.1", 404, _}, _},
request_api(put, api_path("trace/test-name-not-found/stop"), Header, #{})),
?assertMatch(
{error, {"HTTP/1.1", 404, _}, _},
request_api(put, api_path("trace/test-name-not-found/stop"), Header, #{})
),
{ok, List1} = request_api(get, api_path("trace"), Header),
[Data1] = json(List1),
Node = atom_to_binary(node()),
?assertMatch(#{
<<"status">> := <<"stopped">>,
<<"name">> := <<"test-name">>,
<<"log_size">> := #{Node := _},
<<"start_at">> := _,
<<"end_at">> := _,
<<"type">> := <<"topic">>,
<<"topic">> := <<"/x/y/z">>
}, Data1),
?assertMatch(
#{
<<"status">> := <<"stopped">>,
<<"name">> := <<"test-name">>,
<<"log_size">> := #{Node := _},
<<"start_at">> := _,
<<"end_at">> := _,
<<"type">> := <<"topic">>,
<<"topic">> := <<"/x/y/z">>
},
Data1
),
%% delete
{ok, Delete} = request_api(delete, api_path("trace/test-name"), Header),
?assertEqual(<<>>, Delete),
{error, {"HTTP/1.1", 404, "Not Found"}, DeleteNotFound}
= request_api(delete, api_path("trace/test-name"), Header),
?assertEqual(#{<<"code">> => <<"NOT_FOUND">>,
<<"message">> => <<"test-name NOT FOUND">>}, json(DeleteNotFound)),
{error, {"HTTP/1.1", 404, "Not Found"}, DeleteNotFound} =
request_api(delete, api_path("trace/test-name"), Header),
?assertEqual(
#{
<<"code">> => <<"NOT_FOUND">>,
<<"message">> => <<"test-name NOT FOUND">>
},
json(DeleteNotFound)
),
{ok, List2} = request_api(get, api_path("trace"), Header),
?assertEqual([], json(List2)),
@ -123,29 +140,43 @@ t_create_failed(_Config) ->
Trace = [{<<"type">>, <<"topic">>}, {<<"topic">>, <<"/x/y/z">>}],
BadName1 = {<<"name">>, <<"test/bad">>},
?assertMatch({error, {"HTTP/1.1", 400, _}, _},
request_api(post, api_path("trace"), Header, [BadName1 | Trace])),
?assertMatch(
{error, {"HTTP/1.1", 400, _}, _},
request_api(post, api_path("trace"), Header, [BadName1 | Trace])
),
BadName2 = {<<"name">>, list_to_binary(lists:duplicate(257, "t"))},
?assertMatch({error, {"HTTP/1.1", 400, _}, _},
request_api(post, api_path("trace"), Header, [BadName2 | Trace])),
?assertMatch(
{error, {"HTTP/1.1", 400, _}, _},
request_api(post, api_path("trace"), Header, [BadName2 | Trace])
),
%% already_exist
GoodName = {<<"name">>, <<"test-name-0">>},
{ok, Create} = request_api(post, api_path("trace"), Header, [GoodName | Trace]),
?assertMatch(#{<<"name">> := <<"test-name-0">>}, json(Create)),
?assertMatch({error, {"HTTP/1.1", 400, _}, _},
request_api(post, api_path("trace"), Header, [GoodName | Trace])),
?assertMatch(
{error, {"HTTP/1.1", 400, _}, _},
request_api(post, api_path("trace"), Header, [GoodName | Trace])
),
%% MAX Limited
lists:map(fun(Seq) ->
Name0 = list_to_binary("name" ++ integer_to_list(Seq)),
Trace0 = [{name, Name0}, {type, topic},
{topic, list_to_binary("/x/y/" ++ integer_to_list(Seq))}],
{ok, _} = emqx_trace:create(Trace0)
end, lists:seq(1, 30 - ets:info(emqx_trace, size))),
lists:map(
fun(Seq) ->
Name0 = list_to_binary("name" ++ integer_to_list(Seq)),
Trace0 = [
{name, Name0},
{type, topic},
{topic, list_to_binary("/x/y/" ++ integer_to_list(Seq))}
],
{ok, _} = emqx_trace:create(Trace0)
end,
lists:seq(1, 30 - ets:info(emqx_trace, size))
),
GoodName1 = {<<"name">>, <<"test-name-1">>},
?assertMatch({error, {"HTTP/1.1", 400, _}, _},
request_api(post, api_path("trace"), Header, [GoodName1 | Trace])),
?assertMatch(
{error, {"HTTP/1.1", 400, _}, _},
request_api(post, api_path("trace"), Header, [GoodName1 | Trace])
),
unload(),
emqx_trace:clear(),
ok.
@ -158,14 +189,23 @@ t_download_log(_Config) ->
create_trace(Name, ClientId, Now),
{ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]),
{ok, _} = emqtt:connect(Client),
[begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)],
[
begin
_ = emqtt:ping(Client)
end
|| _ <- lists:seq(1, 5)
],
ok = emqx_trace_handler_SUITE:filesync(Name, clientid),
Header = auth_header_(),
{ok, Binary} = request_api(get, api_path("trace/test_client_id/download"), Header),
{ok, [_Comment,
#zip_file{name = ZipName,
info = #file_info{size = Size, type = regular, access = read_write}}]}
= zip:table(Binary),
{ok, [
_Comment,
#zip_file{
name = ZipName,
info = #file_info{size = Size, type = regular, access = read_write}
}
]} =
zip:table(Binary),
?assert(Size > 0),
ZipNamePrefix = lists:flatten(io_lib:format("~s-trace_~s", [node(), Name])),
?assertNotEqual(nomatch, re:run(ZipName, [ZipNamePrefix])),
@ -176,13 +216,18 @@ create_trace(Name, ClientId, Start) ->
?check_trace(
#{timetrap => 900},
begin
{ok, _} = emqx_trace:create([{<<"name">>, Name},
{<<"type">>, clientid}, {<<"clientid">>, ClientId}, {<<"start_at">>, Start}]),
{ok, _} = emqx_trace:create([
{<<"name">>, Name},
{<<"type">>, clientid},
{<<"clientid">>, ClientId},
{<<"start_at">>, Start}
]),
?block_until(#{?snk_kind := update_trace_done})
end,
fun(Trace) ->
?assertMatch([#{}], ?of_kind(update_trace_done, Trace))
end).
end
).
t_stream_log(_Config) ->
application:set_env(emqx, allow_anonymous, true),
@ -194,7 +239,12 @@ t_stream_log(_Config) ->
create_trace(Name, ClientId, Now - 10),
{ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]),
{ok, _} = emqtt:connect(Client),
[begin _ = emqtt:ping(Client) end || _ <- lists:seq(1, 5)],
[
begin
_ = emqtt:ping(Client)
end
|| _ <- lists:seq(1, 5)
],
emqtt:publish(Client, <<"/good">>, #{}, <<"ghood1">>, [{qos, 0}]),
emqtt:publish(Client, <<"/good">>, #{}, <<"ghood2">>, [{qos, 0}]),
ok = emqtt:disconnect(Client),
@ -239,8 +289,9 @@ do_request_api(Method, Request) ->
{error, socket_closed_remotely};
{error, {shutdown, server_closed}} ->
{error, server_closed};
{ok, {{"HTTP/1.1", Code, _}, _Headers, Return}}
when Code =:= 200 orelse Code =:= 201 orelse Code =:= 204 ->
{ok, {{"HTTP/1.1", Code, _}, _Headers, Return}} when
Code =:= 200 orelse Code =:= 201 orelse Code =:= 204
->
{ok, Return};
{ok, {Reason, _Header, Body}} ->
{error, Reason, Body}
@ -250,7 +301,8 @@ api_path(Path) ->
?HOST ++ filename:join([?BASE_PATH, ?API_VERSION, Path]).
json(Data) ->
{ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx.
{ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]),
Jsx.
load() ->
emqx_trace:start_link().

74
apps/emqx_modules/i18n/emqx_modules_schema_i18n.conf Normal file
View File

@ -0,0 +1,74 @@
emqx_modules_schema {
rewrite {
desc {
en: """The topic rewriting function of EMQX supports rewriting topic A to topic B when the client subscribes to topics, publishes messages, and cancels subscriptions according to user-configured rules.
Each rewrite rule consists of three parts: subject filter, regular expression, and target expression.
Under the premise that the subject rewriting function is enabled, when EMQX receives a subject-based MQTT message such as a PUBLISH message,
it will use the subject of the message to sequentially match the subject filter part of the rule in the configuration file. If the match is successful,
the regular expression is used to extract the information in the subject, and then replaced with the target expression to form a new subject.
Variables in the format of $N can be used in the target expression to match the elements extracted from the regular expression.
The value of $N is the Nth element extracted from the regular expression. For example, $1 is the regular expression. The first element extracted by the expression.
It should be noted that EMQX uses reverse order to read the rewrite rules in the configuration file.
When a topic can match the topic filter of multiple topic rewrite rules at the same time, EMQX will only use the first rule it matches. Rewrite.
If the regular expression in this rule does not match the subject of the MQTT message, the rewriting will fail, and no other rules will be attempted for rewriting.
Therefore, users need to carefully design MQTT message topics and topic rewriting rules when using them."""
zh: """EMQX 的主题重写功能支持根据用户配置的规则在客户端订阅主题、发布消息、取消订阅的时候将 A 主题重写为 B 主题。
重写规则分为 Pub 规则和 Sub 规则，Pub 规则匹配 PUSHLISH 报文携带的主题，Sub 规则匹配 SUBSCRIBE、UNSUBSCRIBE 报文携带的主题。
每条重写规则都由主题过滤器、正则表达式、目标表达式三部分组成。
在主题重写功能开启的前提下，EMQX 在收到诸如 PUBLISH 报文等带有主题的 MQTT 报文时，将使用报文中的主题去依次匹配配置文件中规则的主题过滤器部分，一旦成功匹配，则使用正则表达式提取主题中的信息，然后替换至目标表达式以构成新的主题。
目标表达式中可以使用 $N 这种格式的变量匹配正则表达中提取出来的元素，$N 的值为正则表达式中提取出来的第 N 个元素，比如 $1 即为正则表达式提取的第一个元素。
需要注意的是，EMQX 使用倒序读取配置文件中的重写规则，当一条主题可以同时匹配多条主题重写规则的主题过滤器时，EMQX 仅会使用它匹配到的第一条规则进行重写，如果该条规则中的正则表达式与 MQTT 报文主题不匹配，则重写失败，不会再尝试使用其他的规则进行重写。
因此用户在使用时需要谨慎的设计 MQTT 报文主题以及主题重写规则。"""
}
label {
en: """Topic Rewrite"""
zh: """主题重写"""
}
}
tr_source_topic {
desc {
en: """Source topic, specified by the client."""
zh: """源主题，客户端业务指定的主题"""
}
label {
en: """Source Topic"""
zh: """源主题"""
}
}
tr_action {
desc {
en: """subscribe: Rewrite topic when client do subscribe.
publish: Rewrite topic when client do publish.
all: Both"""
zh: """subscribe：订阅时重写主题；
publish：发布时重写主题；
all：全部重写主题"""
}
label {
en: """Action"""
zh: """Action"""
}
}
tr_re {
desc {
en: """Regular expressions"""
zh: """正则表达式"""
}
}
tr_dest_topic {
desc {
en: """Destination topic."""
zh: """目标主题。"""
}
label {
en: """Destination Topic"""
zh: """目标主题"""
}
}
}

25
apps/emqx_modules/i18n/emqx_rewrite_api_i18n.conf Normal file
View File

@ -0,0 +1,25 @@
emqx_rewrite_api {
list_topic_rewrite_api {
desc {
en: """List all rewrite rules"""
zh: """列出全部主题重写规则"""
}
}
update_topic_rewrite_api {
desc {
en: """Update all rewrite rules"""
zh: """更新全部主题重写规则"""
}
}
update_topic_rewrite_api_response413 {
desc {
en: """Rules count exceed max limit"""
zh: """超出主题重写规则数量上限"""
}
}
}

12
apps/emqx_modules/src/emqx_delayed_api.erl
View File

@ -149,9 +149,9 @@ schema("/mqtt/delayed/messages") ->
[
{data, mk(hoconsc:array(ref("message")), #{})},
{meta, [
{page, mk(integer(), #{})},
{limit, mk(integer(), #{})},
{count, mk(integer(), #{})}
{page, mk(pos_integer(), #{})},
{limit, mk(pos_integer(), #{})},
{count, mk(non_neg_integer(), #{})}
]}
]
}
@ -163,11 +163,11 @@ fields("message_without_payload") ->
{msgid, mk(integer(), #{desc => <<"Message Id (MQTT message id hash)">>})},
{node, mk(binary(), #{desc => <<"The node where message from">>})},
{publish_at, mk(binary(), #{desc => <<"Client publish message time, rfc 3339">>})},
{delayed_interval, mk(integer(), #{desc => <<"Delayed interval, second">>})},
{delayed_remaining, mk(integer(), #{desc => <<"Delayed remaining, second">>})},
{delayed_interval, mk(pos_integer(), #{desc => <<"Delayed interval, second">>})},
{delayed_remaining, mk(non_neg_integer(), #{desc => <<"Delayed remaining, second">>})},
{expected_at, mk(binary(), #{desc => <<"Expect publish time, rfc 3339">>})},
{topic, mk(binary(), #{desc => <<"Topic">>, example => <<"/sys/#">>})},
{qos, mk(binary(), #{desc => <<"QoS">>})},
{qos, mk(emqx_schema:qos(), #{desc => <<"QoS">>})},
{from_clientid, mk(binary(), #{desc => <<"From ClientId">>})},
{from_username, mk(binary(), #{desc => <<"From Username">>})}
];

12
apps/emqx_modules/src/emqx_modules_schema.erl
View File

@ -16,6 +16,7 @@
-module(emqx_modules_schema).
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
-behaviour(hocon_schema).
@ -50,17 +51,17 @@ fields("rewrite") ->
{action,
sc(
hoconsc:enum([subscribe, publish, all]),
#{desc => <<"Action">>, example => publish}
#{required => true, desc => ?DESC(tr_action), example => publish}
)},
{source_topic,
sc(
binary(),
#{desc => <<"Origin Topic">>, example => "x/#"}
#{required => true, desc => ?DESC(tr_source_topic), example => "x/#"}
)},
{dest_topic,
sc(
binary(),
#{desc => <<"Destination Topic">>, example => "z/y/$1"}
#{required => true, desc => ?DESC(tr_dest_topic), example => "z/y/$1"}
)},
{re, fun regular_expression/1}
];
@ -72,14 +73,15 @@ desc("telemetry") ->
desc("delayed") ->
"Settings for the delayed module.";
desc("rewrite") ->
"Rewrite rule.";
?DESC(rewrite);
desc("topic_metrics") ->
"";
desc(_) ->
undefined.
regular_expression(type) -> binary();
regular_expression(desc) -> "Regular expressions";
regular_expression(required) -> true;
regular_expression(desc) -> ?DESC(tr_re);
regular_expression(example) -> "^x/y/(.+)$";
regular_expression(validator) -> fun is_re/1;
regular_expression(_) -> undefined.

12
apps/emqx_modules/src/emqx_rewrite_api.erl
View File

@ -16,6 +16,8 @@
-module(emqx_rewrite_api).
-behaviour(minirest_api).
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
-include("emqx_modules.hrl").
@ -38,16 +40,16 @@ schema("/mqtt/topic_rewrite") ->
'operationId' => topic_rewrite,
get => #{
tags => ?API_TAG_MQTT,
description => <<"List rewrite topic.">>,
description => ?DESC(list_topic_rewrite_api),
responses => #{
200 => hoconsc:mk(
hoconsc:array(hoconsc:ref(emqx_modules_schema, "rewrite")),
#{desc => <<"List all rewrite rules">>}
#{desc => ?DESC(list_topic_rewrite_api)}
)
}
},
put => #{
description => <<"Update rewrite topic">>,
description => ?DESC(update_topic_rewrite_api),
tags => ?API_TAG_MQTT,
'requestBody' => hoconsc:mk(
hoconsc:array(
@ -58,11 +60,11 @@ schema("/mqtt/topic_rewrite") ->
responses => #{
200 => hoconsc:mk(
hoconsc:array(hoconsc:ref(emqx_modules_schema, "rewrite")),
#{desc => <<"Update rewrite topic success.">>}
#{desc => ?DESC(update_topic_rewrite_api)}
),
413 => emqx_dashboard_swagger:error_codes(
[?EXCEED_LIMIT],
<<"Rules count exceed max limit">>
?DESC(update_topic_rewrite_api_response413)
)
}
}

10
apps/emqx_plugin_libs/src/emqx_plugin_libs_metrics.erl
View File

@ -307,17 +307,19 @@ calculate_rate(CurrVal, #rate{max = MaxRate0, last_v = LastVal,
%% calculate the max rate since the emqx startup
MaxRate =
if MaxRate0 >= CurrRate -> MaxRate0;
true -> CurrRate
case MaxRate0 >= CurrRate of
true -> MaxRate0;
false -> CurrRate
end,
%% calculate the average rate in last 5 mins
{Last5MinSamples, Acc5Min, Last5Min} =
if Tick =< ?SAMPCOUNT_5M ->
case Tick =< ?SAMPCOUNT_5M of
true ->
Acc = AccRate5Min0 + CurrRate,
{lists:reverse([CurrRate | lists:reverse(Last5MinSamples0)]),
Acc, Acc / Tick};
true ->
false ->
[FirstRate | Rates] = Last5MinSamples0,
Acc = AccRate5Min0 + CurrRate - FirstRate,
{lists:reverse([CurrRate | lists:reverse(Rates)]),

Some files were not shown because too many files have changed in this diff Show More