yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge tag 'v5.0.11' into dev/ee5.0

This commit is contained in:
Zaiming (Stone) Shi 2022-11-28 21:02:21 +01:00
parent c559334e69 05719daff2
commit 7ee53e5319
95 changed files with 2514 additions and 1084 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
.github/workflows/release.yaml vendored
View File

@ -3,6 +3,15 @@ on:
release:
types:
- published
workflow_dispatch:
inputs:
tag:
type: string
required: true
publish_release_artefacts:
type: boolean
required: true
default: false
jobs:
upload:
@ -15,22 +24,35 @@ jobs:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
- name: Get packages
- uses: actions/checkout@v3
with:
ref: ${{ github.event.inputs.tag }}
- name: Detect profile
id: profile
run: |
REF=${{ github.ref_name }}
if git describe --tags --match '[v|e]*' --exact; then
REF=$(git describe --tags --match '[v|e]*' --exact)
else
echo "Only release tags matching '[v|e]*' are supported"
exit 1
fi
case "$REF" in
v*)
s3dir='emqx-ce'
echo "profile=emqx" >> $GITHUB_OUTPUT
echo "version=$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT
echo "s3dir=emqx-ce" >> $GITHUB_OUTPUT
;;
e*)
s3dir='emqx-ee'
;;
*)
echo "tag $REF is not supported"
exit 1
echo "profile=emqx-enterprise" >> $GITHUB_OUTPUT
echo "version=$(./pkg-vsn.sh emqx-enterprise)" >> $GITHUB_OUTPUT
echo "s3dir=emqx-ee" >> $GITHUB_OUTPUT
;;
esac
aws s3 cp --recursive s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} packages
- name: Get packages
run: |
BUCKET=${{ secrets.AWS_S3_BUCKET }}
OUTPUT_DIR=${{ steps.profile.outputs.s3dir }}
aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages
cd packages
DEFAULT_BEAM_PLATFORM='otp24.3.4.2-1'
# all packages including full-name and default-name are uploaded to s3
@ -47,7 +69,7 @@ jobs:
with:
asset_paths: '["packages/*"]'
- name: update to emqx.io
if: github.event_name == 'release'
if: github.event_name == 'release' || inputs.publish_release_artefacts
run: |
set -e -x -u
curl -w %{http_code} \
@ -58,18 +80,8 @@ jobs:
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
${{ secrets.EMQX_IO_RELEASE_API }}
- name: update homebrew packages
if: github.event_name == 'release'
if: steps.profile.outputs.profile == 'emqx' && (github.event_name == 'release' || inputs.publish_release_artefacts)
run: |
REF=${{ github.ref_name }}
case "$REF" in
v*)
BOOL_FLAG_NAME="emqx_ce"
;;
e*)
echo "Not updating homebrew for enterprise eidition"
exit 0
;;
esac
if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
@ -78,30 +90,11 @@ jobs:
-d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
fi
upload-helm:
runs-on: ubuntu-20.04
if: github.event_name == 'release'
strategy:
fail-fast: false
steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.ref }}
- uses: emqx/push-helm-action@v1
if: startsWith(github.ref_name, 'v')
if: github.event_name == 'release' || inputs.publish_release_artefacts
with:
charts_dir: "${{ github.workspace }}/deploy/charts/emqx"
version: ${{ github.ref_name }}
aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws_region: "us-west-2"
aws_bucket_name: "repos-emqx-io"
- uses: emqx/push-helm-action@v1
if: startsWith(github.ref_name, 'e')
with:
charts_dir: "${{ github.workspace }}/deploy/charts/emqx-enterprise"
version: ${{ github.ref_name }}
charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}"
version: ${{ steps.profile.outputs.version }}
aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws_region: "us-west-2"

4
Makefile
View File

@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
export EMQX_DEFAULT_RUNNER = debian:11-slim
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v1.1.1
export EMQX_DASHBOARD_VERSION ?= v1.1.2
export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.5
export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1
@ -23,7 +23,7 @@ PKG_PROFILES := emqx-pkg emqx-enterprise-pkg
PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default
CT_NODE_NAME ?= 'test@127.0.0.1'
CT_READABLE ?= false
CT_READABLE ?= true
export REBAR_GIT_CLONE_OPTIONS += --depth=1

14
apps/emqx/i18n/emqx_schema_i18n.conf
View File

@ -2045,12 +2045,18 @@ Type of the rate limit.
base_listener_enable_authn {
desc {
en: """
Set <code>true</code> (default) to enable client authentication on this listener.
When set to <code>false</code> clients will be allowed to connect without authentication.
Set <code>true</code> (default) to enable client authentication on this listener, the authentication
process goes through the configured authentication chain.
When set to <code>false</code> to allow any clients with or without authentication information such as username or password to log in.
When set to <code>quick_deny_anonymous<code>, it behaves like when set to <code>true</code> but clients will be
denied immediately without going through any authenticators if <code>username</code> is not provided. This is useful to fence off
anonymous clients early.
"""
zh: """
配置 <code>true</code> （默认值）启用客户端进行身份认证。
配置 <code>false</code> 时，将不对客户端做任何认证。
配置 <code>true</code> （默认值）启用客户端进行身份认证，通过检查认配置的认认证器链来决定是否允许接入。
配置 <code>false</code> 时，将不对客户端做任何认证，任何客户端，不论是不是携带用户名等认证信息，都可以接入。
配置 <code>quick_deny_anonymous</code> 时，行为跟 <code>true</code> 类似，但是会对匿名
客户直接拒绝，不做使用任何认证器对客户端进行身份检查。
"""
}
label: {

2
apps/emqx/include/emqx_release.hrl
View File

@ -32,7 +32,7 @@
%% `apps/emqx/src/bpapi/README.md'
%% Community edition
-define(EMQX_RELEASE_CE, "5.0.10").
-define(EMQX_RELEASE_CE, "5.0.11").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.0.0-beta.5").

19
apps/emqx/src/emqx_access_control.erl
View File

@ -38,11 +38,22 @@
| {ok, map(), binary()}
| {continue, map()}
| {continue, binary(), map()}
| {error, term()}.
| {error, not_authorized}.
authenticate(Credential) ->
case run_hooks('client.authenticate', [Credential], {ok, #{is_superuser => false}}) of
%% pre-hook quick authentication or
%% if auth backend returning nothing but just 'ok'
%% it means it's not a superuser, or there is no way to tell.
NotSuperUser = #{is_superuser => false},
case emqx_authentication:pre_hook_authenticate(Credential) of
ok ->
{ok, #{is_superuser => false}};
{ok, NotSuperUser};
continue ->
case run_hooks('client.authenticate', [Credential], {ok, #{is_superuser => false}}) of
ok ->
{ok, NotSuperUser};
Other ->
Other
end;
Other ->
Other
end.
@ -56,7 +67,7 @@ authorize(ClientInfo, PubSub, <<"$delayed/", Data/binary>> = RawTopic) ->
authorize(ClientInfo, PubSub, Topic);
_ ->
?SLOG(warning, #{
msg => "invalid_dealyed_topic_format",
msg => "invalid_delayed_topic_format",
expected_example => "$delayed/1/t/foo",
got => RawTopic
}),

14
apps/emqx/src/emqx_alarm.erl
View File

@ -41,7 +41,8 @@
delete_all_deactivated_alarms/0,
get_alarms/0,
get_alarms/1,
format/1
format/1,
format/2
]).
%% gen_server callbacks
@ -169,12 +170,15 @@ get_alarms(activated) ->
get_alarms(deactivated) ->
gen_server:call(?MODULE, {get_alarms, deactivated}).
format(#activated_alarm{name = Name, message = Message, activate_at = At, details = Details}) ->
format(Alarm) ->
format(node(), Alarm).
format(Node, #activated_alarm{name = Name, message = Message, activate_at = At, details = Details}) ->
Now = erlang:system_time(microsecond),
%% mnesia db stored microsecond for high frequency alarm
%% format for dashboard using millisecond
#{
node => node(),
node => Node,
name => Name,
message => Message,
%% to millisecond
@ -182,7 +186,7 @@ format(#activated_alarm{name = Name, message = Message, activate_at = At, detail
activate_at => to_rfc3339(At),
details => Details
};
format(#deactivated_alarm{
format(Node, #deactivated_alarm{
name = Name,
message = Message,
activate_at = At,
@ -190,7 +194,7 @@ format(#deactivated_alarm{
deactivate_at = DAt
}) ->
#{
node => node(),
node => Node,
name => Name,
message => Message,
%% to millisecond

25
apps/emqx/src/emqx_authentication.erl
View File

@ -29,9 +29,13 @@
-include_lib("stdlib/include/ms_transform.hrl").
-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
-define(IS_UNDEFINED(X), (X =:= undefined orelse X =:= <<>>)).
%% The authentication entrypoint.
-export([authenticate/2]).
-export([
pre_hook_authenticate/1,
authenticate/2
]).
%% Authenticator manager process start/stop
-export([
@ -221,10 +225,23 @@ when
%%------------------------------------------------------------------------------
%% Authenticate
%%------------------------------------------------------------------------------
authenticate(#{enable_authn := false}, _AuthResult) ->
-spec pre_hook_authenticate(emqx_types:clientinfo()) ->
ok | continue | {error, not_authorized}.
pre_hook_authenticate(#{enable_authn := false}) ->
inc_authenticate_metric('authentication.success.anonymous'),
?TRACE_RESULT("authentication_result", ignore, enable_authn_false);
?TRACE_RESULT("authentication_result", ok, enable_authn_false);
pre_hook_authenticate(#{enable_authn := quick_deny_anonymous} = Credential) ->
case maps:get(username, Credential, undefined) of
U when ?IS_UNDEFINED(U) ->
?TRACE_RESULT(
"authentication_result", {error, not_authorized}, enable_authn_false
);
_ ->
continue
end;
pre_hook_authenticate(_) ->
continue.
authenticate(#{listener := Listener, protocol := Protocol} = Credential, _AuthResult) ->
case get_authenticators(Listener, global_chain(Protocol)) of
{ok, ChainName, Authenticators} ->

23
apps/emqx/src/emqx_banned.erl
View File

@ -21,6 +21,7 @@
-include("emqx.hrl").
-include("logger.hrl").
-include("types.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% Mnesia bootstrap
-export([mnesia/1]).
@ -180,7 +181,7 @@ create(#{
create(Banned = #banned{who = Who}) ->
case look_up(Who) of
[] ->
mria:dirty_write(?BANNED_TAB, Banned),
insert_banned(Banned),
{ok, Banned};
[OldBanned = #banned{until = Until}] ->
%% Don't support shorten or extend the until time by overwrite.
@ -190,7 +191,7 @@ create(Banned = #banned{who = Who}) ->
{error, {already_exist, OldBanned}};
%% overwrite expired one is ok.
false ->
mria:dirty_write(?BANNED_TAB, Banned),
insert_banned(Banned),
{ok, Banned}
end
end.
@ -266,3 +267,21 @@ expire_banned_items(Now) ->
ok,
?BANNED_TAB
).
insert_banned(Banned) ->
mria:dirty_write(?BANNED_TAB, Banned),
on_banned(Banned).
on_banned(#banned{who = {clientid, ClientId}}) ->
%% kick the session if the client is banned by clientid
?tp(
warning,
kick_session_due_to_banned,
#{
clientid => ClientId
}
),
emqx_cm:kick_session(ClientId),
ok;
on_banned(_) ->
ok.

8
apps/emqx/src/emqx_channel.erl
View File

@ -2134,10 +2134,14 @@ will_delay_interval(WillMsg) ->
0
).
publish_will_msg(ClientInfo, Msg = #message{topic = Topic}) ->
publish_will_msg(
ClientInfo = #{mountpoint := MountPoint},
Msg = #message{topic = Topic}
) ->
case emqx_access_control:authorize(ClientInfo, publish, Topic) of
allow ->
_ = emqx_broker:publish(Msg),
NMsg = emqx_mountpoint:mount(MountPoint, Msg),
_ = emqx_broker:publish(NMsg),
ok;
deny ->
?tp(

4
apps/emqx/src/emqx_cm.erl
View File

@ -650,8 +650,8 @@ init([]) ->
TabOpts = [public, {write_concurrency, true}],
ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]),
ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]),
ok = emqx_tables:new(?CHAN_INFO_TAB, [set, compressed | TabOpts]),
ok = emqx_tables:new(?CHAN_LIVE_TAB, [set, {write_concurrency, true} | TabOpts]),
ok = emqx_tables:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]),
ok = emqx_tables:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]),
ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0),
State = #{chan_pmon => emqx_pmon:new()},
{ok, State}.

12
apps/emqx/src/emqx_map_lib.erl
View File

@ -23,6 +23,7 @@
deep_force_put/3,
deep_remove/2,
deep_merge/2,
binary_key_map/1,
safe_atom_key_map/1,
unsafe_atom_key_map/1,
jsonable_map/1,
@ -153,6 +154,17 @@ deep_convert(Val, _, _Args) ->
unsafe_atom_key_map(Map) ->
covert_keys_to_atom(Map, fun(K) -> binary_to_atom(K, utf8) end).
-spec binary_key_map(map()) -> map().
binary_key_map(Map) ->
deep_convert(
Map,
fun
(K, V) when is_atom(K) -> {atom_to_binary(K, utf8), V};
(K, V) when is_binary(K) -> {K, V}
end,
[]
).
-spec safe_atom_key_map(#{binary() | atom() => any()}) -> #{atom() => any()}.
safe_atom_key_map(Map) ->
covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end).

17
apps/emqx/src/emqx_misc.erl
View File

@ -54,7 +54,8 @@
pmap/3,
readable_error_msg/1,
safe_to_existing_atom/1,
safe_to_existing_atom/2
safe_to_existing_atom/2,
pub_props_to_packet/1
]).
-export([
@ -568,3 +569,17 @@ ipv6_probe_test() ->
end.
-endif.
pub_props_to_packet(Properties) ->
F = fun
('User-Property', M) ->
case is_map(M) andalso map_size(M) > 0 of
true -> {true, maps:to_list(M)};
false -> false
end;
('User-Property-Pairs', _) ->
false;
(_, _) ->
true
end,
maps:filtermap(F, Properties).

4
apps/emqx/src/emqx_schema.erl
View File

@ -399,7 +399,7 @@ fields("mqtt") ->
sc(
range(1, 65535),
#{
default => 65535,
default => 128,
desc => ?DESC(mqtt_max_topic_levels)
}
)},
@ -1668,7 +1668,7 @@ base_listener(Bind) ->
)},
{"enable_authn",
sc(
boolean(),
hoconsc:enum([true, false, quick_deny_anonymous]),
#{
desc => ?DESC(base_listener_enable_authn),
default => true

15
apps/emqx/src/emqx_trace/emqx_trace.erl
View File

@ -38,7 +38,8 @@
delete/1,
clear/0,
update/2,
check/0
check/0,
now_second/0
]).
-export([
@ -287,7 +288,7 @@ insert_new_trace(Trace) ->
transaction(fun emqx_trace_dl:insert_new_trace/1, [Trace]).
update_trace(Traces) ->
Now = erlang:system_time(second),
Now = now_second(),
{_Waiting, Running, Finished} = classify_by_time(Traces, Now),
disable_finished(Finished),
Started = emqx_trace_handler:running(),
@ -455,7 +456,7 @@ ensure_map(Trace) when is_list(Trace) ->
).
fill_default(Trace = #?TRACE{start_at = undefined}) ->
fill_default(Trace#?TRACE{start_at = erlang:system_time(second)});
fill_default(Trace#?TRACE{start_at = now_second()});
fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) ->
fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60});
fill_default(Trace) ->
@ -493,7 +494,7 @@ to_trace(#{start_at := StartAt} = Trace, Rec) ->
{ok, Sec} = to_system_second(StartAt),
to_trace(maps:remove(start_at, Trace), Rec#?TRACE{start_at = Sec});
to_trace(#{end_at := EndAt} = Trace, Rec) ->
Now = erlang:system_time(second),
Now = now_second(),
case to_system_second(EndAt) of
{ok, Sec} when Sec > Now ->
to_trace(maps:remove(end_at, Trace), Rec#?TRACE{end_at = Sec});
@ -517,8 +518,7 @@ validate_ip_address(IP) ->
end.
to_system_second(Sec) ->
Now = erlang:system_time(second),
{ok, erlang:max(Now, Sec)}.
{ok, erlang:max(now_second(), Sec)}.
zip_dir() ->
filename:join([trace_dir(), "zip"]).
@ -570,3 +570,6 @@ filter_cli_handler(Names) ->
end,
Names
).
now_second() ->
os:system_time(second).

4
apps/emqx/src/emqx_trace/emqx_trace_dl.erl
View File

@ -30,7 +30,7 @@
-include("emqx_trace.hrl").
%%================================================================================
%% API funcions
%% API functions
%%================================================================================
%% Introduced in 5.0
@ -43,7 +43,7 @@ update(Name, Enable) ->
[#?TRACE{enable = Enable}] ->
ok;
[Rec] ->
case erlang:system_time(second) >= Rec#?TRACE.end_at of
case emqx_trace:now_second() >= Rec#?TRACE.end_at of
false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write);
true -> mnesia:abort(finished)
end

67
apps/emqx/test/emqx_access_control_SUITE.erl
View File

@ -20,6 +20,7 @@
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/emqx_hooks.hrl").
-include_lib("eunit/include/eunit.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).
@ -32,12 +33,13 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]).
end_per_testcase(t_delayed_authorize, Config) ->
meck:unload(emqx_access_control),
Config;
end_per_testcase(_, Config) ->
init_per_testcase(_, Config) ->
Config.
end_per_testcase(_, _Config) ->
ok = emqx_hooks:del('client.authorize', {?MODULE, authz_stub}),
ok = emqx_hooks:del('client.authenticate', {?MODULE, quick_deny_anonymous_authn}).
t_authenticate(_) ->
?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())).
@ -46,31 +48,62 @@ t_authorize(_) ->
?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish, <<"t">>)).
t_delayed_authorize(_) ->
RawTopic = "$dealyed/1/foo/2",
InvalidTopic = "$dealyed/1/foo/3",
Topic = "foo/2",
RawTopic = <<"$delayed/1/foo/2">>,
InvalidTopic = <<"$delayed/1/foo/3">>,
Topic = <<"foo/2">>,
ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
ok = meck:expect(
emqx_access_control,
do_authorize,
fun
(_, _, Topic) -> allow;
(_, _, _) -> deny
end
),
ok = emqx_hooks:put('client.authorize', {?MODULE, authz_stub, [Topic]}, ?HP_AUTHZ),
Publish1 = ?PUBLISH_PACKET(?QOS_0, RawTopic, 1, <<"payload">>),
?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish1, RawTopic)),
Publish2 = ?PUBLISH_PACKET(?QOS_0, InvalidTopic, 1, <<"payload">>),
?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish2, InvalidTopic)),
?assertEqual(deny, emqx_access_control:authorize(clientinfo(), Publish2, InvalidTopic)),
ok.
t_quick_deny_anonymous(_) ->
ok = emqx_hooks:put(
'client.authenticate',
{?MODULE, quick_deny_anonymous_authn, []},
?HP_AUTHN
),
RawClient0 = clientinfo(),
RawClient = RawClient0#{username => undefined},
%% No name, No authn
Client1 = RawClient#{enable_authn => false},
?assertMatch({ok, _}, emqx_access_control:authenticate(Client1)),
%% No name, With quick_deny_anonymous
Client2 = RawClient#{enable_authn => quick_deny_anonymous},
?assertMatch({error, _}, emqx_access_control:authenticate(Client2)),
%% Bad name, With quick_deny_anonymous
Client3 = RawClient#{enable_authn => quick_deny_anonymous, username => <<"badname">>},
?assertMatch({error, _}, emqx_access_control:authenticate(Client3)),
%% Good name, With quick_deny_anonymous
Client4 = RawClient#{enable_authn => quick_deny_anonymous, username => <<"goodname">>},
?assertMatch({ok, _}, emqx_access_control:authenticate(Client4)),
%% Name, With authn
Client5 = RawClient#{enable_authn => true, username => <<"badname">>},
?assertMatch({error, _}, emqx_access_control:authenticate(Client5)),
ok.
%%--------------------------------------------------------------------
%% Helper functions
%%--------------------------------------------------------------------
authz_stub(_Client, _PubSub, ValidTopic, _DefaultResult, ValidTopic) -> {stop, #{result => allow}};
authz_stub(_Client, _PubSub, _Topic, _DefaultResult, _ValidTopic) -> {stop, #{result => deny}}.
quick_deny_anonymous_authn(#{username := <<"badname">>}, _AuthResult) ->
{stop, {error, not_authorized}};
quick_deny_anonymous_authn(_ClientInfo, _AuthResult) ->
{stop, {ok, #{is_superuser => false}}}.
clientinfo() -> clientinfo(#{}).
clientinfo(InitProps) ->
maps:merge(

62
apps/emqx/test/emqx_banned_SUITE.erl
View File

@ -21,18 +21,20 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
application:load(emqx),
emqx_common_test_helpers:start_apps([]),
ok = ekka:start(),
Config.
end_per_suite(_Config) ->
ekka:stop(),
mria:stop(),
mria_mnesia:delete_schema().
mria_mnesia:delete_schema(),
emqx_common_test_helpers:stop_apps([]).
t_add_delete(_) ->
Banned = #banned{
@ -95,19 +97,47 @@ t_check(_) ->
?assertEqual(0, emqx_banned:info(size)).
t_unused(_) ->
catch emqx_banned:stop(),
{ok, Banned} = emqx_banned:start_link(),
{ok, _} = emqx_banned:create(#banned{
who = {clientid, <<"BannedClient1">>},
until = erlang:system_time(second)
}),
{ok, _} = emqx_banned:create(#banned{
who = {clientid, <<"BannedClient2">>},
until = erlang:system_time(second) - 1
}),
?assertEqual(ignored, gen_server:call(Banned, unexpected_req)),
?assertEqual(ok, gen_server:cast(Banned, unexpected_msg)),
?assertEqual(ok, Banned ! ok),
Who1 = {clientid, <<"BannedClient1">>},
Who2 = {clientid, <<"BannedClient2">>},
?assertMatch(
{ok, _},
emqx_banned:create(#banned{
who = Who1,
until = erlang:system_time(second)
})
),
?assertMatch(
{ok, _},
emqx_banned:create(#banned{
who = Who2,
until = erlang:system_time(second) - 1
})
),
?assertEqual(ignored, gen_server:call(emqx_banned, unexpected_req)),
?assertEqual(ok, gen_server:cast(emqx_banned, unexpected_msg)),
%% expiry timer
timer:sleep(500),
ok = emqx_banned:stop().
ok = emqx_banned:delete(Who1),
ok = emqx_banned:delete(Who2).
t_kick(_) ->
ClientId = <<"client">>,
snabbkaffe:start_trace(),
Now = erlang:system_time(second),
Who = {clientid, ClientId},
emqx_banned:create(#{
who => Who,
by => <<"test">>,
reason => <<"test">>,
at => Now,
until => Now + 120
}),
Trace = snabbkaffe:collect_trace(),
snabbkaffe:stop(),
emqx_banned:delete(Who),
?assertEqual(1, length(?of_kind(kick_session_due_to_banned, Trace))).

7
apps/emqx/test/emqx_bpapi_static_checks.erl
View File

@ -62,9 +62,10 @@
%% List of business-layer functions that are exempt from the checks:
%% erlfmt-ignore
-define(EXEMPTIONS,
"emqx_mgmt_api:do_query/6" % Reason: legacy code. A fun and a QC query are
% passed in the args, it's futile to try to statically
% check it
% Reason: legacy code. A fun and a QC query are
% passed in the args, it's futile to try to statically
% check it
"emqx_mgmt_api:do_query/2, emqx_mgmt_api:collect_total_from_tail_nodes/3"
).
-define(XREF, myxref).

16
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -728,6 +728,22 @@ t_quota_qos2(_) ->
del_bucket(),
esockd_limiter:stop().
t_mount_will_msg(_) ->
Self = self(),
ClientInfo = clientinfo(#{mountpoint => <<"prefix/">>}),
Msg = emqx_message:make(test, <<"will_topic">>, <<"will_payload">>),
Channel = channel(#{clientinfo => ClientInfo, will_msg => Msg}),
ok = meck:expect(emqx_broker, publish, fun(M) -> Self ! {pub, M} end),
{shutdown, kicked, ok, ?DISCONNECT_PACKET(?RC_ADMINISTRATIVE_ACTION), _} = emqx_channel:handle_call(
kick, Channel
),
receive
{pub, #message{topic = <<"prefix/will_topic">>}} -> ok
after 200 -> exit(will_message_not_published_or_not_correct)
end.
%%--------------------------------------------------------------------
%% Test cases for handle_deliver
%%--------------------------------------------------------------------

37
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -537,21 +537,51 @@ ensure_quic_listener(Name, UdpPort) ->
%% Clusterisation and multi-node testing
%%
-type cluster_spec() :: [node_spec()].
-type node_spec() :: role() | {role(), shortname()} | {role(), shortname(), node_opts()}.
-type role() :: core | replicant.
-type shortname() :: atom().
-type nodename() :: atom().
-type node_opts() :: #{
%% Need to loaded apps. These apps will be loaded once the node started
load_apps => list(),
%% Need to started apps. It is the first arg passed to emqx_common_test_helpers:start_apps/2
apps => list(),
%% Extras app starting handler. It is the second arg passed to emqx_common_test_helpers:start_apps/2
env_handler => fun((AppName :: atom()) -> term()),
%% Application env preset before calling `emqx_common_test_helpers:start_apps/2`
env => {AppName :: atom(), Key :: atom(), Val :: term()},
%% Whether to execute `emqx_config:init_load(SchemaMod)`
%% default: true
load_schema => boolean(),
%% Eval by emqx_config:put/2
conf => [{KeyPath :: list(), Val :: term()}],
%% Fast option to config listener port
%% default rule:
%% - tcp: base_port
%% - ssl: base_port + 1
%% - ws : base_port + 3
%% - wss: base_port + 4
listener_ports => [{Type :: tcp | ssl | ws | wss, inet:port_number()}]
}.
-spec emqx_cluster(cluster_spec()) -> [{shortname(), node_opts()}].
emqx_cluster(Specs) ->
emqx_cluster(Specs, #{}).
-spec emqx_cluster(cluster_spec(), node_opts()) -> [{shortname(), node_opts()}].
emqx_cluster(Specs, CommonOpts) when is_list(CommonOpts) ->
emqx_cluster(Specs, maps:from_list(CommonOpts));
emqx_cluster(Specs0, CommonOpts) ->
Specs1 = lists:zip(Specs0, lists:seq(1, length(Specs0))),
Specs = expand_node_specs(Specs1, CommonOpts),
CoreNodes = [node_name(Name) || {{core, Name, _}, _} <- Specs],
%% Assign grpc ports:
%% Assign grpc ports
GenRpcPorts = maps:from_list([
{node_name(Name), {tcp, gen_rpc_port(base_port(Num))}}
|| {{_, Name, _}, Num} <- Specs
]),
%% Set the default node of the cluster:
CoreNodes = [node_name(Name) || {{core, Name, _}, _} <- Specs],
JoinTo =
case CoreNodes of
[First | _] -> First;
@ -572,6 +602,8 @@ emqx_cluster(Specs0, CommonOpts) ->
].
%% Lower level starting API
-spec start_slave(shortname(), node_opts()) -> nodename().
start_slave(Name, Opts) ->
{ok, Node} = ct_slave:start(
list_to_atom(atom_to_list(Name) ++ "@" ++ host()),
@ -608,6 +640,7 @@ epmd_path() ->
%% Node initialization
-spec setup_node(nodename(), node_opts()) -> ok.
setup_node(Node, Opts) when is_list(Opts) ->
setup_node(Node, maps:from_list(Opts));
setup_node(Node, Opts) when is_map(Opts) ->

2
apps/emqx/test/emqx_trace_SUITE.erl
View File

@ -40,7 +40,7 @@ init_per_suite(Config) ->
?wait_async_action(
emqx_common_test_helpers:start_apps([]),
#{?snk_kind := listener_started, bind := 1883},
timer:seconds(10)
timer:seconds(100)
),
fun(Trace) ->
%% more than one listener

49
apps/emqx_authn/src/emqx_authn_api.erl
View File

@ -30,6 +30,7 @@
-define(BAD_REQUEST, 'BAD_REQUEST').
-define(NOT_FOUND, 'NOT_FOUND').
-define(ALREADY_EXISTS, 'ALREADY_EXISTS').
-define(INTERNAL_ERROR, 'INTERNAL_ERROR').
% Swagger
@ -224,7 +225,8 @@ schema("/authentication/:id/status") ->
hoconsc:ref(emqx_authn_schema, "metrics_status_fields"),
status_metrics_example()
),
400 => error_codes([?BAD_REQUEST], <<"Bad Request">>)
404 => error_codes([?NOT_FOUND], <<"Not Found">>),
500 => error_codes([?INTERNAL_ERROR], <<"Internal Service Error">>)
}
}
};
@ -576,7 +578,11 @@ authenticator(delete, #{bindings := #{id := AuthenticatorID}}) ->
delete_authenticator([authentication], ?GLOBAL, AuthenticatorID).
authenticator_status(get, #{bindings := #{id := AuthenticatorID}}) ->
lookup_from_all_nodes(?GLOBAL, AuthenticatorID).
with_authenticator(
AuthenticatorID,
[authentication],
fun(_) -> lookup_from_all_nodes(?GLOBAL, AuthenticatorID) end
).
listener_authenticators(post, #{bindings := #{listener_id := ListenerID}, body := Config}) ->
with_listener(
@ -647,8 +653,12 @@ listener_authenticator_status(
) ->
with_listener(
ListenerID,
fun(_, _, ChainName) ->
lookup_from_all_nodes(ChainName, AuthenticatorID)
fun(Type, Name, ChainName) ->
with_authenticator(
AuthenticatorID,
[listeners, Type, Name, authentication],
fun(_) -> lookup_from_all_nodes(ChainName, AuthenticatorID) end
)
end
).
@ -774,6 +784,18 @@ listener_authenticator_user(delete, #{
%% Internal functions
%%------------------------------------------------------------------------------
with_authenticator(AuthenticatorID, ConfKeyPath, Fun) ->
case find_authenticator_config(AuthenticatorID, ConfKeyPath) of
{ok, AuthenticatorConfig} ->
Fun(AuthenticatorConfig);
{error, Reason} ->
serialize_error(Reason)
end.
find_authenticator_config(AuthenticatorID, ConfKeyPath) ->
AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath),
find_config(AuthenticatorID, AuthenticatorsConfig).
with_listener(ListenerID, Fun) ->
case find_listener(ListenerID) of
{ok, {BType, BName}} ->
@ -836,13 +858,13 @@ list_authenticators(ConfKeyPath) ->
{200, NAuthenticators}.
list_authenticator(_, ConfKeyPath, AuthenticatorID) ->
AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath),
case find_config(AuthenticatorID, AuthenticatorsConfig) of
{ok, AuthenticatorConfig} ->
{200, maps:put(id, AuthenticatorID, convert_certs(AuthenticatorConfig))};
{error, Reason} ->
serialize_error(Reason)
end.
with_authenticator(
AuthenticatorID,
ConfKeyPath,
fun(AuthenticatorConfig) ->
{200, maps:put(id, AuthenticatorID, convert_certs(AuthenticatorConfig))}
end
).
resource_provider() ->
[
@ -877,7 +899,8 @@ lookup_from_local_node(ChainName, AuthenticatorID) ->
lookup_from_all_nodes(ChainName, AuthenticatorID) ->
Nodes = mria_mnesia:running_nodes(),
case is_ok(emqx_authn_proto_v1:lookup_from_all_nodes(Nodes, ChainName, AuthenticatorID)) of
LookupResult = emqx_authn_proto_v1:lookup_from_all_nodes(Nodes, ChainName, AuthenticatorID),
case is_ok(LookupResult) of
{ok, ResList} ->
{StatusMap, MetricsMap, ResourceMetricsMap, ErrorMap} = make_result_map(ResList),
AggregateStatus = aggregate_status(maps:values(StatusMap)),
@ -901,7 +924,7 @@ lookup_from_all_nodes(ChainName, AuthenticatorID) ->
node_error => HelpFun(maps:map(Fun, ErrorMap), reason)
}};
{error, ErrL} ->
{400, #{
{500, #{
code => <<"INTERNAL_ERROR">>,
message => list_to_binary(io_lib:format("~p", [ErrL]))
}}

52
apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
View File

@ -47,7 +47,8 @@
]).
-export([
query/4,
qs2ms/2,
run_fuzzy_filter/2,
format_user_info/1,
group_match_spec/1
]).
@ -66,7 +67,6 @@
{<<"user_group">>, binary},
{<<"is_superuser">>, atom}
]).
-define(QUERY_FUN, {?MODULE, query}).
-type user_group() :: binary().
@ -262,42 +262,30 @@ lookup_user(UserID, #{user_group := UserGroup}) ->
list_users(QueryString, #{user_group := UserGroup}) ->
NQueryString = QueryString#{<<"user_group">> => UserGroup},
emqx_mgmt_api:node_query(node(), NQueryString, ?TAB, ?AUTHN_QSCHEMA, ?QUERY_FUN).
%%--------------------------------------------------------------------
%% Query Functions
query(Tab, {QString, []}, Continuation, Limit) ->
Ms = ms_from_qstring(QString),
emqx_mgmt_api:select_table_with_count(
Tab,
Ms,
Continuation,
Limit,
fun format_user_info/1
);
query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
Ms = ms_from_qstring(QString),
FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
emqx_mgmt_api:select_table_with_count(
Tab,
{Ms, FuzzyFilterFun},
Continuation,
Limit,
fun format_user_info/1
emqx_mgmt_api:node_query(
node(),
?TAB,
NQueryString,
?AUTHN_QSCHEMA,
fun ?MODULE:qs2ms/2,
fun ?MODULE:format_user_info/1
).
%%--------------------------------------------------------------------
%% Match funcs
%% QueryString to MatchSpec
-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
qs2ms(_Tab, {QString, Fuzzy}) ->
#{
match_spec => ms_from_qstring(QString),
fuzzy_fun => fuzzy_filter_fun(Fuzzy)
}.
%% Fuzzy username funcs
fuzzy_filter_fun([]) ->
undefined;
fuzzy_filter_fun(Fuzzy) ->
fun(MsRaws) when is_list(MsRaws) ->
lists:filter(
fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
MsRaws
)
end.
{fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
run_fuzzy_filter(_, []) ->
true;

4
apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl
View File

@ -365,11 +365,11 @@ verify(JWT, JWKs, VerifyClaims, AclClaimName) ->
acl(Claims, AclClaimName) ->
Acl =
case Claims of
#{<<"exp">> := Expire, AclClaimName := Rules} ->
#{AclClaimName := Rules} ->
#{
acl => #{
rules => Rules,
expire => Expire
expire => maps:get(<<"exp">>, Claims, undefined)
}
};
_ ->

52
apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
View File

@ -49,7 +49,8 @@
]).
-export([
query/4,
qs2ms/2,
run_fuzzy_filter/2,
format_user_info/1,
group_match_spec/1
]).
@ -84,7 +85,6 @@
{<<"user_group">>, binary},
{<<"is_superuser">>, atom}
]).
-define(QUERY_FUN, {?MODULE, query}).
%%------------------------------------------------------------------------------
%% Mnesia bootstrap
@ -288,42 +288,30 @@ lookup_user(UserID, #{user_group := UserGroup}) ->
list_users(QueryString, #{user_group := UserGroup}) ->
NQueryString = QueryString#{<<"user_group">> => UserGroup},
emqx_mgmt_api:node_query(node(), NQueryString, ?TAB, ?AUTHN_QSCHEMA, ?QUERY_FUN).
%%--------------------------------------------------------------------
%% Query Functions
query(Tab, {QString, []}, Continuation, Limit) ->
Ms = ms_from_qstring(QString),
emqx_mgmt_api:select_table_with_count(
Tab,
Ms,
Continuation,
Limit,
fun format_user_info/1
);
query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
Ms = ms_from_qstring(QString),
FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
emqx_mgmt_api:select_table_with_count(
Tab,
{Ms, FuzzyFilterFun},
Continuation,
Limit,
fun format_user_info/1
emqx_mgmt_api:node_query(
node(),
?TAB,
NQueryString,
?AUTHN_QSCHEMA,
fun ?MODULE:qs2ms/2,
fun ?MODULE:format_user_info/1
).
%%--------------------------------------------------------------------
%% Match funcs
%% QueryString to MatchSpec
-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
qs2ms(_Tab, {QString, FuzzyQString}) ->
#{
match_spec => ms_from_qstring(QString),
fuzzy_fun => fuzzy_filter_fun(FuzzyQString)
}.
%% Fuzzy username funcs
fuzzy_filter_fun([]) ->
undefined;
fuzzy_filter_fun(Fuzzy) ->
fun(MsRaws) when is_list(MsRaws) ->
lists:filter(
fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
MsRaws
)
end.
{fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
run_fuzzy_filter(_, []) ->
true;

33
apps/emqx_authn/test/emqx_authn_api_SUITE.erl
View File

@ -39,6 +39,9 @@ all() ->
groups() ->
[].
init_per_testcase(t_authenticator_fail, Config) ->
meck:expect(emqx_authn_proto_v1, lookup_from_all_nodes, 3, [{error, {exception, badarg}}]),
init_per_testcase(default, Config);
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
emqx_authn_test_lib:delete_authenticators(
@ -54,6 +57,12 @@ init_per_testcase(_, Config) ->
{atomic, ok} = mria:clear_table(emqx_authn_mnesia),
Config.
end_per_testcase(t_authenticator_fail, Config) ->
meck:unload(emqx_authn_proto_v1),
Config;
end_per_testcase(_, Config) ->
Config.
init_per_suite(Config) ->
emqx_config:erase(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY),
_ = application:load(emqx_conf),
@ -90,6 +99,21 @@ t_authenticators(_) ->
t_authenticator(_) ->
test_authenticator([]).
t_authenticator_fail(_) ->
ValidConfig0 = emqx_authn_test_lib:http_example(),
{ok, 200, _} = request(
post,
uri([?CONF_NS]),
ValidConfig0
),
?assertMatch(
{ok, 500, _},
request(
get,
uri([?CONF_NS, "password_based:http", "status"])
)
).
t_authenticator_users(_) ->
test_authenticator_users([]).
@ -247,6 +271,15 @@ test_authenticator(PathPrefix) ->
<<"connected">>,
LookFun([<<"status">>])
),
?assertMatch(
{ok, 404, _},
request(
get,
uri(PathPrefix ++ [?CONF_NS, "unknown_auth_chain", "status"])
)
),
{ok, 404, _} = request(
get,
uri(PathPrefix ++ [?CONF_NS, "password_based:redis"])

2
apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
View File

@ -213,7 +213,7 @@ t_list_users(_) ->
#{
data := [#{is_superuser := false, user_id := <<"u3">>}],
meta := #{page := 1, limit := 20, count := 1}
meta := #{page := 1, limit := 20, count := 0}
} = emqx_authn_mnesia:list_users(
#{
<<"page">> => 1,

2
apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl
View File

@ -319,7 +319,7 @@ t_list_users(_) ->
is_superuser := _
}
],
meta := #{page := 1, limit := 3, count := 1}
meta := #{page := 1, limit := 3, count := 0}
} = emqx_enhanced_authn_scram_mnesia:list_users(
#{
<<"page">> => 1,

86
apps/emqx_authz/src/emqx_authz_api_mnesia.erl
View File

@ -24,8 +24,8 @@
-import(hoconsc, [mk/1, mk/2, ref/1, ref/2, array/1, enum/1]).
-define(QUERY_USERNAME_FUN, {?MODULE, query_username}).
-define(QUERY_CLIENTID_FUN, {?MODULE, query_clientid}).
-define(QUERY_USERNAME_FUN, fun ?MODULE:query_username/2).
-define(QUERY_CLIENTID_FUN, fun ?MODULE:query_clientid/2).
-define(ACL_USERNAME_QSCHEMA, [{<<"like_username">>, binary}]).
-define(ACL_CLIENTID_QSCHEMA, [{<<"like_clientid">>, binary}]).
@ -49,12 +49,12 @@
%% query funs
-export([
query_username/4,
query_clientid/4
query_username/2,
query_clientid/2,
run_fuzzy_filter/2,
format_result/1
]).
-export([format_result/1]).
-define(BAD_REQUEST, 'BAD_REQUEST').
-define(NOT_FOUND, 'NOT_FOUND').
-define(ALREADY_EXISTS, 'ALREADY_EXISTS').
@ -405,10 +405,11 @@ users(get, #{query_string := QueryString}) ->
case
emqx_mgmt_api:node_query(
node(),
QueryString,
?ACL_TABLE,
QueryString,
?ACL_USERNAME_QSCHEMA,
?QUERY_USERNAME_FUN
?QUERY_USERNAME_FUN,
fun ?MODULE:format_result/1
)
of
{error, page_limit_invalid} ->
@ -440,10 +441,11 @@ clients(get, #{query_string := QueryString}) ->
case
emqx_mgmt_api:node_query(
node(),
QueryString,
?ACL_TABLE,
QueryString,
?ACL_CLIENTID_QSCHEMA,
?QUERY_CLIENTID_FUN
?QUERY_CLIENTID_FUN,
fun ?MODULE:format_result/1
)
of
{error, page_limit_invalid} ->
@ -574,59 +576,27 @@ purge(delete, _) ->
end.
%%--------------------------------------------------------------------
%% Query Functions
%% QueryString to MatchSpec
query_username(Tab, {_QString, []}, Continuation, Limit) ->
Ms = emqx_authz_mnesia:list_username_rules(),
emqx_mgmt_api:select_table_with_count(
Tab,
Ms,
Continuation,
Limit,
fun format_result/1
);
query_username(Tab, {_QString, FuzzyQString}, Continuation, Limit) ->
Ms = emqx_authz_mnesia:list_username_rules(),
FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
emqx_mgmt_api:select_table_with_count(
Tab,
{Ms, FuzzyFilterFun},
Continuation,
Limit,
fun format_result/1
).
-spec query_username(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
query_username(_Tab, {_QString, FuzzyQString}) ->
#{
match_spec => emqx_authz_mnesia:list_username_rules(),
fuzzy_fun => fuzzy_filter_fun(FuzzyQString)
}.
query_clientid(Tab, {_QString, []}, Continuation, Limit) ->
Ms = emqx_authz_mnesia:list_clientid_rules(),
emqx_mgmt_api:select_table_with_count(
Tab,
Ms,
Continuation,
Limit,
fun format_result/1
);
query_clientid(Tab, {_QString, FuzzyQString}, Continuation, Limit) ->
Ms = emqx_authz_mnesia:list_clientid_rules(),
FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
emqx_mgmt_api:select_table_with_count(
Tab,
{Ms, FuzzyFilterFun},
Continuation,
Limit,
fun format_result/1
).
%%--------------------------------------------------------------------
%% Match funcs
-spec query_clientid(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
query_clientid(_Tab, {_QString, FuzzyQString}) ->
#{
match_spec => emqx_authz_mnesia:list_clientid_rules(),
fuzzy_fun => fuzzy_filter_fun(FuzzyQString)
}.
%% Fuzzy username funcs
fuzzy_filter_fun([]) ->
undefined;
fuzzy_filter_fun(Fuzzy) ->
fun(MsRaws) when is_list(MsRaws) ->
lists:filter(
fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
MsRaws
)
end.
{fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
run_fuzzy_filter(_, []) ->
true;

44
apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl
View File

@ -305,6 +305,50 @@ t_check_expire(_Config) ->
ok = emqtt:disconnect(C).
t_check_no_expire(_Config) ->
Payload = #{
<<"username">> => <<"username">>,
<<"acl">> => #{<<"sub">> => [<<"a/b">>]}
},
JWT = generate_jws(Payload),
{ok, C} = emqtt:start_link(
[
{clean_start, true},
{proto_ver, v5},
{clientid, <<"clientid">>},
{username, <<"username">>},
{password, JWT}
]
),
{ok, _} = emqtt:connect(C),
?assertMatch(
{ok, #{}, [0]},
emqtt:subscribe(C, <<"a/b">>, 0)
),
?assertMatch(
{ok, #{}, [0]},
emqtt:unsubscribe(C, <<"a/b">>)
),
ok = emqtt:disconnect(C).
t_check_undefined_expire(_Config) ->
Acl = #{expire => undefined, rules => #{<<"sub">> => [<<"a/b">>]}},
Client = #{acl => Acl},
?assertMatch(
{matched, allow},
emqx_authz_client_info:authorize(Client, subscribe, <<"a/b">>, undefined)
),
?assertMatch(
{matched, deny},
emqx_authz_client_info:authorize(Client, subscribe, <<"a/bar">>, undefined)
).
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------

3
apps/emqx_bridge/src/emqx_bridge_app.erl
View File

@ -68,7 +68,8 @@ pre_config_update(Path, Conf, _OldConfig) when is_map(Conf) ->
post_config_update(Path, '$remove', _, OldConf, _AppEnvs) ->
_ = emqx_connector_ssl:clear_certs(filename:join(Path), OldConf);
post_config_update(_Path, _Req, _, _OldConf, _AppEnvs) ->
post_config_update(Path, _Req, NewConf, OldConf, _AppEnvs) ->
_ = emqx_connector_ssl:try_clear_certs(filename:join(Path), NewConf, OldConf),
ok.
%% internal functions

29
apps/emqx_bridge/test/data/certs/cafile Normal file
View File

@ -0,0 +1,29 @@
-----BEGIN CERTIFICATE-----
MIIE5DCCAswCCQCF3o0gIdaNDjANBgkqhkiG9w0BAQsFADA0MRIwEAYDVQQKDAlF
TVFYIFRlc3QxHjAcBgNVBAMMFUNlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yMTEy
MzAwODQxMTFaFw00OTA1MTcwODQxMTFaMDQxEjAQBgNVBAoMCUVNUVggVGVzdDEe
MBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF
AAOCAg8AMIICCgKCAgEAqmqSrxyH16j63QhqGLT1UO8I+m6BM3HfnJQM8laQdtJ0
WgHqCh0/OphH3S7v4SfF4fNJDEJWMWuuzJzU9cTqHPLzhvo3+ZHcMIENgtY2p2Cf
7AQjEqFViEDyv2ZWNEe76BJeShntdY5NZr4gIPar99YGG/Ln8YekspleV+DU38rE
EX9WzhgBr02NN9z4NzIxeB+jdvPnxcXs3WpUxzfnUjOQf/T1tManvSdRbFmKMbxl
A8NLYK3oAYm8EbljWUINUNN6loqYhbigKv8bvo5S4xvRqmX86XB7sc0SApngtNcg
O0EKn8z/KVPDskE+8lMfGMiU2e2Tzw6Rph57mQPOPtIp5hPiKRik7ST9n0p6piXW
zRLplJEzSjf40I1u+VHmpXlWI/Fs8b1UkDSMiMVJf0LyWb4ziBSZOY2LtZzWHbWj
LbNgxQcwSS29tKgUwfEFmFcm+iOM59cPfkl2IgqVLh5h4zmKJJbfQKSaYb5fcKRf
50b1qsN40VbR3Pk/0lJ0/WqgF6kZCExmT1qzD5HJES/5grjjKA4zIxmHOVU86xOF
ouWvtilVR4PGkzmkFvwK5yRhBUoGH/A9BurhqOc0QCGay1kqHQFA6se4JJS+9KOS
x8Rn1Nm6Pi7sd6Le3cKmHTlyl5a/ofKqTCX2Qh+v/7y62V1V1wnoh3ipRjdPTnMC
AwEAATANBgkqhkiG9w0BAQsFAAOCAgEARCqaocvlMFUQjtFtepO2vyG1krn11xJ0
e7md26i+g8SxCCYqQ9IqGmQBg0Im8fyNDKRN/LZoj5+A4U4XkG1yya91ZIrPpWyF
KUiRAItchNj3g1kHmI2ckl1N//6Kpx3DPaS7qXZaN3LTExf6Ph+StE1FnS0wVF+s
tsNIf6EaQ+ZewW3pjdlLeAws3jvWKUkROc408Ngvx74zbbKo/zAC4tz8oH9ZcpsT
WD8enVVEeUQKI6ItcpZ9HgTI9TFWgfZ1vYwvkoRwNIeabYI62JKmLEo2vGfGwWKr
c+GjnJ/tlVI2DpPljfWOnQ037/7yyJI/zo65+HPRmGRD6MuW/BdPDYOvOZUTcQKh
kANi5THSbJJgZcG3jb1NLebaUQ1H0zgVjn0g3KhUV+NJQYk8RQ7rHtB+MySqTKlM
kRkRjfTfR0Ykxpks7Mjvsb6NcZENf08ZFPd45+e/ptsxpiKu4e4W4bV7NZDvNKf9
0/aD3oGYNMiP7s+KJ1lRSAjnBuG21Yk8FpzG+yr8wvJhV8aFgNQ5wIH86SuUTmN0
5bVzFEIcUejIwvGoQEctNHBlOwHrb7zmB6OwyZeMapdXBQ+9UDhYg8ehDqdDOdfn
wsBcnjD2MwNhlE1hjL+tZWLNwSHiD6xx3LvNoXZu2HK8Cp3SOrkE69cFghYMIZZb
T+fp6tNL6LE=
-----END CERTIFICATE-----

24
apps/emqx_bridge/test/data/certs/certfile Normal file
View File

@ -0,0 +1,24 @@
-----BEGIN CERTIFICATE-----
MIID/jCCAeagAwIBAgIJAKTICmq1Lg6dMA0GCSqGSIb3DQEBCwUAMDQxEjAQBgNV
BAoMCUVNUVggVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5MB4X
DTIxMTIzMDA4NDExMloXDTQ5MDUxNzA4NDExMlowJTESMBAGA1UECgwJRU1RWCBU
ZXN0MQ8wDQYDVQQDDAZjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDzrujfx6XZTH0MWqLO6kNAeHndUZ+OGaURXvxKMPMF5dA40lxNG6cEzzlq
0Rm61adlv8tF4kRJrs6EnRjEVoMImrdh07vGFdOTYqP01LjiBhErAzyRtSn2X8FT
Te8ExoCRs3x61SPebGY2hOvFxuO6YDPVOSDvbbxvRgqIlM1ZXC8dOvPSSGZ+P8hV
56EPayRthfu1FVptnkW9CyZCRI0gg95Hv8RC7bGG+tuWpkN9ZrRvohhgGR1+bDUi
BNBpncEsSh+UgWaj8KRN8D16H6m/Im6ty467j0at49FvPx5nACL48/ghtYvzgKLc
uKHtokKUuuzebDK/hQxN3mUSAJStAgMBAAGjIjAgMAsGA1UdDwQEAwIFoDARBglg
hkgBhvhCAQEEBAMCB4AwDQYJKoZIhvcNAQELBQADggIBAIlVyPhOpkz3MNzQmjX7
xgJ3vGPK5uK11n/wfjRwe2qXwZbrI2sYLVtTpUgvLDuP0gB73Vwfu7xAMdue6TRm
CKr9z0lkQsVBtgoqzZCjd4PYLfHm4EhsOMi98OGKU5uOGD4g3yLwQWXHhbYtiZMO
Jsj0hebYveYJt/BYTd1syGQcIcYCyVExWvSWjidfpAqjT6EF7whdubaFtuF2kaGF
IO9yn9rWtXB5yK99uCguEmKhx3fAQxomzqweTu3WRvy9axsUH3WAUW9a4DIBSz2+
ZSJNheFn5GktgggygJUGYqpSZHooUJW0UBs/8vX6AP+8MtINmqOGZUawmNwLWLOq
wHyVt2YGD5TXjzzsWNSQ4mqXxM6AXniZVZK0yYNjA4ATikX1AtwunyWBR4IjyE/D
FxYPORdZCOtywRFE1R5KLTUq/C8BNGCkYnoO78DJBO+pT0oagkQGQb0CnmC6C1db
4lWzA9K0i4B0PyooZA+gp+5FFgaLuX1DkyeaY1J204QhHR1z/Vcyl5dpqR9hqnYP
t8raLk9ogMDKqKA9iG0wc3CBNckD4sjVWAEeovXhElG55fD21wwhF+AnDCvX8iVK
cBfKV6z6uxfKjGIxc2I643I5DiIn+V3DnPxYyY74Ln1lWFYmt5JREhAxPu42zq74
e6+eIMYFszB+5gKgt6pa6ZNI
-----END CERTIFICATE-----

27
apps/emqx_bridge/test/data/certs/keyfile Normal file
View File

@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEA867o38el2Ux9DFqizupDQHh53VGfjhmlEV78SjDzBeXQONJc
TRunBM85atEZutWnZb/LReJESa7OhJ0YxFaDCJq3YdO7xhXTk2Kj9NS44gYRKwM8
kbUp9l/BU03vBMaAkbN8etUj3mxmNoTrxcbjumAz1Tkg7228b0YKiJTNWVwvHTrz
0khmfj/IVeehD2skbYX7tRVabZ5FvQsmQkSNIIPeR7/EQu2xhvrblqZDfWa0b6IY
YBkdfmw1IgTQaZ3BLEoflIFmo/CkTfA9eh+pvyJurcuOu49GrePRbz8eZwAi+PP4
IbWL84Ci3Lih7aJClLrs3mwyv4UMTd5lEgCUrQIDAQABAoIBAQDwEbBgznrIwn8r
jZt5x/brbAV7Ea/kOcWSgIaCvQifFdJ2OGAwov5/UXwajNgRZe2d4z7qoUhvYuUY
ZwCAZU6ASpRBr2v9cYFYYURvrqZaHmoJew3P6q/lhl6aqFvC06DUagRHqvXEafyk
13zEAvZVpfNKrBaTawPKiDFWb2qDDc9D6hC07EuJ/DNeehiHvzHrSZSDVV5Ut7Bw
YDm33XygheUPAlHfeCnaixzcs3osiVyFEmVjxcIaM0ZS1NgcSaohSpJHMzvEaohX
e+v9vccraSVlw01AlvFwI2vHYUV8jT6HwglTPKKGOCzK/ace3wPdYSU9qLcqfuHn
EFhNc3tNAoGBAPugLMgbReJg2gpbIPUkYyoMMAAU7llFU1WvPWwXzo1a9EBjBACw
WfCZISNtANXR38zIYXzoH547uXi4YPks1Nne3sYuCDpvuX+iz7fIo4zHf1nFmxH7
eE6GtQr2ubmuuipTc28S0wBMGT1/KybH0e2NKL6GaOkNDmAI0IbEMBrvAoGBAPfr
Y1QYLhPhan6m5g/5s+bQpKtHfNH9TNkk13HuYu72zNuY3qL2GC7oSadR8vTbRXZg
KQqfaO0IGRcdkSFTq/AEhSSqr2Ld5nPadMbKvSGrSCc1s8rFH97jRVQY56yhM7ti
IW4+6cE8ylCMbdYB6wuduK/GIgNpqoF4xs1i2XojAoGACacBUMPLEH4Kny8TupOk
wi4pgTdMVVxVcAoC3yyincWJbRbfRm99Y79cCBHcYFdmsGJXawU0gUtlN/5KqgRQ
PfNQtGV7p1I12XGTakdmDrZwai8sXao52TlNpJgGU9siBRGicfZU5cQFi9he/WPY
57XshDJ/v8DidkigRysrdT0CgYEA5iuO22tblC+KvK1dGOXeZWO+DhrfwuGlcFBp
CaimB2/w/8vsn2VVTG9yujo2E6hj1CQw1mDrfG0xRim4LTXOgpbfugwRqvuTUmo2
Ur21XEX2RhjwpEfhcACWxB4fMUG0krrniMA2K6axupi1/KNpQi6bYe3UdFCs8Wld
QSAOAvsCgYBk/X5PmD44DvndE5FShM2w70YOoMr3Cgl5sdwAFUFE9yDuC14UhVxk
oxnYxwtVI9uVVirET+LczP9JEvcvxnN/Xg3tH/qm0WlIxmTxyYrFFIK9j0rqeu9z
blPu56OzNI2VMrR1GbOBLxQINLTIpaacjNJAlr8XOlegdUJsW/Jwqw==
-----END RSA PRIVATE KEY-----

95
apps/emqx_bridge/test/emqx_bridge_SUITE.erl
View File

@ -148,3 +148,98 @@ setup_fake_telemetry_data() ->
{ok, _} = snabbkaffe_collector:receive_events(Sub),
ok = snabbkaffe:stop(),
ok.
t_update_ssl_conf(_) ->
Path = [bridges, <<"mqtt">>, <<"ssl_update_test">>],
EnableSSLConf = #{
<<"connector">> =>
#{
<<"bridge_mode">> => false,
<<"clean_start">> => true,
<<"keepalive">> => <<"60s">>,
<<"mode">> => <<"cluster_shareload">>,
<<"proto_ver">> => <<"v4">>,
<<"server">> => <<"127.0.0.1:1883">>,
<<"ssl">> =>
#{
<<"cacertfile">> => cert_file("cafile"),
<<"certfile">> => cert_file("certfile"),
<<"enable">> => true,
<<"keyfile">> => cert_file("keyfile"),
<<"verify">> => <<"verify_peer">>
}
},
<<"direction">> => <<"ingress">>,
<<"local_qos">> => 1,
<<"payload">> => <<"${payload}">>,
<<"remote_qos">> => 1,
<<"remote_topic">> => <<"t/#">>,
<<"retain">> => false
},
emqx:update_config(Path, EnableSSLConf),
?assertMatch({ok, [_, _, _]}, list_pem_dir(Path)),
NoSSLConf = #{
<<"connector">> =>
#{
<<"bridge_mode">> => false,
<<"clean_start">> => true,
<<"keepalive">> => <<"60s">>,
<<"max_inflight">> => 32,
<<"mode">> => <<"cluster_shareload">>,
<<"password">> => <<>>,
<<"proto_ver">> => <<"v4">>,
<<"reconnect_interval">> => <<"15s">>,
<<"replayq">> =>
#{<<"offload">> => false, <<"seg_bytes">> => <<"100MB">>},
<<"retry_interval">> => <<"15s">>,
<<"server">> => <<"127.0.0.1:1883">>,
<<"ssl">> =>
#{
<<"ciphers">> => <<>>,
<<"depth">> => 10,
<<"enable">> => false,
<<"reuse_sessions">> => true,
<<"secure_renegotiate">> => true,
<<"user_lookup_fun">> => <<"emqx_tls_psk:lookup">>,
<<"verify">> => <<"verify_peer">>,
<<"versions">> =>
[
<<"tlsv1.3">>,
<<"tlsv1.2">>,
<<"tlsv1.1">>,
<<"tlsv1">>
]
},
<<"username">> => <<>>
},
<<"direction">> => <<"ingress">>,
<<"enable">> => true,
<<"local_qos">> => 1,
<<"payload">> => <<"${payload}">>,
<<"remote_qos">> => 1,
<<"remote_topic">> => <<"t/#">>,
<<"retain">> => false
},
emqx:update_config(Path, NoSSLConf),
?assertMatch({error, not_dir}, list_pem_dir(Path)),
emqx:remove_config(Path),
ok.
list_pem_dir(Path) ->
Dir = filename:join([emqx:mutable_certs_dir() | Path]),
case filelib:is_dir(Dir) of
true ->
file:list_dir(Dir);
_ ->
{error, not_dir}
end.
data_file(Name) ->
Dir = code:lib_dir(emqx_bridge, test),
{ok, Bin} = file:read_file(filename:join([Dir, "data", Name])),
Bin.
cert_file(Name) ->
data_file(filename:join(["certs", Name])).

58
apps/emqx_connector/src/emqx_connector_ssl.erl
View File

@ -16,9 +16,12 @@
-module(emqx_connector_ssl).
-include_lib("emqx/include/logger.hrl").
-export([
convert_certs/2,
clear_certs/2
clear_certs/2,
try_clear_certs/3
]).
%% TODO: rm `connector` case after `dev/ee5.0` merged into `master`.
@ -27,12 +30,12 @@
convert_certs(RltvDir, #{<<"connector">> := Connector} = Config) when
is_map(Connector)
->
SSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
SSL = maps:get(<<"ssl">>, Connector, undefined),
new_ssl_config(RltvDir, Config, SSL);
convert_certs(RltvDir, #{connector := Connector} = Config) when
is_map(Connector)
->
SSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
SSL = maps:get(ssl, Connector, undefined),
new_ssl_config(RltvDir, Config, SSL);
%% for bridges without `connector` field. i.e. webhook
convert_certs(RltvDir, #{<<"ssl">> := SSL} = Config) ->
@ -43,21 +46,37 @@ convert_certs(RltvDir, #{ssl := SSL} = Config) ->
convert_certs(_RltvDir, Config) ->
{ok, Config}.
clear_certs(RltvDir, #{<<"connector">> := Connector} = _Config) when
clear_certs(RltvDir, Config) ->
clear_certs2(RltvDir, normalize_key_to_bin(Config)).
clear_certs2(RltvDir, #{<<"connector">> := Connector} = _Config) when
is_map(Connector)
->
OldSSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
%% TODO remove the 'connector' clause after dev/ee5.0 is merged back to master
%% The `connector` config layer will be removed.
%% for bridges with `connector` field. i.e. `mqtt_source` and `mqtt_sink`
OldSSL = maps:get(<<"ssl">>, Connector, undefined),
ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
clear_certs(RltvDir, #{connector := Connector} = _Config) when
is_map(Connector)
->
OldSSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
clear_certs2(RltvDir, #{<<"ssl">> := OldSSL} = _Config) ->
ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
clear_certs(RltvDir, #{<<"ssl">> := OldSSL} = _Config) ->
ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
clear_certs(RltvDir, #{ssl := OldSSL} = _Config) ->
ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
clear_certs(_RltvDir, _) ->
clear_certs2(_RltvDir, _) ->
ok.
try_clear_certs(RltvDir, NewConf, OldConf) ->
try_clear_certs2(
RltvDir,
normalize_key_to_bin(NewConf),
normalize_key_to_bin(OldConf)
).
try_clear_certs2(RltvDir, #{<<"connector">> := NewConnector}, #{<<"connector">> := OldConnector}) ->
NewSSL = maps:get(<<"ssl">>, NewConnector, undefined),
OldSSL = maps:get(<<"ssl">>, OldConnector, undefined),
try_clear_certs2(RltvDir, NewSSL, OldSSL);
try_clear_certs2(RltvDir, NewSSL, OldSSL) when is_map(NewSSL) andalso is_map(OldSSL) ->
ok = emqx_tls_lib:delete_ssl_files(RltvDir, NewSSL, OldSSL);
try_clear_certs2(RltvDir, NewConf, OldConf) ->
?SLOG(debug, #{msg => "unexpected_conf", path => RltvDir, new => NewConf, OldConf => OldConf}),
ok.
new_ssl_config(RltvDir, Config, SSL) ->
@ -79,12 +98,5 @@ new_ssl_config(#{<<"ssl">> := _} = Config, NewSSL) ->
new_ssl_config(Config, _NewSSL) ->
Config.
map_get_oneof([], _Map, Default) ->
Default;
map_get_oneof([Key | Keys], Map, Default) ->
case maps:find(Key, Map) of
error ->
map_get_oneof(Keys, Map, Default);
{ok, Value} ->
Value
end.
normalize_key_to_bin(Map) ->
emqx_map_lib:binary_key_map(Map).

10
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
View File

@ -81,17 +81,20 @@ to_remote_msg(MapMsg, #{
Payload = process_payload(PayloadToken, MapMsg),
QoS = replace_simple_var(QoSToken, MapMsg),
Retain = replace_simple_var(RetainToken, MapMsg),
PubProps = maps:get(pub_props, MapMsg, #{}),
#mqtt_msg{
qos = QoS,
retain = Retain,
topic = topic(Mountpoint, Topic),
props = #{},
props = emqx_misc:pub_props_to_packet(PubProps),
payload = Payload
};
to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) ->
Msg#message{topic = topic(Mountpoint, Topic)}.
%% published from remote node over a MQTT connection
to_broker_msg(Msg, Vars, undefined) ->
to_broker_msg(Msg, Vars, #{});
to_broker_msg(
#{dup := Dup} = MapMsg,
#{
@ -109,8 +112,9 @@ to_broker_msg(
Payload = process_payload(PayloadToken, MapMsg),
QoS = replace_simple_var(QoSToken, MapMsg),
Retain = replace_simple_var(RetainToken, MapMsg),
PubProps = maps:get(pub_props, MapMsg, #{}),
set_headers(
Props,
Props#{properties => emqx_misc:pub_props_to_packet(PubProps)},
emqx_message:set_flags(
#{dup => Dup, retain => Retain},
emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload)
@ -157,8 +161,6 @@ estimate_size(#{topic := Topic, payload := Payload}) ->
estimate_size(Term) ->
erlang:external_size(Term).
set_headers(undefined, Msg) ->
Msg;
set_headers(Val, Msg) ->
emqx_message:set_headers(Val, Msg).
topic(undefined, Topic) -> Topic;

22
apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
View File

@ -121,13 +121,21 @@ fields(sampler_current) ->
monitor(get, #{query_string := QS, bindings := Bindings}) ->
Latest = maps:get(<<"latest">>, QS, infinity),
Node = binary_to_atom(maps:get(node, Bindings, <<"all">>)),
case emqx_dashboard_monitor:samplers(Node, Latest) of
{badrpc, {Node, Reason}} ->
Message = list_to_binary(io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason])),
{400, 'BAD_RPC', Message};
Samplers ->
{200, Samplers}
RawNode = maps:get(node, Bindings, all),
case emqx_misc:safe_to_existing_atom(RawNode, utf8) of
{ok, Node} ->
case emqx_dashboard_monitor:samplers(Node, Latest) of
{badrpc, {Node, Reason}} ->
Message = list_to_binary(
io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason])
),
{400, 'BAD_RPC', Message};
Samplers ->
{200, Samplers}
end;
_ ->
Message = list_to_binary(io_lib:format("Bad node ~p", [RawNode])),
{400, 'BAD_RPC', Message}
end.
monitor_current(get, #{bindings := Bindings}) ->

7
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -138,7 +138,12 @@ fields(limit) ->
Meta = #{in => query, desc => Desc, default => ?DEFAULT_ROW, example => 50},
[{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}];
fields(count) ->
Meta = #{desc => <<"Results count.">>, required => true},
Desc = <<
"Total number of records counted.<br/>"
"Note: this field is <code>0</code> when the queryed table is empty, "
"or if the query can not be optimized and requires a full table scan."
>>,
Meta = #{desc => Desc, required => true},
[{count, hoconsc:mk(non_neg_integer(), Meta)}];
fields(meta) ->
fields(page) ++ fields(limit) ++ fields(count).

8
apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf
View File

@ -57,6 +57,14 @@ It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto`
}
}
gateway_enable_in_path {
desc {
en: """Whether or not gateway is enabled"""
zh: """是否开启此网关"""
}
}
gateway_status {
desc {
en: """Gateway status"""

2
apps/emqx_gateway/src/emqx_gateway.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway, [
{description, "The Gateway management application"},
{vsn, "0.1.7"},
{vsn, "0.1.8"},
{registered, []},
{mod, {emqx_gateway_app, []}},
{applications, [kernel, stdlib, grpc, emqx, emqx_authn]},

249
apps/emqx_gateway/src/emqx_gateway_api.erl
View File

@ -19,8 +19,6 @@
-include("emqx_gateway_http.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-include_lib("emqx/include/emqx_authentication.hrl").
-behaviour(minirest_api).
@ -34,7 +32,7 @@
]
).
%% minirest/dashbaord_swagger behaviour callbacks
%% minirest/dashboard_swagger behaviour callbacks
-export([
api_spec/0,
paths/0,
@ -49,8 +47,9 @@
%% http handlers
-export([
gateways/2,
gateway/2,
gateway_insta/2
gateway_enable/2
]).
-define(KNOWN_GATEWAY_STATUSES, [<<"running">>, <<"stopped">>, <<"unloaded">>]).
@ -66,13 +65,14 @@ api_spec() ->
paths() ->
emqx_gateway_utils:make_deprecated_paths([
"/gateways",
"/gateways/:name"
"/gateways/:name",
"/gateways/:name/enable/:enable"
]).
%%--------------------------------------------------------------------
%% http handlers
gateway(get, Request) ->
gateways(get, Request) ->
Params = maps:get(query_string, Request, #{}),
Status = maps:get(<<"status">>, Params, <<"all">>),
case lists:member(Status, [<<"all">> | ?KNOWN_GATEWAY_STATUSES]) of
@ -89,84 +89,85 @@ gateway(get, Request) ->
lists:join(", ", ?KNOWN_GATEWAY_STATUSES)
]
)
end;
gateway(post, Request) ->
Body = maps:get(body, Request, #{}),
try
Name0 = maps:get(<<"name">>, Body),
GwName = binary_to_existing_atom(Name0),
case emqx_gateway_registry:lookup(GwName) of
undefined ->
error(badarg);
_ ->
GwConf = maps:without([<<"name">>], Body),
case emqx_gateway_conf:load_gateway(GwName, GwConf) of
{ok, NGwConf} ->
{201, NGwConf};
{error, Reason} ->
emqx_gateway_http:reason2resp(Reason)
end
end
catch
error:{badkey, K} ->
return_http_error(400, [K, " is required"]);
error:{badconf, _} = Reason1 ->
emqx_gateway_http:reason2resp(Reason1);
error:badarg ->
return_http_error(404, "Bad gateway name")
end.
gateway_insta(delete, #{bindings := #{name := Name0}}) ->
with_gateway(Name0, fun(GwName, _) ->
case emqx_gateway_conf:unload_gateway(GwName) of
ok ->
gateway(get, #{bindings := #{name := Name}}) ->
try
GwName = gw_name(Name),
case emqx_gateway:lookup(GwName) of
undefined ->
{200, #{name => GwName, status => unloaded}};
Gateway ->
GwConf = emqx_gateway_conf:gateway(Name),
GwInfo0 = emqx_gateway_utils:unix_ts_to_rfc3339(
[created_at, started_at, stopped_at],
Gateway
),
GwInfo1 = maps:with(
[
name,
status,
created_at,
started_at,
stopped_at
],
GwInfo0
),
{200, maps:merge(GwConf, GwInfo1)}
end
catch
throw:not_found ->
return_http_error(404, <<"NOT FOUND">>)
end;
gateway(put, #{
body := GwConf0,
bindings := #{name := Name}
}) ->
GwConf = maps:without([<<"name">>], GwConf0),
try
GwName = gw_name(Name),
LoadOrUpdateF =
case emqx_gateway:lookup(GwName) of
undefined ->
fun emqx_gateway_conf:load_gateway/2;
_ ->
fun emqx_gateway_conf:update_gateway/2
end,
case LoadOrUpdateF(GwName, GwConf) of
{ok, _} ->
{204};
{error, Reason} ->
emqx_gateway_http:reason2resp(Reason)
end
end);
gateway_insta(get, #{bindings := #{name := Name0}}) ->
try binary_to_existing_atom(Name0) of
GwName ->
case emqx_gateway:lookup(GwName) of
undefined ->
{200, #{name => GwName, status => unloaded}};
Gateway ->
GwConf = emqx_gateway_conf:gateway(Name0),
GwInfo0 = emqx_gateway_utils:unix_ts_to_rfc3339(
[created_at, started_at, stopped_at],
Gateway
),
GwInfo1 = maps:with(
[
name,
status,
created_at,
started_at,
stopped_at
],
GwInfo0
),
{200, maps:merge(GwConf, GwInfo1)}
end
catch
error:badarg ->
return_http_error(404, "Bad gateway name")
end;
gateway_insta(put, #{
body := GwConf0,
bindings := #{name := Name0}
}) ->
with_gateway(Name0, fun(GwName, _) ->
%% XXX: Clear the unused fields
GwConf = maps:without([<<"name">>], GwConf0),
case emqx_gateway_conf:update_gateway(GwName, GwConf) of
{ok, Gateway} ->
{200, Gateway};
{error, Reason} ->
emqx_gateway_http:reason2resp(Reason)
error:{badconf, _} = Reason1 ->
emqx_gateway_http:reason2resp(Reason1);
throw:not_found ->
return_http_error(404, <<"NOT FOUND">>)
end.
gateway_enable(put, #{bindings := #{name := Name, enable := Enable}}) ->
try
GwName = gw_name(Name),
case emqx_gateway:lookup(GwName) of
undefined ->
return_http_error(404, <<"NOT FOUND">>);
_Gateway ->
{ok, _} = emqx_gateway_conf:update_gateway(GwName, #{<<"enable">> => Enable}),
{204}
end
end).
catch
throw:not_found ->
return_http_error(404, <<"NOT FOUND">>)
end.
-spec gw_name(binary()) -> stomp | coap | lwm2m | mqttsn | exproto | no_return().
gw_name(<<"stomp">>) -> stomp;
gw_name(<<"coap">>) -> coap;
gw_name(<<"lwm2m">>) -> lwm2m;
gw_name(<<"mqttsn">>) -> mqttsn;
gw_name(<<"exproto">>) -> exproto;
gw_name(_Else) -> throw(not_found).
%%--------------------------------------------------------------------
%% Swagger defines
@ -174,7 +175,7 @@ gateway_insta(put, #{
schema("/gateways") ->
#{
'operationId' => gateway,
'operationId' => gateways,
get =>
#{
tags => ?TAGS,
@ -182,29 +183,20 @@ schema("/gateways") ->
summary => <<"List All Gateways">>,
parameters => params_gateway_status_in_qs(),
responses =>
?STANDARD_RESP(
#{
200 => emqx_dashboard_swagger:schema_with_example(
hoconsc:array(ref(gateway_overview)),
examples_gateway_overview()
)
}
)
},
post =>
#{
tags => ?TAGS,
desc => ?DESC(enable_gateway),
summary => <<"Enable a Gateway">>,
%% TODO: distinguish create & response swagger schema
'requestBody' => schema_gateways_conf(),
responses =>
?STANDARD_RESP(#{201 => schema_gateways_conf()})
#{
200 => emqx_dashboard_swagger:schema_with_example(
hoconsc:array(ref(gateway_overview)),
examples_gateway_overview()
),
400 => emqx_dashboard_swagger:error_codes(
[?BAD_REQUEST], <<"Bad request">>
)
}
}
};
schema("/gateways/:name") ->
#{
'operationId' => gateway_insta,
'operationId' => gateway,
get =>
#{
tags => ?TAGS,
@ -212,26 +204,41 @@ schema("/gateways/:name") ->
summary => <<"Get the Gateway">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(#{200 => schema_gateways_conf()})
},
delete =>
#{
tags => ?TAGS,
desc => ?DESC(delete_gateway),
summary => <<"Unload the gateway">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(#{204 => <<"Deleted">>})
#{
200 => schema_gateways_conf(),
404 => emqx_dashboard_swagger:error_codes(
[?NOT_FOUND, ?RESOURCE_NOT_FOUND], <<"Not Found">>
)
}
},
put =>
#{
tags => ?TAGS,
desc => ?DESC(update_gateway),
summary => <<"Update the gateway confs">>,
% [FIXME] add proper desc
summary => <<"Load or update the gateway confs">>,
parameters => params_gateway_name_in_path(),
'requestBody' => schema_update_gateways_conf(),
'requestBody' => schema_load_or_update_gateways_conf(),
responses =>
?STANDARD_RESP(#{200 => schema_gateways_conf()})
?STANDARD_RESP(#{204 => <<"Gateway configuration updated">>})
}
};
schema("/gateways/:name/enable/:enable") ->
#{
'operationId' => gateway_enable,
put =>
#{
tags => ?TAGS,
desc => ?DESC(update_gateway),
summary => <<"Enable or disable gateway">>,
parameters => params_gateway_name_in_path() ++ params_gateway_enable_in_path(),
responses =>
#{
204 => <<"Gateway configuration updated">>,
404 => emqx_dashboard_swagger:error_codes(
[?NOT_FOUND, ?RESOURCE_NOT_FOUND], <<"Not Found">>
)
}
}
};
schema(Path) ->
@ -268,6 +275,18 @@ params_gateway_status_in_qs() ->
)}
].
params_gateway_enable_in_path() ->
[
{enable,
mk(
boolean(),
#{
in => path,
desc => ?DESC(gateway_enable_in_path),
example => true
}
)}
].
%%--------------------------------------------------------------------
%% schemas
@ -377,8 +396,6 @@ fields(Gw) when
->
[{name, mk(Gw, #{desc => ?DESC(gateway_name)})}] ++
convert_listener_struct(emqx_gateway_schema:fields(Gw));
fields(update_disable_enable_only) ->
[{enable, mk(boolean(), #{desc => <<"Enable/Disable the gateway">>})}];
fields(Gw) when
Gw == update_stomp;
Gw == update_mqttsn;
@ -431,15 +448,19 @@ fields(Listener) when
fields(gateway_stats) ->
[{key, mk(binary(), #{})}].
schema_update_gateways_conf() ->
schema_load_or_update_gateways_conf() ->
emqx_dashboard_swagger:schema_with_examples(
hoconsc:union([
ref(?MODULE, stomp),
ref(?MODULE, mqttsn),
ref(?MODULE, coap),
ref(?MODULE, lwm2m),
ref(?MODULE, exproto),
ref(?MODULE, update_stomp),
ref(?MODULE, update_mqttsn),
ref(?MODULE, update_coap),
ref(?MODULE, update_lwm2m),
ref(?MODULE, update_exproto),
ref(?MODULE, update_disable_enable_only)
ref(?MODULE, update_exproto)
]),
examples_update_gateway_confs()
).

3
apps/emqx_gateway/src/emqx_gateway_api_authn.erl
View File

@ -30,8 +30,7 @@
[
return_http_error/2,
with_gateway/2,
with_authn/2,
checks/2
with_authn/2
]
).

58
apps/emqx_gateway/src/emqx_gateway_api_clients.erl
View File

@ -55,8 +55,10 @@
%% internal exports (for client query)
-export([
query/4,
format_channel_info/1
qs2ms/2,
run_fuzzy_filter/2,
format_channel_info/1,
format_channel_info/2
]).
-define(TAGS, [<<"Gateway Clients">>]).
@ -97,8 +99,6 @@ paths() ->
{<<"lte_lifetime">>, integer}
]).
-define(QUERY_FUN, {?MODULE, query}).
clients(get, #{
bindings := #{name := Name0},
query_string := QString
@ -109,10 +109,11 @@ clients(get, #{
case maps:get(<<"node">>, QString, undefined) of
undefined ->
emqx_mgmt_api:cluster_query(
QString,
TabName,
QString,
?CLIENT_QSCHEMA,
?QUERY_FUN
fun ?MODULE:qs2ms/2,
fun ?MODULE:format_channel_info/2
);
Node0 ->
case emqx_misc:safe_to_existing_atom(Node0) of
@ -120,10 +121,11 @@ clients(get, #{
QStringWithoutNode = maps:without([<<"node">>], QString),
emqx_mgmt_api:node_query(
Node1,
QStringWithoutNode,
TabName,
QStringWithoutNode,
?CLIENT_QSCHEMA,
?QUERY_FUN
fun ?MODULE:qs2ms/2,
fun ?MODULE:format_channel_info/2
);
{error, _} ->
{error, Node0, {badrpc, <<"invalid node">>}}
@ -264,27 +266,11 @@ extra_sub_props(Props) ->
).
%%--------------------------------------------------------------------
%% query funcs
%% QueryString to MatchSpec
query(Tab, {Qs, []}, Continuation, Limit) ->
Ms = qs2ms(Qs),
emqx_mgmt_api:select_table_with_count(
Tab,
Ms,
Continuation,
Limit,
fun format_channel_info/1
);
query(Tab, {Qs, Fuzzy}, Continuation, Limit) ->
Ms = qs2ms(Qs),
FuzzyFilterFun = fuzzy_filter_fun(Fuzzy),
emqx_mgmt_api:select_table_with_count(
Tab,
{Ms, FuzzyFilterFun},
Continuation,
Limit,
fun format_channel_info/1
).
-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
qs2ms(_Tab, {Qs, Fuzzy}) ->
#{match_spec => qs2ms(Qs), fuzzy_fun => fuzzy_filter_fun(Fuzzy)}.
qs2ms(Qs) ->
{MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}),
@ -339,13 +325,10 @@ ms(lifetime, X) ->
%%--------------------------------------------------------------------
%% Fuzzy filter funcs
fuzzy_filter_fun([]) ->
undefined;
fuzzy_filter_fun(Fuzzy) ->
fun(MsRaws) when is_list(MsRaws) ->
lists:filter(
fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
MsRaws
)
end.
{fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
run_fuzzy_filter(_, []) ->
true;
@ -363,8 +346,11 @@ run_fuzzy_filter(
%%--------------------------------------------------------------------
%% format funcs
format_channel_info({_, Infos, Stats} = R) ->
Node = maps:get(node, Infos, node()),
format_channel_info(ChannInfo) ->
format_channel_info(node(), ChannInfo).
format_channel_info(WhichNode, {_, Infos, Stats} = R) ->
Node = maps:get(node, Infos, WhichNode),
ClientInfo = maps:get(clientinfo, Infos, #{}),
ConnInfo = maps:get(conninfo, Infos, #{}),
SessInfo = maps:get(session, Infos, #{}),

3
apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl
View File

@ -2319,5 +2319,4 @@ returncode_name(?SN_RC2_EXCEED_LIMITATION) -> rejected_exceed_limitation;
returncode_name(?SN_RC2_REACHED_MAX_RETRY) -> reached_max_retry_times;
returncode_name(_) -> accepted.
name_to_returncode(not_authorized) -> ?SN_RC2_NOT_AUTHORIZE;
name_to_returncode(_) -> ?SN_RC2_NOT_AUTHORIZE.
name_to_returncode(not_authorized) -> ?SN_RC2_NOT_AUTHORIZE.

191
apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl
View File

@ -23,7 +23,7 @@
emqx_gateway_test_utils,
[
assert_confs/2,
assert_feilds_apperence/2,
assert_fields_exist/2,
request/2,
request/3,
ssl_server_opts/0,
@ -32,6 +32,7 @@
).
-include_lib("eunit/include/eunit.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% this parses to #{}, will not cause config cleanup
%% so we will need call emqx_config:erase
@ -55,32 +56,68 @@ end_per_suite(Conf) ->
emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]),
Conf.
init_per_testcase(t_gateway_fail, Config) ->
meck:expect(
emqx_gateway_conf,
update_gateway,
fun
(stomp, V) -> {error, {badconf, #{key => gw, value => V, reason => test_error}}};
(coap, V) -> error({badconf, #{key => gw, value => V, reason => test_crash}})
end
),
Config;
init_per_testcase(_, Config) ->
Config.
end_per_testcase(TestCase, Config) ->
case TestCase of
t_gateway_fail -> meck:unload(emqx_gateway_conf);
_ -> ok
end,
[emqx_gateway_conf:unload_gateway(GwName) || GwName <- [stomp, mqttsn, coap, lwm2m, exproto]],
Config.
%%--------------------------------------------------------------------
%% Cases
%%--------------------------------------------------------------------
t_gateway(_) ->
t_gateways(_) ->
{200, Gateways} = request(get, "/gateways"),
lists:foreach(fun assert_gw_unloaded/1, Gateways),
{200, UnloadedGateways} = request(get, "/gateways?status=unloaded"),
lists:foreach(fun assert_gw_unloaded/1, UnloadedGateways),
{200, NoRunningGateways} = request(get, "/gateways?status=running"),
?assertEqual([], NoRunningGateways),
{404, GwNotFoundReq} = request(get, "/gateways/unknown_gateway"),
assert_not_found(GwNotFoundReq),
{400, BadReqInvalidStatus} = request(get, "/gateways?status=invalid_status"),
assert_bad_request(BadReqInvalidStatus),
{400, BadReqUCStatus} = request(get, "/gateways?status=UNLOADED"),
assert_bad_request(BadReqUCStatus),
{201, _} = request(post, "/gateways", #{name => <<"stomp">>}),
{200, StompGw1} = request(get, "/gateways/stomp"),
assert_feilds_apperence(
ok.
t_gateway(_) ->
{404, GwNotFoundReq1} = request(get, "/gateways/not_a_known_atom"),
assert_not_found(GwNotFoundReq1),
{404, GwNotFoundReq2} = request(get, "/gateways/undefined"),
assert_not_found(GwNotFoundReq2),
{204, _} = request(put, "/gateways/stomp", #{}),
{200, StompGw} = request(get, "/gateways/stomp"),
assert_fields_exist(
[name, status, enable, created_at, started_at],
StompGw1
StompGw
),
{204, _} = request(delete, "/gateways/stomp"),
{200, StompGw2} = request(get, "/gateways/stomp"),
assert_gw_unloaded(StompGw2),
{204, _} = request(put, "/gateways/stomp", #{enable => true}),
{200, #{enable := true}} = request(get, "/gateway/stomp"),
{204, _} = request(put, "/gateways/stomp", #{enable => false}),
{200, #{enable := false}} = request(get, "/gateway/stomp"),
{404, _} = request(put, "/gateways/undefined", #{}),
{400, _} = request(put, "/gateways/stomp", #{bad_key => "foo"}),
ok.
t_gateway_fail(_) ->
{204, _} = request(put, "/gateways/stomp", #{}),
{400, _} = request(put, "/gateways/stomp", #{}),
{204, _} = request(put, "/gateways/coap", #{}),
{400, _} = request(put, "/gateways/coap", #{}),
ok.
t_deprecated_gateway(_) ->
@ -88,21 +125,30 @@ t_deprecated_gateway(_) ->
lists:foreach(fun assert_gw_unloaded/1, Gateways),
{404, NotFoundReq} = request(get, "/gateway/uname_gateway"),
assert_not_found(NotFoundReq),
{201, _} = request(post, "/gateway", #{name => <<"stomp">>}),
{200, StompGw1} = request(get, "/gateway/stomp"),
assert_feilds_apperence(
{204, _} = request(put, "/gateway/stomp", #{}),
{200, StompGw} = request(get, "/gateway/stomp"),
assert_fields_exist(
[name, status, enable, created_at, started_at],
StompGw1
StompGw
),
{204, _} = request(delete, "/gateway/stomp"),
{200, StompGw2} = request(get, "/gateway/stomp"),
assert_gw_unloaded(StompGw2),
ok.
t_gateway_enable(_) ->
{204, _} = request(put, "/gateways/stomp", #{}),
{200, #{enable := Enable}} = request(get, "/gateway/stomp"),
NotEnable = not Enable,
{204, _} = request(put, "/gateways/stomp/enable/" ++ atom_to_list(NotEnable), undefined),
{200, #{enable := NotEnable}} = request(get, "/gateway/stomp"),
{204, _} = request(put, "/gateways/stomp/enable/" ++ atom_to_list(Enable), undefined),
{200, #{enable := Enable}} = request(get, "/gateway/stomp"),
{404, _} = request(put, "/gateways/undefined/enable/true", undefined),
{404, _} = request(put, "/gateways/not_a_known_atom/enable/true", undefined),
{404, _} = request(put, "/gateways/coap/enable/true", undefined),
ok.
t_gateway_stomp(_) ->
{200, Gw} = request(get, "/gateways/stomp"),
assert_gw_unloaded(Gw),
%% post
GwConf = #{
name => <<"stomp">>,
frame => #{
@ -114,20 +160,18 @@ t_gateway_stomp(_) ->
#{name => <<"def">>, type => <<"tcp">>, bind => <<"61613">>}
]
},
{201, _} = request(post, "/gateways", GwConf),
{204, _} = request(put, "/gateways/stomp", GwConf),
{200, ConfResp} = request(get, "/gateways/stomp"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{frame => #{max_headers => 10}}),
{200, _} = request(put, "/gateways/stomp", maps:without([name, listeners], GwConf2)),
{204, _} = request(put, "/gateways/stomp", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/stomp"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateways/stomp").
ok.
t_gateway_mqttsn(_) ->
{200, Gw} = request(get, "/gateways/mqttsn"),
assert_gw_unloaded(Gw),
%% post
GwConf = #{
name => <<"mqttsn">>,
gateway_id => 1,
@ -138,20 +182,18 @@ t_gateway_mqttsn(_) ->
#{name => <<"def">>, type => <<"udp">>, bind => <<"1884">>}
]
},
{201, _} = request(post, "/gateways", GwConf),
{204, _} = request(put, "/gateways/mqttsn", GwConf),
{200, ConfResp} = request(get, "/gateways/mqttsn"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{predefined => []}),
{200, _} = request(put, "/gateways/mqttsn", maps:without([name, listeners], GwConf2)),
{204, _} = request(put, "/gateways/mqttsn", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/mqttsn"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateways/mqttsn").
ok.
t_gateway_coap(_) ->
{200, Gw} = request(get, "/gateways/coap"),
assert_gw_unloaded(Gw),
%% post
GwConf = #{
name => <<"coap">>,
heartbeat => <<"60s">>,
@ -160,20 +202,18 @@ t_gateway_coap(_) ->
#{name => <<"def">>, type => <<"udp">>, bind => <<"5683">>}
]
},
{201, _} = request(post, "/gateways", GwConf),
{204, _} = request(put, "/gateways/coap", GwConf),
{200, ConfResp} = request(get, "/gateways/coap"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{heartbeat => <<"10s">>}),
{200, _} = request(put, "/gateways/coap", maps:without([name, listeners], GwConf2)),
{204, _} = request(put, "/gateways/coap", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/coap"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateways/coap").
ok.
t_gateway_lwm2m(_) ->
{200, Gw} = request(get, "/gateways/lwm2m"),
assert_gw_unloaded(Gw),
%% post
GwConf = #{
name => <<"lwm2m">>,
xml_dir => <<"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml">>,
@ -192,20 +232,18 @@ t_gateway_lwm2m(_) ->
#{name => <<"def">>, type => <<"udp">>, bind => <<"5783">>}
]
},
{201, _} = request(post, "/gateways", GwConf),
{204, _} = request(put, "/gateways/lwm2m", GwConf),
{200, ConfResp} = request(get, "/gateways/lwm2m"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}),
{200, _} = request(put, "/gateways/lwm2m", maps:without([name, listeners], GwConf2)),
{204, _} = request(put, "/gateways/lwm2m", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/lwm2m"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateways/lwm2m").
ok.
t_gateway_exproto(_) ->
{200, Gw} = request(get, "/gateways/exproto"),
assert_gw_unloaded(Gw),
%% post
GwConf = #{
name => <<"exproto">>,
server => #{bind => <<"9100">>},
@ -214,15 +252,14 @@ t_gateway_exproto(_) ->
#{name => <<"def">>, type => <<"tcp">>, bind => <<"7993">>}
]
},
{201, _} = request(post, "/gateways", GwConf),
{204, _} = request(put, "/gateways/exproto", GwConf),
{200, ConfResp} = request(get, "/gateways/exproto"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}),
{200, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)),
{204, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/exproto"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateways/exproto").
ok.
t_gateway_exproto_with_ssl(_) ->
{200, Gw} = request(get, "/gateways/exproto"),
@ -230,7 +267,6 @@ t_gateway_exproto_with_ssl(_) ->
SslSvrOpts = ssl_server_opts(),
SslCliOpts = ssl_client_opts(),
%% post
GwConf = #{
name => <<"exproto">>,
server => #{
@ -245,27 +281,22 @@ t_gateway_exproto_with_ssl(_) ->
#{name => <<"def">>, type => <<"tcp">>, bind => <<"7993">>}
]
},
{201, _} = request(post, "/gateways", GwConf),
{204, _} = request(put, "/gateways/exproto", GwConf),
{200, ConfResp} = request(get, "/gateways/exproto"),
assert_confs(GwConf, ConfResp),
%% put
GwConf2 = emqx_map_lib:deep_merge(GwConf, #{
server => #{
bind => <<"9200">>,
ssl_options => SslCliOpts
}
}),
{200, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)),
{204, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)),
{200, ConfResp2} = request(get, "/gateways/exproto"),
assert_confs(GwConf2, ConfResp2),
{204, _} = request(delete, "/gateways/exproto").
ok.
t_authn(_) ->
GwConf = #{name => <<"stomp">>},
{201, _} = request(post, "/gateways", GwConf),
ct:sleep(500),
{204, _} = request(get, "/gateways/stomp/authentication"),
init_gw("stomp"),
AuthConf = #{
mechanism => <<"password_based">>,
backend => <<"built_in_database">>,
@ -283,22 +314,18 @@ t_authn(_) ->
{204, _} = request(delete, "/gateways/stomp/authentication"),
{204, _} = request(get, "/gateways/stomp/authentication"),
{204, _} = request(delete, "/gateways/stomp").
ok.
t_authn_data_mgmt(_) ->
GwConf = #{name => <<"stomp">>},
{201, _} = request(post, "/gateways", GwConf),
ct:sleep(500),
{204, _} = request(get, "/gateways/stomp/authentication"),
init_gw("stomp"),
AuthConf = #{
mechanism => <<"password_based">>,
backend => <<"built_in_database">>,
user_id_type => <<"clientid">>
},
{201, _} = request(post, "/gateways/stomp/authentication", AuthConf),
ct:sleep(500),
{200, ConfResp} = request(get, "/gateways/stomp/authentication"),
{200, ConfResp} =
?retry(10, 10, {200, _} = request(get, "/gateways/stomp/authentication")),
assert_confs(AuthConf, ConfResp),
User1 = #{
@ -358,11 +385,10 @@ t_authn_data_mgmt(_) ->
{204, _} = request(delete, "/gateways/stomp/authentication"),
{204, _} = request(get, "/gateways/stomp/authentication"),
{204, _} = request(delete, "/gateways/stomp").
ok.
t_listeners_tcp(_) ->
GwConf = #{name => <<"stomp">>},
{201, _} = request(post, "/gateways", GwConf),
{204, _} = request(put, "/gateways/stomp", #{}),
{404, _} = request(get, "/gateways/stomp/listeners"),
LisConf = #{
name => <<"def">>,
@ -387,7 +413,7 @@ t_listeners_tcp(_) ->
{204, _} = request(delete, "/gateways/stomp/listeners/stomp:tcp:def"),
{404, _} = request(get, "/gateways/stomp/listeners/stomp:tcp:def"),
{204, _} = request(delete, "/gateways/stomp").
ok.
t_listeners_authn(_) ->
GwConf = #{
@ -400,9 +426,7 @@ t_listeners_authn(_) ->
}
]
},
{201, _} = request(post, "/gateways", GwConf),
ct:sleep(500),
{200, ConfResp} = request(get, "/gateways/stomp"),
ConfResp = init_gw("stomp", GwConf),
assert_confs(GwConf, ConfResp),
AuthConf = #{
@ -424,7 +448,7 @@ t_listeners_authn(_) ->
{204, _} = request(delete, Path),
%% FIXME: 204?
{204, _} = request(get, Path),
{204, _} = request(delete, "/gateways/stomp").
ok.
t_listeners_authn_data_mgmt(_) ->
GwConf = #{
@ -437,7 +461,7 @@ t_listeners_authn_data_mgmt(_) ->
}
]
},
{201, _} = request(post, "/gateways", GwConf),
{204, _} = request(put, "/gateways/stomp", GwConf),
{200, ConfResp} = request(get, "/gateways/stomp"),
assert_confs(GwConf, ConfResp),
@ -514,13 +538,10 @@ t_listeners_authn_data_mgmt(_) ->
{filename, "user-credentials.csv", CSVData}
]),
{204, _} = request(delete, "/gateways/stomp").
ok.
t_authn_fuzzy_search(_) ->
GwConf = #{name => <<"stomp">>},
{201, _} = request(post, "/gateways", GwConf),
{204, _} = request(get, "/gateways/stomp/authentication"),
init_gw("stomp"),
AuthConf = #{
mechanism => <<"password_based">>,
backend => <<"built_in_database">>,
@ -561,7 +582,25 @@ t_authn_fuzzy_search(_) ->
{204, _} = request(delete, "/gateways/stomp/authentication"),
{204, _} = request(get, "/gateways/stomp/authentication"),
{204, _} = request(delete, "/gateways/stomp").
ok.
%%--------------------------------------------------------------------
%% Helpers
init_gw(GwName) ->
init_gw(GwName, #{}).
init_gw(GwName, GwConf) ->
{204, _} = request(put, "/gateways/" ++ GwName, GwConf),
?retry(
10,
10,
begin
{200, #{status := Status} = RespConf} = request(get, "/gateways/" ++ GwName),
false = (Status == <<"unloaded">>),
RespConf
end
).
%%--------------------------------------------------------------------
%% Asserts

2
apps/emqx_gateway/test/emqx_gateway_test_utils.erl
View File

@ -94,7 +94,7 @@ maybe_unconvert_listeners(Conf) when is_map(Conf) ->
maybe_unconvert_listeners(Conf) ->
Conf.
assert_feilds_apperence(Ks, Map) ->
assert_fields_exist(Ks, Map) ->
lists:foreach(
fun(K) ->
_ = maps:get(K, Map)

6
apps/emqx_gateway/test/emqx_stomp_SUITE.erl
View File

@ -25,7 +25,7 @@
-import(
emqx_gateway_test_utils,
[
assert_feilds_apperence/2,
assert_fields_exist/2,
request/2,
request/3
]
@ -730,7 +730,7 @@ t_rest_clienit_info(_) ->
binary_to_list(ClientId),
{200, StompClient1} = request(get, ClientPath),
?assertEqual(StompClient, StompClient1),
assert_feilds_apperence(
assert_fields_exist(
[
proto_name,
awaiting_rel_max,
@ -787,7 +787,7 @@ t_rest_clienit_info(_) ->
{200, Subs} = request(get, ClientPath ++ "/subscriptions"),
?assertEqual(1, length(Subs)),
assert_feilds_apperence([topic, qos], lists:nth(1, Subs)),
assert_fields_exist([topic, qos], lists:nth(1, Subs)),
{201, _} = request(
post,

45
apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf
View File

@ -124,4 +124,49 @@ MQTT 消息发布的错误码，这些错误码也是 MQTT 规范中 PUBACK 消
zh: "失败的详细原因。"
}
}
message_properties {
desc {
en: "The Properties of the PUBLISH message."
zh: "PUBLISH 消息里的 Property 字段。"
}
}
msg_payload_format_indicator {
desc {
en: """0 (0x00) Byte Indicates that the Payload is unspecified bytes, which is equivalent to not sending a Payload Format Indicator.
1 (0x01) Byte Indicates that the Payload is UTF-8 Encoded Character Data. The UTF-8 data in the Payload MUST be well-formed UTF-8 as defined by the Unicode specification and restated in RFC 3629.
"""
zh: "载荷格式指示标识符，0 表示载荷是未指定格式的数据，相当于没有发送载荷格式指示；1 表示载荷是 UTF-8 编码的字符数据，载荷中的 UTF-8 数据必须是按照 Unicode 的规范和 RFC 3629 的标准要求进行编码的。"
}
}
msg_message_expiry_interval {
desc {
en: "Identifier of the Message Expiry Interval. If the Message Expiry Interval has passed and the Server has not managed to start onward delivery to a matching subscriber, then it MUST delete the copy of the message for that subscriber."
zh: "消息过期间隔标识符，以秒为单位。当消失已经过期时，如果服务端还没有开始向匹配的订阅者投递该消息，则服务端会删除该订阅者的消息副本。如果不设置，则消息永远不会过期"
}
}
msg_response_topic {
desc {
en: "Identifier of the Response Topic.The Response Topic MUST be a UTF-8 Encoded, It MUST NOT contain wildcard characters."
zh: "响应主题标识符， UTF-8 编码的字符串，用作响应消息的主题名。响应主题不能包含通配符，也不能包含多个主题，否则将造成协议错误。当存在响应主题时，消息将被视作请求报文。服务端在收到应用消息时必须将响应主题原封不动的发送给所有的订阅者。"
}
}
msg_correlation_data {
desc {
en: "Identifier of the Correlation Data. The Server MUST send the Correlation Data unaltered to all subscribers receiving the Application Message."
zh: "对比数据标识符，服务端在收到应用消息时必须原封不动的把对比数据发送给所有的订阅者。对比数据只对请求消息(Request Message)的发送端和响应消息(Response Message)的接收端有意义。"
}
}
msg_user_properties {
desc {
en: "The User-Property key-value pairs. Note: in case there are duplicated keys, only the last one will be used."
zh: "指定 MQTT 消息的 User Property 键值对。注意，如果出现重复的键，只有最后一个会保留。"
}
}
msg_content_type {
desc {
en: "The Content Type MUST be a UTF-8 Encoded String."
zh: "内容类型标识符，以 UTF-8 格式编码的字符串，用来描述应用消息的内容，服务端必须把收到的应用消息中的内容类型原封不动的发送给所有的订阅者。"
}
}
}

460
apps/emqx_management/src/emqx_mgmt_api.erl
View File

@ -21,6 +21,7 @@
-elvis([{elvis_style, dont_repeat_yourself, #{min_complexity => 100}}]).
-define(FRESH_SELECT, fresh_select).
-define(LONG_QUERY_TIMEOUT, 50000).
-export([
paginate/3,
@ -29,13 +30,34 @@
%% first_next query APIs
-export([
node_query/5,
cluster_query/4,
select_table_with_count/5,
node_query/6,
cluster_query/5,
b2i/1
]).
-export([do_query/6]).
-export_type([
match_spec_and_filter/0
]).
-type query_params() :: list() | map().
-type query_schema() :: [
{Key :: binary(), Type :: atom | binary | integer | timestamp | ip | ip_port}
].
-type query_to_match_spec_fun() :: fun((list(), list()) -> match_spec_and_filter()).
-type match_spec_and_filter() :: #{match_spec := ets:match_spec(), fuzzy_fun := fuzzy_filter_fun()}.
-type fuzzy_filter_fun() :: undefined | {fun(), list()}.
-type format_result_fun() ::
fun((node(), term()) -> term())
| fun((term()) -> term()).
-type query_return() :: #{meta := map(), data := [term()]}.
-export([do_query/2, apply_total_query/1]).
paginate(Tables, Params, {Module, FormatFun}) ->
Qh = query_handle(Tables),
@ -117,171 +139,289 @@ limit(Params) when is_map(Params) ->
limit(Params) ->
proplists:get_value(<<"limit">>, Params, emqx_mgmt:max_row_limit()).
init_meta(Params) ->
Limit = b2i(limit(Params)),
Page = b2i(page(Params)),
#{
page => Page,
limit => Limit,
count => 0
}.
%%--------------------------------------------------------------------
%% Node Query
%%--------------------------------------------------------------------
node_query(Node, QString, Tab, QSchema, QueryFun) ->
{_CodCnt, NQString} = parse_qstring(QString, QSchema),
page_limit_check_query(
init_meta(QString),
{fun do_node_query/5, [Node, Tab, NQString, QueryFun, init_meta(QString)]}
).
-spec node_query(
node(),
atom(),
query_params(),
query_schema(),
query_to_match_spec_fun(),
format_result_fun()
) -> {error, page_limit_invalid} | {error, atom(), term()} | query_return().
node_query(Node, Tab, QString, QSchema, MsFun, FmtFun) ->
case parse_pager_params(QString) of
false ->
{error, page_limit_invalid};
Meta ->
{_CodCnt, NQString} = parse_qstring(QString, QSchema),
ResultAcc = init_query_result(),
QueryState = init_query_state(Tab, NQString, MsFun, Meta),
NResultAcc = do_node_query(
Node, QueryState, ResultAcc
),
format_query_result(FmtFun, Meta, NResultAcc)
end.
%% @private
do_node_query(Node, Tab, QString, QueryFun, Meta) ->
do_node_query(Node, Tab, QString, QueryFun, _Continuation = ?FRESH_SELECT, Meta, _Results = []).
do_node_query(
Node,
Tab,
QString,
QueryFun,
Continuation,
Meta = #{limit := Limit},
Results
QueryState,
ResultAcc
) ->
case do_query(Node, Tab, QString, QueryFun, Continuation, Limit) of
case do_query(Node, QueryState) of
{error, {badrpc, R}} ->
{error, Node, {badrpc, R}};
{Len, Rows, ?FRESH_SELECT} ->
{NMeta, NResults} = sub_query_result(Len, Rows, Limit, Results, Meta),
#{meta => NMeta, data => NResults};
{Len, Rows, NContinuation} ->
{NMeta, NResults} = sub_query_result(Len, Rows, Limit, Results, Meta),
do_node_query(Node, Tab, QString, QueryFun, NContinuation, NMeta, NResults)
{Rows, NQueryState = #{continuation := ?FRESH_SELECT}} ->
{_, NResultAcc} = accumulate_query_rows(Node, Rows, NQueryState, ResultAcc),
NResultAcc;
{Rows, NQueryState} ->
case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of
{enough, NResultAcc} ->
NResultAcc;
{more, NResultAcc} ->
do_node_query(Node, NQueryState, NResultAcc)
end
end.
%%--------------------------------------------------------------------
%% Cluster Query
%%--------------------------------------------------------------------
cluster_query(QString, Tab, QSchema, QueryFun) ->
{_CodCnt, NQString} = parse_qstring(QString, QSchema),
Nodes = mria_mnesia:running_nodes(),
page_limit_check_query(
init_meta(QString),
{fun do_cluster_query/5, [Nodes, Tab, NQString, QueryFun, init_meta(QString)]}
).
-spec cluster_query(
atom(),
query_params(),
query_schema(),
query_to_match_spec_fun(),
format_result_fun()
) -> {error, page_limit_invalid} | {error, atom(), term()} | query_return().
cluster_query(Tab, QString, QSchema, MsFun, FmtFun) ->
case parse_pager_params(QString) of
false ->
{error, page_limit_invalid};
Meta ->
{_CodCnt, NQString} = parse_qstring(QString, QSchema),
Nodes = mria_mnesia:running_nodes(),
ResultAcc = init_query_result(),
QueryState = init_query_state(Tab, NQString, MsFun, Meta),
NResultAcc = do_cluster_query(
Nodes, QueryState, ResultAcc
),
format_query_result(FmtFun, Meta, NResultAcc)
end.
%% @private
do_cluster_query(Nodes, Tab, QString, QueryFun, Meta) ->
do_cluster_query(
Nodes,
Tab,
QString,
QueryFun,
_Continuation = ?FRESH_SELECT,
Meta,
_Results = []
).
do_cluster_query([], _Tab, _QString, _QueryFun, _Continuation, Meta, Results) ->
#{meta => Meta, data => Results};
do_cluster_query([], _QueryState, ResultAcc) ->
ResultAcc;
do_cluster_query(
[Node | Tail] = Nodes,
Tab,
QString,
QueryFun,
Continuation,
Meta = #{limit := Limit},
Results
QueryState,
ResultAcc
) ->
case do_query(Node, Tab, QString, QueryFun, Continuation, Limit) of
case do_query(Node, QueryState) of
{error, {badrpc, R}} ->
{error, Node, {bar_rpc, R}};
{Len, Rows, ?FRESH_SELECT} ->
{NMeta, NResults} = sub_query_result(Len, Rows, Limit, Results, Meta),
do_cluster_query(Tail, Tab, QString, QueryFun, ?FRESH_SELECT, NMeta, NResults);
{Len, Rows, NContinuation} ->
{NMeta, NResults} = sub_query_result(Len, Rows, Limit, Results, Meta),
do_cluster_query(Nodes, Tab, QString, QueryFun, NContinuation, NMeta, NResults)
{error, Node, {badrpc, R}};
{Rows, NQueryState} ->
case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of
{enough, NResultAcc} ->
maybe_collect_total_from_tail_nodes(Tail, NQueryState, NResultAcc);
{more, NResultAcc} ->
NextNodes =
case NQueryState of
#{continuation := ?FRESH_SELECT} -> Tail;
_ -> Nodes
end,
do_cluster_query(NextNodes, NQueryState, NResultAcc)
end
end.
maybe_collect_total_from_tail_nodes([], _QueryState, ResultAcc) ->
ResultAcc;
maybe_collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc) ->
case counting_total_fun(QueryState) of
false ->
ResultAcc;
_Fun ->
collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc)
end.
collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc}) ->
%% XXX: badfun risk? if the FuzzyFun is an anonumous func in local node
case rpc:multicall(Nodes, ?MODULE, apply_total_query, [QueryState], ?LONG_QUERY_TIMEOUT) of
{_, [Node | _]} ->
{error, Node, {badrpc, badnode}};
{ResL0, []} ->
ResL = lists:zip(Nodes, ResL0),
case lists:filter(fun({_, I}) -> not is_integer(I) end, ResL) of
[{Node, {badrpc, Reason}} | _] ->
{error, Node, {badrpc, Reason}};
[] ->
ResultAcc#{total => ResL ++ TotalAcc}
end
end.
%%--------------------------------------------------------------------
%% Do Query (or rpc query)
%%--------------------------------------------------------------------
%% QueryState ::
%% #{continuation := ets:continuation(),
%% page := pos_integer(),
%% limit := pos_integer(),
%% total := [{node(), non_neg_integer()}],
%% table := atom(),
%% qs := {Qs, Fuzzy} %% parsed query params
%% msfun := query_to_match_spec_fun()
%% }
init_query_state(Tab, QString, MsFun, _Meta = #{page := Page, limit := Limit}) ->
#{match_spec := Ms, fuzzy_fun := FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
%% assert FuzzyFun type
_ =
case FuzzyFun of
undefined ->
ok;
{NamedFun, Args} ->
true = is_list(Args),
{type, external} = erlang:fun_info(NamedFun, type)
end,
#{
page => Page,
limit => Limit,
table => Tab,
qs => QString,
msfun => MsFun,
mactch_spec => Ms,
fuzzy_fun => FuzzyFun,
total => [],
continuation => ?FRESH_SELECT
}.
%% @private This function is exempt from BPAPI
do_query(Node, Tab, QString, {M, F}, Continuation, Limit) when Node =:= node() ->
erlang:apply(M, F, [Tab, QString, Continuation, Limit]);
do_query(Node, Tab, QString, QueryFun, Continuation, Limit) ->
do_query(Node, QueryState) when Node =:= node() ->
do_select(Node, QueryState);
do_query(Node, QueryState) ->
case
rpc:call(
Node,
?MODULE,
do_query,
[Node, Tab, QString, QueryFun, Continuation, Limit],
50000
[Node, QueryState],
?LONG_QUERY_TIMEOUT
)
of
{badrpc, _} = R -> {error, R};
Ret -> Ret
end.
sub_query_result(Len, Rows, Limit, Results, Meta) ->
{Flag, NMeta} = judge_page_with_counting(Len, Meta),
NResults =
case Flag of
more ->
[];
cutrows ->
{SubStart, NeedNowNum} = rows_sub_params(Len, NMeta),
ThisRows = lists:sublist(Rows, SubStart, NeedNowNum),
lists:sublist(lists:append(Results, ThisRows), SubStart, Limit);
enough ->
lists:sublist(lists:append(Results, Rows), 1, Limit)
do_select(
Node,
QueryState0 = #{
table := Tab,
mactch_spec := Ms,
fuzzy_fun := FuzzyFun,
continuation := Continuation,
limit := Limit
}
) ->
QueryState = maybe_apply_total_query(Node, QueryState0),
Result =
case Continuation of
?FRESH_SELECT ->
ets:select(Tab, Ms, Limit);
_ ->
%% XXX: Repair is necessary because we pass Continuation back
%% and forth through the nodes in the `do_cluster_query`
ets:select(ets:repair_continuation(Continuation, Ms))
end,
{NMeta, NResults}.
case Result of
'$end_of_table' ->
{[], QueryState#{continuation => ?FRESH_SELECT}};
{Rows, NContinuation} ->
NRows =
case FuzzyFun of
undefined ->
Rows;
{FilterFun, Args0} when is_function(FilterFun), is_list(Args0) ->
lists:filter(
fun(E) -> erlang:apply(FilterFun, [E | Args0]) end,
Rows
)
end,
{NRows, QueryState#{continuation => NContinuation}}
end.
%%--------------------------------------------------------------------
%% Table Select
%%--------------------------------------------------------------------
maybe_apply_total_query(Node, QueryState = #{total := TotalAcc}) ->
case proplists:get_value(Node, TotalAcc, undefined) of
undefined ->
Total = apply_total_query(QueryState),
QueryState#{total := [{Node, Total} | TotalAcc]};
_ ->
QueryState
end.
select_table_with_count(Tab, {Ms, FuzzyFilterFun}, ?FRESH_SELECT, Limit, FmtFun) when
is_function(FuzzyFilterFun) andalso Limit > 0
->
case ets:select(Tab, Ms, Limit) of
'$end_of_table' ->
{0, [], ?FRESH_SELECT};
{RawResult, NContinuation} ->
Rows = FuzzyFilterFun(RawResult),
{length(Rows), lists:map(FmtFun, Rows), NContinuation}
apply_total_query(QueryState = #{table := Tab}) ->
case counting_total_fun(QueryState) of
false ->
%% return a fake total number if the query have any conditions
0;
Fun ->
Fun(Tab)
end.
counting_total_fun(_QueryState = #{qs := {[], []}}) ->
fun(Tab) -> ets:info(Tab, size) end;
counting_total_fun(_QueryState = #{mactch_spec := Ms, fuzzy_fun := undefined}) ->
%% XXX: Calculating the total number of data that match a certain
%% condition under a large table is very expensive because the
%% entire ETS table needs to be scanned.
%%
%% XXX: How to optimize it? i.e, using:
[{MatchHead, Conditions, _Return}] = Ms,
CountingMs = [{MatchHead, Conditions, [true]}],
fun(Tab) ->
ets:select_count(Tab, CountingMs)
end;
select_table_with_count(_Tab, {Ms, FuzzyFilterFun}, Continuation, _Limit, FmtFun) when
is_function(FuzzyFilterFun)
->
case ets:select(ets:repair_continuation(Continuation, Ms)) of
'$end_of_table' ->
{0, [], ?FRESH_SELECT};
{RawResult, NContinuation} ->
Rows = FuzzyFilterFun(RawResult),
{length(Rows), lists:map(FmtFun, Rows), NContinuation}
end;
select_table_with_count(Tab, Ms, ?FRESH_SELECT, Limit, FmtFun) when
Limit > 0
->
case ets:select(Tab, Ms, Limit) of
'$end_of_table' ->
{0, [], ?FRESH_SELECT};
{RawResult, NContinuation} ->
{length(RawResult), lists:map(FmtFun, RawResult), NContinuation}
end;
select_table_with_count(_Tab, Ms, Continuation, _Limit, FmtFun) ->
case ets:select(ets:repair_continuation(Continuation, Ms)) of
'$end_of_table' ->
{0, [], ?FRESH_SELECT};
{RawResult, NContinuation} ->
{length(RawResult), lists:map(FmtFun, RawResult), NContinuation}
counting_total_fun(_QueryState = #{fuzzy_fun := FuzzyFun}) when FuzzyFun =/= undefined ->
%% XXX: Calculating the total number for a fuzzy searching is very very expensive
%% so it is not supported now
false.
%% ResultAcc :: #{count := integer(),
%% cursor := integer(),
%% rows := [{node(), Rows :: list()}],
%% total := [{node() => integer()}]
%% }
init_query_result() ->
#{cursor => 0, count => 0, rows => [], total => []}.
accumulate_query_rows(
Node,
Rows,
_QueryState = #{page := Page, limit := Limit, total := TotalAcc},
ResultAcc = #{cursor := Cursor, count := Count, rows := RowsAcc}
) ->
PageStart = (Page - 1) * Limit + 1,
PageEnd = Page * Limit,
Len = length(Rows),
case Cursor + Len of
NCursor when NCursor < PageStart ->
{more, ResultAcc#{cursor => NCursor, total => TotalAcc}};
NCursor when NCursor < PageEnd ->
{more, ResultAcc#{
cursor => NCursor,
count => Count + length(Rows),
total => TotalAcc,
rows => [{Node, Rows} | RowsAcc]
}};
NCursor when NCursor >= PageEnd ->
SubRows = lists:sublist(Rows, Limit - Count),
{enough, ResultAcc#{
cursor => NCursor,
count => Count + length(SubRows),
total => TotalAcc,
rows => [{Node, SubRows} | RowsAcc]
}}
end.
%%--------------------------------------------------------------------
@ -295,6 +435,7 @@ parse_qstring(QString, QSchema) ->
{length(NQString) + length(FuzzyQString), {NQString, FuzzyQString}}.
do_parse_qstring([], _, Acc1, Acc2) ->
%% remove fuzzy keys if present in accurate query
NAcc2 = [E || E <- Acc2, not lists:keymember(element(1, E), 1, Acc1)],
{lists:reverse(Acc1), lists:reverse(NAcc2)};
do_parse_qstring([{Key, Value} | RestQString], QSchema, Acc1, Acc2) ->
@ -379,40 +520,41 @@ is_fuzzy_key(<<"match_", _/binary>>) ->
is_fuzzy_key(_) ->
false.
page_start(1, _) -> 1;
page_start(Page, Limit) -> (Page - 1) * Limit + 1.
format_query_result(_FmtFun, _Meta, Error = {error, _Node, _Reason}) ->
Error;
format_query_result(
FmtFun, Meta, _ResultAcc = #{total := TotalAcc, rows := RowsAcc}
) ->
Total = lists:foldr(fun({_Node, T}, N) -> N + T end, 0, TotalAcc),
#{
%% The `count` is used in HTTP API to indicate the total number of
%% queries that can be read
meta => Meta#{count => Total},
data => lists:flatten(
lists:foldl(
fun({Node, Rows}, Acc) ->
[lists:map(fun(Row) -> exec_format_fun(FmtFun, Node, Row) end, Rows) | Acc]
end,
[],
RowsAcc
)
)
}.
judge_page_with_counting(Len, Meta = #{page := Page, limit := Limit, count := Count}) ->
PageStart = page_start(Page, Limit),
PageEnd = Page * Limit,
case Count + Len of
NCount when NCount < PageStart ->
{more, Meta#{count => NCount}};
NCount when NCount < PageEnd ->
{cutrows, Meta#{count => NCount}};
NCount when NCount >= PageEnd ->
{enough, Meta#{count => NCount}}
exec_format_fun(FmtFun, Node, Row) ->
case erlang:fun_info(FmtFun, arity) of
{arity, 1} -> FmtFun(Row);
{arity, 2} -> FmtFun(Node, Row)
end.
rows_sub_params(Len, _Meta = #{page := Page, limit := Limit, count := Count}) ->
PageStart = page_start(Page, Limit),
case (Count - Len) < PageStart of
parse_pager_params(Params) ->
Page = b2i(page(Params)),
Limit = b2i(limit(Params)),
case Page > 0 andalso Limit > 0 of
true ->
NeedNowNum = Count - PageStart + 1,
SubStart = Len - NeedNowNum + 1,
{SubStart, NeedNowNum};
#{page => Page, limit => Limit, count => 0};
false ->
{_SubStart = 1, _NeedNowNum = Len}
end.
page_limit_check_query(Meta, {F, A}) ->
case Meta of
#{page := Page, limit := Limit} when
Page < 1; Limit < 1
->
{error, page_limit_invalid};
_ ->
erlang:apply(F, A)
false
end.
%%--------------------------------------------------------------------
@ -458,6 +600,11 @@ to_ip_port(IPAddress) ->
Port = list_to_integer(Port0),
{IP, Port}.
b2i(Bin) when is_binary(Bin) ->
binary_to_integer(Bin);
b2i(Any) ->
Any.
%%--------------------------------------------------------------------
%% EUnits
%%--------------------------------------------------------------------
@ -502,8 +649,3 @@ params2qs_test() ->
{0, {[], []}} = parse_qstring([{not_a_predefined_params, val}], QSchema).
-endif.
b2i(Bin) when is_binary(Bin) ->
binary_to_integer(Bin);
b2i(Any) ->
Any.

26
apps/emqx_management/src/emqx_mgmt_api_alarms.erl
View File

@ -24,12 +24,12 @@
-export([api_spec/0, paths/0, schema/1, fields/1]).
-export([alarms/2]).
-export([alarms/2, format_alarm/2]).
-define(TAGS, [<<"Alarms">>]).
%% internal export (for query)
-export([query/4]).
-export([qs2ms/2]).
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
@ -112,7 +112,15 @@ alarms(get, #{query_string := QString}) ->
true -> ?ACTIVATED_ALARM;
false -> ?DEACTIVATED_ALARM
end,
case emqx_mgmt_api:cluster_query(QString, Table, [], {?MODULE, query}) of
case
emqx_mgmt_api:cluster_query(
Table,
QString,
[],
fun ?MODULE:qs2ms/2,
fun ?MODULE:format_alarm/2
)
of
{error, page_limit_invalid} ->
{400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
{error, Node, {badrpc, R}} ->
@ -128,11 +136,9 @@ alarms(delete, _Params) ->
%%%==============================================================================================
%% internal
query(Table, _QsSpec, Continuation, Limit) ->
Ms = [{'$1', [], ['$1']}],
emqx_mgmt_api:select_table_with_count(Table, Ms, Continuation, Limit, fun format_alarm/1).
-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
qs2ms(_Tab, {_Qs, _Fuzzy}) ->
#{match_spec => [{'$1', [], ['$1']}], fuzzy_fun => undefined}.
format_alarm(Alarms) when is_list(Alarms) ->
[emqx_alarm:format(Alarm) || Alarm <- Alarms];
format_alarm(Alarm) ->
emqx_alarm:format(Alarm).
format_alarm(WhichNode, Alarm) ->
emqx_alarm:format(WhichNode, Alarm).

75
apps/emqx_management/src/emqx_mgmt_api_clients.erl
View File

@ -46,8 +46,10 @@
]).
-export([
query/4,
format_channel_info/1
qs2ms/2,
run_fuzzy_filter/2,
format_channel_info/1,
format_channel_info/2
]).
%% for batch operation
@ -73,7 +75,6 @@
{<<"lte_connected_at">>, timestamp}
]).
-define(QUERY_FUN, {?MODULE, query}).
-define(FORMAT_FUN, {?MODULE, format_channel_info}).
-define(CLIENT_ID_NOT_FOUND,
@ -584,13 +585,13 @@ authz_cache(delete, #{bindings := Bindings}) ->
clean_authz_cache(Bindings).
subscribe(post, #{bindings := #{clientid := ClientID}, body := TopicInfo}) ->
Opts = emqx_map_lib:unsafe_atom_key_map(TopicInfo),
Opts = to_topic_info(TopicInfo),
subscribe(Opts#{clientid => ClientID}).
subscribe_batch(post, #{bindings := #{clientid := ClientID}, body := TopicInfos}) ->
Topics =
[
emqx_map_lib:unsafe_atom_key_map(TopicInfo)
to_topic_info(TopicInfo)
|| TopicInfo <- TopicInfos
],
subscribe_batch(#{clientid => ClientID, topics => Topics}).
@ -642,10 +643,11 @@ list_clients(QString) ->
case maps:get(<<"node">>, QString, undefined) of
undefined ->
emqx_mgmt_api:cluster_query(
QString,
?CLIENT_QTAB,
QString,
?CLIENT_QSCHEMA,
?QUERY_FUN
fun ?MODULE:qs2ms/2,
fun ?MODULE:format_channel_info/2
);
Node0 ->
case emqx_misc:safe_to_existing_atom(Node0) of
@ -653,10 +655,11 @@ list_clients(QString) ->
QStringWithoutNode = maps:without([<<"node">>], QString),
emqx_mgmt_api:node_query(
Node1,
QStringWithoutNode,
?CLIENT_QTAB,
QStringWithoutNode,
?CLIENT_QSCHEMA,
?QUERY_FUN
fun ?MODULE:qs2ms/2,
fun ?MODULE:format_channel_info/2
);
{error, _} ->
{error, Node0, {badrpc, <<"invalid node">>}}
@ -780,32 +783,16 @@ do_unsubscribe(ClientID, Topic) ->
Res
end.
%%--------------------------------------------------------------------
%% Query Functions
query(Tab, {QString, []}, Continuation, Limit) ->
Ms = qs2ms(QString),
emqx_mgmt_api:select_table_with_count(
Tab,
Ms,
Continuation,
Limit,
fun format_channel_info/1
);
query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
Ms = qs2ms(QString),
FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
emqx_mgmt_api:select_table_with_count(
Tab,
{Ms, FuzzyFilterFun},
Continuation,
Limit,
fun format_channel_info/1
).
%%--------------------------------------------------------------------
%% QueryString to Match Spec
-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
qs2ms(_Tab, {QString, FuzzyQString}) ->
#{
match_spec => qs2ms(QString),
fuzzy_fun => fuzzy_filter_fun(FuzzyQString)
}.
-spec qs2ms(list()) -> ets:match_spec().
qs2ms(Qs) ->
{MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}),
@ -855,13 +842,10 @@ ms(created_at, X) ->
%%--------------------------------------------------------------------
%% Match funcs
fuzzy_filter_fun([]) ->
undefined;
fuzzy_filter_fun(Fuzzy) ->
fun(MsRaws) when is_list(MsRaws) ->
lists:filter(
fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
MsRaws
)
end.
{fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
run_fuzzy_filter(_, []) ->
true;
@ -876,12 +860,11 @@ run_fuzzy_filter(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr} |
%%--------------------------------------------------------------------
%% format funcs
format_channel_info({_, ClientInfo0, ClientStats}) ->
Node =
case ClientInfo0 of
#{node := N} -> N;
_ -> node()
end,
format_channel_info(ChannInfo = {_, _ClientInfo, _ClientStats}) ->
format_channel_info(node(), ChannInfo).
format_channel_info(WhichNode, {_, ClientInfo0, ClientStats}) ->
Node = maps:get(node, ClientInfo0, WhichNode),
ClientInfo1 = emqx_map_lib:deep_remove([conninfo, clientid], ClientInfo0),
ClientInfo2 = emqx_map_lib:deep_remove([conninfo, username], ClientInfo1),
StatsMap = maps:without(
@ -973,3 +956,7 @@ format_authz_cache({{PubSub, Topic}, {AuthzResult, Timestamp}}) ->
result => AuthzResult,
updated_time => Timestamp
}.
to_topic_info(Data) ->
M = maps:with([<<"topic">>, <<"qos">>, <<"nl">>, <<"rap">>, <<"rh">>], Data),
emqx_map_lib:safe_atom_key_map(M).

4
apps/emqx_management/src/emqx_mgmt_api_configs.erl
View File

@ -268,7 +268,7 @@ config(put, #{body := Body}, Req) ->
global_zone_configs(get, _Params, _Req) ->
Paths = global_zone_roots(),
Zones = lists:foldl(
fun(Path, Acc) -> Acc#{Path => get_config_with_default([Path])} end,
fun(Path, Acc) -> maps:merge(Acc, get_config_with_default(Path)) end,
#{},
Paths
),
@ -343,7 +343,7 @@ get_full_config() ->
).
get_config_with_default(Path) ->
emqx_config:fill_defaults(emqx:get_raw_config(Path)).
emqx_config:fill_defaults(#{Path => emqx:get_raw_config([Path])}).
conf_path_from_querystr(Req) ->
case proplists:get_value(<<"conf_path">>, cowboy_req:parse_qs(Req)) of

68
apps/emqx_management/src/emqx_mgmt_api_publish.erl
View File

@ -114,6 +114,11 @@ fields(message) ->
required => true,
example => <<"hello emqx api">>
})},
{properties,
hoconsc:mk(hoconsc:ref(?MODULE, message_properties), #{
desc => ?DESC(message_properties),
required => false
})},
{retain,
hoconsc:mk(boolean(), #{
desc => ?DESC(retain),
@ -130,6 +135,43 @@ fields(publish_message) ->
default => plain
})}
] ++ fields(message);
fields(message_properties) ->
[
{'payload_format_indicator',
hoconsc:mk(typerefl:range(0, 1), #{
desc => ?DESC(msg_payload_format_indicator),
required => false,
example => 0
})},
{'message_expiry_interval',
hoconsc:mk(integer(), #{
desc => ?DESC(msg_message_expiry_interval),
required => false
})},
{'response_topic',
hoconsc:mk(binary(), #{
desc => ?DESC(msg_response_topic),
required => false,
example => <<"some_other_topic">>
})},
{'correlation_data',
hoconsc:mk(binary(), #{
desc => ?DESC(msg_correlation_data),
required => false
})},
{'user_properties',
hoconsc:mk(map(), #{
desc => ?DESC(msg_user_properties),
required => false,
example => #{<<"foo">> => <<"bar">>}
})},
{'content_type',
hoconsc:mk(binary(), #{
desc => ?DESC(msg_content_type),
required => false,
example => <<"text/plain">>
})}
];
fields(publish_ok) ->
[
{id,
@ -288,13 +330,23 @@ make_message(Map) ->
QoS = maps:get(<<"qos">>, Map, 0),
Topic = maps:get(<<"topic">>, Map),
Retain = maps:get(<<"retain">>, Map, false),
Headers =
case maps:get(<<"properties">>, Map, #{}) of
Properties when
is_map(Properties) andalso
map_size(Properties) > 0
->
#{properties => to_msg_properties(Properties)};
_ ->
#{}
end,
try
_ = emqx_topic:validate(name, Topic)
catch
error:_Reason ->
throw(invalid_topic_name)
end,
Message = emqx_message:make(From, QoS, Topic, Payload, #{retain => Retain}, #{}),
Message = emqx_message:make(From, QoS, Topic, Payload, #{retain => Retain}, Headers),
Size = emqx_message:estimate_size(Message),
(Size > size_limit()) andalso throw(packet_too_large),
{ok, Message};
@ -302,6 +354,20 @@ make_message(Map) ->
{error, R}
end.
to_msg_properties(Properties) ->
maps:fold(
fun to_property/3,
#{},
Properties
).
to_property(<<"payload_format_indicator">>, V, M) -> M#{'Payload-Format-Indicator' => V};
to_property(<<"message_expiry_interval">>, V, M) -> M#{'Message-Expiry-Interval' => V};
to_property(<<"response_topic">>, V, M) -> M#{'Response-Topic' => V};
to_property(<<"correlation_data">>, V, M) -> M#{'Correlation-Data' => V};
to_property(<<"user_properties">>, V, M) -> M#{'User-Property' => maps:to_list(V)};
to_property(<<"content_type">>, V, M) -> M#{'Content-Type' => V}.
%% get the global packet size limit since HTTP API does not belong to any zone.
size_limit() ->
try

85
apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
View File

@ -32,8 +32,9 @@
-export([subscriptions/2]).
-export([
query/4,
format/1
qs2ms/2,
run_fuzzy_filter/2,
format/2
]).
-define(SUBS_QTABLE, emqx_suboption).
@ -47,8 +48,6 @@
{<<"match_topic">>, binary}
]).
-define(QUERY_FUN, {?MODULE, query}).
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
@ -139,20 +138,22 @@ subscriptions(get, #{query_string := QString}) ->
case maps:get(<<"node">>, QString, undefined) of
undefined ->
emqx_mgmt_api:cluster_query(
QString,
?SUBS_QTABLE,
QString,
?SUBS_QSCHEMA,
?QUERY_FUN
fun ?MODULE:qs2ms/2,
fun ?MODULE:format/2
);
Node0 ->
case emqx_misc:safe_to_existing_atom(Node0) of
{ok, Node1} ->
emqx_mgmt_api:node_query(
Node1,
QString,
?SUBS_QTABLE,
QString,
?SUBS_QSCHEMA,
?QUERY_FUN
fun ?MODULE:qs2ms/2,
fun ?MODULE:format/2
);
{error, _} ->
{error, Node0, {badrpc, <<"invalid node">>}}
@ -168,16 +169,12 @@ subscriptions(get, #{query_string := QString}) ->
{200, Result}
end.
format(Items) when is_list(Items) ->
[format(Item) || Item <- Items];
format({{Subscriber, Topic}, Options}) ->
format({Subscriber, Topic, Options});
format({_Subscriber, Topic, Options}) ->
format(WhichNode, {{_Subscriber, Topic}, Options}) ->
maps:merge(
#{
topic => get_topic(Topic, Options),
clientid => maps:get(subid, Options),
node => node()
node => WhichNode
},
maps:with([qos, nl, rap, rh], Options)
).
@ -190,53 +187,21 @@ get_topic(Topic, _) ->
Topic.
%%--------------------------------------------------------------------
%% Query Function
%% QueryString to MatchSpec
%%--------------------------------------------------------------------
query(Tab, {Qs, []}, Continuation, Limit) ->
Ms = qs2ms(Qs),
emqx_mgmt_api:select_table_with_count(
Tab,
Ms,
Continuation,
Limit,
fun format/1
);
query(Tab, {Qs, Fuzzy}, Continuation, Limit) ->
Ms = qs2ms(Qs),
FuzzyFilterFun = fuzzy_filter_fun(Fuzzy),
emqx_mgmt_api:select_table_with_count(
Tab,
{Ms, FuzzyFilterFun},
Continuation,
Limit,
fun format/1
).
-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
qs2ms(_Tab, {Qs, Fuzzy}) ->
#{match_spec => gen_match_spec(Qs), fuzzy_fun => fuzzy_filter_fun(Fuzzy)}.
fuzzy_filter_fun(Fuzzy) ->
fun(MsRaws) when is_list(MsRaws) ->
lists:filter(
fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
MsRaws
)
end.
run_fuzzy_filter(_, []) ->
true;
run_fuzzy_filter(E = {{_, Topic}, _}, [{topic, match, TopicFilter} | Fuzzy]) ->
emqx_topic:match(Topic, TopicFilter) andalso run_fuzzy_filter(E, Fuzzy).
%%--------------------------------------------------------------------
%% Query String to Match Spec
qs2ms(Qs) ->
MtchHead = qs2ms(Qs, {{'_', '_'}, #{}}),
gen_match_spec(Qs) ->
MtchHead = gen_match_spec(Qs, {{'_', '_'}, #{}}),
[{MtchHead, [], ['$_']}].
qs2ms([], MtchHead) ->
gen_match_spec([], MtchHead) ->
MtchHead;
qs2ms([{Key, '=:=', Value} | More], MtchHead) ->
qs2ms(More, update_ms(Key, Value, MtchHead)).
gen_match_spec([{Key, '=:=', Value} | More], MtchHead) ->
gen_match_spec(More, update_ms(Key, Value, MtchHead)).
update_ms(clientid, X, {{Pid, Topic}, Opts}) ->
{{Pid, Topic}, Opts#{subid => X}};
@ -246,3 +211,13 @@ update_ms(share_group, X, {{Pid, Topic}, Opts}) ->
{{Pid, Topic}, Opts#{share => X}};
update_ms(qos, X, {{Pid, Topic}, Opts}) ->
{{Pid, Topic}, Opts#{qos => X}}.
fuzzy_filter_fun([]) ->
undefined;
fuzzy_filter_fun(Fuzzy) ->
{fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
run_fuzzy_filter(_, []) ->
true;
run_fuzzy_filter(E = {{_, Topic}, _}, [{topic, match, TopicFilter} | Fuzzy]) ->
emqx_topic:match(Topic, TopicFilter) andalso run_fuzzy_filter(E, Fuzzy).

28
apps/emqx_management/src/emqx_mgmt_api_topics.erl
View File

@ -34,7 +34,7 @@
topic/2
]).
-export([query/4]).
-export([qs2ms/2, format/1]).
-define(TOPIC_NOT_FOUND, 'TOPIC_NOT_FOUND').
@ -109,7 +109,12 @@ topic(get, #{bindings := Bindings}) ->
do_list(Params) ->
case
emqx_mgmt_api:node_query(
node(), Params, emqx_route, ?TOPICS_QUERY_SCHEMA, {?MODULE, query}
node(),
emqx_route,
Params,
?TOPICS_QUERY_SCHEMA,
fun ?MODULE:qs2ms/2,
fun ?MODULE:format/1
)
of
{error, page_limit_invalid} ->
@ -138,16 +143,19 @@ generate_topic(Params = #{topic := Topic}) ->
generate_topic(Params) ->
Params.
query(Tab, {Qs, _}, Continuation, Limit) ->
Ms = qs2ms(Qs, [{{route, '_', '_'}, [], ['$_']}]),
emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit, fun format/1).
-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
qs2ms(_Tab, {Qs, _}) ->
#{
match_spec => gen_match_spec(Qs, [{{route, '_', '_'}, [], ['$_']}]),
fuzzy_fun => undefined
}.
qs2ms([], Res) ->
gen_match_spec([], Res) ->
Res;
qs2ms([{topic, '=:=', T} | Qs], [{{route, _, N}, [], ['$_']}]) ->
qs2ms(Qs, [{{route, T, N}, [], ['$_']}]);
qs2ms([{node, '=:=', N} | Qs], [{{route, T, _}, [], ['$_']}]) ->
qs2ms(Qs, [{{route, T, N}, [], ['$_']}]).
gen_match_spec([{topic, '=:=', T} | Qs], [{{route, _, N}, [], ['$_']}]) ->
gen_match_spec(Qs, [{{route, T, N}, [], ['$_']}]);
gen_match_spec([{node, '=:=', N} | Qs], [{{route, T, _}, [], ['$_']}]) ->
gen_match_spec(Qs, [{{route, T, N}, [], ['$_']}]).
format(#route{topic = Topic, dest = {_, Node}}) ->
#{topic => Topic, node => Node};

143
apps/emqx_management/test/emqx_mgmt_api_SUITE.erl Normal file
View File

@ -0,0 +1,143 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_mgmt_api_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
%%--------------------------------------------------------------------
%% setup
%%--------------------------------------------------------------------
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
Config.
end_per_suite(_) ->
ok.
%%--------------------------------------------------------------------
%% cases
%%--------------------------------------------------------------------
t_cluster_query(_Config) ->
net_kernel:start(['master@127.0.0.1', longnames]),
ct:timetrap({seconds, 120}),
snabbkaffe:fix_ct_logging(),
[{Name, Opts}, {Name1, Opts1}] = cluster_specs(),
Node1 = emqx_common_test_helpers:start_slave(Name, Opts),
Node2 = emqx_common_test_helpers:start_slave(Name1, Opts1),
try
process_flag(trap_exit, true),
ClientLs1 = [start_emqtt_client(Node1, I, 2883) || I <- lists:seq(1, 10)],
ClientLs2 = [start_emqtt_client(Node2, I, 3883) || I <- lists:seq(1, 10)],
%% returned list should be the same regardless of which node is requested
{200, ClientsAll} = query_clients(Node1, #{}),
?assertEqual({200, ClientsAll}, query_clients(Node2, #{})),
?assertMatch(
#{page := 1, limit := 100, count := 20},
maps:get(meta, ClientsAll)
),
?assertMatch(20, length(maps:get(data, ClientsAll))),
%% query the first page, counting in entire cluster
{200, ClientsPage1} = query_clients(Node1, #{<<"limit">> => 5}),
?assertMatch(
#{page := 1, limit := 5, count := 20},
maps:get(meta, ClientsPage1)
),
?assertMatch(5, length(maps:get(data, ClientsPage1))),
%% assert: AllPage = Page1 + Page2 + Page3 + Page4
%% !!!Note: this equation requires that the queried tables must be ordered_set
{200, ClientsPage2} = query_clients(Node1, #{<<"page">> => 2, <<"limit">> => 5}),
{200, ClientsPage3} = query_clients(Node2, #{<<"page">> => 3, <<"limit">> => 5}),
{200, ClientsPage4} = query_clients(Node1, #{<<"page">> => 4, <<"limit">> => 5}),
GetClientIds = fun(L) -> lists:map(fun(#{clientid := Id}) -> Id end, L) end,
?assertEqual(
GetClientIds(maps:get(data, ClientsAll)),
GetClientIds(
maps:get(data, ClientsPage1) ++ maps:get(data, ClientsPage2) ++
maps:get(data, ClientsPage3) ++ maps:get(data, ClientsPage4)
)
),
%% exact match can return non-zero total
{200, ClientsNode1} = query_clients(Node2, #{<<"username">> => <<"corenode1@127.0.0.1">>}),
?assertMatch(
#{count := 10},
maps:get(meta, ClientsNode1)
),
%% fuzzy searching can't return total
{200, ClientsNode2} = query_clients(Node2, #{<<"like_username">> => <<"corenode2">>}),
?assertMatch(
#{count := 0},
maps:get(meta, ClientsNode2)
),
?assertMatch(10, length(maps:get(data, ClientsNode2))),
_ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1),
_ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs2)
after
emqx_common_test_helpers:stop_slave(Node1),
emqx_common_test_helpers:stop_slave(Node2)
end,
ok.
%%--------------------------------------------------------------------
%% helpers
%%--------------------------------------------------------------------
cluster_specs() ->
Specs =
%% default listeners port
[
{core, corenode1, #{listener_ports => [{tcp, 2883}]}},
{core, corenode2, #{listener_ports => [{tcp, 3883}]}}
],
CommOpts =
[
{env, [{emqx, boot_modules, all}]},
{apps, []},
{conf, [
{[listeners, ssl, default, enabled], false},
{[listeners, ws, default, enabled], false},
{[listeners, wss, default, enabled], false}
]}
],
emqx_common_test_helpers:emqx_cluster(
Specs,
CommOpts
).
start_emqtt_client(Node0, N, Port) ->
Node = atom_to_binary(Node0),
ClientId = iolist_to_binary([Node, "-", integer_to_binary(N)]),
{ok, C} = emqtt:start_link([{clientid, ClientId}, {username, Node}, {port, Port}]),
{ok, _} = emqtt:connect(C),
C.
query_clients(Node, Qs0) ->
Qs = maps:merge(
#{<<"page">> => 1, <<"limit">> => 100},
Qs0
),
rpc:call(Node, emqx_mgmt_api_clients, clients, [get, #{query_string => Qs}]).

12
apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl
View File

@ -133,6 +133,18 @@ t_global_zone(_Config) ->
BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3),
?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)),
%% Remove max_qos_allowed from raw config, but we still get default value(2).
Mqtt0 = emqx_conf:get_raw([<<"mqtt">>]),
?assertEqual(1, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt0)),
Mqtt1 = maps:remove(<<"max_qos_allowed">>, Mqtt0),
ok = emqx_config:put_raw([<<"mqtt">>], Mqtt1),
Mqtt2 = emqx_conf:get_raw([<<"mqtt">>]),
?assertNot(maps:is_key(<<"max_qos_allowed">>, Mqtt2), Mqtt2),
{ok, #{<<"mqtt">> := Mqtt3}} = get_global_zone(),
%% the default value is 2
?assertEqual(2, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt3)),
ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0),
ok.
get_global_zone() ->

99
apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl
View File

@ -20,9 +20,7 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-define(CLIENTID, <<"api_clientid">>).
-define(USERNAME, <<"api_username">>).
-include_lib("common_test/include/ct.hrl").
-define(TOPIC1, <<"api_topic1">>).
-define(TOPIC2, <<"api_topic2">>).
@ -44,25 +42,56 @@ end_per_testcase(Case, Config) ->
?MODULE:Case({'end', Config}).
t_publish_api({init, Config}) ->
Config;
t_publish_api({'end', _Config}) ->
ok;
t_publish_api(_) ->
{ok, Client} = emqtt:start_link(#{
username => <<"api_username">>, clientid => <<"api_clientid">>
}),
{ok, Client} = emqtt:start_link(
#{
username => <<"api_username">>,
clientid => <<"api_clientid">>,
proto_ver => v5
}
),
{ok, _} = emqtt:connect(Client),
{ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1),
{ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2),
[{client, Client} | Config];
t_publish_api({'end', Config}) ->
Client = ?config(client, Config),
emqtt:stop(Client),
ok;
t_publish_api(_) ->
Payload = <<"hello">>,
Path = emqx_mgmt_api_test_util:api_path(["publish"]),
Auth = emqx_mgmt_api_test_util:auth_header_(),
Body = #{topic => ?TOPIC1, payload => Payload},
UserProperties = #{<<"foo">> => <<"bar">>},
Properties =
#{
<<"payload_format_indicator">> => 0,
<<"message_expiry_interval">> => 1000,
<<"response_topic">> => ?TOPIC2,
<<"correlation_data">> => <<"some_correlation_id">>,
<<"user_properties">> => UserProperties,
<<"content_type">> => <<"application/json">>
},
Body = #{topic => ?TOPIC1, payload => Payload, properties => Properties},
{ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body),
ResponseMap = decode_json(Response),
?assertEqual([<<"id">>], lists:sort(maps:keys(ResponseMap))),
?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)),
emqtt:stop(Client).
{ok, Message} = receive_assert(?TOPIC1, 0, Payload),
RecvProperties = maps:get(properties, Message),
UserPropertiesList = maps:to_list(UserProperties),
#{
'Payload-Format-Indicator' := 0,
'Message-Expiry-Interval' := RecvMessageExpiry,
'Correlation-Data' := <<"some_correlation_id">>,
'User-Property' := UserPropertiesList,
'Content-Type' := <<"application/json">>
} = RecvProperties,
?assert(RecvMessageExpiry =< 1000),
%% note: without props this time
Body2 = #{topic => ?TOPIC2, payload => Payload},
{ok, Response2} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body2),
ResponseMap2 = decode_json(Response2),
?assertEqual([<<"id">>], lists:sort(maps:keys(ResponseMap2))),
?assertEqual(ok, element(1, receive_assert(?TOPIC2, 0, Payload))).
t_publish_no_subscriber({init, Config}) ->
Config;
@ -163,16 +192,18 @@ t_publish_bad_topic_bulk(_Config) ->
).
t_publish_bulk_api({init, Config}) ->
Config;
t_publish_bulk_api({'end', _Config}) ->
ok;
t_publish_bulk_api(_) ->
{ok, Client} = emqtt:start_link(#{
username => <<"api_username">>, clientid => <<"api_clientid">>
}),
{ok, _} = emqtt:connect(Client),
{ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1),
{ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2),
[{client, Client} | Config];
t_publish_bulk_api({'end', Config}) ->
Client = ?config(client, Config),
emqtt:stop(Client),
ok;
t_publish_bulk_api(_) ->
Payload = <<"hello">>,
Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]),
Auth = emqx_mgmt_api_test_util:auth_header_(),
@ -199,9 +230,8 @@ t_publish_bulk_api(_) ->
end,
ResponseList
),
?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)),
?assertEqual(ok, receive_assert(?TOPIC2, 0, Payload)),
emqtt:stop(Client).
?assertEqual(ok, element(1, receive_assert(?TOPIC1, 0, Payload))),
?assertEqual(ok, element(1, receive_assert(?TOPIC2, 0, Payload))).
t_publish_no_subscriber_bulk({init, Config}) ->
Config;
@ -232,8 +262,8 @@ t_publish_no_subscriber_bulk(_) ->
],
ResponseList
),
?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)),
?assertEqual(ok, receive_assert(?TOPIC2, 0, Payload)),
?assertEqual(ok, element(1, receive_assert(?TOPIC1, 0, Payload))),
?assertEqual(ok, element(1, receive_assert(?TOPIC2, 0, Payload))),
emqtt:stop(Client).
t_publish_bulk_dispatch_one_message_invalid_topic({init, Config}) ->
@ -267,17 +297,19 @@ t_publish_bulk_dispatch_one_message_invalid_topic(Config) when is_list(Config) -
t_publish_bulk_dispatch_failure({init, Config}) ->
meck:new(emqx, [no_link, passthrough, no_history]),
meck:expect(emqx, is_running, fun() -> false end),
Config;
t_publish_bulk_dispatch_failure({'end', _Config}) ->
meck:unload(emqx),
ok;
t_publish_bulk_dispatch_failure(Config) when is_list(Config) ->
{ok, Client} = emqtt:start_link(#{
username => <<"api_username">>, clientid => <<"api_clientid">>
}),
{ok, _} = emqtt:connect(Client),
{ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1),
{ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2),
[{client, Client} | Config];
t_publish_bulk_dispatch_failure({'end', Config}) ->
meck:unload(emqx),
Client = ?config(client, Config),
emqtt:stop(Client),
ok;
t_publish_bulk_dispatch_failure(Config) when is_list(Config) ->
Payload = <<"hello">>,
Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]),
Auth = emqx_mgmt_api_test_util:auth_header_(),
@ -303,8 +335,7 @@ t_publish_bulk_dispatch_failure(Config) when is_list(Config) ->
#{<<"reason_code">> := ?RC_NO_MATCHING_SUBSCRIBERS}
],
decode_json(ResponseBody)
),
emqtt:stop(Client).
).
receive_assert(Topic, Qos, Payload) ->
receive
@ -312,12 +343,12 @@ receive_assert(Topic, Qos, Payload) ->
ReceiveTopic = maps:get(topic, Message),
ReceiveQos = maps:get(qos, Message),
ReceivePayload = maps:get(payload, Message),
?assertEqual(ReceiveTopic, Topic),
?assertEqual(ReceiveQos, Qos),
?assertEqual(ReceivePayload, Payload),
ok
?assertEqual(Topic, ReceiveTopic),
?assertEqual(Qos, ReceiveQos),
?assertEqual(Payload, ReceivePayload),
{ok, Message}
after 5000 ->
timeout
{error, timeout}
end.
decode_json(In) ->

4
apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
View File

@ -93,6 +93,7 @@ t_subscription_api(_) ->
{"match_topic", "t/#"}
]),
Headers = emqx_mgmt_api_test_util:auth_header_(),
{ok, ResponseTopic2} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers),
DataTopic2 = emqx_json:decode(ResponseTopic2, [return_maps]),
Meta2 = maps:get(<<"meta">>, DataTopic2),
@ -114,7 +115,8 @@ t_subscription_api(_) ->
MatchMeta = maps:get(<<"meta">>, MatchData),
?assertEqual(1, maps:get(<<"page">>, MatchMeta)),
?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta)),
?assertEqual(1, maps:get(<<"count">>, MatchMeta)),
%% count equals 0 in fuzzy searching
?assertEqual(0, maps:get(<<"count">>, MatchMeta)),
MatchSubs = maps:get(<<"data">>, MatchData),
?assertEqual(1, length(MatchSubs)),

24
apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
View File

@ -31,6 +31,7 @@ end_per_suite(_) ->
emqx_mgmt_api_test_util:end_suite().
t_nodes_api(_) ->
Node = atom_to_binary(node(), utf8),
Topic = <<"test_topic">>,
{ok, Client} = emqtt:start_link(#{
username => <<"routes_username">>, clientid => <<"routes_cid">>
@ -49,11 +50,30 @@ t_nodes_api(_) ->
Data = maps:get(<<"data">>, RoutesData),
Route = erlang:hd(Data),
?assertEqual(Topic, maps:get(<<"topic">>, Route)),
?assertEqual(atom_to_binary(node(), utf8), maps:get(<<"node">>, Route)),
?assertEqual(Node, maps:get(<<"node">>, Route)),
%% exact match
Topic2 = <<"test_topic_2">>,
{ok, _, _} = emqtt:subscribe(Client, Topic2),
QS = uri_string:compose_query([
{"topic", Topic2},
{"node", atom_to_list(node())}
]),
Headers = emqx_mgmt_api_test_util:auth_header_(),
{ok, MatchResponse} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers),
MatchData = emqx_json:decode(MatchResponse, [return_maps]),
?assertMatch(
#{<<"count">> := 1, <<"page">> := 1, <<"limit">> := 100},
maps:get(<<"meta">>, MatchData)
),
?assertMatch(
[#{<<"topic">> := Topic2, <<"node">> := Node}],
maps:get(<<"data">>, MatchData)
),
%% get topics/:topic
RoutePath = emqx_mgmt_api_test_util:api_path(["topics", Topic]),
{ok, RouteResponse} = emqx_mgmt_api_test_util:request_api(get, RoutePath),
RouteData = emqx_json:decode(RouteResponse, [return_maps]),
?assertEqual(Topic, maps:get(<<"topic">>, RouteData)),
?assertEqual(atom_to_binary(node(), utf8), maps:get(<<"node">>, RouteData)).
?assertEqual(Node, maps:get(<<"node">>, RouteData)).

39
apps/emqx_modules/src/emqx_delayed.erl
View File

@ -56,16 +56,20 @@
get_delayed_message/2,
delete_delayed_message/1,
delete_delayed_message/2,
cluster_list/1,
cluster_query/4
cluster_list/1
]).
%% exports for query
-export([
qs2ms/2,
format_delayed/1,
format_delayed/2
]).
-export([
post_config_update/5
]).
-export([format_delayed/1]).
%% exported for `emqx_telemetry'
-export([get_basic_usage_info/0]).
@ -166,16 +170,29 @@ list(Params) ->
emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN).
cluster_list(Params) ->
emqx_mgmt_api:cluster_query(Params, ?TAB, [], {?MODULE, cluster_query}).
emqx_mgmt_api:cluster_query(
?TAB,
Params,
[],
fun ?MODULE:qs2ms/2,
fun ?MODULE:format_delayed/2
).
cluster_query(Table, _QsSpec, Continuation, Limit) ->
Ms = [{'$1', [], ['$1']}],
emqx_mgmt_api:select_table_with_count(Table, Ms, Continuation, Limit, fun format_delayed/1).
-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
qs2ms(_Table, {_Qs, _Fuzzy}) ->
#{
match_spec => [{'$1', [], ['$1']}],
fuzzy_fun => undefined
}.
format_delayed(Delayed) ->
format_delayed(Delayed, false).
format_delayed(node(), Delayed).
format_delayed(WhichNode, Delayed) ->
format_delayed(WhichNode, Delayed, false).
format_delayed(
WhichNode,
#delayed_message{
key = {ExpectTimeStamp, Id},
delayed = Delayed,
@ -195,7 +212,7 @@ format_delayed(
RemainingTime = ExpectTimeStamp - ?NOW,
Result = #{
msgid => emqx_guid:to_hexstr(Id),
node => node(),
node => WhichNode,
publish_at => PublishTime,
delayed_interval => Delayed,
delayed_remaining => RemainingTime div 1000,
@ -222,7 +239,7 @@ get_delayed_message(Id) ->
{error, not_found};
Rows ->
Message = hd(Rows),
{ok, format_delayed(Message, true)}
{ok, format_delayed(node(), Message, true)}
end.
get_delayed_message(Node, Id) when Node =:= node() ->

2
apps/emqx_modules/src/emqx_modules.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_modules, [
{description, "EMQX Modules"},
{vsn, "5.0.6"},
{vsn, "5.0.7"},
{modules, []},
{applications, [kernel, stdlib, emqx]},
{mod, {emqx_modules_app, []}},

3
apps/emqx_modules/test/emqx_delayed_SUITE.erl
View File

@ -37,8 +37,7 @@
}).
all() ->
[t_banned_delayed].
%% emqx_common_test_helpers:all(?MODULE).
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF, #{

2
apps/emqx_retainer/src/emqx_retainer.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_retainer, [
{description, "EMQX Retainer"},
% strict semver, bump manually!
{vsn, "5.0.6"},
{vsn, "5.0.7"},
{modules, []},
{registered, [emqx_retainer_sup]},
{applications, [kernel, stdlib, emqx]},

16
apps/emqx_retainer/src/emqx_retainer_dispatcher.erl
View File

@ -20,6 +20,7 @@
-include("emqx_retainer.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% API
-export([
@ -286,7 +287,20 @@ do_deliver(Msgs, DeliverNum, Pid, Topic, Limiter) ->
end.
do_deliver([Msg | T], Pid, Topic) ->
Pid ! {deliver, Topic, Msg},
case emqx_banned:look_up({clientid, Msg#message.from}) of
[] ->
Pid ! {deliver, Topic, Msg},
ok;
_ ->
?tp(
notice,
ignore_retained_message_deliver,
#{
reason => "client is banned",
clientid => Msg#message.from
}
)
end,
do_deliver(T, Pid, Topic);
do_deliver([], _, _) ->
ok.

40
apps/emqx_retainer/test/emqx_retainer_SUITE.erl
View File

@ -639,6 +639,46 @@ test_disable_then_start(_Config) ->
?assertNotEqual([], gproc_pool:active_workers(emqx_retainer_dispatcher)),
ok.
t_deliver_when_banned(_) ->
Client1 = <<"c1">>,
Client2 = <<"c2">>,
{ok, C1} = emqtt:start_link([{clientid, Client1}, {clean_start, true}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(C1),
lists:foreach(
fun(I) ->
Topic = erlang:list_to_binary(io_lib:format("retained/~p", [I])),
Msg = emqx_message:make(Client2, 0, Topic, <<"this is a retained message">>),
Msg2 = emqx_message:set_flag(retain, Msg),
emqx:publish(Msg2)
end,
lists:seq(1, 3)
),
Now = erlang:system_time(second),
Who = {clientid, Client2},
emqx_banned:create(#{
who => Who,
by => <<"test">>,
reason => <<"test">>,
at => Now,
until => Now + 120
}),
timer:sleep(100),
snabbkaffe:start_trace(),
{ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/+">>, [{qos, 0}, {rh, 0}]),
timer:sleep(500),
Trace = snabbkaffe:collect_trace(),
?assertEqual(3, length(?of_kind(ignore_retained_message_deliver, Trace))),
snabbkaffe:stop(),
emqx_banned:delete(Who),
{ok, #{}, [0]} = emqtt:unsubscribe(C1, <<"retained/+">>),
ok = emqtt:disconnect(C1).
%%--------------------------------------------------------------------
%% Helper functions
%%--------------------------------------------------------------------

26
apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf
View File

@ -218,7 +218,7 @@ Defaults to ${payload}. If variable ${payload} is not found from the selected re
of the rule, then the string "undefined" is used.
"""
zh: """
要重新发布的消息的有效负载。允许使用带有变量的模板，请参阅“republish_args”的描述。。
要重新发布的消息的有效负载。允许使用带有变量的模板，请参阅“republish_args”的描述。
默认为 ${payload}。 如果从所选结果中未找到变量 ${payload}，则使用字符串 "undefined"。
"""
}
@ -227,6 +227,30 @@ of the rule, then the string "undefined" is used.
zh: "消息负载"
}
}
republish_args_user_properties {
desc {
en: """
From which variable should the MQTT message's User-Property pairs be taken from.
The value must be a map.
You may configure it to <code>${pub_props.'User-Property'}</code> or
use <code>SELECT *,pub_props.'User-Property' as user_properties</code>
to forward the original user properties to the republished message.
You may also call <code>map_put</code> function like
<code>map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties</code>
to inject user properties.
NOTE: MQTT spec allows duplicated user property names, but EMQX Rule-Engine does not.
"""
zh: """
指定使用哪个变量来填充 MQTT 消息的 User-Property 列表。这个变量的值必须是一个 map 类型。
可以设置成 <code>${pub_props.'User-Property'}</code> 或者
使用 <code>SELECT *,pub_props.'User-Property' as user_properties</code> 来把源 MQTT 消息
的 User-Property 列表用于填充。
也可以使用 <code>map_put</code> 函数来添加新的 User-Property，
<code>map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties</code>
注意：MQTT 协议允许一个消息中出现多次同一个 property 名，但是 EMQX 的规则引擎不允许。
"""
}
}
rule_engine_ignore_sys_message {
desc {

81
apps/emqx_rule_engine/src/emqx_rule_actions.erl
View File

@ -37,6 +37,8 @@
-callback pre_process_action_args(FuncName :: atom(), action_fun_args()) -> action_fun_args().
-define(ORIGINAL_USER_PROPERTIES, original).
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
@ -57,7 +59,8 @@ pre_process_action_args(
topic := Topic,
qos := QoS,
retain := Retain,
payload := Payload
payload := Payload,
user_properties := UserProperties
} = Args
) ->
Args#{
@ -65,7 +68,8 @@ pre_process_action_args(
topic => emqx_plugin_libs_rule:preproc_tmpl(Topic),
qos => preproc_vars(QoS),
retain => preproc_vars(Retain),
payload => emqx_plugin_libs_rule:preproc_tmpl(Payload)
payload => emqx_plugin_libs_rule:preproc_tmpl(Payload),
user_properties => preproc_user_properties(UserProperties)
}
};
pre_process_action_args(_, Args) ->
@ -93,16 +97,16 @@ republish(
_Args
) ->
?SLOG(error, #{msg => "recursive_republish_detected", topic => Topic});
%% republish a PUBLISH message
republish(
Selected,
#{flags := Flags, metadata := #{rule_id := RuleId}},
#{metadata := #{rule_id := RuleId}} = Env,
#{
preprocessed_tmpl := #{
qos := QoSTks,
retain := RetainTks,
topic := TopicTks,
payload := PayloadTks
payload := PayloadTks,
user_properties := UserPropertiesTks
}
}
) ->
@ -110,27 +114,22 @@ republish(
Payload = format_msg(PayloadTks, Selected),
QoS = replace_simple_var(QoSTks, Selected, 0),
Retain = replace_simple_var(RetainTks, Selected, false),
?TRACE("RULE", "republish_message", #{topic => Topic, payload => Payload}),
safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload);
%% in case this is a "$events/" event
republish(
Selected,
#{metadata := #{rule_id := RuleId}},
#{
preprocessed_tmpl := #{
qos := QoSTks,
retain := RetainTks,
topic := TopicTks,
payload := PayloadTks
%% 'flags' is set for message re-publishes or message related
%% events such as message.acked and message.dropped
Flags0 = maps:get(flags, Env, #{}),
Flags = Flags0#{retain => Retain},
PubProps = format_pub_props(UserPropertiesTks, Selected, Env),
?TRACE(
"RULE",
"republish_message",
#{
flags => Flags,
topic => Topic,
payload => Payload,
pub_props => PubProps
}
}
) ->
Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected),
Payload = format_msg(PayloadTks, Selected),
QoS = replace_simple_var(QoSTks, Selected, 0),
Retain = replace_simple_var(RetainTks, Selected, false),
?TRACE("RULE", "republish_message_with_flags", #{topic => Topic, payload => Payload}),
safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload).
),
safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps).
%%--------------------------------------------------------------------
%% internal functions
@ -168,13 +167,16 @@ pre_process_args(Mod, Func, Args) ->
false -> Args
end.
safe_publish(RuleId, Topic, QoS, Flags, Payload) ->
safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) ->
Msg = #message{
id = emqx_guid:gen(),
qos = QoS,
from = RuleId,
flags = Flags,
headers = #{republish_by => RuleId},
headers = #{
republish_by => RuleId,
properties => emqx_misc:pub_props_to_packet(PubProps)
},
topic = Topic,
payload = Payload,
timestamp = erlang:system_time(millisecond)
@ -187,6 +189,19 @@ preproc_vars(Data) when is_binary(Data) ->
preproc_vars(Data) ->
Data.
preproc_user_properties(<<"${pub_props.'User-Property'}">>) ->
%% keep the original
%% avoid processing this special variable because
%% we do not want to force users to select the value
%% the value will be taken from Env.pub_props directly
?ORIGINAL_USER_PROPERTIES;
preproc_user_properties(<<"${", _/binary>> = V) ->
%% use a variable
emqx_plugin_libs_rule:preproc_tmpl(V);
preproc_user_properties(_) ->
%% invalid, discard
undefined.
replace_simple_var(Tokens, Data, Default) when is_list(Tokens) ->
[Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}),
case Var of
@ -201,3 +216,15 @@ format_msg([], Selected) ->
emqx_json:encode(Selected);
format_msg(Tokens, Selected) ->
emqx_plugin_libs_rule:proc_tmpl(Tokens, Selected).
format_pub_props(UserPropertiesTks, Selected, Env) ->
UserProperties =
case UserPropertiesTks of
?ORIGINAL_USER_PROPERTIES ->
maps:get('User-Property', maps:get(pub_props, Env, #{}), #{});
undefined ->
#{};
_ ->
replace_simple_var(UserPropertiesTks, Selected, #{})
end,
#{'User-Property' => UserProperties}.

71
apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
View File

@ -34,7 +34,7 @@
-export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]).
%% query callback
-export([query/4]).
-export([qs2ms/2, run_fuzzy_match/2, format_rule_resp/1]).
-define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))).
-define(ERR_BADARGS(REASON), begin
@ -274,10 +274,11 @@ param_path_id() ->
case
emqx_mgmt_api:node_query(
node(),
QueryString,
?RULE_TAB,
QueryString,
?RULE_QS_SCHEMA,
{?MODULE, query}
fun ?MODULE:qs2ms/2,
fun ?MODULE:format_rule_resp/1
)
of
{error, page_limit_invalid} ->
@ -552,38 +553,40 @@ filter_out_request_body(Conf) ->
],
maps:without(ExtraConfs, Conf).
query(Tab, {Qs, Fuzzy}, Start, Limit) ->
Ms = qs2ms(),
FuzzyFun = fuzzy_match_fun(Qs, Ms, Fuzzy),
emqx_mgmt_api:select_table_with_count(
Tab, {Ms, FuzzyFun}, Start, Limit, fun format_rule_resp/1
).
%% rule is not a record, so everything is fuzzy filter.
qs2ms() ->
[{'_', [], ['$_']}].
fuzzy_match_fun(Qs, Ms, Fuzzy) ->
MsC = ets:match_spec_compile(Ms),
fun(Rows) ->
Ls = ets:match_spec_run(Rows, MsC),
lists:filter(
fun(E) ->
run_qs_match(E, Qs) andalso
run_fuzzy_match(E, Fuzzy)
end,
Ls
)
-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
qs2ms(_Tab, {Qs, Fuzzy}) ->
case lists:keytake(from, 1, Qs) of
false ->
#{match_spec => generate_match_spec(Qs), fuzzy_fun => fuzzy_match_fun(Fuzzy)};
{value, {from, '=:=', From}, Ls} ->
#{
match_spec => generate_match_spec(Ls),
fuzzy_fun => fuzzy_match_fun([{from, '=:=', From} | Fuzzy])
}
end.
run_qs_match(_, []) ->
true;
run_qs_match(E = {_Id, #{enable := Enable}}, [{enable, '=:=', Pattern} | Qs]) ->
Enable =:= Pattern andalso run_qs_match(E, Qs);
run_qs_match(E = {_Id, #{from := From}}, [{from, '=:=', Pattern} | Qs]) ->
lists:member(Pattern, From) andalso run_qs_match(E, Qs);
run_qs_match(E, [_ | Qs]) ->
run_qs_match(E, Qs).
generate_match_spec(Qs) ->
{MtchHead, Conds} = generate_match_spec(Qs, 2, {#{}, []}),
[{{'_', MtchHead}, Conds, ['$_']}].
generate_match_spec([], _, {MtchHead, Conds}) ->
{MtchHead, lists:reverse(Conds)};
generate_match_spec([Qs | Rest], N, {MtchHead, Conds}) ->
Holder = binary_to_atom(iolist_to_binary(["$", integer_to_list(N)]), utf8),
NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(element(1, Qs), Holder)),
NConds = put_conds(Qs, Holder, Conds),
generate_match_spec(Rest, N + 1, {NMtchHead, NConds}).
put_conds({_, Op, V}, Holder, Conds) ->
[{Op, Holder, V} | Conds].
ms(enable, X) ->
#{enable => X}.
fuzzy_match_fun([]) ->
undefined;
fuzzy_match_fun(Fuzzy) ->
{fun ?MODULE:run_fuzzy_match/2, [Fuzzy]}.
run_fuzzy_match(_, []) ->
true;
@ -591,6 +594,8 @@ run_fuzzy_match(E = {Id, _}, [{id, like, Pattern} | Fuzzy]) ->
binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = {_Id, #{description := Desc}}, [{description, like, Pattern} | Fuzzy]) ->
binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = {_, #{from := Topics}}, [{from, '=:=', Pattern} | Fuzzy]) ->
lists:member(Pattern, Topics) /= false andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy]) ->
lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) andalso
run_fuzzy_match(E, Fuzzy);

9
apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
View File

@ -173,6 +173,15 @@ fields("republish_args") ->
default => <<"${payload}">>,
example => <<"${payload}">>
}
)},
{user_properties,
?HOCON(
binary(),
#{
desc => ?DESC("republish_args_user_properties"),
default => <<"${user_properties}">>,
example => <<"${pub_props.'User-Property'}">>
}
)}
].

2
apps/emqx_rule_engine/src/emqx_rule_events.erl
View File

@ -1060,7 +1060,7 @@ printable_maps(Headers) ->
(K, V, AccIn) ->
AccIn#{K => V}
end,
#{},
#{'User-Property' => #{}},
Headers
).

155
apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
View File

@ -61,11 +61,14 @@ groups() ->
t_sqlselect_0,
t_sqlselect_00,
t_sqlselect_001,
t_sqlselect_inject_props,
t_sqlselect_01,
t_sqlselect_02,
t_sqlselect_1,
t_sqlselect_2,
t_sqlselect_3,
t_sqlselect_message_publish_event_keep_original_props_1,
t_sqlselect_message_publish_event_keep_original_props_2,
t_sqlparse_event_1,
t_sqlparse_event_2,
t_sqlparse_event_3,
@ -1037,9 +1040,10 @@ t_sqlselect_001(_Config) ->
)
).
t_sqlselect_01(_Config) ->
t_sqlselect_inject_props(_Config) ->
SQL =
"SELECT json_decode(payload) as p, payload "
"SELECT json_decode(payload) as p, payload, "
"map_put('inject_key', 'inject_val', user_properties) as user_properties "
"FROM \"t3/#\", \"t1\" "
"WHERE p.x = 1",
Repub = republish_action(<<"t2">>),
@ -1050,34 +1054,64 @@ t_sqlselect_01(_Config) ->
actions => [Repub]
}
),
{ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
Props = user_properties(#{<<"inject_key">> => <<"inject_val">>}),
{ok, Client} = emqtt:start_link([{username, <<"emqx">>}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0),
ct:sleep(100),
emqtt:publish(Client, <<"t1">>, #{}, <<"{\"x\":1}">>, [{qos, 0}]),
receive
{publish, #{topic := T, payload := Payload, properties := Props2}} ->
?assertEqual(Props, Props2),
?assertEqual(<<"t2">>, T),
?assertEqual(<<"{\"x\":1}">>, Payload)
after 2000 ->
ct:fail(wait_for_t2)
end,
emqtt:stop(Client),
delete_rule(TopicRule1).
t_sqlselect_01(_Config) ->
SQL =
"SELECT json_decode(payload) as p, payload "
"FROM \"t3/#\", \"t1\" "
"WHERE p.x = 1",
Repub = republish_action(<<"t2">>, <<"${payload}">>, <<"${pub_props.'User-Property'}">>),
{ok, TopicRule1} = emqx_rule_engine:create_rule(
#{
sql => SQL,
id => ?TMP_RULEID,
actions => [Repub]
}
),
Props = user_properties(#{<<"mykey">> => <<"myval">>}),
{ok, Client} = emqtt:start_link([{username, <<"emqx">>}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
emqtt:publish(Client, <<"t1">>, Props, <<"{\"x\":1}">>, [{qos, 0}]),
receive
{publish, #{topic := T, payload := Payload}} ->
?assertEqual(<<"t2">>, T),
?assertEqual(<<"{\"x\":1}">>, Payload)
after 1000 ->
after 2000 ->
ct:fail(wait_for_t2)
end,
emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0),
emqtt:publish(Client, <<"t1">>, Props, <<"{\"x\":2}">>, [{qos, 0}]),
receive
{publish, #{topic := <<"t2">>, payload := _}} ->
ct:fail(unexpected_t2)
after 1000 ->
after 2000 ->
ok
end,
emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0),
emqtt:publish(Client, <<"t3/a">>, Props, <<"{\"x\":1}">>, [{qos, 0}]),
receive
{publish, #{topic := T3, payload := Payload3}} ->
{publish, #{topic := T3, payload := Payload3, properties := Props2}} ->
?assertEqual(Props, Props2),
?assertEqual(<<"t2">>, T3),
?assertEqual(<<"{\"x\":1}">>, Payload3)
after 1000 ->
ct:fail(wait_for_t2)
after 2000 ->
ct:fail(wait_for_t3)
end,
emqtt:stop(Client),
@ -1145,13 +1179,12 @@ t_sqlselect_1(_Config) ->
{ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
ct:sleep(200),
emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":2}">>, 0),
receive
{publish, #{topic := T, payload := Payload}} ->
?assertEqual(<<"t2">>, T),
?assertEqual(<<"{\"x\":1,\"y\":2}">>, Payload)
after 1000 ->
after 2000 ->
ct:fail(wait_for_t2)
end,
@ -1214,14 +1247,13 @@ t_sqlselect_3(_Config) ->
{ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
ct:sleep(200),
{ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]),
{ok, _} = emqtt:connect(Client1),
receive
{publish, #{topic := T, payload := Payload}} ->
?assertEqual(<<"t2">>, T),
?assertEqual(<<"clientid=c_emqx1">>, Payload)
after 1000 ->
after 2000 ->
ct:fail(wait_for_t2)
end,
@ -1236,6 +1268,82 @@ t_sqlselect_3(_Config) ->
emqtt:stop(Client),
delete_rule(TopicRule).
t_sqlselect_message_publish_event_keep_original_props_1(_Config) ->
%% republish the client.connected msg
Topic = <<"foo/bar/1">>,
SQL = <<
"SELECT clientid "
"FROM \"$events/message_dropped\" "
>>,
%"WHERE topic = \"", Topic/binary, "\"">>,
Repub = republish_action(
<<"t2">>,
<<"clientid=${clientid}">>,
<<"${pub_props.'User-Property'}">>
),
{ok, TopicRule} = emqx_rule_engine:create_rule(
#{
sql => SQL,
id => ?TMP_RULEID,
actions => [Repub]
}
),
{ok, Client1} = emqtt:start_link([{clientid, <<"sub-01">>}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(Client1),
{ok, _, _} = emqtt:subscribe(Client1, <<"t2">>, 1),
{ok, Client2} = emqtt:start_link([{clientid, <<"pub-02">>}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(Client2),
Props = user_properties(#{<<"mykey">> => <<"111111">>}),
emqtt:publish(Client2, Topic, Props, <<"{\"x\":1}">>, [{qos, 1}]),
receive
{publish, #{topic := T, payload := Payload, properties := Props1}} ->
?assertEqual(Props1, Props),
?assertEqual(<<"t2">>, T),
?assertEqual(<<"clientid=pub-02">>, Payload)
after 2000 ->
ct:fail(wait_for_t2)
end,
emqtt:stop(Client2),
emqtt:stop(Client1),
delete_rule(TopicRule).
t_sqlselect_message_publish_event_keep_original_props_2(_Config) ->
%% republish the client.connected msg
Topic = <<"foo/bar/1">>,
SQL = <<
"SELECT clientid, pub_props.'User-Property' as user_properties "
"FROM \"$events/message_dropped\" "
>>,
%"WHERE topic = \"", Topic/binary, "\"">>,
Repub = republish_action(<<"t2">>, <<"clientid=${clientid}">>),
{ok, TopicRule} = emqx_rule_engine:create_rule(
#{
sql => SQL,
id => ?TMP_RULEID,
actions => [Repub]
}
),
{ok, Client1} = emqtt:start_link([{clientid, <<"sub-01">>}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(Client1),
{ok, _, _} = emqtt:subscribe(Client1, <<"t2">>, 1),
{ok, Client2} = emqtt:start_link([{clientid, <<"pub-02">>}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(Client2),
Props = user_properties(#{<<"mykey">> => <<"222222222222">>}),
emqtt:publish(Client2, Topic, Props, <<"{\"x\":1}">>, [{qos, 1}]),
receive
{publish, #{topic := T, payload := Payload, properties := Props1}} ->
?assertEqual(Props1, Props),
?assertEqual(<<"t2">>, T),
?assertEqual(<<"clientid=pub-02">>, Payload)
after 2000 ->
ct:fail(wait_for_t2)
end,
emqtt:stop(Client2),
emqtt:stop(Client1),
delete_rule(TopicRule).
t_sqlparse_event_1(_Config) ->
Sql =
"select topic as tp "
@ -2581,10 +2689,20 @@ t_get_basic_usage_info_1(_Config) ->
republish_action(Topic) ->
republish_action(Topic, <<"${payload}">>).
republish_action(Topic, Payload) ->
republish_action(Topic, Payload, <<"${user_properties}">>).
republish_action(Topic, Payload, UserProperties) ->
#{
function => republish,
args => #{payload => Payload, topic => Topic, qos => 0, retain => false}
args => #{
payload => Payload,
topic => Topic,
qos => 0,
retain => false,
user_properties => UserProperties
}
}.
make_simple_rule_with_ts(RuleId, Ts) when is_binary(RuleId) ->
@ -2970,6 +3088,9 @@ verify_ipaddr(IPAddrS) ->
init_events_counters() ->
ets:new(events_record_tab, [named_table, bag, public]).
user_properties(PairsMap) ->
#{'User-Property' => maps:to_list(PairsMap)}.
%%------------------------------------------------------------------------------
%% Start Apps
%%------------------------------------------------------------------------------

12
apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl
View File

@ -133,23 +133,23 @@ t_list_rule_api(_Config) ->
QueryStr2 = #{query_string => #{<<"like_description">> => <<"也能"/utf8>>}},
{200, Result2} = emqx_rule_engine_api:'/rules'(get, QueryStr2),
?assertEqual(Result1, Result2),
?assertEqual(maps:get(data, Result1), maps:get(data, Result2)),
QueryStr3 = #{query_string => #{<<"from">> => <<"t/1">>}},
{200, #{meta := #{count := Count3}}} = emqx_rule_engine_api:'/rules'(get, QueryStr3),
?assertEqual(19, Count3),
{200, #{data := Data3}} = emqx_rule_engine_api:'/rules'(get, QueryStr3),
?assertEqual(19, length(Data3)),
QueryStr4 = #{query_string => #{<<"like_from">> => <<"t/1/+">>}},
{200, Result4} = emqx_rule_engine_api:'/rules'(get, QueryStr4),
?assertEqual(Result1, Result4),
?assertEqual(maps:get(data, Result1), maps:get(data, Result4)),
QueryStr5 = #{query_string => #{<<"match_from">> => <<"t/+/+">>}},
{200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5),
?assertEqual(Result1, Result5),
?assertEqual(maps:get(data, Result1), maps:get(data, Result5)),
QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}},
{200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6),
?assertEqual(Result1, Result6),
?assertEqual(maps:get(data, Result1), maps:get(data, Result6)),
%% clean up
lists:foreach(

6
apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf
View File

@ -45,6 +45,12 @@ emqx_statsd_schema {
zh: """指标的推送间隔。"""
}
}
tags {
desc {
en: """The tags for metrics."""
zh: """指标的标签。"""
}
}
enable {
desc {

5
apps/emqx_statsd/include/emqx_statsd.hrl
View File

@ -1,5 +1,2 @@
-define(APP, emqx_statsd).
-define(DEFAULT_SAMPLE_TIME_INTERVAL, 10000).
-define(DEFAULT_FLUSH_TIME_INTERVAL, 10000).
-define(DEFAULT_HOST, "127.0.0.1").
-define(DEFAULT_PORT, 8125).
-define(STATSD, [statsd]).

4
apps/emqx_statsd/src/emqx_statsd.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_statsd, [
{description, "An OTP application"},
{vsn, "5.0.2"},
{description, "EMQX Statsd"},
{vsn, "5.0.3"},
{registered, []},
{mod, {emqx_statsd_app, []}},
{applications, [

116
apps/emqx_statsd/src/emqx_statsd.erl
View File

@ -28,18 +28,17 @@
-include_lib("emqx/include/logger.hrl").
-export([
update/1,
start/0,
stop/0,
restart/0,
%% for rpc
%% for rpc: remove after 5.1.x
do_start/0,
do_stop/0,
do_restart/0
]).
%% Interface
-export([start_link/1]).
-export([start_link/0]).
%% Internal Exports
-export([
@ -51,40 +50,15 @@
terminate/2
]).
-record(state, {
timer :: reference() | undefined,
sample_time_interval :: pos_integer(),
flush_time_interval :: pos_integer(),
estatsd_pid :: pid()
}).
update(Config) ->
case
emqx_conf:update(
[statsd],
Config,
#{rawconf_with_defaults => true, override_to => cluster}
)
of
{ok, #{raw_config := NewConfigRows}} ->
ok = stop(),
case maps:get(<<"enable">>, Config, true) of
true ->
ok = restart();
false ->
ok = stop()
end,
{ok, NewConfigRows};
{error, Reason} ->
{error, Reason}
end.
-define(SAMPLE_TIMEOUT, sample_timeout).
%% Remove after 5.1.x
start() -> check_multicall_result(emqx_statsd_proto_v1:start(mria_mnesia:running_nodes())).
stop() -> check_multicall_result(emqx_statsd_proto_v1:stop(mria_mnesia:running_nodes())).
restart() -> check_multicall_result(emqx_statsd_proto_v1:restart(mria_mnesia:running_nodes())).
do_start() ->
emqx_statsd_sup:ensure_child_started(?APP, emqx_conf:get([statsd], #{})).
emqx_statsd_sup:ensure_child_started(?APP).
do_stop() ->
emqx_statsd_sup:ensure_child_stopped(?APP).
@ -94,59 +68,51 @@ do_restart() ->
ok = do_start(),
ok.
start_link(Opts) ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []).
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
init([Opts]) ->
init([]) ->
process_flag(trap_exit, true),
Tags = tags(maps:get(tags, Opts, #{})),
{Host, Port} = maps:get(server, Opts, {?DEFAULT_HOST, ?DEFAULT_PORT}),
Opts1 = maps:without(
[
sample_time_interval,
flush_time_interval
],
Opts#{
tags => Tags,
host => Host,
port => Port,
prefix => <<"emqx">>
}
),
{ok, Pid} = estatsd:start_link(maps:to_list(Opts1)),
SampleTimeInterval = maps:get(sample_time_interval, Opts, ?DEFAULT_FLUSH_TIME_INTERVAL),
FlushTimeInterval = maps:get(flush_time_interval, Opts, ?DEFAULT_FLUSH_TIME_INTERVAL),
#{
tags := TagsRaw,
server := {Host, Port},
sample_time_interval := SampleTimeInterval,
flush_time_interval := FlushTimeInterval
} = emqx_conf:get([statsd]),
Tags = maps:fold(fun(K, V, Acc) -> [{to_bin(K), to_bin(V)} | Acc] end, [], TagsRaw),
Opts = [{tags, Tags}, {host, Host}, {port, Port}, {prefix, <<"emqx">>}],
{ok, Pid} = estatsd:start_link(Opts),
{ok,
ensure_timer(#state{
sample_time_interval = SampleTimeInterval,
flush_time_interval = FlushTimeInterval,
estatsd_pid = Pid
ensure_timer(#{
sample_time_interval => SampleTimeInterval,
flush_time_interval => FlushTimeInterval,
estatsd_pid => Pid
})}.
handle_call(_Req, _From, State) ->
{noreply, State}.
{reply, ignore, State}.
handle_cast(_Msg, State) ->
{noreply, State}.
handle_info(
{timeout, Ref, sample_timeout},
State = #state{
sample_time_interval = SampleTimeInterval,
flush_time_interval = FlushTimeInterval,
estatsd_pid = Pid,
timer = Ref
{timeout, Ref, ?SAMPLE_TIMEOUT},
State = #{
sample_time_interval := SampleTimeInterval,
flush_time_interval := FlushTimeInterval,
estatsd_pid := Pid,
timer := Ref
}
) ->
Metrics = emqx_metrics:all() ++ emqx_stats:getstats() ++ emqx_vm_data(),
SampleRate = SampleTimeInterval / FlushTimeInterval,
StatsdMetrics = [
{gauge, trans_metrics_name(Name), Value, SampleRate, []}
{gauge, Name, Value, SampleRate, []}
|| {Name, Value} <- Metrics
],
estatsd:submit(Pid, StatsdMetrics),
{noreply, ensure_timer(State)};
handle_info({'EXIT', Pid, Error}, State = #state{estatsd_pid = Pid}) ->
ok = estatsd:submit(Pid, StatsdMetrics),
{noreply, ensure_timer(State), hibernate};
handle_info({'EXIT', Pid, Error}, State = #{estatsd_pid := Pid}) ->
{stop, {shutdown, Error}, State};
handle_info(_Msg, State) ->
{noreply, State}.
@ -154,16 +120,13 @@ handle_info(_Msg, State) ->
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
terminate(_Reason, #state{estatsd_pid = Pid}) ->
terminate(_Reason, #{estatsd_pid := Pid}) ->
estatsd:stop(Pid),
ok.
%%------------------------------------------------------------------------------
%% Internal function
%%------------------------------------------------------------------------------
trans_metrics_name(Name) ->
Name0 = atom_to_binary(Name, utf8),
binary_to_atom(<<"emqx.", Name0/binary>>, utf8).
emqx_vm_data() ->
Idle =
@ -179,12 +142,8 @@ emqx_vm_data() ->
{cpu_use, 100 - Idle}
] ++ emqx_vm:mem_info().
tags(Map) ->
Tags = maps:to_list(Map),
[{atom_to_binary(Key, utf8), Value} || {Key, Value} <- Tags].
ensure_timer(State = #state{sample_time_interval = SampleTimeInterval}) ->
State#state{timer = emqx_misc:start_timer(SampleTimeInterval, sample_timeout)}.
ensure_timer(State = #{sample_time_interval := SampleTimeInterval}) ->
State#{timer => emqx_misc:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}.
check_multicall_result({Results, []}) ->
case
@ -201,3 +160,8 @@ check_multicall_result({Results, []}) ->
end;
check_multicall_result({_, _}) ->
error(multicall_failed).
to_bin(B) when is_binary(B) -> B;
to_bin(I) when is_integer(I) -> integer_to_binary(I);
to_bin(L) when is_list(L) -> list_to_binary(L);
to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8).

9
apps/emqx_statsd/src/emqx_statsd_api.erl
View File

@ -77,15 +77,16 @@ statsd_config_schema() ->
statsd_example() ->
#{
enable => true,
flush_time_interval => "32s",
sample_time_interval => "32s",
server => "127.0.0.1:8125"
flush_time_interval => "30s",
sample_time_interval => "30s",
server => "127.0.0.1:8125",
tags => #{}
}.
statsd(get, _Params) ->
{200, emqx:get_raw_config([<<"statsd">>], #{})};
statsd(put, #{body := Body}) ->
case emqx_statsd:update(Body) of
case emqx_statsd_config:update(Body) of
{ok, NewConfig} ->
{200, NewConfig};
{error, Reason} ->

11
apps/emqx_statsd/src/emqx_statsd_app.erl
View File

@ -27,15 +27,8 @@
start(_StartType, _StartArgs) ->
{ok, Sup} = emqx_statsd_sup:start_link(),
maybe_enable_statsd(),
emqx_statsd_config:add_handler(),
{ok, Sup}.
stop(_) ->
emqx_statsd_config:remove_handler(),
ok.
maybe_enable_statsd() ->
case emqx_conf:get([statsd, enable], false) of
true ->
emqx_statsd_sup:ensure_child_started(?APP, emqx_conf:get([statsd], #{}));
false ->
ok
end.

54
apps/emqx_statsd/src/emqx_statsd_config.erl Normal file
View File

@ -0,0 +1,54 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_statsd_config).
-behaviour(emqx_config_handler).
-include("emqx_statsd.hrl").
-export([add_handler/0, remove_handler/0]).
-export([post_config_update/5]).
-export([update/1]).
update(Config) ->
case
emqx_conf:update(
?STATSD,
Config,
#{rawconf_with_defaults => true, override_to => cluster}
)
of
{ok, #{raw_config := NewConfigRows}} ->
{ok, NewConfigRows};
{error, Reason} ->
{error, Reason}
end.
add_handler() ->
ok = emqx_config_handler:add_handler(?STATSD, ?MODULE),
ok.
remove_handler() ->
ok = emqx_config_handler:remove_handler(?STATSD),
ok.
post_config_update(?STATSD, _Req, #{enable := true}, _Old, _AppEnvs) ->
emqx_statsd_sup:ensure_child_stopped(?APP),
emqx_statsd_sup:ensure_child_started(?APP);
post_config_update(?STATSD, _Req, #{enable := false}, _Old, _AppEnvs) ->
emqx_statsd_sup:ensure_child_stopped(?APP);
post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) ->
ok.

35
apps/emqx_statsd/src/emqx_statsd_schema.erl
View File

@ -25,7 +25,8 @@
namespace/0,
roots/0,
fields/1,
desc/1
desc/1,
validations/0
]).
namespace() -> "statsd".
@ -45,7 +46,8 @@ fields("statsd") ->
)},
{server, fun server/1},
{sample_time_interval, fun sample_interval/1},
{flush_time_interval, fun flush_interval/1}
{flush_time_interval, fun flush_interval/1},
{tags, fun tags/1}
].
desc("statsd") -> ?DESC(statsd);
@ -59,12 +61,37 @@ server(_) -> undefined.
sample_interval(type) -> emqx_schema:duration_ms();
sample_interval(required) -> true;
sample_interval(default) -> "10s";
sample_interval(default) -> "30s";
sample_interval(desc) -> ?DESC(?FUNCTION_NAME);
sample_interval(_) -> undefined.
flush_interval(type) -> emqx_schema:duration_ms();
flush_interval(required) -> true;
flush_interval(default) -> "10s";
flush_interval(default) -> "30s";
flush_interval(desc) -> ?DESC(?FUNCTION_NAME);
flush_interval(_) -> undefined.
tags(type) -> map();
tags(required) -> false;
tags(default) -> #{};
tags(desc) -> ?DESC(?FUNCTION_NAME);
tags(_) -> undefined.
validations() ->
[
{check_interval, fun check_interval/1}
].
check_interval(Conf) ->
case hocon_maps:get("statsd.sample_time_interval", Conf) of
undefined ->
ok;
Sample ->
Flush = hocon_maps:get("statsd.flush_time_interval", Conf),
case Sample =< Flush of
true ->
true;
false ->
{bad_interval, #{sample_time_interval => Sample, flush_time_interval => Flush}}
end
end.

21
apps/emqx_statsd/src/emqx_statsd_sup.erl
View File

@ -10,7 +10,6 @@
-export([
start_link/0,
ensure_child_started/1,
ensure_child_started/2,
ensure_child_stopped/1
]).
@ -19,7 +18,7 @@
%% Helper macro for declaring children of supervisor
-define(CHILD(Mod, Opts), #{
id => Mod,
start => {Mod, start_link, [Opts]},
start => {Mod, start_link, Opts},
restart => permanent,
shutdown => 5000,
type => worker,
@ -29,13 +28,9 @@
start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-spec ensure_child_started(supervisor:child_spec()) -> ok.
ensure_child_started(ChildSpec) when is_map(ChildSpec) ->
assert_started(supervisor:start_child(?MODULE, ChildSpec)).
-spec ensure_child_started(atom(), map()) -> ok.
ensure_child_started(Mod, Opts) when is_atom(Mod) andalso is_map(Opts) ->
assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Opts))).
-spec ensure_child_started(atom()) -> ok.
ensure_child_started(Mod) when is_atom(Mod) ->
assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, []))).
%% @doc Stop the child worker process.
-spec ensure_child_stopped(any()) -> ok.
@ -50,13 +45,17 @@ ensure_child_stopped(ChildId) ->
end.
init([]) ->
{ok, {{one_for_one, 10, 3600}, []}}.
Children =
case emqx_conf:get([statsd, enable], false) of
true -> [?CHILD(emqx_statsd, [])];
false -> []
end,
{ok, {{one_for_one, 100, 3600}, Children}}.
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
assert_started({ok, _Pid}) -> ok;
assert_started({ok, _Pid, _Info}) -> ok;
assert_started({error, {already_started, _Pid}}) -> ok;
assert_started({error, Reason}) -> erlang:error(Reason).

90
apps/emqx_statsd/test/emqx_statsd_SUITE.erl
View File

@ -5,28 +5,104 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("eunit/include/eunit.hrl").
-import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
-define(BASE_CONF, <<
"\n"
"statsd {\n"
"enable = true\n"
"flush_time_interval = 4s\n"
"sample_time_interval = 4s\n"
"server = \"127.0.0.1:8126\"\n"
"tags {\"t1\" = \"good\", test = 100}\n"
"}\n"
>>).
init_per_suite(Config) ->
emqx_common_test_helpers:start_apps([emqx_statsd]),
emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_dashboard, emqx_statsd],
fun set_special_configs/1
),
ok = emqx_common_test_helpers:load_config(emqx_statsd_schema, ?BASE_CONF, #{
raw_with_default => true
}),
Config.
end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([emqx_statsd]).
emqx_common_test_helpers:stop_apps([emqx_statsd, emqx_dashboard, emqx_conf]).
set_special_configs(emqx_dashboard) ->
emqx_dashboard_api_test_helpers:set_default_config();
set_special_configs(_) ->
ok.
all() ->
emqx_common_test_helpers:all(?MODULE).
t_statsd(_) ->
{ok, Socket} = gen_udp:open(8125),
{ok, Socket} = gen_udp:open(8126, [{active, true}]),
receive
{udp, _Socket, _Host, _Port, Bin} ->
?assert(length(Bin) > 50)
after 11 * 1000 ->
?assert(true, failed)
{udp, Socket1, Host, Port, Data} ->
ct:pal("receive:~p~n", [{Socket, Socket1, Host, Port}]),
?assert(length(Data) > 50),
?assert(nomatch =/= string:find(Data, "\nemqx.cpu_use:"))
after 10 * 1000 ->
error(timeout)
end,
gen_udp:close(Socket).
t_management(_) ->
?assertMatch(ok, emqx_statsd:start()),
?assertMatch(ok, emqx_statsd:start()),
?assertMatch(ok, emqx_statsd:stop()),
?assertMatch(ok, emqx_statsd:stop()),
?assertMatch(ok, emqx_statsd:restart()).
t_rest_http(_) ->
{ok, Res0} = request(get),
?assertEqual(
#{
<<"enable">> => true,
<<"flush_time_interval">> => <<"4s">>,
<<"sample_time_interval">> => <<"4s">>,
<<"server">> => <<"127.0.0.1:8126">>,
<<"tags">> => #{<<"t1">> => <<"good">>, <<"test">> => 100}
},
Res0
),
{ok, Res1} = request(put, #{enable => false}),
?assertMatch(#{<<"enable">> := false}, Res1),
?assertEqual(maps:remove(<<"enable">>, Res0), maps:remove(<<"enable">>, Res1)),
{ok, Res2} = request(get),
?assertEqual(Res1, Res2),
?assertEqual(
error, request(put, #{sample_time_interval => "11s", flush_time_interval => "10s"})
),
{ok, _} = request(put, #{enable => true}),
ok.
t_kill_exit(_) ->
{ok, _} = request(put, #{enable => true}),
Pid = erlang:whereis(emqx_statsd),
?assertEqual(ignore, gen_server:call(Pid, whatever)),
?assertEqual(ok, gen_server:cast(Pid, whatever)),
?assertEqual(Pid, erlang:whereis(emqx_statsd)),
#{estatsd_pid := Estatsd} = sys:get_state(emqx_statsd),
?assert(erlang:exit(Estatsd, kill)),
?assertEqual(false, is_process_alive(Estatsd)),
ct:sleep(150),
Pid1 = erlang:whereis(emqx_statsd),
?assertNotEqual(Pid, Pid1),
#{estatsd_pid := Estatsd1} = sys:get_state(emqx_statsd),
?assertNotEqual(Estatsd, Estatsd1),
ok.
request(Method) -> request(Method, []).
request(Method, Body) ->
case request(Method, uri(["statsd"]), Body) of
{ok, 200, Res} ->
{ok, emqx_json:decode(Res, [return_maps])};
{ok, _Status, _} ->
error
end.

2
bin/emqx
View File

@ -396,7 +396,7 @@ remsh() {
# Generate a random id
relx_gen_id() {
od -t x -N 4 /dev/urandom | head -n1 | awk '{print $2}'
od -t u -N 4 /dev/urandom | head -n1 | awk '{print $2 % 1000}'
}
call_nodetool() {

9
bin/nodetool
View File

@ -226,9 +226,14 @@ nodename(Name) ->
this_node_name(longnames, Name) ->
[Node, Host] = re:split(Name, "@", [{return, list}, unicode]),
list_to_atom(lists:concat(["remsh_maint_", Node, os:getpid(), "@", Host]));
list_to_atom(lists:concat(["remsh_maint_", Node, node_name_suffix_id(), "@", Host]));
this_node_name(shortnames, Name) ->
list_to_atom(lists:concat(["remsh_maint_", Name, os:getpid()])).
list_to_atom(lists:concat(["remsh_maint_", Name, node_name_suffix_id()])).
%% use the reversed value that from pid mod 1000 as the node name suffix
node_name_suffix_id() ->
Pid = os:getpid(),
string:slice(string:reverse(Pid), 0, 3).
%% For windows???
create_mnesia_dir(DataDir, NodeName) ->

1
changes/v5.0.10-en.md
View File

@ -50,4 +50,3 @@
- Fix query string parameter 'node' to `/configs` resource being ignored, return 404 if node does not exist [#9310](https://github.com/emqx/emqx/pull/9310/).
- Avoid re-dispatching shared-subscription session messages when a session is kicked or taken-over (to a new session) [#9123](https://github.com/emqx/emqx/pull/9123).

40
changes/v5.0.11-en.md
View File

@ -2,5 +2,45 @@
## Enhancements
- Security enhancement for retained messages [#9326](https://github.com/emqx/emqx/pull/9326).
The retained messages will not be published if the publisher client is banned.
- Security enhancement for the `subscribe` API [#9355](https://github.com/emqx/emqx/pull/9355).
- Enhance the `banned` feature [#9367](https://github.com/emqx/emqx/pull/9367).
Now the corresponding session will be kicked when client is banned by `clientid`.
- Redesign `/gateways` API [9364](https://github.com/emqx/emqx/pull/9364).
Use `PUT /gateways/{name}` instead of `POST /gateways`, gateway gets 'loaded'
automatically if needed. Use `PUT /gateways/{name}/enable/{true|false}` to
enable or disable gateway. No more `DELETE /gateways/{name}`.
- Support `statsd {tags: {"user-defined-tag" = "tag-value"}` configure and improve stability of `emqx_statsd` [#9363](http://github.com/emqx/emqx/pull/9363).
- Improve node name generation rules to avoid potential atom table overflow risk [#9387](https://github.com/emqx/emqx/pull/9387).
- Set the default value for the maximum level of a topic to 128 [#9406](https://github.com/emqx/emqx/pull/9406).
- Keep MQTT v5 User-Property pairs from bridge ingested MQTT messsages to bridge target [#9398](https://github.com/emqx/emqx/pull/9398).
- Add a new config `quick_deny_anonymous` to allow quick deny of anonymous clients (without username) so the auth backend checks can be skipped [#8516](https://github.com/emqx/emqx/pull/8516).
- Support message properties in `/publish` API [#9401](https://github.com/emqx/emqx/pull/9401).
- Optimize client query performance for HTTP APIs [#9374](https://github.com/emqx/emqx/pull/9374).
## Bug fixes
- Fix `ssl.existingName` option of helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307).
- Fix create trace sometime failed by end_at time has already passed. [#9303](https://github.com/emqx/emqx/pull/9303)
- Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
- Fix that JWT ACL rules are only applied if an `exp` claim is set [#9368](https://github.com/emqx/emqx/pull/9368).
- Fix that `/configs/global_zone` API cannot get the default value of the configuration [#9392](https://github.com/emqx/emqx/pull/9392).
- Fix mountpoint not working for will-msg [#9399](https://github.com/emqx/emqx/pull/9399).
- Fix that the obsolete SSL files aren't deleted after the bridge configuration update [#9411](https://github.com/emqx/emqx/pull/9411).

39
changes/v5.0.11-zh.md
View File

@ -2,4 +2,43 @@
## 增强
- 增强 `保留消息` 的安全性 [#9332](https://github.com/emqx/emqx/pull/9332)。
现在投递保留消息前，会先过滤掉来源客户端被封禁了的那些消息。
- 增强订阅 API 的安全性 [#9355](https://github.com/emqx/emqx/pull/9355)。
- 增加 `封禁` 功能 [#9367](https://github.com/emqx/emqx/pull/9367)。
现在客户端通过 `clientid` 被封禁时将会踢掉对应的会话。
- 重新设计了 /gateways API [9364](https://github.com/emqx/emqx/pull/9364)。
使用 PUT /gateways/{name} 代替了 POST /gateways，现在网关将在需要时自动加载，然后删除了 DELETE /gateways/{name}，之后可以使用 PUT /gateways/{name}/enable/{true|false} 来开启或禁用网关。
- 支持 `statsd {tags: {"user-defined-tag" = "tag-value"}` 配置，并提升 `emqx_statsd` 的稳定性 [#9363](http://github.com/emqx/emqx/pull/9363)。
- 改进了节点名称生成规则，以避免潜在的原子表溢出风险 [#9387](https://github.com/emqx/emqx/pull/9387)。
- 将主题的最大层级限制的默认值设置为128 [#9406](https://github.com/emqx/emqx/pull/9406)。
- 为桥接收到的 MQTT v5 消息再转发时保留 User-Property 列表 [#9398](https://github.com/emqx/emqx/pull/9398)。
- 添加了一个名为 `quick_deny_anonymous` 的新配置，用来在不调用认证链的情况下，快速的拒绝掉匿名用户，从而提高认证效率 [#8516](https://github.com/emqx/emqx/pull/8516)。
- 支持在 /publish API 中添加消息属性 [#9401](https://github.com/emqx/emqx/pull/9401)。
- 优化查询客户端列表的 HTTP API 性能 [#9374](https://github.com/emqx/emqx/pull/9374)。
## 修复
- 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。
- 修复创建追踪日志时偶尔会报`end_at time has already passed`错误，导致创建失败。[#9303](https://github.com/emqx/emqx/pull/9303)
- 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。
- 修复 JWT ACL 规则只在设置了超期时间时才生效的问题 [#9368](https://github.com/emqx/emqx/pull/9368)。
- 修复 `/configs/global_zone` API 无法正确获取配置的默认值问题 [#9392](https://github.com/emqx/emqx/pull/9392)。
- 修复 mountpoint 配置未对遗嘱消息生效的问题 [#9399](https://github.com/emqx/emqx/pull/9399)
- 修复桥接配置更新 SSL 相关配置后，过时的 SSL 文件没有被删除的问题 [#9411](https://github.com/emqx/emqx/pull/9411)。

2
deploy/charts/emqx-enterprise/templates/StatefulSet.yaml
View File

@ -56,7 +56,7 @@ spec:
{{- if .Values.ssl.enabled }}
- name: ssl-cert
secret:
secretName: {{ include "emqx.fullname" . }}-tls
secretName: {{ include "emqx.ssl.secretName" . }}
{{- end }}
{{- if not .Values.persistence.enabled }}
- name: emqx-data

12
deploy/charts/emqx-enterprise/templates/_helpers.tpl
View File

@ -30,3 +30,15 @@ Create chart name and version as used by the chart label.
{{- define "emqx.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Get ssl secret name .
*/}}
{{- define "emqx.ssl.secretName" -}}
{{- if and .Values.ssl.useExisting .Values.ssl.existingName -}}
{{ .Values.ssl.existingName }}
{{- else -}}
{{ include "emqx.fullname" . }}-tls
{{- end -}}
{{- end -}}

4
deploy/charts/emqx/Chart.yaml
View File

@ -14,8 +14,8 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
version: 5.0.10
version: 5.0.11
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application.
appVersion: 5.0.10
appVersion: 5.0.11

2
deploy/charts/emqx/templates/StatefulSet.yaml
View File

@ -56,7 +56,7 @@ spec:
{{- if .Values.ssl.enabled }}
- name: ssl-cert
secret:
secretName: {{ include "emqx.fullname" . }}-tls
secretName: {{ include "emqx.ssl.secretName" . }}
{{- end }}
{{- if not .Values.persistence.enabled }}
- name: emqx-data

12
deploy/charts/emqx/templates/_helpers.tpl
View File

@ -30,3 +30,15 @@ Create chart name and version as used by the chart label.
{{- define "emqx.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Get ssl secret name .
*/}}
{{- define "emqx.ssl.secretName" -}}
{{- if and .Values.ssl.useExisting .Values.ssl.existingName -}}
{{ .Values.ssl.existingName }}
{{- else -}}
{{ include "emqx.fullname" . }}-tls
{{- end -}}
{{- end -}}

37
scripts/macos-sign-binaries.sh
View File

@ -42,14 +42,29 @@ for keychain in ${keychains}; do
done
security -v list-keychains -s "${keychain_names[@]}" "${KEYCHAIN}"
# sign
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/asn1-*/priv/lib/asn1rt_nif.so
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/bcrypt-*/priv/bcrypt_nif.so
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/crypto-*/priv/lib/{crypto.so,otp_test_engine.so}
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/jiffy-*/priv/jiffy.so
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/jq-*/priv/{jq_nif1.so,libjq.1.dylib,libonig.4.dylib,erlang_jq_port}
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/rocksdb-*/priv/liberocksdb.so
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
find "${REL_DIR}/lib/" -name libquicer_nif.so -exec codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime {} \;
# known runtime executables and binaries
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
"${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
"${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
"${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
"${REL_DIR}"/lib/jq-*/priv/{jq_nif1.so,libjq.1.dylib,libonig.4.dylib,erlang_jq_port}
# other files from runtime and dependencies
for f in \
asn1rt_nif.so \
bcrypt_nif.so \
crc32cer_nif.so \
crypto.so \
crypto_callback.so \
jiffy.so \
liberocksdb.so \
libquicer_nif.so \
odbcserver \
otp_test_engine.so \
sasl_auth.so \
snappyer.so \
; do
find "${REL_DIR}"/lib/ -name "$f" -exec codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime {} \;
done