yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #8010 from terry-xiaoyu/copy_of_main-v4.3 

Copy of main v4.3
This commit is contained in:
Xinyu Liu 2022-05-23 13:22:10 +08:00 committed by GitHub
parent b1d642bd6a 98a06f25b0
commit 6c3059f891
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
86 changed files with 1403 additions and 606 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
CHANGES-4.3.md
View File

@ -18,11 +18,24 @@ File format:
* Add support for JWT authorization [#7596]
Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists.
* Better randomisation of app screts (changed from timestamp seeded sha hash (uuid) to crypto:strong_rand_bytes)
* Return a client_identifier_not_valid error when username is empty and username_as_clientid is set to true [#7862]
* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/4 [#7894]
* Add proto_name and proto_ver fields for $event/client_disconnected event.
* Mnesia auth/acl http api support multiple condition queries.
* Inflight QoS1 Messages for shared topics are now redispatched to another alive subscribers upon chosen subscriber session termination.
* Make auth metrics name more understandable.
### Bug fixes
* List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions.
* SSL closed error bug fixed for redis client.
* Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message
* Rule-engine function hexstr2bin/1 support half byte [#7977]
* Add rule-engine function float2str/2, user can specify the float output precision [#7991]
* Improved resilience against autocluster partitioning during cluster
startup. [#7876]
[ekka-158](https://github.com/emqx/ekka/pull/158)
* Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979]
## v4.3.14

13
apps/emqx_auth_http/include/emqx_auth_http.hrl
View File

@ -1,14 +1 @@
-define(APP, emqx_auth_http).
-record(auth_metrics, {
success = 'client.auth.success',
failure = 'client.auth.failure',
ignore = 'client.auth.ignore'
}).
-define(METRICS(Type), tl(tuple_to_list(#Type{}))).
-define(METRICS(Type, K), #Type{}#Type.K).
-define(AUTH_METRICS, ?METRICS(auth_metrics)).
-define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).

2
apps/emqx_auth_http/src/emqx_auth_http.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_auth_http,
[{description, "EMQ X Authentication/ACL with HTTP API"},
{vsn, "4.3.5"}, % strict semver, bump manually!
{vsn, "4.3.6"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_auth_http_sup]},
{applications, [kernel,stdlib,ehttpc]},

21
apps/emqx_auth_http/src/emqx_auth_http.appup.src
View File

@ -1,27 +1,38 @@
%% -*- mode: erlang -*-
{VSN,
[{"4.3.4",
[{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}
]},
[{"4.3.5",
[{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
{"4.3.4",
[{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
{"4.3.3",
[{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
{"4.3.2",
[{apply,{application,stop,[emqx_auth_http]}},
{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]},
{<<"4.3.[0-1]">>,
[{restart_application,emqx_auth_http}]},
{<<".*">>,[]}],
[{"4.3.4",
[{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}]},
[{"4.3.5",
[{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
{"4.3.4",
[{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
{"4.3.3",
[{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
{"4.3.2",
[{apply,{application,stop,[emqx_auth_http]}},
{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]},
{<<"4.3.[0-1]">>,

12
apps/emqx_auth_http/src/emqx_auth_http.erl
View File

@ -30,22 +30,16 @@
]).
%% Callbacks
-export([ register_metrics/0
, check/3
-export([ check/3
, description/0
]).
-spec(register_metrics() -> ok).
register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
check(ClientInfo, AuthResult, #{auth := AuthParms = #{path := Path},
super := SuperParams}) ->
case authenticate(AuthParms, ClientInfo) of
{ok, 200, <<"ignore">>} ->
emqx_metrics:inc(?AUTH_METRICS(ignore)), ok;
ok;
{ok, 200, Body} ->
emqx_metrics:inc(?AUTH_METRICS(success)),
IsSuperuser = is_superuser(SuperParams, ClientInfo),
{stop, AuthResult#{is_superuser => IsSuperuser,
auth_result => success,
@ -54,12 +48,10 @@ check(ClientInfo, AuthResult, #{auth := AuthParms = #{path := Path},
{ok, Code, _Body} ->
?LOG(error, "Deny connection from path: ~s, response http code: ~p",
[Path, Code]),
emqx_metrics:inc(?AUTH_METRICS(failure)),
{stop, AuthResult#{auth_result => http_to_connack_error(Code),
anonymous => false}};
{error, Error} ->
?LOG(error, "Request auth path: ~s, error: ~p", [Path, Error]),
emqx_metrics:inc(?AUTH_METRICS(failure)),
%%FIXME later: server_unavailable is not right.
{stop, AuthResult#{auth_result => server_unavailable,
anonymous => false}}

2
apps/emqx_auth_http/src/emqx_auth_http_app.erl
View File

@ -112,7 +112,6 @@ load_hooks() ->
case application:get_env(?APP, auth_req) of
undefined -> ok;
{ok, AuthReq} ->
ok = emqx_auth_http:register_metrics(),
PoolOpts = proplists:get_value(pool_opts, AuthReq),
PoolName = proplists:get_value(pool_name, AuthReq),
{ok, _} = ehttpc_sup:start_pool(PoolName, PoolOpts),
@ -160,4 +159,3 @@ path(#{path := ""}) ->
"/";
path(#{path := Path}) ->
Path.

57
apps/emqx_auth_jwt/src/emqx_auth_jwt.erl
View File

@ -21,28 +21,11 @@
-logger_header("[JWT]").
-export([ register_metrics/0
, check_auth/3
-export([ check_auth/3
, check_acl/5
, description/0
]).
-record(auth_metrics, {
success = 'client.auth.success',
failure = 'client.auth.failure',
ignore = 'client.auth.ignore'
}).
-define(METRICS(Type), tl(tuple_to_list(#Type{}))).
-define(METRICS(Type, K), #Type{}#Type.K).
-define(AUTH_METRICS, ?METRICS(auth_metrics)).
-define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-spec(register_metrics() -> ok).
register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
%%--------------------------------------------------------------------
%% Authentication callbacks
%%--------------------------------------------------------------------
@ -50,17 +33,16 @@ register_metrics() ->
check_auth(ClientInfo, AuthResult, #{from := From, checklists := Checklists}) ->
case maps:find(From, ClientInfo) of
error ->
ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
ok;
{ok, undefined} ->
ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
ok;
{ok, Token} ->
case emqx_auth_jwt_svr:verify(Token) of
{error, not_found} ->
ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
ok;
{error, not_token} ->
ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
ok;
{error, Reason} ->
ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
{stop, AuthResult#{auth_result => Reason, anonymous => false}};
{ok, Claims} ->
{stop, maps:merge(AuthResult, verify_claims(Checklists, Claims, ClientInfo))}
@ -72,14 +54,33 @@ check_acl(ClientInfo = #{jwt_claims := Claims},
Topic,
_NoMatchAction,
#{acl_claim_name := AclClaimName}) ->
Deadline = erlang:system_time(second),
case Claims of
#{AclClaimName := Acl, <<"exp">> := Exp}
when is_integer(Exp) andalso Exp >= Deadline ->
#{AclClaimName := Acl, <<"exp">> := Exp} ->
try is_expired(Exp) of
true ->
?DEBUG("acl_deny_due_to_jwt_expired", []),
deny;
false ->
verify_acl(ClientInfo, Acl, PubSub, Topic)
catch
_:_ ->
?DEBUG("acl_deny_due_to_invalid_jwt_exp", []),
deny
end;
#{AclClaimName := Acl} ->
verify_acl(ClientInfo, Acl, PubSub, Topic);
_ -> ignore
_ ->
?DEBUG("no_acl_jwt_claim", []),
ignore
end.
is_expired(Exp) when is_binary(Exp) ->
ExpInt = binary_to_integer(Exp),
is_expired(ExpInt);
is_expired(Exp) ->
Now = erlang:system_time(second),
Now > Exp.
description() -> "Authentication with JWT".
%%------------------------------------------------------------------------------
@ -102,10 +103,8 @@ verify_acl(ClientInfo, [AclTopic | AclTopics], Topic) ->
verify_claims(Checklists, Claims, ClientInfo) ->
case do_verify_claims(feedvar(Checklists, ClientInfo), Claims) of
{error, Reason} ->
ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
#{auth_result => Reason, anonymous => false};
ok ->
ok = emqx_metrics:inc(?AUTH_METRICS(success)),
#{auth_result => success, anonymous => false, jwt_claims => Claims}
end.

1
apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl
View File

@ -32,7 +32,6 @@ start(_Type, _Args) ->
{ok, Sup} = supervisor:start_link({local, ?MODULE}, ?MODULE, []),
{ok, _} = start_auth_server(jwks_svr_options()),
ok = emqx_auth_jwt:register_metrics(),
AuthEnv = auth_env(),
_ = emqx:hook('client.authenticate', {emqx_auth_jwt, check_auth, [AuthEnv]}),

32
apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl
View File

@ -203,18 +203,32 @@ do_verify(JwsCompacted, [Jwk|More]) ->
check_claims(Claims) ->
Now = os:system_time(seconds),
Checker = [{<<"exp">>, fun(ExpireTime) ->
Now < ExpireTime
end},
{<<"iat">>, fun(IssueAt) ->
IssueAt =< Now
end},
{<<"nbf">>, fun(NotBefore) ->
NotBefore =< Now
end}
Checker = [{<<"exp">>, with_int_value(
fun(ExpireTime) -> Now < ExpireTime end)},
{<<"iat">>, with_int_value(
fun(IssueAt) -> IssueAt =< Now end)},
{<<"nbf">>, with_int_value(
fun(NotBefore) -> NotBefore =< Now end)}
],
do_check_claim(Checker, Claims).
with_int_value(Fun) ->
fun(Value) ->
case Value of
Int when is_integer(Int) -> Fun(Int);
Bin when is_binary(Bin) ->
case string:to_integer(Bin) of
{Int, <<>>} -> Fun(Int);
_ -> false
end;
Str when is_list(Str) ->
case string:to_integer(Str) of
{Int, ""} -> Fun(Int);
_ -> false
end
end
end.
do_check_claim([], Claims) ->
Claims;
do_check_claim([{K, F}|More], Claims) ->

69
apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl
View File

@ -93,6 +93,52 @@ t_check_auth(_Config) ->
?assertEqual({error, invalid_signature}, Result2),
?assertMatch({error, _}, emqx_access_control:authenticate(Plain#{password => <<"asd">>})).
t_check_auth_invalid_exp(init, _Config) ->
application:unset_env(emqx_auth_jwt, verify_claims).
t_check_auth_invalid_exp(_Config) ->
Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
Jwt0 = sign([{clientid, <<"client1">>},
{username, <<"plain">>},
{exp, [{foo, bar}]}], <<"HS256">>, <<"emqxsecret">>),
ct:pal("Jwt: ~p~n", [Jwt0]),
Result0 = emqx_access_control:authenticate(Plain#{password => Jwt0}),
ct:pal("Auth result: ~p~n", [Result0]),
?assertMatch({error, _}, Result0),
Jwt1 = sign([{clientid, <<"client1">>},
{username, <<"plain">>},
{exp, <<"foobar">>}], <<"HS256">>, <<"emqxsecret">>),
ct:pal("Jwt: ~p~n", [Jwt1]),
Result1 = emqx_access_control:authenticate(Plain#{password => Jwt1}),
ct:pal("Auth result: ~p~n", [Result1]),
?assertMatch({error, _}, Result1).
t_check_auth_str_exp(init, _Config) ->
application:unset_env(emqx_auth_jwt, verify_claims).
t_check_auth_str_exp(_Config) ->
Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
Exp = integer_to_binary(os:system_time(seconds) + 3),
Jwt0 = sign([{clientid, <<"client1">>},
{username, <<"plain">>},
{exp, Exp}], <<"HS256">>, <<"emqxsecret">>),
ct:pal("Jwt: ~p~n", [Jwt0]),
Result0 = emqx_access_control:authenticate(Plain#{password => Jwt0}),
ct:pal("Auth result: ~p~n", [Result0]),
?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
Jwt1 = sign([{clientid, <<"client1">>},
{username, <<"plain">>},
{exp, <<"0">>}], <<"HS256">>, <<"emqxsecret">>),
ct:pal("Jwt: ~p~n", [Jwt1]),
Result1 = emqx_access_control:authenticate(Plain#{password => Jwt1}),
ct:pal("Auth result: ~p~n", [Result1]),
?assertMatch({error, _}, Result1).
t_check_claims(init, _Config) ->
application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
t_check_claims(_Config) ->
@ -276,3 +322,26 @@ t_check_jwt_acl_expire(_Config) ->
emqtt:subscribe(C, <<"a/b">>, 0)),
ok = emqtt:disconnect(C).
t_check_jwt_acl_no_exp(init, _Config) ->
application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
t_check_jwt_acl_no_exp(_Config) ->
Jwt = sign([{client_id, <<"client1">>},
{username, <<"plain">>},
{sub, value},
{acl, [{sub, [<<"a/b">>]}]}],
<<"HS256">>,
<<"emqxsecret">>),
{ok, C} = emqtt:start_link(
[{clean_start, true},
{proto_ver, v5},
{client_id, <<"client1">>},
{password, Jwt}]),
{ok, _} = emqtt:connect(C),
?assertMatch(
{ok, #{}, [0]},
emqtt:subscribe(C, <<"a/b">>, 0)),
ok = emqtt:disconnect(C).

13
apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl
View File

@ -1,14 +1 @@
-define(APP, emqx_auth_ldap).
-record(auth_metrics, {
success = 'client.auth.success',
failure = 'client.auth.failure',
ignore = 'client.auth.ignore'
}).
-define(METRICS(Type), tl(tuple_to_list(#Type{}))).
-define(METRICS(Type, K), #Type{}#Type.K).
-define(AUTH_METRICS, ?METRICS(auth_metrics)).
-define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).

2
apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_auth_ldap,
[{description, "EMQ X Authentication/ACL with LDAP"},
{vsn, "4.3.4"}, % strict semver, bump manually!
{vsn, "4.3.5"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_auth_ldap_sup]},
{applications, [kernel,stdlib,eldap2,ecpool]},

63
apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src
View File

@ -1,40 +1,29 @@
%% -*-: erlang -*-
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[ {"4.3.3", [
%% There are only changes to the schema file, so we don't need
%% any commands here.
]},
{"4.3.0",
[ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
, {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []}
, {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
]},
{"4.3.1",
[ {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []}
, {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
, {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
]},
[{<<"4\\.3\\.[3-4]">>,
[{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]},
{"4.3.2",
[ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
, {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
]},
{<<".*">>, []}
],
[ {"4.3.3", []},
{"4.3.0",
[ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
, {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []}
, {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
]},
{"4.3.1",
[ {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []}
, {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
, {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
]},
[{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[0-1]">>,
[{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{<<"4\\.3\\.[3-4]">>,
[{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]},
{"4.3.2",
[ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
, {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
]},
{<<".*">>, []}
]
}.
[{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[0-1]">>,
[{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}]}.

11
apps/emqx_auth_ldap/src/emqx_auth_ldap.erl
View File

@ -26,17 +26,12 @@
-import(emqx_auth_ldap_cli, [search/3]).
-export([ register_metrics/0
, check/3
-export([ check/3
, description/0
, prepare_filter/4
, replace_vars/2
]).
-spec(register_metrics() -> ok).
register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
check(ClientInfo = #{username := Username, password := Password}, AuthResult,
State = #{password_attr := PasswdAttr, bind_as_user := BindAsUserRequired, pool := Pool}) ->
CheckResult =
@ -63,12 +58,10 @@ check(ClientInfo = #{username := Username, password := Password}, AuthResult,
end,
case CheckResult of
ok ->
ok = emqx_metrics:inc(?AUTH_METRICS(success)),
{stop, AuthResult#{auth_result => success, anonymous => false}};
{error, not_found} ->
emqx_metrics:inc(?AUTH_METRICS(ignore));
ok;
{error, ResultCode} ->
ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
?LOG(error, "[LDAP] Auth from ldap failed: ~p", [ResultCode]),
{stop, AuthResult#{auth_result => ResultCode, anonymous => false}}
end.

1
apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl
View File

@ -49,7 +49,6 @@ stop(_State) ->
ok.
load_auth_hook(DeviceDn) ->
ok = emqx_auth_ldap:register_metrics(),
Params = maps:from_list(DeviceDn),
emqx:hook('client.authenticate', fun emqx_auth_ldap:check/3, [Params#{pool => ?APP}]).

4
apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl
View File

@ -48,7 +48,9 @@ init_per_group(GrpName, Cfg) ->
Cfg.
end_per_group(_GrpName, _Cfg) ->
emqx_ct_helpers:stop_apps([emqx_auth_ldap]).
emqx_ct_helpers:stop_apps([emqx_auth_ldap]),
%% clear the application envs to avoid cross-suite testcase failure
application:unload(emqx_auth_ldap).
%%--------------------------------------------------------------------
%% Cases

4
apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl
View File

@ -40,7 +40,9 @@ init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
emqx_ct_helpers:stop_apps([emqx_auth_ldap]).
emqx_ct_helpers:stop_apps([emqx_auth_ldap]),
%% clear the application envs to avoid cross-suite testcase failure
application:unload(emqx_auth_ldap).
check_auth(_) ->
MqttUser1 = #{clientid => <<"mqttuser1">>,

12
apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl
View File

@ -41,15 +41,3 @@
}).
-type(acl_record() :: {acl_target(), emqx_topic:topic(), action(), access(), created_at()}).
-record(auth_metrics, {
success = 'client.auth.success',
failure = 'client.auth.failure',
ignore = 'client.auth.ignore'
}).
-define(METRICS(Type), tl(tuple_to_list(#Type{}))).
-define(METRICS(Type, K), #Type{}#Type.K).
-define(AUTH_METRICS, ?METRICS(auth_metrics)).
-define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).

50
apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl
View File

@ -96,18 +96,24 @@
, delete/2
]).
-define(CLIENTID_SCHEMA, [{<<"clientid">>, binary}, {<<"_like_clientid">>, binary}] ++ ?COMMON_SCHEMA).
-define(USERNAME_SCHEMA, [{<<"username">>, binary}, {<<"_like_username">>, binary}] ++ ?COMMON_SCHEMA).
-define(COMMON_SCHEMA, [{<<"topic">>, binary}, {<<"action">>, atom}, {<<"access">>, atom}]).
list_clientid(_Bindings, Params) ->
Table = emqx_acl_mnesia_db:login_acl_table(clientid),
return({ok, emqx_auth_mnesia_api:paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
{_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?CLIENTID_SCHEMA),
Table = emqx_acl_mnesia_db:login_acl_table(clientid, Params1),
return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
list_username(_Bindings, Params) ->
Table = emqx_acl_mnesia_db:login_acl_table(username),
return({ok, emqx_auth_mnesia_api:paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
{_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?USERNAME_SCHEMA),
Table = emqx_acl_mnesia_db:login_acl_table(username, Params1),
return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
list_all(_Bindings, Params) ->
Table = emqx_acl_mnesia_db:login_acl_table(all),
return({ok, emqx_auth_mnesia_api:paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
{_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?COMMON_SCHEMA),
Table = emqx_acl_mnesia_db:login_acl_table(all, Params1),
return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
lookup(#{clientid := Clientid}, _Params) ->
return({ok, format(emqx_acl_mnesia_db:lookup_acl({clientid, urldecode(Clientid)}))});
@ -170,7 +176,11 @@ delete(#{topic := Topic}, _) ->
%%------------------------------------------------------------------------------
count(QH) ->
qlc:fold(fun(_, Count) -> Count + 1 end, 0, QH).
Count = qlc:fold(fun(_, Sum) -> Sum + 1 end, 0, QH),
case is_integer(Count) of
true -> Count;
false -> 0
end.
format({{clientid, Clientid}, Topic, Action, Access, _CreatedAt}) ->
#{clientid => Clientid, topic => Topic, action => Action, access => Access};
@ -222,3 +232,27 @@ format_msg(Message) when is_tuple(Message) ->
urldecode(S) ->
emqx_http_lib:uri_decode(S).
paginate_qh(Qh, Count, Params, ComparingFun, RowFun) ->
Page = page(Params),
Limit = limit(Params),
Cursor = qlc:cursor(Qh),
case Page > 1 of
true ->
_ = qlc:next_answers(Cursor, (Page - 1) * Limit),
ok;
false -> ok
end,
Rows = qlc:next_answers(Cursor, Limit),
qlc:delete_cursor(Cursor),
#{meta => #{page => Page, limit => Limit, count => Count},
data => [RowFun(Row) || Row <- lists:sort(ComparingFun, Rows)]}.
page(Params) ->
binary_to_integer(proplists:get_value(<<"_page">>, Params, <<"1">>)).
limit(Params) ->
case proplists:get_value(<<"_limit">>, Params) of
undefined -> 50;
Size -> binary_to_integer(Size)
end.

97
apps/emqx_auth_mnesia/src/emqx_acl_mnesia_db.erl
View File

@ -33,6 +33,7 @@
, remove_acl/2
, merge_acl_records/3
, login_acl_table/1
, login_acl_table/2
, is_migration_started/0
]).
@ -124,7 +125,7 @@ all_acls_export() ->
{atomic, Records} = mnesia:transaction(
fun() ->
QH = acl_table(MatchSpecNew, MatchSpecOld, fun mnesia:table/2, fun lookup_mnesia/2),
QH = acl_table(MatchSpecNew, MatchSpecOld, {#{}, #{}}, fun mnesia:table/2, fun lookup_mnesia/2),
qlc:eval(QH)
end),
Records.
@ -132,9 +133,15 @@ all_acls_export() ->
%% @doc QLC table of logins matching spec
-spec(login_acl_table(acl_target_type()) -> qlc:query_handle()).
login_acl_table(AclTargetType) ->
MatchSpecNew = login_match_spec_new(AclTargetType),
MatchSpecOld = login_match_spec_old(AclTargetType),
acl_table(MatchSpecNew, MatchSpecOld, fun ets:table/2, fun lookup_ets/2).
login_acl_table(AclTargetType, {[], []}).
login_acl_table(AclTargetType, {Qs, Fuzzy}) ->
ToMap = fun({Type, Symbol, Val}, Acc) -> Acc#{{Type, Symbol} => Val} end,
Qs1 = lists:foldl(ToMap, #{}, Qs),
Fuzzy1 = lists:foldl(ToMap, #{}, Fuzzy),
MatchSpecNew = login_match_spec_new(AclTargetType, Qs1),
MatchSpecOld = login_match_spec_old(AclTargetType, Qs1),
acl_table(MatchSpecNew, MatchSpecOld, {Qs1, Fuzzy1}, fun ets:table/2, fun lookup_ets/2).
%% @doc Combine old `emqx_acl` ACL records with a new `emqx_acl2` ACL record for a given login
-spec(merge_acl_records(acl_target(), [#?ACL_TABLE{}], [#?ACL_TABLE2{}]) -> #?ACL_TABLE2{}).
@ -223,27 +230,39 @@ comparing({_, _, _, _, CreatedAt1},
{_, _, _, _, CreatedAt2}) ->
CreatedAt1 >= CreatedAt2.
login_match_spec_old(all) ->
login_match_spec_old(Type) -> login_match_spec_old(Type, #{}).
login_match_spec_old(all, _) ->
ets:fun2ms(fun(#?ACL_TABLE{filter = {all, _}} = Record) ->
Record
end);
login_match_spec_old(Type) when (Type =:= username) or (Type =:= clientid) ->
ets:fun2ms(fun(#?ACL_TABLE{filter = {{RecordType, _}, _}} = Record)
when RecordType =:= Type -> Record
end).
login_match_spec_old(Type, Params) when (Type =:= username) orelse (Type =:= clientid) ->
case maps:get({Type, '=:='}, Params, undefined) of
undefined ->
ets:fun2ms(fun(#?ACL_TABLE{filter = {{RType, _}, _}} = Rec) when RType =:= Type -> Rec end);
Val ->
ets:fun2ms(fun(#?ACL_TABLE{filter = {{RType, RVal}, _}} = Rec)
when RType =:= Type andalso RVal =:= Val -> Rec end)
end.
login_match_spec_new(all) ->
login_match_spec_new(Type) -> login_match_spec_new(Type, #{}).
login_match_spec_new(all, _) ->
ets:fun2ms(fun(#?ACL_TABLE2{who = all} = Record) ->
Record
end);
login_match_spec_new(Type) when (Type =:= username) or (Type =:= clientid) ->
ets:fun2ms(fun(#?ACL_TABLE2{who = {RecordType, _}} = Record)
when RecordType =:= Type -> Record
end).
login_match_spec_new(Type, Params) when (Type =:= username) orelse (Type =:= clientid) ->
case maps:get({Type, '=:='}, Params, undefined) of
undefined ->
ets:fun2ms(fun(#?ACL_TABLE2{who = {RType, _}} = Rec) when RType =:= Type -> Rec end);
Val ->
ets:fun2ms(fun(#?ACL_TABLE2{who = {RType, RVal}} = Rec)
when RType =:= Type andalso RVal =:= Val -> Rec end)
end.
acl_table(MatchSpecNew, MatchSpecOld, TableFun, LookupFun) ->
acl_table(MatchSpecNew, MatchSpecOld, Params, TableFun, LookupFun) ->
TraverseFun =
fun() ->
CursorNew =
@ -252,7 +271,7 @@ acl_table(MatchSpecNew, MatchSpecOld, TableFun, LookupFun) ->
CursorOld =
qlc:cursor(
TableFun(?ACL_TABLE, [{traverse, {select, MatchSpecOld}}])),
traverse_new(CursorNew, CursorOld, #{}, LookupFun)
traverse_new(CursorNew, CursorOld, Params, #{}, LookupFun)
end,
qlc:table(TraverseFun, []).
@ -265,12 +284,12 @@ acl_table(MatchSpecNew, MatchSpecOld, TableFun, LookupFun) ->
% After migration, number of such logins is zero, so traversing starts working in
% constant memory.
traverse_new(CursorNew, CursorOld, FoundKeys, LookupFun) ->
traverse_new(CursorNew, CursorOld, Params, FoundKeys, LookupFun) ->
Acls = qlc:next_answers(CursorNew, 1),
case Acls of
[] ->
qlc:delete_cursor(CursorNew),
traverse_old(CursorOld, FoundKeys);
traverse_old(CursorOld, Params, FoundKeys);
[#?ACL_TABLE2{who = Login, rules = Rules} = Acl] ->
Keys = lists:usort([{Login, Topic} || {_, _, Topic, _} <- Rules]),
OldRecs = lists:flatmap(fun(Key) -> LookupFun(?ACL_TABLE, Key) end, Keys),
@ -281,27 +300,57 @@ traverse_new(CursorNew, CursorOld, FoundKeys, LookupFun) ->
OldRecs),
case acl_to_list(MergedAcl) of
[] ->
traverse_new(CursorNew, CursorOld, NewFoundKeys, LookupFun);
traverse_new(CursorNew, CursorOld, Params, NewFoundKeys, LookupFun);
List ->
List ++ fun() -> traverse_new(CursorNew, CursorOld, NewFoundKeys, LookupFun) end
filter_params(List, Params) ++
fun() -> traverse_new(CursorNew, CursorOld, Params, NewFoundKeys, LookupFun) end
end
end.
traverse_old(CursorOld, FoundKeys) ->
filter_params(List, {Qs, Fuzzy}) ->
case maps:size(Qs) =:= 0 andalso maps:size(Fuzzy) =:= 0 of
false ->
Topic = maps:get({topic, '=:='}, Qs, undefined),
Action = maps:get({action, '=:='}, Qs, undefined),
Access = maps:get({access, '=:='}, Qs, undefined),
lists:filter(fun({Target, Topic0, Action0, Access0, _CreatedAt}) ->
CheckList = [{Topic, Topic0}, {Action, Action0}, {Access, Access0}],
case lists:all(fun is_match/1, CheckList) of
true ->
case Target of
{Type, Login} ->
case maps:get({Type, 'like'}, Fuzzy, <<>>) of
<<>> -> true;
LikeSchema -> binary:match(Login, LikeSchema) =/= nomatch
end;
all -> true
end;
false -> false
end
end, List);
true -> List
end.
is_match({Schema, Val}) ->
Schema =:= undefined orelse Schema =:= Val.
traverse_old(CursorOld, Params, FoundKeys) ->
OldAcls = qlc:next_answers(CursorOld),
case OldAcls of
[] ->
qlc:delete_cursor(CursorOld),
[];
_ ->
Records = [ {Login, Topic, Action, Access, CreatedAt}
Records = [{Login, Topic, Action, Access, CreatedAt}
|| #?ACL_TABLE{filter = {Login, Topic}, action = LegacyAction, access = Access, created_at = CreatedAt} <- OldAcls,
{_, Action, _, _} <- normalize_rule({Access, LegacyAction, Topic, CreatedAt}),
not maps:is_key({Login, Topic}, FoundKeys)
],
case Records of
[] -> traverse_old(CursorOld, FoundKeys);
List -> List ++ fun() -> traverse_old(CursorOld, FoundKeys) end
[] -> traverse_old(CursorOld, Params, FoundKeys);
List ->
filter_params(List, Params)
++ fun() -> traverse_old(CursorOld, Params, FoundKeys) end
end
end.

2
apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_auth_mnesia,
[{description, "EMQ X Authentication with Mnesia"},
{vsn, "4.3.6"}, % strict semver, bump manually
{vsn, "4.3.7"}, % strict semver, bump manually
{modules, []},
{registered, []},
{applications, [kernel,stdlib,mnesia]},

22
apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src
View File

@ -1,9 +1,12 @@
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[{"4.3.5",
[{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
[{<<"4\\.3\\.[5-6]">>,
[{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[0-3]">>,
[{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
@ -19,13 +22,18 @@
{"4.3.4",
[{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.3.5",
[{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
[{<<"4\\.3\\.[5-6]">>,
[{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[0-3]">>,
[{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
@ -40,8 +48,10 @@
{delete_module,emqx_acl_mnesia_db}]},
{"4.3.4",
[{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}]}.

8
apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl
View File

@ -27,7 +27,6 @@
-define(TABLE, emqx_user).
%% Auth callbacks
-export([ init/1
, register_metrics/0
, check/3
, description/0
]).
@ -51,10 +50,6 @@ init(#{clientid_list := ClientidList, username_list := UsernameList}) ->
ok = ekka_mnesia:copy_table(?TABLE, disc_copies).
-spec(register_metrics() -> ok).
register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
hash_type() ->
application:get_env(emqx_auth_mnesia, password_hash, sha256).
@ -67,17 +62,14 @@ check(ClientInfo = #{ clientid := Clientid
end),
case ets:select(?TABLE, MatchSpec) of
[] ->
emqx_metrics:inc(?AUTH_METRICS(ignore)),
ok;
List ->
case match_password(NPassword, HashType, List) of
false ->
Info = maps:without([password], ClientInfo),
?LOG(info, "[Mnesia] Auth from mnesia failed: ~p", [Info]),
emqx_metrics:inc(?AUTH_METRICS(failure)),
{stop, AuthResult#{anonymous => false, auth_result => password_error}};
_ ->
emqx_metrics:inc(?AUTH_METRICS(success)),
{stop, AuthResult#{anonymous => false, auth_result => success}}
end
end.

96
apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl
View File

@ -18,18 +18,20 @@
-include_lib("stdlib/include/qlc.hrl").
-include_lib("stdlib/include/ms_transform.hrl").
-include("emqx_auth_mnesia.hrl").
-define(TABLE, emqx_user).
-import(proplists, [get_value/2]).
-import(minirest, [return/1]).
-export([paginate_qh/5]).
-export([ list_clientid/2
, lookup_clientid/2
, add_clientid/2
, update_clientid/2
, delete_clientid/2
, query_clientid/3
, query_username/3
]).
-rest_api(#{name => list_clientid,
@ -109,13 +111,28 @@
descr => "Delete username in the cluster"
}).
-define(CLIENTID_SCHEMA, {?TABLE,
[
{<<"clientid">>, binary},
{<<"_like_clientid">>, binary}
]}).
-define(USERNAME_SCHEMA, {?TABLE,
[
{<<"username">>, binary},
{<<"_like_username">>, binary}
]}).
-define(query_clientid, {?MODULE, query_clientid}).
-define(query_username, {?MODULE, query_username}).
%%------------------------------------------------------------------------------
%% Auth Clientid Api
%%------------------------------------------------------------------------------
list_clientid(_Bindings, Params) ->
MatchSpec = ets:fun2ms(fun({?TABLE, {clientid, Clientid}, Password, CreatedAt}) -> {?TABLE, {clientid, Clientid}, Password, CreatedAt} end),
return({ok, paginate(?TABLE, MatchSpec, Params, fun emqx_auth_mnesia_cli:comparing/2, fun({?TABLE, {clientid, X}, _, _}) -> #{clientid => X} end)}).
SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun)}).
lookup_clientid(#{clientid := Clientid}, _Params) ->
return({ok, format(emqx_auth_mnesia_cli:lookup_user({clientid, urldecode(Clientid)}))}).
@ -164,8 +181,8 @@ delete_clientid(#{clientid := Clientid}, _) ->
%%------------------------------------------------------------------------------
list_username(_Bindings, Params) ->
MatchSpec = ets:fun2ms(fun({?TABLE, {username, Username}, Password, CreatedAt}) -> {?TABLE, {username, Username}, Password, CreatedAt} end),
return({ok, paginate(?TABLE, MatchSpec, Params, fun emqx_auth_mnesia_cli:comparing/2, fun({?TABLE, {username, X}, _, _}) -> #{username => X} end)}).
SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun)}).
lookup_username(#{username := Username}, _Params) ->
return({ok, format(emqx_auth_mnesia_cli:lookup_user({username, urldecode(Username)}))}).
@ -211,57 +228,52 @@ delete_username(#{username := Username}, _) ->
%%------------------------------------------------------------------------------
%% Paging Query
%%------------------------------------------------------------------------------
query_clientid(Qs, Start, Limit) -> query(clientid, Qs, Start, Limit).
query_username(Qs, Start, Limit) -> query(username, Qs, Start, Limit).
paginate(Table, MatchSpec, Params, ComparingFun, RowFun) ->
Qh = query_handle(Table, MatchSpec),
Count = count(Table, MatchSpec),
paginate_qh(Qh, Count, Params, ComparingFun, RowFun).
query(Type, {Qs, []}, Start, Limit) ->
Ms = qs2ms(Type, Qs),
emqx_mgmt_api:select_table(?TABLE, Ms, Start, Limit, fun format/1);
paginate_qh(Qh, Count, Params, ComparingFun, RowFun) ->
Page = page(Params),
Limit = limit(Params),
Cursor = qlc:cursor(Qh),
case Page > 1 of
true ->
_ = qlc:next_answers(Cursor, (Page - 1) * Limit),
ok;
false -> ok
end,
Rows = qlc:next_answers(Cursor, Limit),
qlc:delete_cursor(Cursor),
#{meta => #{page => Page, limit => Limit, count => Count},
data => [RowFun(Row) || Row <- lists:sort(ComparingFun, Rows)]}.
query(Type, {Qs, Fuzzy}, Start, Limit) ->
Ms = qs2ms(Type, Qs),
MatchFun = match_fun(Ms, Fuzzy),
emqx_mgmt_api:traverse_table(?TABLE, MatchFun, Start, Limit, fun format/1).
query_handle(Table, MatchSpec) when is_atom(Table) ->
Options = {traverse, {select, MatchSpec}},
qlc:q([R || R <- ets:table(Table, Options)]).
-spec qs2ms(clientid | username, list()) -> ets:match_spec().
qs2ms(Type, Qs) ->
Init = #?TABLE{login = {Type, '_'}, password = '_', created_at = '_'},
MatchHead = lists:foldl(fun(Q, Acc) -> match_ms(Q, Acc) end, Init, Qs),
[{MatchHead, [], ['$_']}].
count(Table, MatchSpec) when is_atom(Table) ->
[{MatchPattern, Where, _Re}] = MatchSpec,
NMatchSpec = [{MatchPattern, Where, [true]}],
ets:select_count(Table, NMatchSpec).
match_ms({Type, '=:=', Value}, MatchHead) -> MatchHead#?TABLE{login = {Type, Value}};
match_ms(_, MatchHead) -> MatchHead.
page(Params) ->
binary_to_integer(proplists:get_value(<<"_page">>, Params, <<"1">>)).
limit(Params) ->
case proplists:get_value(<<"_limit">>, Params) of
undefined -> 10;
Size -> binary_to_integer(Size)
match_fun(Ms, Fuzzy) ->
MsC = ets:match_spec_compile(Ms),
fun(Rows) ->
Ls = ets:match_spec_run(Rows, MsC),
lists:filter(fun(E) -> run_fuzzy_match(E, Fuzzy) end, Ls)
end.
run_fuzzy_match(_, []) -> true;
run_fuzzy_match(E = #?TABLE{login = {Key, Str}}, [{Key, like, SubStr}|Fuzzy]) ->
binary:match(Str, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(_E, [{_Key, like, _SubStr}| _Fuzzy]) -> false.
%%------------------------------------------------------------------------------
%% Interval Funcs
%%------------------------------------------------------------------------------
format([{?TABLE, {clientid, ClientId}, _Password, _InterTime}]) ->
#{clientid => ClientId};
format([{?TABLE, {clientid, ClientId}, _Password, CreatedAt}]) ->
#{clientid => ClientId, created_at => CreatedAt};
format([{?TABLE, {username, Username}, _Password, _InterTime}]) ->
#{username => Username};
format([{?TABLE, {username, Username}, _Password, CreatedAt}]) ->
#{username => Username, created_at => CreatedAt};
format([]) ->
#{}.
#{};
format(User) -> format([User]).
validate([], []) ->
ok;

1
apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl
View File

@ -56,7 +56,6 @@ load_auth_hook() ->
ClientidList = application:get_env(?APP, clientid_list, []),
UsernameList = application:get_env(?APP, username_list, []),
ok = emqx_auth_mnesia:init(#{clientid_list => ClientidList, username_list => UsernameList}),
ok = emqx_auth_mnesia:register_metrics(),
Params = #{hash_type => emqx_auth_mnesia:hash_type()},
emqx:hook('client.authenticate', fun emqx_auth_mnesia:check/3, [Params]).

59
apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl
View File

@ -25,6 +25,7 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-import(emqx_ct_http, [ request_api/3
, request_api/4
, request_api/5
, get_http_data/1
, create_default_app/0
@ -358,13 +359,31 @@ t_rest_api(_Config) ->
<<"topic">> => <<"topic/C">>,
<<"action">> => <<"pubsub">>,
<<"access">> => <<"deny">>
},
#{<<"clientid">> => <<"good_clientid1">>,
<<"topic">> => <<"topic/D">>,
<<"action">> => <<"pubsub">>,
<<"access">> => <<"deny">>
}],
{ok, _} = request_http_rest_add([], Params1),
{ok, Re1} = request_http_rest_list(["clientid", "test_clientid"]),
?assertMatch(4, length(get_http_data(Re1))),
{ok, Re11} = request_http_rest_list(["clientid"], "_like_clientid=good"),
?assertMatch(2, length(get_http_data(Re11))),
{ok, Re12} = request_http_rest_list(["clientid"], "_like_clientid=clientid"),
?assertMatch(6, length(get_http_data(Re12))),
{ok, Re13} = request_http_rest_list(["clientid"], "_like_clientid=clientid&action=pub"),
?assertMatch(3, length(get_http_data(Re13))),
{ok, Re14} = request_http_rest_list(["clientid"], "_like_clientid=clientid&access=deny"),
?assertMatch(4, length(get_http_data(Re14))),
{ok, Re15} = request_http_rest_list(["clientid"], "_like_clientid=clientid&topic=topic/A"),
?assertMatch(1, length(get_http_data(Re15))),
{ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/A"]),
{ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/B"]),
{ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/C"]),
{ok, _} = request_http_rest_delete(["clientid", "good_clientid1", "topic", "topic/D"]),
{ok, Res1} = request_http_rest_list(["clientid"]),
?assertMatch([], get_http_data(Res1)),
@ -382,13 +401,30 @@ t_rest_api(_Config) ->
<<"topic">> => <<"topic/C">>,
<<"action">> => <<"pubsub">>,
<<"access">> => <<"deny">>
},
#{<<"username">> => <<"good_username">>,
<<"topic">> => <<"topic/D">>,
<<"action">> => <<"pubsub">>,
<<"access">> => <<"deny">>
}],
{ok, _} = request_http_rest_add([], Params2),
{ok, Re2} = request_http_rest_list(["username", "test_username"]),
?assertMatch(4, length(get_http_data(Re2))),
{ok, Re21} = request_http_rest_list(["username"], "_like_username=good"),
?assertMatch(2, length(get_http_data(Re21))),
{ok, Re22} = request_http_rest_list(["username"], "_like_username=username"),
?assertMatch(6, length(get_http_data(Re22))),
{ok, Re23} = request_http_rest_list(["username"], "_like_username=username&action=pub"),
?assertMatch(3, length(get_http_data(Re23))),
{ok, Re24} = request_http_rest_list(["username"], "_like_username=username&access=deny"),
?assertMatch(4, length(get_http_data(Re24))),
{ok, Re25} = request_http_rest_list(["username"], "_like_username=username&topic=topic/A"),
?assertMatch(1, length(get_http_data(Re25))),
{ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/A"]),
{ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/B"]),
{ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/C"]),
{ok, _} = request_http_rest_delete(["username", "good_username", "topic", "topic/D"]),
{ok, Res2} = request_http_rest_list(["username"]),
?assertMatch([], get_http_data(Res2)),
@ -403,13 +439,29 @@ t_rest_api(_Config) ->
#{<<"topic">> => <<"topic/C">>,
<<"action">> => <<"pubsub">>,
<<"access">> => <<"deny">>
}],
},
#{<<"topic">> => <<"topic/D">>,
<<"action">> => <<"pubsub">>,
<<"access">> => <<"deny">>
}
],
{ok, _} = request_http_rest_add([], Params3),
{ok, Re3} = request_http_rest_list(["$all"]),
?assertMatch(4, length(get_http_data(Re3))),
?assertMatch(6, length(get_http_data(Re3))),
{ok, Re31} = request_http_rest_list(["$all"], "topic=topic/A"),
?assertMatch(1, length(get_http_data(Re31))),
{ok, Re32} = request_http_rest_list(["$all"], "action=sub"),
?assertMatch(3, length(get_http_data(Re32))),
{ok, Re33} = request_http_rest_list(["$all"], "access=deny"),
?assertMatch(4, length(get_http_data(Re33))),
{ok, Re34} = request_http_rest_list(["$all"], "action=sub&access=deny"),
?assertMatch(2, length(get_http_data(Re34))),
{ok, _} = request_http_rest_delete(["$all", "topic", "topic/A"]),
{ok, _} = request_http_rest_delete(["$all", "topic", "topic/B"]),
{ok, _} = request_http_rest_delete(["$all", "topic", "topic/C"]),
{ok, _} = request_http_rest_delete(["$all", "topic", "topic/D"]),
{ok, Res3} = request_http_rest_list(["$all"]),
?assertMatch([], get_http_data(Res3)).
@ -443,6 +495,9 @@ combined_conflicting_records() ->
request_http_rest_list(Path) ->
request_api(get, uri(Path), default_auth_header()).
request_http_rest_list(Path, Qs) ->
request_api(get, uri(Path), Qs, default_auth_header()).
request_http_rest_lookup(Path) ->
request_api(get, uri(Path), default_auth_header()).

23
apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl
View File

@ -286,20 +286,26 @@ t_clientid_rest_api(_Config) ->
Params3 = [ #{<<"clientid">> => ?CLIENTID, <<"password">> => ?PASSWORD}
, #{<<"clientid">> => <<"clientid1">>, <<"password">> => ?PASSWORD}
, #{<<"clientid">> => <<"clientid2">>, <<"password">> => ?PASSWORD}
, #{<<"clientid">> => <<"client2">>, <<"password">> => ?PASSWORD}
],
{ok, Result3} = request_http_rest_add(["auth_clientid"], Params3),
?assertMatch(#{ ?CLIENTID := <<"{error,existed}">>
, <<"clientid1">> := <<"ok">>
, <<"clientid2">> := <<"ok">>
, <<"client2">> := <<"ok">>
}, get_http_data(Result3)),
{ok, Result4} = request_http_rest_list(["auth_clientid"]),
?assertEqual(3, length(get_http_data(Result4))),
{ok, Result5} = request_http_rest_list(["auth_clientid?_like_clientid=id"]),
?assertEqual(2, length(get_http_data(Result5))),
{ok, Result6} = request_http_rest_list(["auth_clientid?_like_clientid=x"]),
?assertEqual(0, length(get_http_data(Result6))),
{ok, _} = request_http_rest_delete(Path),
{ok, Result5} = request_http_rest_lookup(Path),
?assertMatch(#{}, get_http_data(Result5)).
{ok, Result7} = request_http_rest_lookup(Path),
?assertMatch(#{}, get_http_data(Result7)).
t_username_rest_api(_Config) ->
clean_all_users(),
@ -330,9 +336,14 @@ t_username_rest_api(_Config) ->
{ok, Result4} = request_http_rest_list(["auth_username"]),
?assertEqual(3, length(get_http_data(Result4))),
{ok, Result5} = request_http_rest_list(["auth_username?_like_username=for"]),
?assertEqual(1, length(get_http_data(Result5))),
{ok, Result6} = request_http_rest_list(["auth_username?_like_username=x"]),
?assertEqual(0, length(get_http_data(Result6))),
{ok, _} = request_http_rest_delete(Path),
{ok, Result5} = request_http_rest_lookup([Path]),
?assertMatch(#{}, get_http_data(Result5)).
{ok, Result7} = request_http_rest_lookup([Path]),
?assertMatch(#{}, get_http_data(Result7)).
t_password_hash(_) ->
clean_all_users(),

13
apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl
View File

@ -1,4 +1,3 @@
-define(APP, emqx_auth_mongo).
-define(DEFAULT_SELECTORS, [{<<"username">>, <<"%u">>}]).
@ -14,15 +13,3 @@
-record(aclquery, {collection = <<"mqtt_acl">>,
selector = {<<"username">>, <<"%u">>}}).
-record(auth_metrics, {
success = 'client.auth.success',
failure = 'client.auth.failure',
ignore = 'client.auth.ignore'
}).
-define(METRICS(Type), tl(tuple_to_list(#Type{}))).
-define(METRICS(Type, K), #Type{}#Type.K).
-define(AUTH_METRICS, ?METRICS(auth_metrics)).
-define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).

2
apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_auth_mongo,
[{description, "EMQ X Authentication/ACL with MongoDB"},
{vsn, "4.4.3"}, % strict semver, bump manually!
{vsn, "4.4.4"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_auth_mongo_sup]},
{applications, [kernel,stdlib,mongodb,ecpool]},

26
apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src
View File

@ -1,29 +1,15 @@
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[{"4.4.2",
[{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]},
{"4.4.1",
[{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
[{<<"4.4.[0-3]">>,
[{load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]},
{"4.4.0",
[{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.4.2",
[{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]},
{"4.4.1",
[{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
[{<<"4.4.[0-3]">>,
[{load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]},
{"4.4.0",
[{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}]}.

12
apps/emqx_auth_mongo/src/emqx_auth_mongo.erl
View File

@ -23,8 +23,7 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/types.hrl").
-export([ register_metrics/0
, check/3
-export([ check/3
, description/0
]).
@ -39,20 +38,15 @@
, available/3
]).
-spec(register_metrics() -> ok).
register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
check(ClientInfo = #{password := Password}, AuthResult,
Env = #{authquery := AuthQuery, superquery := SuperQuery}) ->
#authquery{collection = Collection, field = Fields,
hash = HashType, selector = Selector} = AuthQuery,
Pool = maps:get(pool, Env, ?APP),
case query(Pool, Collection, maps:from_list(replvars(Selector, ClientInfo))) of
undefined -> emqx_metrics:inc(?AUTH_METRICS(ignore));
undefined -> ok;
{error, Reason} ->
?LOG(error, "[MongoDB] Can't connect to MongoDB server: ~0p", [Reason]),
ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
{stop, AuthResult#{auth_result => not_authorized, anonymous => false}};
UserMap ->
Result = case [maps:get(Field, UserMap, undefined) || Field <- Fields] of
@ -64,13 +58,11 @@ check(ClientInfo = #{password := Password}, AuthResult,
end,
case Result of
ok ->
ok = emqx_metrics:inc(?AUTH_METRICS(success)),
{stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo),
anonymous => false,
auth_result => success}};
{error, Error} ->
?LOG(error, "[MongoDB] check auth fail: ~p", [Error]),
ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
{stop, AuthResult#{auth_result => Error, anonymous => false}}
end
end.

2
apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl
View File

@ -68,7 +68,6 @@ safe_start() ->
reg_authmod(AuthQuery) ->
case emqx_auth_mongo:available(?APP, AuthQuery) of
ok ->
emqx_auth_mongo:register_metrics(),
HookFun = fun emqx_auth_mongo:check/3,
HookOptions = #{authquery => AuthQuery, superquery => undefined, pool => ?APP},
case r(super_query, application:get_env(?APP, super_query, undefined)) of
@ -122,4 +121,3 @@ r(auth_query, Config) ->
r(acl_query, Config) ->
#aclquery{collection = list_to_binary(get_value(collection, Config, "mqtt_acl")),
selector = get_value(selector, Config, [?DEFAULT_SELECTORS])}.

13
apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl
View File

@ -1,14 +1 @@
-define(APP, emqx_auth_mysql).
-record(auth_metrics, {
success = 'client.auth.success',
failure = 'client.auth.failure',
ignore = 'client.auth.ignore'
}).
-define(METRICS(Type), tl(tuple_to_list(#Type{}))).
-define(METRICS(Type, K), #Type{}#Type.K).
-define(AUTH_METRICS, ?METRICS(auth_metrics)).
-define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).

2
apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_auth_mysql,
[{description, "EMQ X Authentication/ACL with MySQL"},
{vsn, "4.3.2"}, % strict semver, bump manually!
{vsn, "4.3.3"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_auth_mysql_sup]},
{applications, [kernel,stdlib,mysql,ecpool]},

13
apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src
View File

@ -1,16 +1,19 @@
%% -*- mode: erlang -*-
{VSN,
[{"4.3.1", [
%% There are only changes to the schema file, so we don't need
%% any commands here.
]},
[{<<"4\\.3\\.[1-2]">>,
[{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}]},
{"4.3.0",
[{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.3.1", []},
[{<<"4\\.3\\.[1-2]">>,
[{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}]},
{"4.3.0",
[{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}]
}.

12
apps/emqx_auth_mysql/src/emqx_auth_mysql.erl
View File

@ -22,17 +22,12 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/types.hrl").
-export([ register_metrics/0
, check/3
-export([ check/3
, description/0
]).
-define(EMPTY(Username), (Username =:= undefined orelse Username =:= <<>>)).
-spec(register_metrics() -> ok).
register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
check(ClientInfo = #{password := Password}, AuthResult,
#{auth_query := {AuthSql, AuthParams},
super_query := SuperQuery,
@ -51,15 +46,13 @@ check(ClientInfo = #{password := Password}, AuthResult,
end,
case CheckPass of
ok ->
emqx_metrics:inc(?AUTH_METRICS(success)),
{stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo),
anonymous => false,
auth_result => success}};
{error, not_found} ->
emqx_metrics:inc(?AUTH_METRICS(ignore)), ok;
ok;
{error, ResultCode} ->
?LOG(error, "[MySQL] Auth from mysql failed: ~p", [ResultCode]),
emqx_metrics:inc(?AUTH_METRICS(failure)),
{stop, AuthResult#{auth_result => ResultCode, anonymous => false}}
end.
@ -88,4 +81,3 @@ check_pass(Password, HashType) ->
end.
description() -> "Authentication with MySQL".

1
apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl
View File

@ -50,7 +50,6 @@ stop(_State) ->
ok.
load_auth_hook(AuthQuery) ->
ok = emqx_auth_mysql:register_metrics(),
SuperQuery = parse_query(application:get_env(?APP, super_query, undefined)),
{ok, HashType} = application:get_env(?APP, password_hash),
Params = #{auth_query => AuthQuery,

12
apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl
View File

@ -1,13 +1 @@
-define(APP, emqx_auth_pgsql).
-record(auth_metrics, {
success = 'client.auth.success',
failure = 'client.auth.failure',
ignore = 'client.auth.ignore'
}).
-define(METRICS(Type), tl(tuple_to_list(#Type{}))).
-define(METRICS(Type, K), #Type{}#Type.K).
-define(AUTH_METRICS, ?METRICS(auth_metrics)).
-define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).

2
apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_auth_pgsql,
[{description, "EMQ X Authentication/ACL with PostgreSQL"},
{vsn, "4.4.2"}, % strict semver, bump manually!
{vsn, "4.4.3"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_auth_pgsql_sup]},
{applications, [kernel,stdlib,epgsql,ecpool]},

28
apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src
View File

@ -1,19 +1,15 @@
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[{"4.4.1", [
%% There are only changes to the schema file, so we don't need
%% any commands here.
]},
{"4.4.0",
[{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]},
[{<<"4\\.4\\.[0-2]">>,
%% epgsql 4.4.0 -> 4.6.0.
%% epgsql has no appup, so we can only restart it.
[{restart_application,epgsql},
{restart_application,emqx_auth_pgsql}]},
{<<".*">>,[]}],
[{"4.4.1", [
%% There are only changes to the schema file, so we don't need
%% any commands here.
]},
{"4.4.0",
[{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}]
}.
[{<<"4\\.4\\.[0-2]">>,
%% epgsql 4.4.0 -> 4.6.0.
%% epgsql has no appup, so we can only restart it.
[{restart_application,epgsql},
{restart_application,emqx_auth_pgsql}]},
{<<".*">>,[]}]}.

12
apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl
View File

@ -21,15 +21,10 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl").
-export([ register_metrics/0
, check/3
-export([ check/3
, description/0
]).
-spec(register_metrics() -> ok).
register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
%%--------------------------------------------------------------------
%% Auth Module Callbacks
%%--------------------------------------------------------------------
@ -50,15 +45,13 @@ check(ClientInfo = #{password := Password}, AuthResult,
end,
case CheckPass of
ok ->
emqx_metrics:inc(?AUTH_METRICS(success)),
{stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo),
anonymous => false,
auth_result => success}};
{error, not_found} ->
emqx_metrics:inc(?AUTH_METRICS(ignore)), ok;
ok;
{error, ResultCode} ->
?LOG(error, "[Postgres] Auth from pgsql failed: ~p", [ResultCode]),
emqx_metrics:inc(?AUTH_METRICS(failure)),
{stop, AuthResult#{auth_result => ResultCode, anonymous => false}}
end.
@ -88,4 +81,3 @@ check_pass(Password, HashType) ->
end.
description() -> "Authentication with PostgreSQL".

2
apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl
View File

@ -42,7 +42,6 @@ start(_StartType, _StartArgs) ->
super_query => SuperQuery,
hash_type => HashType,
pool => ?APP},
ok = emqx_auth_pgsql:register_metrics(),
ok = emqx:hook('client.authenticate', fun emqx_auth_pgsql:check/3, [AuthEnv])
end),
if_enabled(acl_query, fun(AclQuery) ->
@ -59,4 +58,3 @@ if_enabled(Par, Fun) ->
{ok, Query} -> Fun(parse_query(Par, Query));
undefined -> ok
end.

13
apps/emqx_auth_redis/include/emqx_auth_redis.hrl
View File

@ -1,14 +1 @@
-define(APP, emqx_auth_redis).
-record(auth_metrics, {
success = 'client.auth.success',
failure = 'client.auth.failure',
ignore = 'client.auth.ignore'
}).
-define(METRICS(Type), tl(tuple_to_list(#Type{}))).
-define(METRICS(Type, K), #Type{}#Type.K).
-define(AUTH_METRICS, ?METRICS(auth_metrics)).
-define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).

2
apps/emqx_auth_redis/src/emqx_auth_redis.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_auth_redis,
[{description, "EMQ X Authentication/ACL with Redis"},
{vsn, "4.3.2"}, % strict semver, bump manually!
{vsn, "4.3.3"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_auth_redis_sup]},
{applications, [kernel,stdlib,eredis,eredis_cluster,ecpool]},

13
apps/emqx_auth_redis/src/emqx_auth_redis.appup.src
View File

@ -1,16 +1,19 @@
%% -*- mode: erlang -*-
{VSN,
[{"4.3.1", [
%% There are only changes to the schema file, so we don't need
%% any commands here.
]},
[{<<"4\\.3\\.[1-2]">>,
[{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}]},
{"4.3.0",
[{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_redis,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.3.1", []},
[{<<"4\\.3\\.[1-2]">>,
[{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}]},
{"4.3.0",
[{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_redis,brutal_purge,soft_purge,[]},
{load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}]
}.

12
apps/emqx_auth_redis/src/emqx_auth_redis.erl
View File

@ -21,15 +21,10 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl").
-export([ register_metrics/0
, check/3
-export([ check/3
, description/0
]).
-spec(register_metrics() -> ok).
register_metrics() ->
lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
check(ClientInfo = #{password := Password}, AuthResult,
#{auth_cmd := AuthCmd,
super_cmd := SuperCmd,
@ -52,15 +47,13 @@ check(ClientInfo = #{password := Password}, AuthResult,
end,
case CheckPass of
ok ->
ok = emqx_metrics:inc(?AUTH_METRICS(success)),
IsSuperuser = is_superuser(Pool, Type, SuperCmd, ClientInfo, Timeout),
{stop, AuthResult#{is_superuser => IsSuperuser,
anonymous => false,
auth_result => success}};
{error, not_found} ->
ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
ok;
{error, ResultCode} ->
ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
?LOG(error, "[Redis] Auth from redis failed: ~p", [ResultCode]),
{stop, AuthResult#{auth_result => ResultCode, anonymous => false}}
end.
@ -82,4 +75,3 @@ check_pass(Password, HashType) ->
ok -> ok;
{error, _Reason} -> {error, not_authorized}
end.

2
apps/emqx_auth_redis/src/emqx_auth_redis_app.erl
View File

@ -49,7 +49,6 @@ load_auth_hook(AuthCmd) ->
timeout => Timeout,
type => Type,
pool => ?APP},
ok = emqx_auth_redis:register_metrics(),
emqx:hook('client.authenticate', fun emqx_auth_redis:check/3, [Config]).
load_acl_hook(AclCmd) ->
@ -66,4 +65,3 @@ if_cmd_enabled(Par, Fun) ->
{ok, Cmd} -> Fun(Cmd);
undefined -> ok
end.

2
apps/emqx_exproto/src/emqx_exproto.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_exproto,
[{description, "EMQ X Extension for Protocol"},
{vsn, "4.3.7"}, %% 4.3.3 is used by ee
{vsn, "4.3.8"}, %% 4.3.3 is used by ee
{modules, []},
{registered, []},
{mod, {emqx_exproto_app, []}},

21
apps/emqx_exproto/src/emqx_exproto.appup.src
View File

@ -1,14 +1,9 @@
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[
{"4.3.6",
[ %% There are only changes to the schema file, so we don't need any
%% commands here
]},
{<<"4\\.3\\.[4-5]">>,
[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[2-3]">>,
[{<<"4\\.3\\.[6-7]">>,
[{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[2-5]">>,
[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[0-1]">>,
@ -17,11 +12,9 @@
{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.3.6", []},
{<<"4\\.3\\.[4-5]">>,
[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[2-3]">>,
[{<<"4\\.3\\.[6-7]">>,
[{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[2-5]">>,
[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
{<<"4\\.3\\.[0-1]">>,

5
apps/emqx_exproto/src/emqx_exproto_channel.erl
View File

@ -299,8 +299,6 @@ handle_call({auth, RequestedClientInfo, Password},
case emqx_access_control:authenticate(ClientInfo1#{password => Password}) of
{ok, AuthResult} ->
emqx_logger:set_metadata_clientid(ClientId),
is_anonymous(AuthResult) andalso
emqx_metrics:inc('client.auth.anonymous'),
NClientInfo = maps:merge(ClientInfo1, AuthResult),
NChannel = Channel1#channel{clientinfo = NClientInfo},
case emqx_cm:open_session(true, NClientInfo, NConnInfo) of
@ -424,9 +422,6 @@ terminate(Reason, Channel) ->
Req = #{reason => stringfy(Reason)},
try_dispatch(on_socket_closed, wrap(Req), Channel).
is_anonymous(#{anonymous := true}) -> true;
is_anonymous(_AuthResult) -> false.
packet_to_message(Topic, Qos, Payload,
#channel{
conninfo = #{proto_ver := ProtoVer},

10
apps/emqx_management/src/emqx_mgmt_api.erl
View File

@ -23,6 +23,7 @@
%% first_next query APIs
-export([ params2qs/2
, node_query/4
, node_query/5
, cluster_query/3
, traverse_table/5
, select_table/5
@ -111,6 +112,9 @@ limit(Params) ->
%%--------------------------------------------------------------------
node_query(Node, Params, {Tab, QsSchema}, QueryFun) ->
node_query(Node, Params, {Tab, QsSchema}, QueryFun, undefined).
node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun) ->
{CodCnt, Qs} = params2qs(Params, QsSchema),
Limit = limit(Params),
Page = page(Params),
@ -123,7 +127,11 @@ node_query(Node, Params, {Tab, QsSchema}, QueryFun) ->
true -> Meta#{count => count(Tab), hasnext => length(Rows) > Limit};
_ -> Meta#{count => -1, hasnext => length(Rows) > Limit}
end,
#{meta => NMeta, data => lists:sublist(Rows, Limit)}.
NRows = case SortFun of
undefined -> Rows;
_ -> lists:sort(SortFun, Rows)
end,
#{meta => NMeta, data => lists:sublist(NRows, Limit)}.
%% @private
do_query(Node, Qs, {M,F}, Start, Limit) when Node =:= node() ->

112
apps/emqx_management/src/emqx_mgmt_api_clients.erl
View File

@ -35,6 +35,10 @@
{<<"_gte_created_at">>, timestamp},
{<<"_lte_created_at">>, timestamp},
{<<"_gte_connected_at">>, timestamp},
{<<"_gte_mqueue_len">>, integer},
{<<"_lte_mqueue_len">>, integer},
{<<"_gte_mqueue_dropped">>, integer},
{<<"_lte_mqueue_dropped">>, integer},
{<<"_lte_connected_at">>, timestamp}]}).
-rest_api(#{name => list_clients,
@ -356,15 +360,14 @@ format_acl_cache({{PubSub, Topic}, {AclResult, Timestamp}}) ->
%% Query Functions
%%--------------------------------------------------------------------
query({Qs, []}, Start, Limit) ->
Ms = qs2ms(Qs),
emqx_mgmt_api:select_table(emqx_channel_info, Ms, Start, Limit, fun format_channel_info/1);
query({Qs, Fuzzy}, Start, Limit) ->
Ms = qs2ms(Qs),
MatchFun = match_fun(Ms, Fuzzy),
emqx_mgmt_api:traverse_table(emqx_channel_info, MatchFun,
Start, Limit, fun format_channel_info/1).
case qs2ms(Qs) of
{Ms, []} when Fuzzy =:= [] ->
emqx_mgmt_api:select_table(emqx_channel_info, Ms, Start, Limit, fun format_channel_info/1);
{Ms, FuzzyStats} ->
MatchFun = match_fun(Ms, Fuzzy ++ FuzzyStats),
emqx_mgmt_api:traverse_table(emqx_channel_info, MatchFun, Start, Limit, fun format_channel_info/1)
end.
%%--------------------------------------------------------------------
%% Match funcs
@ -388,27 +391,48 @@ run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr} |
undefined -> <<>>;
V -> V
end,
binary:match(Val, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy).
binary:match(Val, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = {_, _, Stats}, [{Key, '>=', Int}|Fuzzy]) ->
case lists:keyfind(Key, 1, Stats) of
{_, Val} when Val >= Int -> run_fuzzy_match(E, Fuzzy);
_ -> false
end;
run_fuzzy_match(E = {_, _, Stats}, [{Key, '=<', Int}|Fuzzy]) ->
case lists:keyfind(Key, 1, Stats) of
{_, Val} when Val =< Int -> run_fuzzy_match(E, Fuzzy);
_ -> false
end.
%%--------------------------------------------------------------------
%% QueryString to Match Spec
-spec qs2ms(list()) -> ets:match_spec().
qs2ms(Qs) ->
{MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}),
[{{'$1', MtchHead, '_'}, Conds, ['$_']}].
{MatchHead, Conds, FuzzyStats} = qs2ms(Qs, 2, #{}, [], []),
{[{{'$1', MatchHead, '_'}, Conds, ['$_']}], FuzzyStats}.
qs2ms([], _, {MtchHead, Conds}) ->
{MtchHead, lists:reverse(Conds)};
qs2ms([], _, MatchHead, Conds, FuzzyStats) ->
{MatchHead, lists:reverse(Conds), FuzzyStats};
qs2ms([{Key, '=:=', Value} | Rest], N, {MtchHead, Conds}) ->
NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(Key, Value)),
qs2ms(Rest, N, {NMtchHead, Conds});
qs2ms([Qs | Rest], N, {MtchHead, Conds}) ->
qs2ms([{Key, '=:=', Value} | Rest], N, MatchHead, Conds, FuzzyStats) ->
NMatchHead = emqx_mgmt_util:merge_maps(MatchHead, ms(Key, Value)),
qs2ms(Rest, N, NMatchHead, Conds, FuzzyStats);
qs2ms([Qs | Rest], N, MatchHead, Conds, FuzzyStats) ->
Holder = binary_to_atom(iolist_to_binary(["$", integer_to_list(N)]), utf8),
NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(element(1, Qs), Holder)),
NConds = put_conds(Qs, Holder, Conds),
qs2ms(Rest, N+1, {NMtchHead, NConds}).
case ms(element(1, Qs), Holder) of
fuzzy_stats ->
FuzzyStats1 =
case Qs of
{_Key, _Symbol, _Val} -> [Qs | FuzzyStats];
{Key, Symbol1, Val1, Symbol2, Val2} ->
[{Key, Symbol1, Val1}, {Key, Symbol2, Val2} | FuzzyStats]
end,
qs2ms(Rest, N, MatchHead, Conds, FuzzyStats1);
Ms ->
NMatchHead = emqx_mgmt_util:merge_maps(MatchHead, Ms),
NConds = put_conds(Qs, Holder, Conds),
qs2ms(Rest, N+1, NMatchHead, NConds, FuzzyStats)
end.
put_conds({_, Op, V}, Holder, Conds) ->
[{Op, Holder, V} | Conds];
@ -435,7 +459,11 @@ ms(proto_ver, X) ->
ms(connected_at, X) ->
#{conninfo => #{connected_at => X}};
ms(created_at, X) ->
#{session => #{created_at => X}}.
#{session => #{created_at => X}};
ms(mqueue_len, _X) ->
fuzzy_stats;
ms(mqueue_dropped, _X) ->
fuzzy_stats.
filter_ratelimit_params(P) ->
[{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined].
@ -461,6 +489,10 @@ params2qs_test() ->
{<<"_lte_created_at">>, 5},
{<<"_gte_connected_at">>, 1},
{<<"_lte_connected_at">>, 5},
{<<"_lte_mqueue_len">>, 10},
{<<"_gte_mqueue_len">>, 5},
{<<"_lte_mqueue_dropped">>, 100},
{<<"_gte_mqueue_dropped">>, 50},
{<<"_like_clientid">>, <<"a">>},
{<<"_like_username">>, <<"e">>}
],
@ -480,12 +512,17 @@ params2qs_test() ->
{'=<','$2', 5000},
{'>=','$3', 1000},
{'=<','$3', 5000}],
{10, {Qs1, []}} = emqx_mgmt_api:params2qs(Params, QsSchema),
[{{'$1', MtchHead, _}, Condi, _}] = qs2ms(Qs1),
ExpectedFuzzyStats = [{mqueue_dropped,'=<',100},
{mqueue_dropped,'>=',50},
{mqueue_len,'=<',10},
{mqueue_len,'>=',5}],
{12, {Qs1, []}} = emqx_mgmt_api:params2qs(Params, QsSchema),
{[{{'$1', MtchHead, _}, Condi, _}], FuzzyStats} = qs2ms(Qs1),
?assertEqual(ExpectedMtchHead, MtchHead),
?assertEqual(ExpectedCondi, Condi),
?assertEqual(ExpectedFuzzyStats, lists:sort(FuzzyStats)),
[{{'$1', #{}, '_'}, [], ['$_']}] = qs2ms([]).
{[{{'$1', #{}, '_'}, [], ['$_']}], []} = qs2ms([]).
fuzzy_match_test() ->
Info = {emqx_channel_info,
@ -507,4 +544,31 @@ fuzzy_match_test() ->
true = run_fuzzy_match(Info, [{clientid, like, <<"de">>},
{username, like, <<"[]">>}]).
fuzzy_stats_test() ->
Fun = fun(Len, Dropped) ->
{emqx_channel_info,
#{clientinfo =>
#{ clientid => <<"abcde">>, username => <<"abc\\name*[]()">>}},
[{mqueue_len, Len}, {mqueue_max,1000}, {mqueue_dropped, Dropped}]
}
end,
false = run_fuzzy_match(Fun(0, 100), [{mqueue_len, '>=', 1}]),
true = run_fuzzy_match(Fun(1, 100), [{mqueue_len, '>=', 1}]),
false = run_fuzzy_match(Fun(1, 100), [{mqueue_len, '>=', 2}]),
true = run_fuzzy_match(Fun(99, 100), [{mqueue_len, '=<', 100}, {mqueue_len, '>=', 98}]),
false = run_fuzzy_match(Fun(99, 100), [{mqueue_len, '=<', 101}, {mqueue_len, '>=', 100}]),
false = run_fuzzy_match(Fun(1000, 0), [{mqueue_dropped, '>=', 1}]),
true = run_fuzzy_match(Fun(1000, 1), [{mqueue_dropped, '>=', 1}]),
false = run_fuzzy_match(Fun(1000, 1), [{mqueue_dropped, '>=', 2}]),
true = run_fuzzy_match(Fun(1000, 99), [{mqueue_dropped, '=<', 100}, {mqueue_dropped, '>=', 98}]),
false = run_fuzzy_match(Fun(1000, 99), [{mqueue_dropped, '=<', 98}, {mqueue_dropped, '>=', 97}]),
false = run_fuzzy_match(Fun(1000, 102), [{mqueue_dropped, '=<', 104}, {mqueue_dropped, '>=', 103}]),
true = run_fuzzy_match(Fun(1000, 103), [{mqueue_dropped, '=<', 104}, {mqueue_dropped, '>=', 103},
{mqueue_len, '>=', 1000}]),
false = run_fuzzy_match(Fun(1000, 199), [{mqueue_dropped, '>=', 198},
{mqueue_len, '=<', 99}, {mqueue_len, '>=', 1}]),
ok.
-endif.

2
apps/emqx_prometheus/src/emqx_prometheus.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_prometheus,
[{description, "Prometheus for EMQ X"},
{vsn, "4.3.0"}, % strict semver, bump manually!
{vsn, "4.3.1"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_prometheus_sup]},
{applications, [kernel,stdlib,prometheus]},

9
apps/emqx_prometheus/src/emqx_prometheus.appup.src Normal file
View File

@ -0,0 +1,9 @@
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[{"4.3.0",
[{load_module,emqx_prometheus,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.3.0",
[{load_module,emqx_prometheus,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}]}.

6
apps/emqx_prometheus/src/emqx_prometheus.erl
View File

@ -412,8 +412,8 @@ emqx_collect(emqx_client_connected, Stats) ->
counter_metric(?C('client.connected', Stats));
emqx_collect(emqx_client_authenticate, Stats) ->
counter_metric(?C('client.authenticate', Stats));
emqx_collect(emqx_client_auth_anonymous, Stats) ->
counter_metric(?C('client.auth.anonymous', Stats));
emqx_collect(emqx_client_auth_success_anonymous, Stats) ->
counter_metric(?C('client.auth.success.anonymous', Stats));
emqx_collect(emqx_client_check_acl, Stats) ->
counter_metric(?C('client.check_acl', Stats));
emqx_collect(emqx_client_subscribe, Stats) ->
@ -566,7 +566,7 @@ emqx_metrics_delivery() ->
emqx_metrics_client() ->
[ emqx_client_connected
, emqx_client_authenticate
, emqx_client_auth_anonymous
, emqx_client_auth_success_anonymous
, emqx_client_check_acl
, emqx_client_subscribe
, emqx_client_unsubscribe

248
apps/emqx_rule_engine/src/emqx_rule_date.erl Normal file
View File

@ -0,0 +1,248 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_rule_date).
-export([date/3, date/4, parse_date/4]).
-export([
is_int_char/1,
is_symbol_char/1,
is_m_char/1
]).
-record(result, {
%%year()
year = "1970" :: string(),
%%month()
month = "1" :: string(),
%%day()
day = "1" :: string(),
%%hour()
hour = "0" :: string(),
%%minute() %% epoch in millisecond precision
minute = "0" :: string(),
%%second() %% epoch in millisecond precision
second = "0" :: string(),
%%integer() %% zone maybe some value
zone = "+00:00" :: string()
}).
%% -type time_unit() :: 'microsecond'
%% | 'millisecond'
%% | 'nanosecond'
%% | 'second'.
%% -type offset() :: [byte()] | (Time :: integer()).
date(TimeUnit, Offset, FormatString) ->
date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)).
date(TimeUnit, Offset, FormatString, TimeEpoch) ->
[Head | Other] = string:split(FormatString, "%", all),
R = create_tag([{st, Head}], Other),
Res = lists:map(
fun(Expr) ->
eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr)
end,
R
),
lists:concat(Res).
parse_date(TimeUnit, Offset, FormatString, InputString) ->
[Head | Other] = string:split(FormatString, "%", all),
R = create_tag([{st, Head}], Other),
IsZ = fun(V) ->
case V of
{tag, $Z} -> true;
_ -> false
end
end,
R1 = lists:filter(IsZ, R),
IfFun = fun(Con, A, B) ->
case Con of
[] -> A;
_ -> B
end
end,
Res = parse_input(FormatString, InputString),
Str =
Res#result.year ++ "-" ++
Res#result.month ++ "-" ++
Res#result.day ++ "T" ++
Res#result.hour ++ ":" ++
Res#result.minute ++ ":" ++
Res#result.second ++
IfFun(R1, Offset, Res#result.zone),
calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]).
mlist(R) ->
%% %H Shows hour in 24-hour format [15]
[
{$H, R#result.hour},
%% %M Displays minutes [00-59]
{$M, R#result.minute},
%% %S Displays seconds [00-59]
{$S, R#result.second},
%% %y Displays year YYYY [2021]
{$y, R#result.year},
%% %m Displays the number of the month [01-12]
{$m, R#result.month},
%% %d Displays the number of the month [01-12]
{$d, R#result.day},
%% %Z Displays Time zone
{$Z, R#result.zone}
].
rmap(Result) ->
maps:from_list(mlist(Result)).
support_char() -> "HMSymdZ".
create_tag(Head, []) ->
Head;
create_tag(Head, [Val1 | RVal]) ->
case Val1 of
[] ->
create_tag(Head ++ [{st, [$%]}], RVal);
[H | Other] ->
case lists:member(H, support_char()) of
true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal);
false -> create_tag(Head ++ [{st, [$% | Val1]}], RVal)
end
end.
eval_tag(_, {st, Str}) ->
Str;
eval_tag(Map, {tag, Char}) ->
maps:get(Char, Map, "undefined").
%% make_time(TimeUnit, Offset) ->
%% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)).
make_time(TimeUnit, Offset, TimeEpoch) ->
Res = calendar:system_time_to_rfc3339(TimeEpoch,
[{unit, TimeUnit}, {offset, Offset}]),
[Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T,
H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = Res,
IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end,
{FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr),
#result{
year = [Y1, Y2, Y3, Y4]
, month = [Mon1, Mon2]
, day = [D1, D2]
, hour = [H1, H2]
, minute = [Min1, Min2]
, second = [S1, S2] ++ FractionStr
, zone = UtcOffset
}.
is_int_char(C) ->
C >= $0 andalso C =< $9.
is_symbol_char(C) ->
C =:= $- orelse C =:= $+.
is_m_char(C) ->
C =:= $:.
parse_char_with_fun(_, []) ->
error(null_input);
parse_char_with_fun(ValidFun, [C | Other]) ->
Res =
case erlang:is_function(ValidFun) of
true -> ValidFun(C);
false -> erlang:apply(emqx_rule_date, ValidFun, [C])
end,
case Res of
true -> {C, Other};
false -> error({unexpected, [C | Other]})
end.
parse_string([], Input) ->
{[], Input};
parse_string([C | Other], Input) ->
{C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input),
{Res, Input2} = parse_string(Other, Input1),
{[C1 | Res], Input2}.
parse_times(0, _, Input) ->
{[], Input};
parse_times(Times, Fun, Input) ->
{C1, Input1} = parse_char_with_fun(Fun, Input),
{Res, Input2} = parse_times((Times - 1), Fun, Input1),
{[C1 | Res], Input2}.
parse_int_times(Times, Input) ->
parse_times(Times, is_int_char, Input).
parse_fraction(Input) ->
IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end,
lists:splitwith(IsFractionChar, Input).
parse_second(Input) ->
{M, Input1} = parse_int_times(2, Input),
{M1, Input2} = parse_fraction(Input1),
{M ++ M1, Input2}.
parse_zone(Input) ->
{S, Input1} = parse_char_with_fun(is_symbol_char, Input),
{M, Input2} = parse_int_times(2, Input1),
{C, Input3} = parse_char_with_fun(is_m_char, Input2),
{V, Input4} = parse_int_times(2, Input3),
{[S | M ++ [C | V]], Input4}.
mlist1() ->
maps:from_list(
%% %H Shows hour in 24-hour format [15]
[
{$H, fun(Input) -> parse_int_times(2, Input) end},
%% %M Displays minutes [00-59]
{$M, fun(Input) -> parse_int_times(2, Input) end},
%% %S Displays seconds [00-59]
{$S, fun(Input) -> parse_second(Input) end},
%% %y Displays year YYYY [2021]
{$y, fun(Input) -> parse_int_times(4, Input) end},
%% %m Displays the number of the month [01-12]
{$m, fun(Input) -> parse_int_times(2, Input) end},
%% %d Displays the number of the month [01-12]
{$d, fun(Input) -> parse_int_times(2, Input) end},
%% %Z Displays Time zone
{$Z, fun(Input) -> parse_zone(Input) end}
]
).
update_result($H, Res, Str) -> Res#result{hour = Str};
update_result($M, Res, Str) -> Res#result{minute = Str};
update_result($S, Res, Str) -> Res#result{second = Str};
update_result($y, Res, Str) -> Res#result{year = Str};
update_result($m, Res, Str) -> Res#result{month = Str};
update_result($d, Res, Str) -> Res#result{day = Str};
update_result($Z, Res, Str) -> Res#result{zone = Str}.
parse_tag(Res, {st, St}, InputString) ->
{_A, B} = parse_string(St, InputString),
{Res, B};
parse_tag(Res, {tag, St}, InputString) ->
Fun = maps:get(St, mlist1()),
{A, B} = Fun(InputString),
NRes = update_result(St, Res, A),
{NRes, B}.
parse_tags(Res, [], _) ->
Res;
parse_tags(Res, [Tag | Others], InputString) ->
{NRes, B} = parse_tag(Res, Tag, InputString),
parse_tags(NRes, Others, B).
parse_input(FormatString, InputString) ->
[Head | Other] = string:split(FormatString, "%", all),
R = create_tag([{st, Head}], Other),
parse_tags(#result{}, R, InputString).

32
apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
View File

@ -2,17 +2,24 @@
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[{"4.4.3",
[{load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
[{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
{add_module,emqx_rule_date},
{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}]},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
{"4.4.2",
[{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
[{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
{add_module,emqx_rule_date},
{load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
@ -27,6 +34,7 @@
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
{add_module,emqx_rule_date},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
{"4.4.0",
[{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
@ -34,6 +42,7 @@
{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
{add_module,emqx_rule_date},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{update,emqx_rule_metrics,{advanced,["4.4.0"]}},
{load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@ -42,17 +51,24 @@
{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.4.3",
[{load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
[{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
{delete_module,emqx_rule_date},
{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}]},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
{"4.4.2",
[{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
[{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
{delete_module,emqx_rule_date},
{load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
@ -67,6 +83,7 @@
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
{delete_module,emqx_rule_date},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
{"4.4.0",
[{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
@ -79,5 +96,6 @@
{load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
{delete_module,emqx_rule_date}]},
{<<".*">>,[]}]}.

85
apps/emqx_rule_engine/src/emqx_rule_engine.erl
View File

@ -37,7 +37,9 @@
, test_resource/1
, start_resource/1
, get_resource_status/1
, is_source_alive/1
, is_resource_alive/1
, is_resource_alive/2
, is_resource_alive/3
, get_resource_params/1
, ensure_resource_deleted/1
, delete_resource/1
@ -46,7 +48,7 @@
-export([ init_resource/4
, init_action/4
, clear_resource/3
, clear_resource/4
, clear_rule/1
, clear_actions/1
, clear_action/3
@ -55,6 +57,13 @@
-export([ restore_action_metrics/2
]).
-export([ fetch_resource_status/3
]).
-ifdef(TEST).
-export([alarm_name_of_resource_down/2]).
-endif.
-type(rule() :: #rule{}).
-type(action() :: #action{}).
-type(resource() :: #resource{}).
@ -340,11 +349,11 @@ test_resource(#{type := Type} = Params) ->
try
case create_resource(maps:put(id, ResId, Params), no_retry) of
{ok, _} ->
case is_source_alive(ResId) of
case is_resource_alive(ResId, #{fetch => true}) of
true ->
ok;
false ->
%% in is_source_alive, the cluster-call RPC logs errors
%% in is_resource_alive, the cluster-call RPC logs errors
%% so we do not log anything here
{error, {resource_down, ResId}}
end;
@ -362,18 +371,56 @@ test_resource(#{type := Type} = Params) ->
{error, {resource_type_not_found, Type}}
end.
is_source_alive(ResId) ->
case rpc:multicall(ekka_mnesia:running_nodes(), ?MODULE, get_resource_status, [ResId], 5000) of
{ResL, []} ->
is_source_alive_(ResL);
{_, _Errors} ->
false
is_resource_alive(ResId) ->
is_resource_alive(ResId, #{fetch => false}).
is_resource_alive(ResId, Opts) ->
is_resource_alive(ekka_mnesia:running_nodes(), ResId, Opts).
-spec(is_resource_alive(list(node()) | node(), resource_id(), #{fetch := boolean()}) -> boolean()).
is_resource_alive(Node, ResId, Opts) when is_atom(Node) ->
is_resource_alive([Node], ResId, Opts);
is_resource_alive(Nodes, ResId, _Opts = #{fetch := true}) ->
try
case emqx_rule_registry:find_resource(ResId) of
{ok, #resource{type = ResType}} ->
{ok, #resource_type{on_status = {Mod, OnStatus}}}
= emqx_rule_registry:find_resource_type(ResType),
case rpc:multicall(Nodes,
?MODULE, fetch_resource_status, [Mod, OnStatus, ResId], 5000) of
{ResL, []} ->
is_resource_alive_(ResL);
{_, _Error} ->
false
end;
not_found ->
false
end
catch E:R:S ->
?LOG(warning, "is_resource_alive failed, ~0p:~0p ~0p", [E, R, S]),
false
end;
is_resource_alive(Nodes, ResId, _Opts = #{fetch := false}) ->
try
case rpc:multicall(Nodes, ?MODULE, get_resource_status, [ResId], 5000) of
{ResL, []} ->
is_resource_alive_(ResL);
{_, _Errors} ->
false
end
catch E:R:S ->
?LOG(warning, "is_resource_alive failed, ~0p:~0p ~0p", [E, R, S]),
false
end.
is_source_alive_([]) -> true;
is_source_alive_([{ok, #{is_alive := true}} | ResL]) -> is_source_alive_(ResL);
is_source_alive_([{ok, #{is_alive := false}} | _ResL]) -> false;
is_source_alive_([_Error | _ResL]) -> false.
%% fetch_resource_status -> #{is_alive => boolean()}
%% get_resource_status -> {ok, #{is_alive => boolean()}}
is_resource_alive_([]) -> true;
is_resource_alive_([#{is_alive := true} | ResL]) -> is_resource_alive_(ResL);
is_resource_alive_([#{is_alive := false} | _ResL]) -> false;
is_resource_alive_([{ok, #{is_alive := true}} | ResL]) -> is_resource_alive_(ResL);
is_resource_alive_([{ok, #{is_alive := false}} | _ResL]) -> false;
is_resource_alive_([_Error | _ResL]) -> false.
-spec(get_resource_status(resource_id()) -> {ok, resource_status()} | {error, Reason :: term()}).
get_resource_status(ResId) ->
@ -407,7 +454,7 @@ delete_resource(ResId) ->
try
case emqx_rule_registry:remove_resource(ResId) of
ok ->
_ = ?CLUSTER_CALL(clear_resource, [ModD, Destroy, ResId]),
_ = ?CLUSTER_CALL(clear_resource, [ModD, Destroy, ResId, ResType]),
ok;
{error, _} = R -> R
end
@ -609,9 +656,13 @@ init_action(Module, OnCreate, ActionInstId, Params) ->
#action_instance_params{id = ActionInstId, params = Params, apply = Apply})
end.
clear_resource(_Module, undefined, ResId) ->
clear_resource(_Module, undefined, Type, ResId) ->
Name = alarm_name_of_resource_down(Type, ResId),
_ = emqx_alarm:deactivate(Name),
ok = emqx_rule_registry:remove_resource_params(ResId);
clear_resource(Module, Destroy, ResId) ->
clear_resource(Module, Destroy, Type, ResId) ->
Name = alarm_name_of_resource_down(Type, ResId),
_ = emqx_alarm:deactivate(Name),
case emqx_rule_registry:find_resource_params(ResId) of
{ok, #resource_params{params = Params}} ->
?RAISE(Module:Destroy(ResId, Params),

18
apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
View File

@ -321,7 +321,7 @@ do_create_resource(Create, ParsedParams) ->
list_resources(#{}, _Params) ->
Data0 = lists:foldr(fun maybe_record_to_map/2, [], emqx_rule_registry:get_resources()),
Data = lists:map(fun(Res = #{id := ResId}) ->
Status = emqx_rule_engine:is_source_alive(ResId),
Status = emqx_rule_engine:is_resource_alive(ResId),
maps:put(status, Status, Res)
end, Data0),
return({ok, Data}).
@ -332,14 +332,14 @@ list_resources_by_type(#{type := Type}, _Params) ->
show_resource(#{id := Id}, _Params) ->
case emqx_rule_registry:find_resource(Id) of
{ok, R} ->
Status =
lists:concat(
[ case rpc:call(Node, emqx_rule_engine, get_resource_status, [Id]) of
{badrpc, _} -> [];
{ok, St} -> [maps:put(node, Node, St)];
{error, _} -> [maps:put(node, Node, #{is_alive => false})]
end
|| Node <- ekka_mnesia:running_nodes()]),
StatusFun =
fun(Node) ->
#{
node => Node,
is_alive => emqx_rule_engine:is_resource_alive(Node, Id, #{fetch => false})
}
end,
Status = [StatusFun(Node) || Node <- ekka_mnesia:running_nodes()],
return({ok, maps:put(status, Status, record_to_map(R))});
not_found ->
return({error, 404, <<"Not Found">>})

6
apps/emqx_rule_engine/src/emqx_rule_events.erl
View File

@ -224,6 +224,8 @@ eventmsg_disconnected(_ClientInfo = #{
ConnInfo = #{
peername := PeerName,
sockname := SockName,
proto_name := ProtoName,
proto_ver := ProtoVer,
disconnected_at := DisconnectedAt
}, Reason) ->
with_basic_columns('client.disconnected',
@ -232,6 +234,8 @@ eventmsg_disconnected(_ClientInfo = #{
username => Username,
peername => ntoa(PeerName),
sockname => ntoa(SockName),
proto_name => ProtoName,
proto_ver => ProtoVer,
disconn_props => printable_maps(maps:get(disconn_props, ConnInfo, #{})),
disconnected_at => DisconnectedAt
}).
@ -686,6 +690,8 @@ columns_with_exam('client.disconnected') ->
, {<<"username">>, <<"u_emqx">>}
, {<<"peername">>, <<"192.168.0.10:56431">>}
, {<<"sockname">>, <<"0.0.0.0:1883">>}
, {<<"proto_name">>, <<"MQTT">>}
, {<<"proto_ver">>, 5}
, {<<"disconnected_at">>, erlang:system_time(millisecond)}
, columns_example_props(disconn_props)
, {<<"timestamp">>, erlang:system_time(millisecond)}

26
apps/emqx_rule_engine/src/emqx_rule_funcs.erl
View File

@ -96,6 +96,7 @@
, bool/1
, int/1
, float/1
, float2str/2
, map/1
, bin2hexstr/1
, hexstr2bin/1
@ -201,6 +202,9 @@
, now_rfc3339/1
, unix_ts_to_rfc3339/1
, unix_ts_to_rfc3339/2
, format_date/3
, format_date/4
, date_to_unix_ts/4
, rfc3339_to_unix_ts/1
, rfc3339_to_unix_ts/2
, now_timestamp/0
@ -538,6 +542,9 @@ int(Data) ->
float(Data) ->
emqx_rule_utils:float(Data).
float2str(Float, Precision) ->
emqx_rule_utils:float2str(Float, Precision).
map(Data) ->
emqx_rule_utils:map(Data).
@ -924,6 +931,25 @@ time_unit(<<"millisecond">>) -> millisecond;
time_unit(<<"microsecond">>) -> microsecond;
time_unit(<<"nanosecond">>) -> nanosecond.
format_date(TimeUnit, Offset, FormatString) ->
emqx_rule_utils:bin(
emqx_rule_date:date(time_unit(TimeUnit),
emqx_rule_utils:str(Offset),
emqx_rule_utils:str(FormatString))).
format_date(TimeUnit, Offset, FormatString, TimeEpoch) ->
emqx_rule_utils:bin(
emqx_rule_date:date(time_unit(TimeUnit),
emqx_rule_utils:str(Offset),
emqx_rule_utils:str(FormatString),
TimeEpoch)).
date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) ->
emqx_rule_date:parse_date(time_unit(TimeUnit),
emqx_rule_utils:str(Offset),
emqx_rule_utils:str(FormatString),
emqx_rule_utils:str(InputString)).
mongo_date() ->
erlang:timestamp().

4
apps/emqx_rule_engine/src/emqx_rule_utils.erl
View File

@ -32,6 +32,7 @@
%% type converting
-export([ str/1
, float2str/2
, bin/1
, bool/1
, int/1
@ -265,6 +266,9 @@ str(List) when is_list(List) ->
end;
str(Data) -> error({invalid_str, Data}).
float2str(Float, Precision) when is_float(Float) and is_integer(Precision)->
float_to_binary(Float, [{decimals, Precision}, compact]).
utf8_bin(Str) when is_binary(Str); is_list(Str) ->
unicode:characters_to_binary(Str);
utf8_bin(Str) ->

20
apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
View File

@ -165,7 +165,7 @@ end_per_suite(_Config) ->
on_resource_create(_id, _) -> #{}.
on_resource_destroy(_id, _) -> ok.
on_get_resource_status(_id, _) -> #{}.
on_get_resource_status(_id, _) -> #{is_alive => true}.
%%------------------------------------------------------------------------------
%% Group specific setup/teardown
@ -329,6 +329,24 @@ t_create_resource(_Config) ->
emqx_rule_registry:remove_resource(ResId),
ok.
t_clean_resource_alarms(_Config) ->
ok = emqx_rule_engine:load_providers(),
{ok, #resource{id = ResId}} = emqx_rule_engine:create_resource(
#{type => built_in,
config => #{},
description => <<"debug resource">>}),
?assert(true, is_binary(ResId)),
Name = emqx_rule_engine:alarm_name_of_resource_down(built_in, ResId),
_ = emqx_alarm:activate(Name, #{id => ResId, type => built_in}),
AlarmExist = fun(#{name := AName}) -> AName == Name end,
Len = length(lists:filter(AlarmExist, emqx_alarm:get_alarms())),
?assert(Len == 1),
ok = emqx_rule_engine:unload_providers(),
emqx_rule_registry:remove_resource(ResId),
LenAfterRemove = length(lists:filter(AlarmExist, emqx_alarm:get_alarms())),
?assert(LenAfterRemove == 0),
ok.
%%------------------------------------------------------------------------------
%% Test cases for rule actions
%%------------------------------------------------------------------------------

34
apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl
View File

@ -125,6 +125,13 @@ t_float(_) ->
?assertError({invalid_number, {a, v}}, emqx_rule_funcs:float({a, v})),
?assertError(_, emqx_rule_funcs:float("a")).
t_float2str(_) ->
?assertEqual(<<"20.2">>, emqx_rule_funcs:float2str(20.2, 1)),
?assertEqual(<<"20.2">>, emqx_rule_funcs:float2str(20.2, 10)),
?assertEqual(<<"20.199999999999999">>, emqx_rule_funcs:float2str(20.2, 15)),
?assertEqual(<<"20.1999999999999993">>, emqx_rule_funcs:float2str(20.2, 16)).
t_map(_) ->
?assertEqual(#{ver => <<"1.0">>, name => "emqx"}, emqx_rule_funcs:map([{ver, <<"1.0">>}, {name, "emqx"}])),
?assertEqual(#{<<"a">> => 1}, emqx_rule_funcs:map(<<"{\"a\":1}">>)),
@ -159,8 +166,12 @@ t_term_encode(_) ->
end, TestData).
t_hexstr2bin(_) ->
?assertEqual(<<1,2>>, emqx_rule_funcs:hexstr2bin(<<"0102">>)),
?assertEqual(<<17,33>>, emqx_rule_funcs:hexstr2bin(<<"1121">>)).
?assertEqual(<<6, 54, 79>>, emqx_rule_funcs:hexstr2bin(<<"6364f">>)),
?assertEqual(<<10>>, emqx_rule_funcs:hexstr2bin(<<"a">>)),
?assertEqual(<<15>>, emqx_rule_funcs:hexstr2bin(<<"f">>)),
?assertEqual(<<5>>, emqx_rule_funcs:hexstr2bin(<<"5">>)),
?assertEqual(<<1, 2>>, emqx_rule_funcs:hexstr2bin(<<"0102">>)),
?assertEqual(<<17, 33>>, emqx_rule_funcs:hexstr2bin(<<"1121">>)).
t_bin2hexstr(_) ->
?assertEqual(<<"0102">>, emqx_rule_funcs:bin2hexstr(<<1,2>>)),
@ -698,6 +709,25 @@ t_rfc3339_to_unix_ts(_) ->
?assertEqual(Epoch, emqx_rule_funcs:rfc3339_to_unix_ts(DateTime, BUnit))
end || Unit <- [second,millisecond,microsecond,nanosecond]].
t_format_date_funcs(_) ->
?PROPTEST(prop_format_date_fun).
prop_format_date_fun() ->
Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>],
?FORALL(S, erlang:system_time(second),
S == apply_func(date_to_unix_ts,
Args1 ++ [apply_func(format_date,
Args1 ++ [S])])),
Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>],
?FORALL(S, erlang:system_time(millisecond),
S == apply_func(date_to_unix_ts,
Args2 ++ [apply_func(format_date,
Args2 ++ [S])])),
Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>],
?FORALL(S, erlang:system_time(second),
S == apply_func(date_to_unix_ts,
Args ++ [apply_func(format_date,
Args ++ [S])])).
%%------------------------------------------------------------------------------
%% Utility functions
%%------------------------------------------------------------------------------

2
apps/emqx_sn/src/emqx_sn.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_sn,
[{description, "EMQ X MQTT-SN Plugin"},
{vsn, "4.3.6"}, % strict semver, bump manually!
{vsn, "4.3.7"}, % strict semver, bump manually!
{modules, []},
{registered, []},
{applications, [kernel,stdlib,esockd]},

6
apps/emqx_sn/src/emqx_sn.appup.src
View File

@ -1,6 +1,9 @@
%% -*- mode: erlang -*-
{VSN,
[
{"4.3.6",[
{load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]}
]},
{"4.3.5",[
{load_module,emqx_sn_frame,brutal_purge,soft_purge,[]},
{load_module,emqx_sn_app,brutal_purge,soft_purge,[]},
@ -26,6 +29,9 @@
{<<"4\\.3\\.[0-1]">>, [{restart_application,emqx_sn}]}
],
[
{"4.3.6",[
{load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]}
]},
{"4.3.5",[
{load_module,emqx_sn_frame,brutal_purge,soft_purge,[]},
{load_module,emqx_sn_app,brutal_purge,soft_purge,[]},

18
apps/emqx_sn/src/emqx_sn_gateway.erl
View File

@ -831,13 +831,15 @@ mqtt2sn(?CONNACK_PACKET(0, _SessPresent), _State) ->
mqtt2sn(?CONNACK_PACKET(_ReturnCode, _SessPresent), _State) ->
?SN_CONNACK_MSG(?SN_RC_CONGESTION);
mqtt2sn(?PUBREC_PACKET(MsgId), _State) ->
mqtt2sn(?PUBACK_PACKET(MsgId, _ReasonCode), _State) ->
TopicIdFinal = get_topic_id(puback, MsgId),
?SN_PUBACK_MSG(TopicIdFinal, MsgId, ?SN_RC_ACCEPTED);
mqtt2sn(?PUBREC_PACKET(MsgId, _ReturnCode), _State) ->
?SN_PUBREC_MSG(?SN_PUBREC, MsgId);
mqtt2sn(?PUBREL_PACKET(MsgId), _State) ->
mqtt2sn(?PUBREL_PACKET(MsgId, _ReturnCode), _State) ->
?SN_PUBREC_MSG(?SN_PUBREL, MsgId);
mqtt2sn(?PUBCOMP_PACKET(MsgId), _State) ->
mqtt2sn(?PUBCOMP_PACKET(MsgId, _ReturnCode), _State) ->
?SN_PUBREC_MSG(?SN_PUBCOMP, MsgId);
mqtt2sn(?UNSUBACK_PACKET(MsgId), _State)->
@ -884,11 +886,7 @@ mqtt2sn(?SUBACK_PACKET(MsgId, ReturnCodes), _State)->
{?QOS_0, get_topic_id(suback, MsgId), ?SN_RC_NOT_SUPPORTED}
end,
Flags = #mqtt_sn_flags{qos = QoS},
?SN_SUBACK_MSG(Flags, TopicId, MsgId, NewReturnCode);
mqtt2sn(?PUBACK_PACKET(MsgId, _ReasonCode), _State) ->
TopicIdFinal = get_topic_id(puback, MsgId),
?SN_PUBACK_MSG(TopicIdFinal, MsgId, ?SN_RC_ACCEPTED).
?SN_SUBACK_MSG(Flags, TopicId, MsgId, NewReturnCode).
send_register(TopicName, TopicId, MsgId, State) ->
send_message(?SN_REGISTER_MSG(TopicId, MsgId, TopicName), State).

38
apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl
View File

@ -36,7 +36,7 @@
-define(FLAG_RETAIN(X),X).
-define(FLAG_SESSION(X),X).
-define(LOG(Format, Args), ct:print("TEST: " ++ Format, Args)).
-define(LOG(Format, Args), ct:log("TEST: " ++ Format, Args)).
-define(MAX_PRED_TOPIC_ID, 2).
-define(PREDEF_TOPIC_ID1, 1).
@ -949,6 +949,42 @@ t_publish_qos2_case03(_) ->
?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
gen_udp:close(Socket).
t_publish_qos2_re_sent(_) ->
Dup = 0,
QoS = 2,
Retain = 0,
Will = 0,
CleanSession = 0,
MsgId = 7,
TopicId0 = 0,
{ok, Socket} = gen_udp:open(0, [binary]),
send_connect_msg(Socket, <<"test">>),
?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
send_subscribe_msg_normal_topic(Socket, QoS, <<"/#">>, MsgId),
?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>,
receive_response(Socket)),
Payload1 = <<20, 21, 22, 23>>,
send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/a">>, Payload1),
?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)),
?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC :2, <<"/a">>/binary, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)),
%% re-sent qos2 PUBLISH
send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/a">>, Payload1),
%% still receive PUBREC normally
?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)),
send_pubrel_msg(Socket, MsgId),
?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)),
timer:sleep(100),
send_disconnect_msg(Socket, undefined),
%% note: receiving DISCONNECT packet here means that the qos2 is not duplicated
%% publish to broker
?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
gen_udp:close(Socket).
t_delivery_qos1_register_invalid_topic_id(_) ->
Dup = 0,
QoS = 1,

4
bin/emqx
View File

@ -530,6 +530,10 @@ fi
NAME_TYPE="$(echo "$NAME_ARG" | awk '{print $1}')"
NAME="$(echo "$NAME_ARG" | awk '{print $2}')"
NODENAME="$(echo "$NAME" | awk -F'@' '{print $1}')"
if ! (echo "$NODENAME" | grep -q '^[0-9A-Za-z_\-]\+$'); then
echo "Invalid node name, should be of format '^[0-9A-Za-z_-]+$'."
exit 1
fi
export ESCRIPT_NAME="$NODENAME"
PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"

25
bin/install_upgrade.escript
View File

@ -5,6 +5,7 @@
-define(TIMEOUT, 300000).
-define(INFO(Fmt,Args), io:format(Fmt++"~n",Args)).
-define(SEMVER_RE, <<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">>).
-mode(compile).
@ -54,6 +55,7 @@ unpack(_, Args) ->
install({RelName, NameTypeArg, NodeName, Cookie}, Opts) ->
TargetNode = start_distribution(NodeName, NameTypeArg, Cookie),
Version = proplists:get_value(version, Opts),
validate_target_version(Version, TargetNode),
case unpack_release(RelName, TargetNode, Version) of
{ok, Vsn} ->
?INFO("Unpacked successfully: ~p.", [Vsn]),
@ -427,6 +429,29 @@ erts_vsn() ->
[ErtsVsn, _] = string:tokens(binary_to_list(Str), " "),
ErtsVsn.
validate_target_version(TargetVersion, TargetNode) ->
CurrentVersion = current_release_version(TargetNode),
case {get_major_minor_vsn(CurrentVersion), get_major_minor_vsn(TargetVersion)} of
{{Major, Minor}, {Major, Minor}} -> ok;
_ ->
?INFO("Cannot upgrade/downgrade to ~s from ~s~n"
"We only support relup between patch versions",
[TargetVersion, CurrentVersion]),
error({relup_not_allowed, unsupported_target_version})
end.
get_major_minor_vsn(Version) ->
Parts = parse_semver(Version),
[Major | Rem0] = Parts,
[Minor | _Rem1] = Rem0,
{Major, Minor}.
parse_semver(Version) ->
case re:run(Version, ?SEMVER_RE, [{capture, all_but_first, binary}]) of
{match, Parts} -> Parts;
nomatch -> error({invalid_semver, Version})
end.
str(A) when is_atom(A) ->
atom_to_list(A);
str(A) when is_binary(A) ->

2
etc/emqx_cloud/vm.args
View File

@ -115,4 +115,4 @@
-shutdown_time 30000
## patches dir
-pa {{ platform_data_dir }}/patches
-pa "{{ platform_data_dir }}/patches"

2
etc/emqx_edge/vm.args
View File

@ -113,4 +113,4 @@
-shutdown_time 10000
## patches dir
-pa {{ platform_data_dir }}/patches
-pa "{{ platform_data_dir }}/patches"

3
lib-ce/emqx_dashboard/priv/emqx_dashboard.schema
View File

@ -83,7 +83,7 @@
]}.
{mapping, "dashboard.listener.https.verify", "emqx_dashboard.listeners", [
{datatype, string}
{datatype, atom}
]}.
{mapping, "dashboard.listener.https.fail_if_no_peer_cert", "emqx_dashboard.listeners", [
@ -149,4 +149,3 @@
end
end, [http, https]))
end}.

2
lib-ce/emqx_telemetry/src/emqx_telemetry.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_telemetry,
[{description, "EMQ X Telemetry"},
{vsn, "4.3.2"}, % strict semver, bump manually!
{vsn, "4.3.3"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_telemetry_sup]},
{applications, [kernel,stdlib]},

4
lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src
View File

@ -1,13 +1,13 @@
%% -*- mode: erlang -*-
{VSN,
[
{<<"4\\.3\\.[0-1]">>, [
{<<"4\\.3\\.[0-2]">>, [
{load_module, emqx_telemetry, brutal_purge, soft_purge, []}
]},
{<<".*">>, []}
],
[
{<<"4\\.3\\.[0-1]">>, [
{<<"4\\.3\\.[0-2]">>, [
{load_module, emqx_telemetry, brutal_purge, soft_purge, []}
]},
{<<".*">>, []}

4
lib-ce/emqx_telemetry/src/emqx_telemetry.erl
View File

@ -367,7 +367,9 @@ report_telemetry(State = #state{url = URL}) ->
end.
httpc_request(Method, URL, Headers, Body) ->
httpc:request(Method, {URL, Headers, "application/json", Body}, [], []).
HTTPOptions = [{timeout, timer:seconds(10)}, {ssl, [{verify, verify_none}]}],
Options = [],
httpc:request(Method, {URL, Headers, "application/json", Body}, HTTPOptions, Options).
ignore_lib_apps(Apps) ->
LibApps = [kernel, stdlib, sasl, appmon, eldap, erts,

2
rebar.config
View File

@ -45,7 +45,7 @@
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.9"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.10"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.0"}}}
, {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}}

40
src/emqx.appup.src
View File

@ -2,14 +2,27 @@
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[{"4.4.3",
[{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
[{load_module,emqx_session,brutal_purge,soft_purge,[]},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
{load_module,emqx_frame,brutal_purge,soft_purge,[]},
{load_module,emqx_channel,brutal_purge,soft_purge,[]},
{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
{load_module,emqx_metrics,brutal_purge,soft_purge,[]},
{apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}},
{load_module,emqx_access_control,brutal_purge,soft_purge,[]},
{load_module,emqx_app,brutal_purge,soft_purge,[]},
{load_module,emqx_relup}]},
{"4.4.2",
[{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
[{load_module,emqx_session,brutal_purge,soft_purge,[]},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
{load_module,emqx_metrics,brutal_purge,soft_purge,[]},
{apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}},
{load_module,emqx_access_control,brutal_purge,soft_purge,[]},
{load_module,emqx_plugins,brutal_purge,soft_purge,[]},
{load_module,emqx_frame,brutal_purge,soft_purge,[]},
{load_module,emqx_channel,brutal_purge,soft_purge,[]},
{load_module,emqx_sys,brutal_purge,soft_purge,[]},
{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
{load_module,emqx,brutal_purge,soft_purge,[]},
@ -18,6 +31,8 @@
{load_module,emqx_relup}]},
{"4.4.1",
[{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
{load_module,emqx_metrics,brutal_purge,soft_purge,[]},
{apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}},
{load_module,emqx,brutal_purge,soft_purge,[]},
{load_module,emqx_hooks,brutal_purge,soft_purge,[]},
{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
@ -61,6 +76,7 @@
{load_module,emqx_connection,brutal_purge,soft_purge,[]},
{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
{load_module,emqx_metrics,brutal_purge,soft_purge,[]},
{apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}},
{apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}},
{load_module,emqx_access_control,brutal_purge,soft_purge,[]},
{load_module,emqx_alarm,brutal_purge,soft_purge,[]},
@ -72,14 +88,25 @@
{load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.4.3",
[{load_module,emqx_relup,brutal_purge,soft_purge,[]},
{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
[{load_module,emqx_session,brutal_purge,soft_purge,[]},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
{load_module,emqx_frame,brutal_purge,soft_purge,[]},
{load_module,emqx_app,brutal_purge,soft_purge,[]}]},
{load_module,emqx_channel,brutal_purge,soft_purge,[]},
{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
{load_module,emqx_metrics,brutal_purge,soft_purge,[]},
{load_module,emqx_access_control,brutal_purge,soft_purge,[]},
{load_module,emqx_app,brutal_purge,soft_purge,[]},
{load_module,emqx_relup}]},
{"4.4.2",
[{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
[{load_module,emqx_session,brutal_purge,soft_purge,[]},
{load_module,emqx_misc,brutal_purge,soft_purge,[]},
{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
{load_module,emqx_metrics,brutal_purge,soft_purge,[]},
{load_module,emqx_access_control,brutal_purge,soft_purge,[]},
{load_module,emqx_plugins,brutal_purge,soft_purge,[]},
{load_module,emqx_frame,brutal_purge,soft_purge,[]},
{load_module,emqx_channel,brutal_purge,soft_purge,[]},
{load_module,emqx_sys,brutal_purge,soft_purge,[]},
{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
{load_module,emqx,brutal_purge,soft_purge,[]},
@ -88,6 +115,7 @@
{load_module,emqx_relup}]},
{"4.4.1",
[{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
{load_module,emqx_metrics,brutal_purge,soft_purge,[]},
{load_module,emqx,brutal_purge,soft_purge,[]},
{load_module,emqx_hooks,brutal_purge,soft_purge,[]},
{load_module,emqx_listeners,brutal_purge,soft_purge,[]},

55
src/emqx_access_control.erl
View File

@ -34,11 +34,15 @@
-spec(authenticate(emqx_types:clientinfo()) -> {ok, result()} | {error, term()}).
authenticate(ClientInfo = #{zone := Zone}) ->
AuthResult = default_auth_result(Zone),
case emqx_zone:get_env(Zone, bypass_auth_plugins, false) of
case
begin ok = emqx_metrics:inc('client.authenticate'),
emqx_zone:get_env(Zone, bypass_auth_plugins, false)
end
of
true ->
return_auth_result(AuthResult);
false ->
return_auth_result(run_hooks('client.authenticate', [ClientInfo], AuthResult))
return_auth_result(emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult))
end.
%% @doc Check ACL
@ -52,6 +56,10 @@ check_acl(ClientInfo, PubSub, Topic) ->
inc_acl_metrics(Result),
Result.
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
%% ACL
check_acl_cache(ClientInfo, PubSub, Topic) ->
case emqx_acl_cache:get_acl_cache(PubSub, Topic) of
not_found ->
@ -66,23 +74,14 @@ check_acl_cache(ClientInfo, PubSub, Topic) ->
do_check_acl(ClientInfo = #{zone := Zone}, PubSub, Topic) ->
Default = emqx_zone:get_env(Zone, acl_nomatch, deny),
Result = case run_hooks('client.check_acl', [ClientInfo, PubSub, Topic], Default) of
allow -> allow;
_Other -> deny
ok = emqx_metrics:inc('client.check_acl'),
Result = case emqx_hooks:run_fold('client.check_acl', [ClientInfo, PubSub, Topic], Default) of
allow -> allow;
_Other -> deny
end,
emqx:run_hook('client.check_acl_complete', [ClientInfo, PubSub, Topic, Result, false]),
Result.
default_auth_result(Zone) ->
case emqx_zone:get_env(Zone, allow_anonymous, false) of
true -> #{auth_result => success, anonymous => true};
false -> #{auth_result => not_authorized, anonymous => false}
end.
-compile({inline, [run_hooks/3]}).
run_hooks(Name, Args, Acc) ->
ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc).
-compile({inline, [inc_acl_metrics/1]}).
inc_acl_metrics(allow) ->
emqx_metrics:inc('client.acl.allow');
@ -91,8 +90,26 @@ inc_acl_metrics(deny) ->
inc_acl_metrics(cache_hit) ->
emqx_metrics:inc('client.acl.cache_hit').
%% Auth
default_auth_result(Zone) ->
case emqx_zone:get_env(Zone, allow_anonymous, false) of
true -> #{auth_result => success, anonymous => true};
false -> #{auth_result => not_authorized, anonymous => false}
end.
-compile({inline, [return_auth_result/1]}).
return_auth_result(Result = #{auth_result := success}) ->
{ok, Result};
return_auth_result(Result) ->
{error, maps:get(auth_result, Result, unknown_error)}.
return_auth_result(AuthResult = #{auth_result := success}) ->
inc_auth_success_metrics(AuthResult),
{ok, AuthResult};
return_auth_result(AuthResult) ->
emqx_metrics:inc('client.auth.failure'),
{error, maps:get(auth_result, AuthResult, unknown_error)}.
-compile({inline, [inc_auth_success_metrics/1]}).
inc_auth_success_metrics(AuthResult) ->
is_anonymous(AuthResult) andalso
emqx_metrics:inc('client.auth.success.anonymous'),
emqx_metrics:inc('client.auth.success').
is_anonymous(#{anonymous := true}) -> true;
is_anonymous(_AuthResult) -> false.

27
src/emqx_channel.erl
View File

@ -1254,13 +1254,18 @@ check_connect(ConnPkt, #channel{clientinfo = #{zone := Zone}}) ->
%% Enrich Client Info
enrich_client(ConnPkt, Channel = #channel{clientinfo = ClientInfo}) ->
{ok, NConnPkt, NClientInfo} = pipeline([fun set_username/2,
fun set_bridge_mode/2,
fun maybe_username_as_clientid/2,
fun maybe_assign_clientid/2,
fun fix_mountpoint/2
], ConnPkt, ClientInfo),
{ok, NConnPkt, Channel#channel{clientinfo = NClientInfo}}.
Pipe = pipeline([fun set_username/2,
fun set_bridge_mode/2,
fun maybe_username_as_clientid/2,
fun maybe_assign_clientid/2,
fun fix_mountpoint/2
], ConnPkt, ClientInfo),
case Pipe of
{ok, NConnPkt, NClientInfo} ->
{ok, NConnPkt, Channel#channel{clientinfo = NClientInfo}};
{error, ReasonCode, NClientInfo} ->
{error, ReasonCode, Channel#channel{clientinfo = NClientInfo}}
end.
set_username(#mqtt_packet_connect{username = Username},
ClientInfo = #{username := undefined}) ->
@ -1275,7 +1280,8 @@ maybe_username_as_clientid(_ConnPkt, ClientInfo = #{username := undefined}) ->
{ok, ClientInfo};
maybe_username_as_clientid(_ConnPkt, ClientInfo = #{zone := Zone, username := Username}) ->
case emqx_zone:use_username_as_clientid(Zone) of
true -> {ok, ClientInfo#{clientid => Username}};
true when Username =/= <<>> -> {ok, ClientInfo#{clientid => Username}};
true -> {error, ?RC_CLIENT_IDENTIFIER_NOT_VALID, ClientInfo};
false -> ok
end.
@ -1317,8 +1323,6 @@ auth_connect(#mqtt_packet_connect{password = Password},
username := Username} = ClientInfo,
case emqx_access_control:authenticate(ClientInfo#{password => Password}) of
{ok, AuthResult} ->
is_anonymous(AuthResult) andalso
emqx_metrics:inc('client.auth.anonymous'),
NClientInfo = maps:merge(ClientInfo, AuthResult),
{ok, Channel#channel{clientinfo = NClientInfo}};
{error, Reason} ->
@ -1327,9 +1331,6 @@ auth_connect(#mqtt_packet_connect{password = Password},
{error, emqx_reason_codes:connack_error(Reason)}
end.
is_anonymous(#{anonymous := true}) -> true;
is_anonymous(_AuthResult) -> false.
%%--------------------------------------------------------------------
%% Enhanced Authentication

44
src/emqx_metrics.erl
View File

@ -68,8 +68,10 @@
%% BACKW
-export([%% v4.3.0
upgrade_retained_delayed_counter_type/0,
%% e4.4.0, e4.3.0-e4.3.6, v4.3.0-v4.3.11
assign_acl_stats_from_ets_to_counter/0
%% v4.3.0-v4.3.11, e4.3.0-e4.3.6; v4.4.0, e4.4.0
assign_acl_stats_from_ets_to_counter/0,
%% v4.3.0-v4.3.14, e4.3.0-e4.3.9; v4.4.0-v4.4.3, e4.4.0-e4.4.3,
assign_auth_stats_from_ets_to_counter/0
]).
-export_type([metric_idx/0]).
@ -174,7 +176,6 @@
{counter, 'client.connack'},
{counter, 'client.connected'},
{counter, 'client.authenticate'},
{counter, 'client.auth.anonymous'},
{counter, 'client.check_acl'},
{counter, 'client.subscribe'},
{counter, 'client.unsubscribe'},
@ -189,8 +190,16 @@
{counter, 'session.discarded'},
{counter, 'session.terminated'}
]).
%% Statistic metrics for ACL checking
-define(STASTS_ACL_METRICS,
%% Statistic metrics for auth checking
-define(STATS_AUTH_METRICS,
[ {counter, 'client.auth.success'},
{counter, 'client.auth.success.anonymous'},
{counter, 'client.auth.failure'}
]).
%% Statistic metrics for ACL checking stats
-define(STATS_ACL_METRICS,
[ {counter, 'client.acl.allow'},
{counter, 'client.acl.deny'},
{counter, 'client.acl.cache_hit'}
@ -228,6 +237,21 @@ assign_acl_stats_from_ets_to_counter() ->
ok = counters:put(CRef, Idx, Val)
end, Names).
%% BACKW: %% v4.3.0-v4.3.14, e4.3.0-e4.3.9; v4.4.0-v4.4.3, e4.4.0-e4.4.3,
assign_auth_stats_from_ets_to_counter() ->
CRef = persistent_term:get(?MODULE),
Names = ['client.auth.success', 'client.auth.success.anonymous', 'client.auth.failure'],
lists:foreach(fun(Name) ->
Val = case emqx_metrics:val(Name) of
undefined -> 0;
Val0 -> Val0
end,
Idx = reserved_idx(Name),
Metric = #metric{name = Name, type = counter, idx = Idx},
ok = gen_server:call(?SERVER, {set, Metric}),
ok = counters:put(CRef, Idx, Val)
end, Names).
%%--------------------------------------------------------------------
%% Metrics API
%%--------------------------------------------------------------------
@ -458,7 +482,8 @@ init([]) ->
?DELIVERY_METRICS,
?CLIENT_METRICS,
?SESSION_METRICS,
?STASTS_ACL_METRICS
?STATS_AUTH_METRICS,
?STATS_ACL_METRICS
]),
% Store reserved indices
ok = lists:foreach(fun({Type, Name}) ->
@ -592,7 +617,6 @@ reserved_idx('client.connack') -> 201;
reserved_idx('client.connected') -> 202;
reserved_idx('client.authenticate') -> 203;
reserved_idx('client.enhanced_authenticate') -> 204;
reserved_idx('client.auth.anonymous') -> 205;
reserved_idx('client.check_acl') -> 206;
reserved_idx('client.subscribe') -> 207;
reserved_idx('client.unsubscribe') -> 208;
@ -604,9 +628,13 @@ reserved_idx('session.takeovered') -> 222;
reserved_idx('session.discarded') -> 223;
reserved_idx('session.terminated') -> 224;
%% Stats metrics
%% ACL
reserved_idx('client.acl.allow') -> 300;
reserved_idx('client.acl.deny') -> 301;
reserved_idx('client.acl.cache_hit') -> 302;
%% Auth
reserved_idx('client.auth.success') -> 310;
reserved_idx('client.auth.success.anonymous') -> 311;
reserved_idx('client.auth.failure') -> 312;
reserved_idx(_) -> undefined.

14
src/emqx_misc.erl
View File

@ -312,7 +312,19 @@ int2hexchar(I, lower) -> I - 10 + $a.
-spec(hexstr2bin(binary()) -> binary()).
hexstr2bin(B) when is_binary(B) ->
<< <<(hexchar2int(H)*16 + hexchar2int(L))>> || <<H:8, L:8>> <= B>>.
hexstr2bin(B, erlang:bit_size(B)).
hexstr2bin(B, Size) when is_binary(B) ->
case Size rem 16 of
0 ->
make_binary(B);
8 ->
make_binary(<<"0", B/binary>>);
_ ->
throw({unsupport_hex_string, B, Size})
end.
make_binary(B) -> <<<<(hexchar2int(H) * 16 + hexchar2int(L))>> || <<H:8, L:8>> <= B>>.
hexchar2int(I) when I >= $0 andalso I =< $9 -> I - $0;
hexchar2int(I) when I >= $A andalso I =< $F -> I - $A + 10;

60
src/emqx_session.erl
View File

@ -491,10 +491,13 @@ deliver_msg(ClientInfo, Msg = #message{qos = QoS}, Session =
end,
{ok, Session1};
false ->
%% Note that we publish message without shared ack header
%% But add to inflight with ack headers
%% This ack header is required for redispatch-on-terminate feature to work
Publish = {PacketId, maybe_ack(Msg)},
Msg2 = mark_begin_deliver(Msg),
Session1 = await(PacketId, Msg2, Session),
{ok, [Publish], next_pkt_id(Session1)}
Inflight1 = emqx_inflight:insert(PacketId, with_ts(Msg2), Inflight),
{ok, [Publish], next_pkt_id(Session#session{inflight = Inflight1})}
end.
-spec(enqueue(emqx_types:clientinfo(), list(emqx_types:deliver()) | emqx_types:message(),
@ -532,14 +535,10 @@ enrich_delivers({deliver, Topic, Msg}, Session = #session{subscriptions = Subs})
enrich_subopts(get_subopts(Topic, Subs), Msg, Session).
maybe_ack(Msg) ->
case emqx_shared_sub:is_ack_required(Msg) of
true -> emqx_shared_sub:maybe_ack(Msg);
false -> Msg
end.
emqx_shared_sub:maybe_ack(Msg).
maybe_nack(Msg) ->
emqx_shared_sub:is_ack_required(Msg)
andalso (ok == emqx_shared_sub:maybe_nack_dropped(Msg)).
emqx_shared_sub:maybe_nack_dropped(Msg).
get_subopts(Topic, SubMap) ->
case maps:find(Topic, SubMap) of
@ -572,14 +571,6 @@ enrich_subopts([{subid, SubId} | Opts], Msg, Session) ->
Msg1 = emqx_message:set_header(properties, Props#{'Subscription-Identifier' => SubId}, Msg),
enrich_subopts(Opts, Msg1, Session).
%%--------------------------------------------------------------------
%% Awaiting ACK for QoS1/QoS2 Messages
%%--------------------------------------------------------------------
await(PacketId, Msg, Session = #session{inflight = Inflight}) ->
Inflight1 = emqx_inflight:insert(PacketId, with_ts(Msg), Inflight),
Session#session{inflight = Inflight1}.
%%--------------------------------------------------------------------
%% Retry Delivery
%%--------------------------------------------------------------------
@ -679,16 +670,43 @@ replay(ClientInfo, Session = #session{inflight = Inflight}) ->
end.
-spec(terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok).
terminate(ClientInfo, discarded, Session) ->
run_hook('session.discarded', [ClientInfo, info(Session)]);
terminate(ClientInfo, takeovered, Session) ->
run_hook('session.takeovered', [ClientInfo, info(Session)]);
terminate(ClientInfo, Reason, Session) ->
run_terminate_hooks(ClientInfo, Reason, Session),
redispatch_shared_messages(Session),
ok.
run_terminate_hooks(ClientInfo, discarded, Session) ->
run_hook('session.discarded', [ClientInfo, info(Session)]);
run_terminate_hooks(ClientInfo, takeovered, Session) ->
run_hook('session.takeovered', [ClientInfo, info(Session)]);
run_terminate_hooks(ClientInfo, Reason, Session) ->
run_hook('session.terminated', [ClientInfo, Reason, info(Session)]).
redispatch_shared_messages(#session{inflight = Inflight}) ->
InflightList = emqx_inflight:to_list(Inflight),
lists:foreach(fun
%% Only QoS1 messages get redispatched, because QoS2 messages
%% must be sent to the same client, once they're in flight
({_, {#message{qos = ?QOS_2} = Msg, _}}) ->
?LOG(warning, "Not redispatching qos2 msg: ~s", [emqx_message:format(Msg)]);
({_, {#message{topic = Topic, qos = ?QOS_1} = Msg, _}}) ->
case emqx_shared_sub:get_group(Msg) of
{ok, Group} ->
%% Note that dispatch is called with self() in failed subs
%% This is done to avoid dispatching back to caller
Delivery = #delivery{sender = self(), message = Msg},
emqx_shared_sub:dispatch(Group, Topic, Delivery, [self()]);
_ ->
false
end;
(_) ->
ok
end, InflightList).
-compile({inline, [run_hook/2]}).
run_hook(Name, Args) ->
ok = emqx_metrics:inc(Name), emqx_hooks:run(Name, Args).
ok = emqx_metrics:inc(Name),
emqx_hooks:run(Name, Args).
%%--------------------------------------------------------------------
%% Inc message/delivery expired counter

70
src/emqx_shared_sub.erl
View File

@ -38,12 +38,15 @@
, unsubscribe/3
]).
-export([dispatch/3]).
-export([ dispatch/3
, dispatch/4
]).
-export([ maybe_ack/1
, maybe_nack_dropped/1
, nack_no_connection/1
, is_ack_required/1
, get_group/1
]).
%% for testing
@ -131,7 +134,7 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) ->
false ->
{error, no_subscribers};
{Type, SubPid} ->
case do_dispatch(SubPid, Topic, Msg, Type) of
case do_dispatch(SubPid, Group, Topic, Msg, Type) of
ok -> {ok, 1};
{error, _Reason} ->
%% Failed to dispatch to this sub, try next.
@ -153,36 +156,33 @@ strategy(Group) ->
ack_enabled() ->
emqx:get_env(shared_dispatch_ack_enabled, false).
do_dispatch(SubPid, Topic, Msg, _Type) when SubPid =:= self() ->
do_dispatch(SubPid, _Group, Topic, Msg, _Type) when SubPid =:= self() ->
%% Deadlock otherwise
_ = erlang:send(SubPid, {deliver, Topic, Msg}),
SubPid ! {deliver, Topic, Msg},
ok;
do_dispatch(SubPid, Topic, Msg, Type) ->
dispatch_per_qos(SubPid, Topic, Msg, Type).
%% return either 'ok' (when everything is fine) or 'error'
dispatch_per_qos(SubPid, Topic, #message{qos = ?QOS_0} = Msg, _Type) ->
do_dispatch(SubPid, _Group, Topic, #message{qos = ?QOS_0} = Msg, _Type) ->
%% For QoS 0 message, send it as regular dispatch
_ = erlang:send(SubPid, {deliver, Topic, Msg}),
SubPid ! {deliver, Topic, Msg},
ok;
dispatch_per_qos(SubPid, Topic, Msg, retry) ->
do_dispatch(SubPid, _Group, Topic, Msg, retry) ->
%% Retry implies all subscribers nack:ed, send again without ack
_ = erlang:send(SubPid, {deliver, Topic, Msg}),
SubPid ! {deliver, Topic, Msg},
ok;
dispatch_per_qos(SubPid, Topic, Msg, fresh) ->
do_dispatch(SubPid, Group, Topic, Msg, fresh) ->
case ack_enabled() of
true ->
dispatch_with_ack(SubPid, Topic, Msg);
dispatch_with_ack(SubPid, Group, Topic, Msg);
false ->
_ = erlang:send(SubPid, {deliver, Topic, Msg}),
SubPid ! {deliver, Topic, Msg},
ok
end.
dispatch_with_ack(SubPid, Topic, Msg) ->
dispatch_with_ack(SubPid, Group, Topic, Msg) ->
%% For QoS 1/2 message, expect an ack
Ref = erlang:monitor(process, SubPid),
Sender = self(),
_ = erlang:send(SubPid, {deliver, Topic, with_ack_ref(Msg, {Sender, Ref})}),
SubPid ! {deliver, Topic, with_group_ack(Msg, Group, Sender, Ref)},
Timeout = case Msg#message.qos of
?QOS_1 -> timer:seconds(?SHARED_SUB_QOS1_DISPATCH_TIMEOUT_SECONDS);
?QOS_2 -> infinity
@ -204,24 +204,32 @@ dispatch_with_ack(SubPid, Topic, Msg) ->
_ = erlang:demonitor(Ref, [flush])
end.
with_ack_ref(Msg, SenderRef) ->
emqx_message:set_headers(#{shared_dispatch_ack => SenderRef}, Msg).
with_group_ack(Msg, Group, Sender, Ref) ->
emqx_message:set_headers(#{shared_dispatch_ack => {Group, Sender, Ref}}, Msg).
without_ack_ref(Msg) ->
-spec(without_group_ack(emqx_types:message()) -> emqx_types:message()).
without_group_ack(Msg) ->
emqx_message:set_headers(#{shared_dispatch_ack => ?NO_ACK}, Msg).
get_ack_ref(Msg) ->
get_group_ack(Msg) ->
emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK).
-spec(get_group(emqx_types:message()) -> {ok, any()} | error).
get_group(Msg) ->
case get_group_ack(Msg) of
?NO_ACK -> error;
{Group, _Sender, _Ref} -> {ok, Group}
end.
-spec(is_ack_required(emqx_types:message()) -> boolean()).
is_ack_required(Msg) -> ?NO_ACK =/= get_ack_ref(Msg).
is_ack_required(Msg) -> ?NO_ACK =/= get_group_ack(Msg).
%% @doc Negative ack dropped message due to inflight window or message queue being full.
-spec(maybe_nack_dropped(emqx_types:message()) -> ok).
-spec(maybe_nack_dropped(emqx_types:message()) -> boolean()).
maybe_nack_dropped(Msg) ->
case get_ack_ref(Msg) of
?NO_ACK -> ok;
{Sender, Ref} -> nack(Sender, Ref, dropped)
case get_group_ack(Msg) of
?NO_ACK -> false;
{_Group, Sender, Ref} -> ok == nack(Sender, Ref, dropped)
end.
%% @doc Negative ack message due to connection down.
@ -229,22 +237,22 @@ maybe_nack_dropped(Msg) ->
%% i.e is_ack_required returned true.
-spec(nack_no_connection(emqx_types:message()) -> ok).
nack_no_connection(Msg) ->
{Sender, Ref} = get_ack_ref(Msg),
{_Group, Sender, Ref} = get_group_ack(Msg),
nack(Sender, Ref, no_connection).
-spec(nack(pid(), reference(), dropped | no_connection) -> ok).
nack(Sender, Ref, Reason) ->
erlang:send(Sender, {Ref, ?NACK(Reason)}),
Sender ! {Ref, ?NACK(Reason)},
ok.
-spec(maybe_ack(emqx_types:message()) -> emqx_types:message()).
maybe_ack(Msg) ->
case get_ack_ref(Msg) of
case get_group_ack(Msg) of
?NO_ACK ->
Msg;
{Sender, Ref} ->
erlang:send(Sender, {Ref, ?ACK}),
without_ack_ref(Msg)
{_Group, Sender, Ref} ->
Sender ! {Ref, ?ACK},
without_group_ack(Msg)
end.
pick(sticky, ClientId, SourceTopic, Group, Topic, FailedSubs) ->

12
test/emqx_client_SUITE.erl
View File

@ -278,9 +278,15 @@ t_username_as_clientid(_) ->
{ok, C} = emqtt:start_link([{username, Username}]),
{ok, _} = emqtt:connect(C),
#{clientinfo := #{clientid := Username}} = emqx_cm:get_chan_info(Username),
emqtt:disconnect(C).
emqtt:disconnect(C),
erlang:process_flag(trap_exit, true),
{ok, C1} = emqtt:start_link([{username, <<>>}]),
?assertEqual({error, {client_identifier_not_valid, undefined}}, emqtt:connect(C1)),
receive
{'EXIT', _, {shutdown, client_identifier_not_valid}} -> ok
after 100 ->
throw({error, "expect_client_identifier_not_valid"})
end.
t_certcn_as_clientid_default_config_tls(_) ->
tls_certcn_as_clientid(default).

49
test/emqx_shared_sub_SUITE.erl
View File

@ -47,20 +47,20 @@ t_is_ack_required(_) ->
?assertEqual(false, emqx_shared_sub:is_ack_required(#message{headers = #{}})).
t_maybe_nack_dropped(_) ->
?assertEqual(ok, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})),
Msg = #message{headers = #{shared_dispatch_ack => {self(), for_test}}},
?assertEqual(ok, emqx_shared_sub:maybe_nack_dropped(Msg)),
?assertEqual(false, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})),
Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
?assertEqual(true, emqx_shared_sub:maybe_nack_dropped(Msg)),
?assertEqual(ok,receive {for_test, {shared_sub_nack, dropped}} -> ok after 100 -> timeout end).
t_nack_no_connection(_) ->
Msg = #message{headers = #{shared_dispatch_ack => {self(), for_test}}},
Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
?assertEqual(ok, emqx_shared_sub:nack_no_connection(Msg)),
?assertEqual(ok,receive {for_test, {shared_sub_nack, no_connection}} -> ok
after 100 -> timeout end).
t_maybe_ack(_) ->
?assertEqual(#message{headers = #{}}, emqx_shared_sub:maybe_ack(#message{headers = #{}})),
Msg = #message{headers = #{shared_dispatch_ack => {self(), for_test}}},
Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
?assertEqual(#message{headers = #{shared_dispatch_ack => ?no_ack}},
emqx_shared_sub:maybe_ack(Msg)),
?assertEqual(ok,receive {for_test, ?ack} -> ok after 100 -> timeout end).
@ -284,11 +284,14 @@ test_two_messages(Strategy, Group) ->
ok.
last_message(ExpectedPayload, Pids) ->
last_message(ExpectedPayload, Pids, 100).
last_message(ExpectedPayload, Pids, Timeout) ->
receive
{publish, #{client_pid := Pid, payload := ExpectedPayload}} ->
ct:pal("last_message: ~p ====== ~p, payload=~p", [Pids, Pid, ExpectedPayload]),
{true, Pid}
after 100 ->
after Timeout ->
ct:pal("not yet"),
<<"not yet?">>
end.
@ -410,6 +413,39 @@ t_local_fallback(_) ->
?assertEqual(UsedSubPid1, UsedSubPid2),
ok.
%% This one tests that broker tries to select another shared subscriber
%% If the first one doesn't return an ACK
t_redispatch(_) ->
ok = ensure_config(sticky, true),
application:set_env(emqx, shared_dispatch_ack_enabled, true),
Group = <<"group1">>,
Topic = <<"foo/bar">>,
ClientId1 = <<"ClientId1">>,
ClientId2 = <<"ClientId2">>,
{ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]),
{ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, false}]),
{ok, _} = emqtt:connect(ConnPid1),
{ok, _} = emqtt:connect(ConnPid2),
emqtt:subscribe(ConnPid1, {<<"$share/", Group/binary, "/foo/bar">>, 1}),
emqtt:subscribe(ConnPid2, {<<"$share/", Group/binary, "/foo/bar">>, 1}),
Message = emqx_message:make(ClientId1, 1, Topic, <<"hello1">>),
emqx:publish(Message),
{true, UsedSubPid1} = last_message(<<"hello1">>, [ConnPid1, ConnPid2]),
ok = emqtt:stop(UsedSubPid1),
Res = last_message(<<"hello1">>, [ConnPid1, ConnPid2], 6000),
?assertMatch({true, Pid} when Pid =/= UsedSubPid1, Res),
{true, UsedSubPid2} = Res,
emqtt:stop(UsedSubPid2),
ok.
%%--------------------------------------------------------------------
%% help functions
%%--------------------------------------------------------------------
@ -481,6 +517,7 @@ setup_node(Node, Port) ->
opts => [{zone,internal}],
proto => tcp}]),
application:set_env(gen_rpc, port_discovery, manual),
application:set_env(gen_rpc, tcp_server_port, Port * 2),
ok;
(_) ->
ok