diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index aeade59c6..0245e23ab 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -18,11 +18,24 @@ File format: * Add support for JWT authorization [#7596] Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists. * Better randomisation of app screts (changed from timestamp seeded sha hash (uuid) to crypto:strong_rand_bytes) +* Return a client_identifier_not_valid error when username is empty and username_as_clientid is set to true [#7862] +* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/4 [#7894] +* Add proto_name and proto_ver fields for $event/client_disconnected event. +* Mnesia auth/acl http api support multiple condition queries. +* Inflight QoS1 Messages for shared topics are now redispatched to another alive subscribers upon chosen subscriber session termination. +* Make auth metrics name more understandable. ### Bug fixes * List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. - * SSL closed error bug fixed for redis client. +* Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message +* Rule-engine function hexstr2bin/1 support half byte [#7977] +* Add rule-engine function float2str/2, user can specify the float output precision [#7991] + +* Improved resilience against autocluster partitioning during cluster + startup. [#7876] + [ekka-158](https://github.com/emqx/ekka/pull/158) +* Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979] ## v4.3.14 diff --git a/apps/emqx_auth_http/include/emqx_auth_http.hrl b/apps/emqx_auth_http/include/emqx_auth_http.hrl index b62f02bcd..0eaa59daf 100644 --- a/apps/emqx_auth_http/include/emqx_auth_http.hrl +++ b/apps/emqx_auth_http/include/emqx_auth_http.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_http). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index fd8d1e046..fba56740f 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_http, [{description, "EMQ X Authentication/ACL with HTTP API"}, - {vsn, "4.3.5"}, % strict semver, bump manually! + {vsn, "4.3.6"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_http_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.appup.src b/apps/emqx_auth_http/src/emqx_auth_http.appup.src index 256bd7566..519604d24 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.appup.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.appup.src @@ -1,27 +1,38 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.4", - [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]} - ]}, + [{"4.3.5", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, + {"4.3.4", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{apply,{application,stop,[emqx_auth_http]}}, {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-1]">>, [{restart_application,emqx_auth_http}]}, {<<".*">>,[]}], - [{"4.3.4", - [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}]}, + [{"4.3.5", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, + {"4.3.4", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{apply,{application,stop,[emqx_auth_http]}}, {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-1]">>, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.erl b/apps/emqx_auth_http/src/emqx_auth_http.erl index f97276849..98a897a8c 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http.erl @@ -30,22 +30,16 @@ ]). %% Callbacks --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo, AuthResult, #{auth := AuthParms = #{path := Path}, super := SuperParams}) -> case authenticate(AuthParms, ClientInfo) of {ok, 200, <<"ignore">>} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; + ok; {ok, 200, Body} -> - emqx_metrics:inc(?AUTH_METRICS(success)), IsSuperuser = is_superuser(SuperParams, ClientInfo), {stop, AuthResult#{is_superuser => IsSuperuser, auth_result => success, @@ -54,12 +48,10 @@ check(ClientInfo, AuthResult, #{auth := AuthParms = #{path := Path}, {ok, Code, _Body} -> ?LOG(error, "Deny connection from path: ~s, response http code: ~p", [Path, Code]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => http_to_connack_error(Code), anonymous => false}}; {error, Error} -> ?LOG(error, "Request auth path: ~s, error: ~p", [Path, Error]), - emqx_metrics:inc(?AUTH_METRICS(failure)), %%FIXME later: server_unavailable is not right. {stop, AuthResult#{auth_result => server_unavailable, anonymous => false}} diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index fb23f9c0a..9e4b842d8 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -112,7 +112,6 @@ load_hooks() -> case application:get_env(?APP, auth_req) of undefined -> ok; {ok, AuthReq} -> - ok = emqx_auth_http:register_metrics(), PoolOpts = proplists:get_value(pool_opts, AuthReq), PoolName = proplists:get_value(pool_name, AuthReq), {ok, _} = ehttpc_sup:start_pool(PoolName, PoolOpts), @@ -160,4 +159,3 @@ path(#{path := ""}) -> "/"; path(#{path := Path}) -> Path. - diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl index 46451d1bb..040f9b629 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl @@ -21,28 +21,11 @@ -logger_header("[JWT]"). --export([ register_metrics/0 - , check_auth/3 +-export([ check_auth/3 , check_acl/5 , description/0 ]). --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). - --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - %%-------------------------------------------------------------------- %% Authentication callbacks %%-------------------------------------------------------------------- @@ -50,17 +33,16 @@ register_metrics() -> check_auth(ClientInfo, AuthResult, #{from := From, checklists := Checklists}) -> case maps:find(From, ClientInfo) of error -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {ok, undefined} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {ok, Token} -> case emqx_auth_jwt_svr:verify(Token) of {error, not_found} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, not_token} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, Reason} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => Reason, anonymous => false}}; {ok, Claims} -> {stop, maps:merge(AuthResult, verify_claims(Checklists, Claims, ClientInfo))} @@ -72,14 +54,33 @@ check_acl(ClientInfo = #{jwt_claims := Claims}, Topic, _NoMatchAction, #{acl_claim_name := AclClaimName}) -> - Deadline = erlang:system_time(second), case Claims of - #{AclClaimName := Acl, <<"exp">> := Exp} - when is_integer(Exp) andalso Exp >= Deadline -> + #{AclClaimName := Acl, <<"exp">> := Exp} -> + try is_expired(Exp) of + true -> + ?DEBUG("acl_deny_due_to_jwt_expired", []), + deny; + false -> + verify_acl(ClientInfo, Acl, PubSub, Topic) + catch + _:_ -> + ?DEBUG("acl_deny_due_to_invalid_jwt_exp", []), + deny + end; + #{AclClaimName := Acl} -> verify_acl(ClientInfo, Acl, PubSub, Topic); - _ -> ignore + _ -> + ?DEBUG("no_acl_jwt_claim", []), + ignore end. +is_expired(Exp) when is_binary(Exp) -> + ExpInt = binary_to_integer(Exp), + is_expired(ExpInt); +is_expired(Exp) -> + Now = erlang:system_time(second), + Now > Exp. + description() -> "Authentication with JWT". %%------------------------------------------------------------------------------ @@ -102,10 +103,8 @@ verify_acl(ClientInfo, [AclTopic | AclTopics], Topic) -> verify_claims(Checklists, Claims, ClientInfo) -> case do_verify_claims(feedvar(Checklists, ClientInfo), Claims) of {error, Reason} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), #{auth_result => Reason, anonymous => false}; ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), #{auth_result => success, anonymous => false, jwt_claims => Claims} end. diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl index c94e8aa3c..d4e0e4960 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl @@ -32,7 +32,6 @@ start(_Type, _Args) -> {ok, Sup} = supervisor:start_link({local, ?MODULE}, ?MODULE, []), {ok, _} = start_auth_server(jwks_svr_options()), - ok = emqx_auth_jwt:register_metrics(), AuthEnv = auth_env(), _ = emqx:hook('client.authenticate', {emqx_auth_jwt, check_auth, [AuthEnv]}), diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl index 05e45cec3..c018bf1bd 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl @@ -203,18 +203,32 @@ do_verify(JwsCompacted, [Jwk|More]) -> check_claims(Claims) -> Now = os:system_time(seconds), - Checker = [{<<"exp">>, fun(ExpireTime) -> - Now < ExpireTime - end}, - {<<"iat">>, fun(IssueAt) -> - IssueAt =< Now - end}, - {<<"nbf">>, fun(NotBefore) -> - NotBefore =< Now - end} + Checker = [{<<"exp">>, with_int_value( + fun(ExpireTime) -> Now < ExpireTime end)}, + {<<"iat">>, with_int_value( + fun(IssueAt) -> IssueAt =< Now end)}, + {<<"nbf">>, with_int_value( + fun(NotBefore) -> NotBefore =< Now end)} ], do_check_claim(Checker, Claims). +with_int_value(Fun) -> + fun(Value) -> + case Value of + Int when is_integer(Int) -> Fun(Int); + Bin when is_binary(Bin) -> + case string:to_integer(Bin) of + {Int, <<>>} -> Fun(Int); + _ -> false + end; + Str when is_list(Str) -> + case string:to_integer(Str) of + {Int, ""} -> Fun(Int); + _ -> false + end + end + end. + do_check_claim([], Claims) -> Claims; do_check_claim([{K, F}|More], Claims) -> diff --git a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl index abfd7b3a8..2136d5384 100644 --- a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl @@ -93,6 +93,52 @@ t_check_auth(_Config) -> ?assertEqual({error, invalid_signature}, Result2), ?assertMatch({error, _}, emqx_access_control:authenticate(Plain#{password => <<"asd">>})). +t_check_auth_invalid_exp(init, _Config) -> + application:unset_env(emqx_auth_jwt, verify_claims). +t_check_auth_invalid_exp(_Config) -> + Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external}, + Jwt0 = sign([{clientid, <<"client1">>}, + {username, <<"plain">>}, + {exp, [{foo, bar}]}], <<"HS256">>, <<"emqxsecret">>), + ct:pal("Jwt: ~p~n", [Jwt0]), + + Result0 = emqx_access_control:authenticate(Plain#{password => Jwt0}), + ct:pal("Auth result: ~p~n", [Result0]), + ?assertMatch({error, _}, Result0), + + Jwt1 = sign([{clientid, <<"client1">>}, + {username, <<"plain">>}, + {exp, <<"foobar">>}], <<"HS256">>, <<"emqxsecret">>), + ct:pal("Jwt: ~p~n", [Jwt1]), + + Result1 = emqx_access_control:authenticate(Plain#{password => Jwt1}), + ct:pal("Auth result: ~p~n", [Result1]), + ?assertMatch({error, _}, Result1). + +t_check_auth_str_exp(init, _Config) -> + application:unset_env(emqx_auth_jwt, verify_claims). +t_check_auth_str_exp(_Config) -> + Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external}, + Exp = integer_to_binary(os:system_time(seconds) + 3), + + Jwt0 = sign([{clientid, <<"client1">>}, + {username, <<"plain">>}, + {exp, Exp}], <<"HS256">>, <<"emqxsecret">>), + ct:pal("Jwt: ~p~n", [Jwt0]), + + Result0 = emqx_access_control:authenticate(Plain#{password => Jwt0}), + ct:pal("Auth result: ~p~n", [Result0]), + ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0), + + Jwt1 = sign([{clientid, <<"client1">>}, + {username, <<"plain">>}, + {exp, <<"0">>}], <<"HS256">>, <<"emqxsecret">>), + ct:pal("Jwt: ~p~n", [Jwt1]), + + Result1 = emqx_access_control:authenticate(Plain#{password => Jwt1}), + ct:pal("Auth result: ~p~n", [Result1]), + ?assertMatch({error, _}, Result1). + t_check_claims(init, _Config) -> application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]). t_check_claims(_Config) -> @@ -276,3 +322,26 @@ t_check_jwt_acl_expire(_Config) -> emqtt:subscribe(C, <<"a/b">>, 0)), ok = emqtt:disconnect(C). + +t_check_jwt_acl_no_exp(init, _Config) -> + application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]). +t_check_jwt_acl_no_exp(_Config) -> + Jwt = sign([{client_id, <<"client1">>}, + {username, <<"plain">>}, + {sub, value}, + {acl, [{sub, [<<"a/b">>]}]}], + <<"HS256">>, + <<"emqxsecret">>), + + {ok, C} = emqtt:start_link( + [{clean_start, true}, + {proto_ver, v5}, + {client_id, <<"client1">>}, + {password, Jwt}]), + {ok, _} = emqtt:connect(C), + + ?assertMatch( + {ok, #{}, [0]}, + emqtt:subscribe(C, <<"a/b">>, 0)), + + ok = emqtt:disconnect(C). diff --git a/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl b/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl index 97a7d0cfc..0c573e0e2 100644 --- a/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl +++ b/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_ldap). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index a072234d7..6875fca30 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_ldap, [{description, "EMQ X Authentication/ACL with LDAP"}, - {vsn, "4.3.4"}, % strict semver, bump manually! + {vsn, "4.3.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_ldap_sup]}, {applications, [kernel,stdlib,eldap2,ecpool]}, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src index ea44d7a1a..fbb59a176 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src @@ -1,40 +1,29 @@ -%% -*-: erlang -*- +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ {"4.3.3", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, - {"4.3.0", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {"4.3.1", - [ {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, + [{<<"4\\.3\\.[3-4]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], - [ {"4.3.3", []}, - {"4.3.0", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {"4.3.1", - [ {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-1]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{<<"4\\.3\\.[3-4]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-1]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl index f2b6140a2..da932c2fe 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl @@ -26,17 +26,12 @@ -import(emqx_auth_ldap_cli, [search/3]). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 , prepare_filter/4 , replace_vars/2 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{username := Username, password := Password}, AuthResult, State = #{password_attr := PasswdAttr, bind_as_user := BindAsUserRequired, pool := Pool}) -> CheckResult = @@ -63,12 +58,10 @@ check(ClientInfo = #{username := Username, password := Password}, AuthResult, end, case CheckResult of ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{auth_result => success, anonymous => false}}; {error, not_found} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, ResultCode} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), ?LOG(error, "[LDAP] Auth from ldap failed: ~p", [ResultCode]), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl index 63fde463b..14c31f1bd 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl @@ -49,7 +49,6 @@ stop(_State) -> ok. load_auth_hook(DeviceDn) -> - ok = emqx_auth_ldap:register_metrics(), Params = maps:from_list(DeviceDn), emqx:hook('client.authenticate', fun emqx_auth_ldap:check/3, [Params#{pool => ?APP}]). diff --git a/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl index 5de24d913..8bf7d532b 100644 --- a/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl @@ -48,7 +48,9 @@ init_per_group(GrpName, Cfg) -> Cfg. end_per_group(_GrpName, _Cfg) -> - emqx_ct_helpers:stop_apps([emqx_auth_ldap]). + emqx_ct_helpers:stop_apps([emqx_auth_ldap]), + %% clear the application envs to avoid cross-suite testcase failure + application:unload(emqx_auth_ldap). %%-------------------------------------------------------------------- %% Cases diff --git a/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl index 4f56ab32a..99f733d78 100644 --- a/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl @@ -40,7 +40,9 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_auth_ldap]). + emqx_ct_helpers:stop_apps([emqx_auth_ldap]), + %% clear the application envs to avoid cross-suite testcase failure + application:unload(emqx_auth_ldap). check_auth(_) -> MqttUser1 = #{clientid => <<"mqttuser1">>, diff --git a/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl b/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl index 2d5a6eb6d..f8569ef8a 100644 --- a/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl +++ b/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl @@ -41,15 +41,3 @@ }). -type(acl_record() :: {acl_target(), emqx_topic:topic(), action(), access(), created_at()}). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl index a88b83d0c..163abf075 100644 --- a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl +++ b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl @@ -96,18 +96,24 @@ , delete/2 ]). +-define(CLIENTID_SCHEMA, [{<<"clientid">>, binary}, {<<"_like_clientid">>, binary}] ++ ?COMMON_SCHEMA). +-define(USERNAME_SCHEMA, [{<<"username">>, binary}, {<<"_like_username">>, binary}] ++ ?COMMON_SCHEMA). +-define(COMMON_SCHEMA, [{<<"topic">>, binary}, {<<"action">>, atom}, {<<"access">>, atom}]). + list_clientid(_Bindings, Params) -> - Table = emqx_acl_mnesia_db:login_acl_table(clientid), - return({ok, emqx_auth_mnesia_api:paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}). + {_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?CLIENTID_SCHEMA), + Table = emqx_acl_mnesia_db:login_acl_table(clientid, Params1), + return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}). list_username(_Bindings, Params) -> - Table = emqx_acl_mnesia_db:login_acl_table(username), - return({ok, emqx_auth_mnesia_api:paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}). + {_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?USERNAME_SCHEMA), + Table = emqx_acl_mnesia_db:login_acl_table(username, Params1), + return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}). list_all(_Bindings, Params) -> - Table = emqx_acl_mnesia_db:login_acl_table(all), - return({ok, emqx_auth_mnesia_api:paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}). - + {_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?COMMON_SCHEMA), + Table = emqx_acl_mnesia_db:login_acl_table(all, Params1), + return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}). lookup(#{clientid := Clientid}, _Params) -> return({ok, format(emqx_acl_mnesia_db:lookup_acl({clientid, urldecode(Clientid)}))}); @@ -170,7 +176,11 @@ delete(#{topic := Topic}, _) -> %%------------------------------------------------------------------------------ count(QH) -> - qlc:fold(fun(_, Count) -> Count + 1 end, 0, QH). + Count = qlc:fold(fun(_, Sum) -> Sum + 1 end, 0, QH), + case is_integer(Count) of + true -> Count; + false -> 0 + end. format({{clientid, Clientid}, Topic, Action, Access, _CreatedAt}) -> #{clientid => Clientid, topic => Topic, action => Action, access => Access}; @@ -222,3 +232,27 @@ format_msg(Message) when is_tuple(Message) -> urldecode(S) -> emqx_http_lib:uri_decode(S). + +paginate_qh(Qh, Count, Params, ComparingFun, RowFun) -> + Page = page(Params), + Limit = limit(Params), + Cursor = qlc:cursor(Qh), + case Page > 1 of + true -> + _ = qlc:next_answers(Cursor, (Page - 1) * Limit), + ok; + false -> ok + end, + Rows = qlc:next_answers(Cursor, Limit), + qlc:delete_cursor(Cursor), + #{meta => #{page => Page, limit => Limit, count => Count}, + data => [RowFun(Row) || Row <- lists:sort(ComparingFun, Rows)]}. + +page(Params) -> + binary_to_integer(proplists:get_value(<<"_page">>, Params, <<"1">>)). + +limit(Params) -> + case proplists:get_value(<<"_limit">>, Params) of + undefined -> 50; + Size -> binary_to_integer(Size) + end. diff --git a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_db.erl b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_db.erl index 23f2447bc..8fcbb8f9a 100644 --- a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_db.erl +++ b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_db.erl @@ -33,6 +33,7 @@ , remove_acl/2 , merge_acl_records/3 , login_acl_table/1 + , login_acl_table/2 , is_migration_started/0 ]). @@ -124,7 +125,7 @@ all_acls_export() -> {atomic, Records} = mnesia:transaction( fun() -> - QH = acl_table(MatchSpecNew, MatchSpecOld, fun mnesia:table/2, fun lookup_mnesia/2), + QH = acl_table(MatchSpecNew, MatchSpecOld, {#{}, #{}}, fun mnesia:table/2, fun lookup_mnesia/2), qlc:eval(QH) end), Records. @@ -132,9 +133,15 @@ all_acls_export() -> %% @doc QLC table of logins matching spec -spec(login_acl_table(acl_target_type()) -> qlc:query_handle()). login_acl_table(AclTargetType) -> - MatchSpecNew = login_match_spec_new(AclTargetType), - MatchSpecOld = login_match_spec_old(AclTargetType), - acl_table(MatchSpecNew, MatchSpecOld, fun ets:table/2, fun lookup_ets/2). + login_acl_table(AclTargetType, {[], []}). + +login_acl_table(AclTargetType, {Qs, Fuzzy}) -> + ToMap = fun({Type, Symbol, Val}, Acc) -> Acc#{{Type, Symbol} => Val} end, + Qs1 = lists:foldl(ToMap, #{}, Qs), + Fuzzy1 = lists:foldl(ToMap, #{}, Fuzzy), + MatchSpecNew = login_match_spec_new(AclTargetType, Qs1), + MatchSpecOld = login_match_spec_old(AclTargetType, Qs1), + acl_table(MatchSpecNew, MatchSpecOld, {Qs1, Fuzzy1}, fun ets:table/2, fun lookup_ets/2). %% @doc Combine old `emqx_acl` ACL records with a new `emqx_acl2` ACL record for a given login -spec(merge_acl_records(acl_target(), [#?ACL_TABLE{}], [#?ACL_TABLE2{}]) -> #?ACL_TABLE2{}). @@ -223,27 +230,39 @@ comparing({_, _, _, _, CreatedAt1}, {_, _, _, _, CreatedAt2}) -> CreatedAt1 >= CreatedAt2. -login_match_spec_old(all) -> +login_match_spec_old(Type) -> login_match_spec_old(Type, #{}). + +login_match_spec_old(all, _) -> ets:fun2ms(fun(#?ACL_TABLE{filter = {all, _}} = Record) -> Record end); -login_match_spec_old(Type) when (Type =:= username) or (Type =:= clientid) -> - ets:fun2ms(fun(#?ACL_TABLE{filter = {{RecordType, _}, _}} = Record) - when RecordType =:= Type -> Record - end). +login_match_spec_old(Type, Params) when (Type =:= username) orelse (Type =:= clientid) -> + case maps:get({Type, '=:='}, Params, undefined) of + undefined -> + ets:fun2ms(fun(#?ACL_TABLE{filter = {{RType, _}, _}} = Rec) when RType =:= Type -> Rec end); + Val -> + ets:fun2ms(fun(#?ACL_TABLE{filter = {{RType, RVal}, _}} = Rec) + when RType =:= Type andalso RVal =:= Val -> Rec end) + end. -login_match_spec_new(all) -> +login_match_spec_new(Type) -> login_match_spec_new(Type, #{}). + +login_match_spec_new(all, _) -> ets:fun2ms(fun(#?ACL_TABLE2{who = all} = Record) -> Record end); -login_match_spec_new(Type) when (Type =:= username) or (Type =:= clientid) -> - ets:fun2ms(fun(#?ACL_TABLE2{who = {RecordType, _}} = Record) - when RecordType =:= Type -> Record - end). +login_match_spec_new(Type, Params) when (Type =:= username) orelse (Type =:= clientid) -> + case maps:get({Type, '=:='}, Params, undefined) of + undefined -> + ets:fun2ms(fun(#?ACL_TABLE2{who = {RType, _}} = Rec) when RType =:= Type -> Rec end); + Val -> + ets:fun2ms(fun(#?ACL_TABLE2{who = {RType, RVal}} = Rec) + when RType =:= Type andalso RVal =:= Val -> Rec end) + end. -acl_table(MatchSpecNew, MatchSpecOld, TableFun, LookupFun) -> +acl_table(MatchSpecNew, MatchSpecOld, Params, TableFun, LookupFun) -> TraverseFun = fun() -> CursorNew = @@ -252,7 +271,7 @@ acl_table(MatchSpecNew, MatchSpecOld, TableFun, LookupFun) -> CursorOld = qlc:cursor( TableFun(?ACL_TABLE, [{traverse, {select, MatchSpecOld}}])), - traverse_new(CursorNew, CursorOld, #{}, LookupFun) + traverse_new(CursorNew, CursorOld, Params, #{}, LookupFun) end, qlc:table(TraverseFun, []). @@ -265,12 +284,12 @@ acl_table(MatchSpecNew, MatchSpecOld, TableFun, LookupFun) -> % After migration, number of such logins is zero, so traversing starts working in % constant memory. -traverse_new(CursorNew, CursorOld, FoundKeys, LookupFun) -> +traverse_new(CursorNew, CursorOld, Params, FoundKeys, LookupFun) -> Acls = qlc:next_answers(CursorNew, 1), case Acls of [] -> qlc:delete_cursor(CursorNew), - traverse_old(CursorOld, FoundKeys); + traverse_old(CursorOld, Params, FoundKeys); [#?ACL_TABLE2{who = Login, rules = Rules} = Acl] -> Keys = lists:usort([{Login, Topic} || {_, _, Topic, _} <- Rules]), OldRecs = lists:flatmap(fun(Key) -> LookupFun(?ACL_TABLE, Key) end, Keys), @@ -281,27 +300,57 @@ traverse_new(CursorNew, CursorOld, FoundKeys, LookupFun) -> OldRecs), case acl_to_list(MergedAcl) of [] -> - traverse_new(CursorNew, CursorOld, NewFoundKeys, LookupFun); + traverse_new(CursorNew, CursorOld, Params, NewFoundKeys, LookupFun); List -> - List ++ fun() -> traverse_new(CursorNew, CursorOld, NewFoundKeys, LookupFun) end + filter_params(List, Params) ++ + fun() -> traverse_new(CursorNew, CursorOld, Params, NewFoundKeys, LookupFun) end end end. -traverse_old(CursorOld, FoundKeys) -> +filter_params(List, {Qs, Fuzzy}) -> + case maps:size(Qs) =:= 0 andalso maps:size(Fuzzy) =:= 0 of + false -> + Topic = maps:get({topic, '=:='}, Qs, undefined), + Action = maps:get({action, '=:='}, Qs, undefined), + Access = maps:get({access, '=:='}, Qs, undefined), + lists:filter(fun({Target, Topic0, Action0, Access0, _CreatedAt}) -> + CheckList = [{Topic, Topic0}, {Action, Action0}, {Access, Access0}], + case lists:all(fun is_match/1, CheckList) of + true -> + case Target of + {Type, Login} -> + case maps:get({Type, 'like'}, Fuzzy, <<>>) of + <<>> -> true; + LikeSchema -> binary:match(Login, LikeSchema) =/= nomatch + end; + all -> true + end; + false -> false + end + end, List); + true -> List + end. + +is_match({Schema, Val}) -> + Schema =:= undefined orelse Schema =:= Val. + +traverse_old(CursorOld, Params, FoundKeys) -> OldAcls = qlc:next_answers(CursorOld), case OldAcls of [] -> qlc:delete_cursor(CursorOld), []; _ -> - Records = [ {Login, Topic, Action, Access, CreatedAt} + Records = [{Login, Topic, Action, Access, CreatedAt} || #?ACL_TABLE{filter = {Login, Topic}, action = LegacyAction, access = Access, created_at = CreatedAt} <- OldAcls, {_, Action, _, _} <- normalize_rule({Access, LegacyAction, Topic, CreatedAt}), not maps:is_key({Login, Topic}, FoundKeys) ], case Records of - [] -> traverse_old(CursorOld, FoundKeys); - List -> List ++ fun() -> traverse_old(CursorOld, FoundKeys) end + [] -> traverse_old(CursorOld, Params, FoundKeys); + List -> + filter_params(List, Params) + ++ fun() -> traverse_old(CursorOld, Params, FoundKeys) end end end. diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src index d8968c05f..592535f46 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mnesia, [{description, "EMQ X Authentication with Mnesia"}, - {vsn, "4.3.6"}, % strict semver, bump manually + {vsn, "4.3.7"}, % strict semver, bump manually {modules, []}, {registered, []}, {applications, [kernel,stdlib,mnesia]}, diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index f4c6ebadd..a849002d6 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -1,9 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.5", - [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]}, + [{<<"4\\.3\\.[5-6]">>, + [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-3]">>, [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, @@ -19,13 +22,18 @@ {"4.3.4", [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.5", - [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]}, + [{<<"4\\.3\\.[5-6]">>, + [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-3]">>, [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, @@ -40,8 +48,10 @@ {delete_module,emqx_acl_mnesia_db}]}, {"4.3.4", [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl index 386adba4b..e14de3cf5 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl @@ -27,7 +27,6 @@ -define(TABLE, emqx_user). %% Auth callbacks -export([ init/1 - , register_metrics/0 , check/3 , description/0 ]). @@ -51,10 +50,6 @@ init(#{clientid_list := ClientidList, username_list := UsernameList}) -> ok = ekka_mnesia:copy_table(?TABLE, disc_copies). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - hash_type() -> application:get_env(emqx_auth_mnesia, password_hash, sha256). @@ -67,17 +62,14 @@ check(ClientInfo = #{ clientid := Clientid end), case ets:select(?TABLE, MatchSpec) of [] -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; List -> case match_password(NPassword, HashType, List) of false -> Info = maps:without([password], ClientInfo), ?LOG(info, "[Mnesia] Auth from mnesia failed: ~p", [Info]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{anonymous => false, auth_result => password_error}}; _ -> - emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{anonymous => false, auth_result => success}} end end. diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl index e400452ec..331d14a92 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl @@ -18,18 +18,20 @@ -include_lib("stdlib/include/qlc.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). +-include("emqx_auth_mnesia.hrl"). -define(TABLE, emqx_user). -import(proplists, [get_value/2]). -import(minirest, [return/1]). --export([paginate_qh/5]). -export([ list_clientid/2 , lookup_clientid/2 , add_clientid/2 , update_clientid/2 , delete_clientid/2 + , query_clientid/3 + , query_username/3 ]). -rest_api(#{name => list_clientid, @@ -109,13 +111,28 @@ descr => "Delete username in the cluster" }). +-define(CLIENTID_SCHEMA, {?TABLE, + [ + {<<"clientid">>, binary}, + {<<"_like_clientid">>, binary} + ]}). + +-define(USERNAME_SCHEMA, {?TABLE, + [ + {<<"username">>, binary}, + {<<"_like_username">>, binary} + ]}). + +-define(query_clientid, {?MODULE, query_clientid}). +-define(query_username, {?MODULE, query_username}). + %%------------------------------------------------------------------------------ %% Auth Clientid Api %%------------------------------------------------------------------------------ list_clientid(_Bindings, Params) -> - MatchSpec = ets:fun2ms(fun({?TABLE, {clientid, Clientid}, Password, CreatedAt}) -> {?TABLE, {clientid, Clientid}, Password, CreatedAt} end), - return({ok, paginate(?TABLE, MatchSpec, Params, fun emqx_auth_mnesia_cli:comparing/2, fun({?TABLE, {clientid, X}, _, _}) -> #{clientid => X} end)}). + SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, + return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun)}). lookup_clientid(#{clientid := Clientid}, _Params) -> return({ok, format(emqx_auth_mnesia_cli:lookup_user({clientid, urldecode(Clientid)}))}). @@ -164,8 +181,8 @@ delete_clientid(#{clientid := Clientid}, _) -> %%------------------------------------------------------------------------------ list_username(_Bindings, Params) -> - MatchSpec = ets:fun2ms(fun({?TABLE, {username, Username}, Password, CreatedAt}) -> {?TABLE, {username, Username}, Password, CreatedAt} end), - return({ok, paginate(?TABLE, MatchSpec, Params, fun emqx_auth_mnesia_cli:comparing/2, fun({?TABLE, {username, X}, _, _}) -> #{username => X} end)}). + SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, + return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun)}). lookup_username(#{username := Username}, _Params) -> return({ok, format(emqx_auth_mnesia_cli:lookup_user({username, urldecode(Username)}))}). @@ -211,57 +228,52 @@ delete_username(#{username := Username}, _) -> %%------------------------------------------------------------------------------ %% Paging Query %%------------------------------------------------------------------------------ +query_clientid(Qs, Start, Limit) -> query(clientid, Qs, Start, Limit). +query_username(Qs, Start, Limit) -> query(username, Qs, Start, Limit). -paginate(Table, MatchSpec, Params, ComparingFun, RowFun) -> - Qh = query_handle(Table, MatchSpec), - Count = count(Table, MatchSpec), - paginate_qh(Qh, Count, Params, ComparingFun, RowFun). +query(Type, {Qs, []}, Start, Limit) -> + Ms = qs2ms(Type, Qs), + emqx_mgmt_api:select_table(?TABLE, Ms, Start, Limit, fun format/1); -paginate_qh(Qh, Count, Params, ComparingFun, RowFun) -> - Page = page(Params), - Limit = limit(Params), - Cursor = qlc:cursor(Qh), - case Page > 1 of - true -> - _ = qlc:next_answers(Cursor, (Page - 1) * Limit), - ok; - false -> ok - end, - Rows = qlc:next_answers(Cursor, Limit), - qlc:delete_cursor(Cursor), - #{meta => #{page => Page, limit => Limit, count => Count}, - data => [RowFun(Row) || Row <- lists:sort(ComparingFun, Rows)]}. +query(Type, {Qs, Fuzzy}, Start, Limit) -> + Ms = qs2ms(Type, Qs), + MatchFun = match_fun(Ms, Fuzzy), + emqx_mgmt_api:traverse_table(?TABLE, MatchFun, Start, Limit, fun format/1). -query_handle(Table, MatchSpec) when is_atom(Table) -> - Options = {traverse, {select, MatchSpec}}, - qlc:q([R || R <- ets:table(Table, Options)]). +-spec qs2ms(clientid | username, list()) -> ets:match_spec(). +qs2ms(Type, Qs) -> + Init = #?TABLE{login = {Type, '_'}, password = '_', created_at = '_'}, + MatchHead = lists:foldl(fun(Q, Acc) -> match_ms(Q, Acc) end, Init, Qs), + [{MatchHead, [], ['$_']}]. -count(Table, MatchSpec) when is_atom(Table) -> - [{MatchPattern, Where, _Re}] = MatchSpec, - NMatchSpec = [{MatchPattern, Where, [true]}], - ets:select_count(Table, NMatchSpec). +match_ms({Type, '=:=', Value}, MatchHead) -> MatchHead#?TABLE{login = {Type, Value}}; +match_ms(_, MatchHead) -> MatchHead. -page(Params) -> - binary_to_integer(proplists:get_value(<<"_page">>, Params, <<"1">>)). - -limit(Params) -> - case proplists:get_value(<<"_limit">>, Params) of - undefined -> 10; - Size -> binary_to_integer(Size) +match_fun(Ms, Fuzzy) -> + MsC = ets:match_spec_compile(Ms), + fun(Rows) -> + Ls = ets:match_spec_run(Rows, MsC), + lists:filter(fun(E) -> run_fuzzy_match(E, Fuzzy) end, Ls) end. +run_fuzzy_match(_, []) -> true; +run_fuzzy_match(E = #?TABLE{login = {Key, Str}}, [{Key, like, SubStr}|Fuzzy]) -> + binary:match(Str, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(_E, [{_Key, like, _SubStr}| _Fuzzy]) -> false. + %%------------------------------------------------------------------------------ %% Interval Funcs %%------------------------------------------------------------------------------ -format([{?TABLE, {clientid, ClientId}, _Password, _InterTime}]) -> - #{clientid => ClientId}; +format([{?TABLE, {clientid, ClientId}, _Password, CreatedAt}]) -> + #{clientid => ClientId, created_at => CreatedAt}; -format([{?TABLE, {username, Username}, _Password, _InterTime}]) -> - #{username => Username}; +format([{?TABLE, {username, Username}, _Password, CreatedAt}]) -> + #{username => Username, created_at => CreatedAt}; format([]) -> - #{}. + #{}; +format(User) -> format([User]). validate([], []) -> ok; diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl index 467b6cf7c..ee38b7165 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl @@ -56,7 +56,6 @@ load_auth_hook() -> ClientidList = application:get_env(?APP, clientid_list, []), UsernameList = application:get_env(?APP, username_list, []), ok = emqx_auth_mnesia:init(#{clientid_list => ClientidList, username_list => UsernameList}), - ok = emqx_auth_mnesia:register_metrics(), Params = #{hash_type => emqx_auth_mnesia:hash_type()}, emqx:hook('client.authenticate', fun emqx_auth_mnesia:check/3, [Params]). diff --git a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl index c91deb5c1..96e66d3c0 100644 --- a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl @@ -25,6 +25,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -import(emqx_ct_http, [ request_api/3 + , request_api/4 , request_api/5 , get_http_data/1 , create_default_app/0 @@ -358,13 +359,31 @@ t_rest_api(_Config) -> <<"topic">> => <<"topic/C">>, <<"action">> => <<"pubsub">>, <<"access">> => <<"deny">> + }, + #{<<"clientid">> => <<"good_clientid1">>, + <<"topic">> => <<"topic/D">>, + <<"action">> => <<"pubsub">>, + <<"access">> => <<"deny">> }], {ok, _} = request_http_rest_add([], Params1), + {ok, Re1} = request_http_rest_list(["clientid", "test_clientid"]), ?assertMatch(4, length(get_http_data(Re1))), + {ok, Re11} = request_http_rest_list(["clientid"], "_like_clientid=good"), + ?assertMatch(2, length(get_http_data(Re11))), + {ok, Re12} = request_http_rest_list(["clientid"], "_like_clientid=clientid"), + ?assertMatch(6, length(get_http_data(Re12))), + {ok, Re13} = request_http_rest_list(["clientid"], "_like_clientid=clientid&action=pub"), + ?assertMatch(3, length(get_http_data(Re13))), + {ok, Re14} = request_http_rest_list(["clientid"], "_like_clientid=clientid&access=deny"), + ?assertMatch(4, length(get_http_data(Re14))), + {ok, Re15} = request_http_rest_list(["clientid"], "_like_clientid=clientid&topic=topic/A"), + ?assertMatch(1, length(get_http_data(Re15))), + {ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/A"]), {ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/B"]), {ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/C"]), + {ok, _} = request_http_rest_delete(["clientid", "good_clientid1", "topic", "topic/D"]), {ok, Res1} = request_http_rest_list(["clientid"]), ?assertMatch([], get_http_data(Res1)), @@ -382,13 +401,30 @@ t_rest_api(_Config) -> <<"topic">> => <<"topic/C">>, <<"action">> => <<"pubsub">>, <<"access">> => <<"deny">> + }, + #{<<"username">> => <<"good_username">>, + <<"topic">> => <<"topic/D">>, + <<"action">> => <<"pubsub">>, + <<"access">> => <<"deny">> }], {ok, _} = request_http_rest_add([], Params2), {ok, Re2} = request_http_rest_list(["username", "test_username"]), ?assertMatch(4, length(get_http_data(Re2))), + {ok, Re21} = request_http_rest_list(["username"], "_like_username=good"), + ?assertMatch(2, length(get_http_data(Re21))), + {ok, Re22} = request_http_rest_list(["username"], "_like_username=username"), + ?assertMatch(6, length(get_http_data(Re22))), + {ok, Re23} = request_http_rest_list(["username"], "_like_username=username&action=pub"), + ?assertMatch(3, length(get_http_data(Re23))), + {ok, Re24} = request_http_rest_list(["username"], "_like_username=username&access=deny"), + ?assertMatch(4, length(get_http_data(Re24))), + {ok, Re25} = request_http_rest_list(["username"], "_like_username=username&topic=topic/A"), + ?assertMatch(1, length(get_http_data(Re25))), + {ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/A"]), {ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/B"]), {ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/C"]), + {ok, _} = request_http_rest_delete(["username", "good_username", "topic", "topic/D"]), {ok, Res2} = request_http_rest_list(["username"]), ?assertMatch([], get_http_data(Res2)), @@ -403,13 +439,29 @@ t_rest_api(_Config) -> #{<<"topic">> => <<"topic/C">>, <<"action">> => <<"pubsub">>, <<"access">> => <<"deny">> - }], + }, + #{<<"topic">> => <<"topic/D">>, + <<"action">> => <<"pubsub">>, + <<"access">> => <<"deny">> + } + ], {ok, _} = request_http_rest_add([], Params3), + {ok, Re3} = request_http_rest_list(["$all"]), - ?assertMatch(4, length(get_http_data(Re3))), + ?assertMatch(6, length(get_http_data(Re3))), + {ok, Re31} = request_http_rest_list(["$all"], "topic=topic/A"), + ?assertMatch(1, length(get_http_data(Re31))), + {ok, Re32} = request_http_rest_list(["$all"], "action=sub"), + ?assertMatch(3, length(get_http_data(Re32))), + {ok, Re33} = request_http_rest_list(["$all"], "access=deny"), + ?assertMatch(4, length(get_http_data(Re33))), + {ok, Re34} = request_http_rest_list(["$all"], "action=sub&access=deny"), + ?assertMatch(2, length(get_http_data(Re34))), + {ok, _} = request_http_rest_delete(["$all", "topic", "topic/A"]), {ok, _} = request_http_rest_delete(["$all", "topic", "topic/B"]), {ok, _} = request_http_rest_delete(["$all", "topic", "topic/C"]), + {ok, _} = request_http_rest_delete(["$all", "topic", "topic/D"]), {ok, Res3} = request_http_rest_list(["$all"]), ?assertMatch([], get_http_data(Res3)). @@ -443,6 +495,9 @@ combined_conflicting_records() -> request_http_rest_list(Path) -> request_api(get, uri(Path), default_auth_header()). +request_http_rest_list(Path, Qs) -> + request_api(get, uri(Path), Qs, default_auth_header()). + request_http_rest_lookup(Path) -> request_api(get, uri(Path), default_auth_header()). diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index 26a2c1589..1dd979379 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -286,20 +286,26 @@ t_clientid_rest_api(_Config) -> Params3 = [ #{<<"clientid">> => ?CLIENTID, <<"password">> => ?PASSWORD} , #{<<"clientid">> => <<"clientid1">>, <<"password">> => ?PASSWORD} - , #{<<"clientid">> => <<"clientid2">>, <<"password">> => ?PASSWORD} + , #{<<"clientid">> => <<"client2">>, <<"password">> => ?PASSWORD} ], {ok, Result3} = request_http_rest_add(["auth_clientid"], Params3), ?assertMatch(#{ ?CLIENTID := <<"{error,existed}">> , <<"clientid1">> := <<"ok">> - , <<"clientid2">> := <<"ok">> + , <<"client2">> := <<"ok">> }, get_http_data(Result3)), {ok, Result4} = request_http_rest_list(["auth_clientid"]), + ?assertEqual(3, length(get_http_data(Result4))), + {ok, Result5} = request_http_rest_list(["auth_clientid?_like_clientid=id"]), + ?assertEqual(2, length(get_http_data(Result5))), + {ok, Result6} = request_http_rest_list(["auth_clientid?_like_clientid=x"]), + ?assertEqual(0, length(get_http_data(Result6))), + {ok, _} = request_http_rest_delete(Path), - {ok, Result5} = request_http_rest_lookup(Path), - ?assertMatch(#{}, get_http_data(Result5)). + {ok, Result7} = request_http_rest_lookup(Path), + ?assertMatch(#{}, get_http_data(Result7)). t_username_rest_api(_Config) -> clean_all_users(), @@ -330,9 +336,14 @@ t_username_rest_api(_Config) -> {ok, Result4} = request_http_rest_list(["auth_username"]), ?assertEqual(3, length(get_http_data(Result4))), + {ok, Result5} = request_http_rest_list(["auth_username?_like_username=for"]), + ?assertEqual(1, length(get_http_data(Result5))), + {ok, Result6} = request_http_rest_list(["auth_username?_like_username=x"]), + ?assertEqual(0, length(get_http_data(Result6))), + {ok, _} = request_http_rest_delete(Path), - {ok, Result5} = request_http_rest_lookup([Path]), - ?assertMatch(#{}, get_http_data(Result5)). + {ok, Result7} = request_http_rest_lookup([Path]), + ?assertMatch(#{}, get_http_data(Result7)). t_password_hash(_) -> clean_all_users(), diff --git a/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl b/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl index 69044e920..68a97ad33 100644 --- a/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl +++ b/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl @@ -1,4 +1,3 @@ - -define(APP, emqx_auth_mongo). -define(DEFAULT_SELECTORS, [{<<"username">>, <<"%u">>}]). @@ -14,15 +13,3 @@ -record(aclquery, {collection = <<"mqtt_acl">>, selector = {<<"username">>, <<"%u">>}}). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src index 857c9d146..13e83387c 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mongo, [{description, "EMQ X Authentication/ACL with MongoDB"}, - {vsn, "4.4.3"}, % strict semver, bump manually! + {vsn, "4.4.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mongo_sup]}, {applications, [kernel,stdlib,mongodb,ecpool]}, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src index 79578d334..67fc5c1db 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src @@ -1,29 +1,15 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.2", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, - {"4.4.1", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + [{<<"4.4.[0-3]">>, + [{load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, - {"4.4.0", - [{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.2", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, - {"4.4.1", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, + [{<<"4.4.[0-3]">>, + [{load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, - {"4.4.0", - [{load_module,emqx_auth_mongo_sup,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl index 0df342295..bfb911707 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl @@ -23,8 +23,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/types.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). @@ -39,20 +38,15 @@ , available/3 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{password := Password}, AuthResult, Env = #{authquery := AuthQuery, superquery := SuperQuery}) -> #authquery{collection = Collection, field = Fields, hash = HashType, selector = Selector} = AuthQuery, Pool = maps:get(pool, Env, ?APP), case query(Pool, Collection, maps:from_list(replvars(Selector, ClientInfo))) of - undefined -> emqx_metrics:inc(?AUTH_METRICS(ignore)); + undefined -> ok; {error, Reason} -> ?LOG(error, "[MongoDB] Can't connect to MongoDB server: ~0p", [Reason]), - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => not_authorized, anonymous => false}}; UserMap -> Result = case [maps:get(Field, UserMap, undefined) || Field <- Fields] of @@ -64,13 +58,11 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case Result of ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo), anonymous => false, auth_result => success}}; {error, Error} -> ?LOG(error, "[MongoDB] check auth fail: ~p", [Error]), - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => Error, anonymous => false}} end end. diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl index 92c025621..a63aa8193 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl @@ -68,7 +68,6 @@ safe_start() -> reg_authmod(AuthQuery) -> case emqx_auth_mongo:available(?APP, AuthQuery) of ok -> - emqx_auth_mongo:register_metrics(), HookFun = fun emqx_auth_mongo:check/3, HookOptions = #{authquery => AuthQuery, superquery => undefined, pool => ?APP}, case r(super_query, application:get_env(?APP, super_query, undefined)) of @@ -122,4 +121,3 @@ r(auth_query, Config) -> r(acl_query, Config) -> #aclquery{collection = list_to_binary(get_value(collection, Config, "mqtt_acl")), selector = get_value(selector, Config, [?DEFAULT_SELECTORS])}. - diff --git a/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl b/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl index 56da35401..b7c185fcf 100644 --- a/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl +++ b/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_mysql). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src index 6df4545fd..a0ddc4dd4 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mysql, [{description, "EMQ X Authentication/ACL with MySQL"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mysql_sup]}, {applications, [kernel,stdlib,mysql,ecpool]}, diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src index 88ff1b38e..a116d7dbb 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src @@ -1,16 +1,19 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.1", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.1", []}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}] }. diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl b/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl index 17d618838..31d9a007f 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl @@ -22,17 +22,12 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/types.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). -define(EMPTY(Username), (Username =:= undefined orelse Username =:= <<>>)). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{password := Password}, AuthResult, #{auth_query := {AuthSql, AuthParams}, super_query := SuperQuery, @@ -51,15 +46,13 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case CheckPass of ok -> - emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo), anonymous => false, auth_result => success}}; {error, not_found} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; + ok; {error, ResultCode} -> ?LOG(error, "[MySQL] Auth from mysql failed: ~p", [ResultCode]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. @@ -88,4 +81,3 @@ check_pass(Password, HashType) -> end. description() -> "Authentication with MySQL". - diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl b/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl index 716cb7a7a..e58f62a90 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl @@ -50,7 +50,6 @@ stop(_State) -> ok. load_auth_hook(AuthQuery) -> - ok = emqx_auth_mysql:register_metrics(), SuperQuery = parse_query(application:get_env(?APP, super_query, undefined)), {ok, HashType} = application:get_env(?APP, password_hash), Params = #{auth_query => AuthQuery, diff --git a/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl b/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl index 92b971667..565aa4668 100644 --- a/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl +++ b/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl @@ -1,13 +1 @@ -define(APP, emqx_auth_pgsql). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src index b3246f0e6..7313747ee 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_pgsql, [{description, "EMQ X Authentication/ACL with PostgreSQL"}, - {vsn, "4.4.2"}, % strict semver, bump manually! + {vsn, "4.4.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_pgsql_sup]}, {applications, [kernel,stdlib,epgsql,ecpool]}, diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src index f67a55355..35f180341 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src @@ -1,19 +1,15 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.1", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, - {"4.4.0", - [{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, + [{<<"4\\.4\\.[0-2]">>, + %% epgsql 4.4.0 -> 4.6.0. + %% epgsql has no appup, so we can only restart it. + [{restart_application,epgsql}, + {restart_application,emqx_auth_pgsql}]}, {<<".*">>,[]}], - [{"4.4.1", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, - {"4.4.0", - [{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}] -}. + [{<<"4\\.4\\.[0-2]">>, + %% epgsql 4.4.0 -> 4.6.0. + %% epgsql has no appup, so we can only restart it. + [{restart_application,epgsql}, + {restart_application,emqx_auth_pgsql}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl index f38552085..f673e07e4 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl @@ -21,15 +21,10 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - %%-------------------------------------------------------------------- %% Auth Module Callbacks %%-------------------------------------------------------------------- @@ -50,15 +45,13 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case CheckPass of ok -> - emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo), anonymous => false, auth_result => success}}; {error, not_found} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; + ok; {error, ResultCode} -> ?LOG(error, "[Postgres] Auth from pgsql failed: ~p", [ResultCode]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. @@ -88,4 +81,3 @@ check_pass(Password, HashType) -> end. description() -> "Authentication with PostgreSQL". - diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl index c658c4d11..dbfeb8423 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl @@ -42,7 +42,6 @@ start(_StartType, _StartArgs) -> super_query => SuperQuery, hash_type => HashType, pool => ?APP}, - ok = emqx_auth_pgsql:register_metrics(), ok = emqx:hook('client.authenticate', fun emqx_auth_pgsql:check/3, [AuthEnv]) end), if_enabled(acl_query, fun(AclQuery) -> @@ -59,4 +58,3 @@ if_enabled(Par, Fun) -> {ok, Query} -> Fun(parse_query(Par, Query)); undefined -> ok end. - diff --git a/apps/emqx_auth_redis/include/emqx_auth_redis.hrl b/apps/emqx_auth_redis/include/emqx_auth_redis.hrl index fe488c37f..075d649ec 100644 --- a/apps/emqx_auth_redis/include/emqx_auth_redis.hrl +++ b/apps/emqx_auth_redis/include/emqx_auth_redis.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_redis). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src index ea6aaefcf..e9e37a463 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_redis, [{description, "EMQ X Authentication/ACL with Redis"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_redis_sup]}, {applications, [kernel,stdlib,eredis,eredis_cluster,ecpool]}, diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src b/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src index 83f5d46be..9036d77a8 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src @@ -1,16 +1,19 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.1", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.1", []}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}] }. diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.erl b/apps/emqx_auth_redis/src/emqx_auth_redis.erl index 04d3542f0..d432e012b 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.erl @@ -21,15 +21,10 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{password := Password}, AuthResult, #{auth_cmd := AuthCmd, super_cmd := SuperCmd, @@ -52,15 +47,13 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case CheckPass of ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), IsSuperuser = is_superuser(Pool, Type, SuperCmd, ClientInfo, Timeout), {stop, AuthResult#{is_superuser => IsSuperuser, anonymous => false, auth_result => success}}; {error, not_found} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, ResultCode} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), ?LOG(error, "[Redis] Auth from redis failed: ~p", [ResultCode]), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. @@ -82,4 +75,3 @@ check_pass(Password, HashType) -> ok -> ok; {error, _Reason} -> {error, not_authorized} end. - diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl b/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl index c6de8b80a..cbea448f9 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl @@ -49,7 +49,6 @@ load_auth_hook(AuthCmd) -> timeout => Timeout, type => Type, pool => ?APP}, - ok = emqx_auth_redis:register_metrics(), emqx:hook('client.authenticate', fun emqx_auth_redis:check/3, [Config]). load_acl_hook(AclCmd) -> @@ -66,4 +65,3 @@ if_cmd_enabled(Par, Fun) -> {ok, Cmd} -> Fun(Cmd); undefined -> ok end. - diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index fe566ae52..41c19f752 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.7"}, %% 4.3.3 is used by ee + {vsn, "4.3.8"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index 0da87e289..33bd4386c 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,14 +1,9 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {"4.3.6", - [ %% There are only changes to the schema file, so we don't need any - %% commands here - ]}, - {<<"4\\.3\\.[4-5]">>, - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, - {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]">>, + [{<<"4\\.3\\.[6-7]">>, + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[2-5]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, @@ -17,11 +12,9 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.6", []}, - {<<"4\\.3\\.[4-5]">>, - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, - {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]">>, + [{<<"4\\.3\\.[6-7]">>, + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[2-5]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 0e39b40a7..dce3b36e1 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -299,8 +299,6 @@ handle_call({auth, RequestedClientInfo, Password}, case emqx_access_control:authenticate(ClientInfo1#{password => Password}) of {ok, AuthResult} -> emqx_logger:set_metadata_clientid(ClientId), - is_anonymous(AuthResult) andalso - emqx_metrics:inc('client.auth.anonymous'), NClientInfo = maps:merge(ClientInfo1, AuthResult), NChannel = Channel1#channel{clientinfo = NClientInfo}, case emqx_cm:open_session(true, NClientInfo, NConnInfo) of @@ -424,9 +422,6 @@ terminate(Reason, Channel) -> Req = #{reason => stringfy(Reason)}, try_dispatch(on_socket_closed, wrap(Req), Channel). -is_anonymous(#{anonymous := true}) -> true; -is_anonymous(_AuthResult) -> false. - packet_to_message(Topic, Qos, Payload, #channel{ conninfo = #{proto_ver := ProtoVer}, diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index eaa27d3c9..3459c2b25 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -23,6 +23,7 @@ %% first_next query APIs -export([ params2qs/2 , node_query/4 + , node_query/5 , cluster_query/3 , traverse_table/5 , select_table/5 @@ -111,6 +112,9 @@ limit(Params) -> %%-------------------------------------------------------------------- node_query(Node, Params, {Tab, QsSchema}, QueryFun) -> + node_query(Node, Params, {Tab, QsSchema}, QueryFun, undefined). + +node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun) -> {CodCnt, Qs} = params2qs(Params, QsSchema), Limit = limit(Params), Page = page(Params), @@ -123,7 +127,11 @@ node_query(Node, Params, {Tab, QsSchema}, QueryFun) -> true -> Meta#{count => count(Tab), hasnext => length(Rows) > Limit}; _ -> Meta#{count => -1, hasnext => length(Rows) > Limit} end, - #{meta => NMeta, data => lists:sublist(Rows, Limit)}. + NRows = case SortFun of + undefined -> Rows; + _ -> lists:sort(SortFun, Rows) + end, + #{meta => NMeta, data => lists:sublist(NRows, Limit)}. %% @private do_query(Node, Qs, {M,F}, Start, Limit) when Node =:= node() -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index a07ffc36c..e0e5af39d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -35,6 +35,10 @@ {<<"_gte_created_at">>, timestamp}, {<<"_lte_created_at">>, timestamp}, {<<"_gte_connected_at">>, timestamp}, + {<<"_gte_mqueue_len">>, integer}, + {<<"_lte_mqueue_len">>, integer}, + {<<"_gte_mqueue_dropped">>, integer}, + {<<"_lte_mqueue_dropped">>, integer}, {<<"_lte_connected_at">>, timestamp}]}). -rest_api(#{name => list_clients, @@ -356,15 +360,14 @@ format_acl_cache({{PubSub, Topic}, {AclResult, Timestamp}}) -> %% Query Functions %%-------------------------------------------------------------------- -query({Qs, []}, Start, Limit) -> - Ms = qs2ms(Qs), - emqx_mgmt_api:select_table(emqx_channel_info, Ms, Start, Limit, fun format_channel_info/1); - query({Qs, Fuzzy}, Start, Limit) -> - Ms = qs2ms(Qs), - MatchFun = match_fun(Ms, Fuzzy), - emqx_mgmt_api:traverse_table(emqx_channel_info, MatchFun, - Start, Limit, fun format_channel_info/1). + case qs2ms(Qs) of + {Ms, []} when Fuzzy =:= [] -> + emqx_mgmt_api:select_table(emqx_channel_info, Ms, Start, Limit, fun format_channel_info/1); + {Ms, FuzzyStats} -> + MatchFun = match_fun(Ms, Fuzzy ++ FuzzyStats), + emqx_mgmt_api:traverse_table(emqx_channel_info, MatchFun, Start, Limit, fun format_channel_info/1) + end. %%-------------------------------------------------------------------- %% Match funcs @@ -388,27 +391,48 @@ run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr} | undefined -> <<>>; V -> V end, - binary:match(Val, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy). + binary:match(Val, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_, _, Stats}, [{Key, '>=', Int}|Fuzzy]) -> + case lists:keyfind(Key, 1, Stats) of + {_, Val} when Val >= Int -> run_fuzzy_match(E, Fuzzy); + _ -> false + end; +run_fuzzy_match(E = {_, _, Stats}, [{Key, '=<', Int}|Fuzzy]) -> + case lists:keyfind(Key, 1, Stats) of + {_, Val} when Val =< Int -> run_fuzzy_match(E, Fuzzy); + _ -> false + end. %%-------------------------------------------------------------------- %% QueryString to Match Spec -spec qs2ms(list()) -> ets:match_spec(). qs2ms(Qs) -> - {MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}), - [{{'$1', MtchHead, '_'}, Conds, ['$_']}]. + {MatchHead, Conds, FuzzyStats} = qs2ms(Qs, 2, #{}, [], []), + {[{{'$1', MatchHead, '_'}, Conds, ['$_']}], FuzzyStats}. -qs2ms([], _, {MtchHead, Conds}) -> - {MtchHead, lists:reverse(Conds)}; +qs2ms([], _, MatchHead, Conds, FuzzyStats) -> + {MatchHead, lists:reverse(Conds), FuzzyStats}; -qs2ms([{Key, '=:=', Value} | Rest], N, {MtchHead, Conds}) -> - NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(Key, Value)), - qs2ms(Rest, N, {NMtchHead, Conds}); -qs2ms([Qs | Rest], N, {MtchHead, Conds}) -> +qs2ms([{Key, '=:=', Value} | Rest], N, MatchHead, Conds, FuzzyStats) -> + NMatchHead = emqx_mgmt_util:merge_maps(MatchHead, ms(Key, Value)), + qs2ms(Rest, N, NMatchHead, Conds, FuzzyStats); +qs2ms([Qs | Rest], N, MatchHead, Conds, FuzzyStats) -> Holder = binary_to_atom(iolist_to_binary(["$", integer_to_list(N)]), utf8), - NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(element(1, Qs), Holder)), - NConds = put_conds(Qs, Holder, Conds), - qs2ms(Rest, N+1, {NMtchHead, NConds}). + case ms(element(1, Qs), Holder) of + fuzzy_stats -> + FuzzyStats1 = + case Qs of + {_Key, _Symbol, _Val} -> [Qs | FuzzyStats]; + {Key, Symbol1, Val1, Symbol2, Val2} -> + [{Key, Symbol1, Val1}, {Key, Symbol2, Val2} | FuzzyStats] + end, + qs2ms(Rest, N, MatchHead, Conds, FuzzyStats1); + Ms -> + NMatchHead = emqx_mgmt_util:merge_maps(MatchHead, Ms), + NConds = put_conds(Qs, Holder, Conds), + qs2ms(Rest, N+1, NMatchHead, NConds, FuzzyStats) + end. put_conds({_, Op, V}, Holder, Conds) -> [{Op, Holder, V} | Conds]; @@ -435,7 +459,11 @@ ms(proto_ver, X) -> ms(connected_at, X) -> #{conninfo => #{connected_at => X}}; ms(created_at, X) -> - #{session => #{created_at => X}}. + #{session => #{created_at => X}}; +ms(mqueue_len, _X) -> + fuzzy_stats; +ms(mqueue_dropped, _X) -> + fuzzy_stats. filter_ratelimit_params(P) -> [{K, parse_ratelimit_str(V)} || {K, V} <- P, V =/= undefined]. @@ -461,6 +489,10 @@ params2qs_test() -> {<<"_lte_created_at">>, 5}, {<<"_gte_connected_at">>, 1}, {<<"_lte_connected_at">>, 5}, + {<<"_lte_mqueue_len">>, 10}, + {<<"_gte_mqueue_len">>, 5}, + {<<"_lte_mqueue_dropped">>, 100}, + {<<"_gte_mqueue_dropped">>, 50}, {<<"_like_clientid">>, <<"a">>}, {<<"_like_username">>, <<"e">>} ], @@ -480,12 +512,17 @@ params2qs_test() -> {'=<','$2', 5000}, {'>=','$3', 1000}, {'=<','$3', 5000}], - {10, {Qs1, []}} = emqx_mgmt_api:params2qs(Params, QsSchema), - [{{'$1', MtchHead, _}, Condi, _}] = qs2ms(Qs1), + ExpectedFuzzyStats = [{mqueue_dropped,'=<',100}, + {mqueue_dropped,'>=',50}, + {mqueue_len,'=<',10}, + {mqueue_len,'>=',5}], + {12, {Qs1, []}} = emqx_mgmt_api:params2qs(Params, QsSchema), + {[{{'$1', MtchHead, _}, Condi, _}], FuzzyStats} = qs2ms(Qs1), ?assertEqual(ExpectedMtchHead, MtchHead), ?assertEqual(ExpectedCondi, Condi), + ?assertEqual(ExpectedFuzzyStats, lists:sort(FuzzyStats)), - [{{'$1', #{}, '_'}, [], ['$_']}] = qs2ms([]). + {[{{'$1', #{}, '_'}, [], ['$_']}], []} = qs2ms([]). fuzzy_match_test() -> Info = {emqx_channel_info, @@ -507,4 +544,31 @@ fuzzy_match_test() -> true = run_fuzzy_match(Info, [{clientid, like, <<"de">>}, {username, like, <<"[]">>}]). +fuzzy_stats_test() -> + Fun = fun(Len, Dropped) -> + {emqx_channel_info, + #{clientinfo => + #{ clientid => <<"abcde">>, username => <<"abc\\name*[]()">>}}, + [{mqueue_len, Len}, {mqueue_max,1000}, {mqueue_dropped, Dropped}] + } + end, + false = run_fuzzy_match(Fun(0, 100), [{mqueue_len, '>=', 1}]), + true = run_fuzzy_match(Fun(1, 100), [{mqueue_len, '>=', 1}]), + false = run_fuzzy_match(Fun(1, 100), [{mqueue_len, '>=', 2}]), + true = run_fuzzy_match(Fun(99, 100), [{mqueue_len, '=<', 100}, {mqueue_len, '>=', 98}]), + false = run_fuzzy_match(Fun(99, 100), [{mqueue_len, '=<', 101}, {mqueue_len, '>=', 100}]), + + false = run_fuzzy_match(Fun(1000, 0), [{mqueue_dropped, '>=', 1}]), + true = run_fuzzy_match(Fun(1000, 1), [{mqueue_dropped, '>=', 1}]), + false = run_fuzzy_match(Fun(1000, 1), [{mqueue_dropped, '>=', 2}]), + true = run_fuzzy_match(Fun(1000, 99), [{mqueue_dropped, '=<', 100}, {mqueue_dropped, '>=', 98}]), + false = run_fuzzy_match(Fun(1000, 99), [{mqueue_dropped, '=<', 98}, {mqueue_dropped, '>=', 97}]), + false = run_fuzzy_match(Fun(1000, 102), [{mqueue_dropped, '=<', 104}, {mqueue_dropped, '>=', 103}]), + + true = run_fuzzy_match(Fun(1000, 103), [{mqueue_dropped, '=<', 104}, {mqueue_dropped, '>=', 103}, + {mqueue_len, '>=', 1000}]), + false = run_fuzzy_match(Fun(1000, 199), [{mqueue_dropped, '>=', 198}, + {mqueue_len, '=<', 99}, {mqueue_len, '>=', 1}]), + ok. + -endif. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index b96608edb..ded1bf46d 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -1,6 +1,6 @@ {application, emqx_prometheus, [{description, "Prometheus for EMQ X"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel,stdlib,prometheus]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.appup.src b/apps/emqx_prometheus/src/emqx_prometheus.appup.src new file mode 100644 index 000000000..a06d65fe1 --- /dev/null +++ b/apps/emqx_prometheus/src/emqx_prometheus.appup.src @@ -0,0 +1,9 @@ +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! +{VSN, + [{"4.3.0", + [{load_module,emqx_prometheus,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.3.0", + [{load_module,emqx_prometheus,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 57c54446f..a7c450ba6 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -412,8 +412,8 @@ emqx_collect(emqx_client_connected, Stats) -> counter_metric(?C('client.connected', Stats)); emqx_collect(emqx_client_authenticate, Stats) -> counter_metric(?C('client.authenticate', Stats)); -emqx_collect(emqx_client_auth_anonymous, Stats) -> - counter_metric(?C('client.auth.anonymous', Stats)); +emqx_collect(emqx_client_auth_success_anonymous, Stats) -> + counter_metric(?C('client.auth.success.anonymous', Stats)); emqx_collect(emqx_client_check_acl, Stats) -> counter_metric(?C('client.check_acl', Stats)); emqx_collect(emqx_client_subscribe, Stats) -> @@ -566,7 +566,7 @@ emqx_metrics_delivery() -> emqx_metrics_client() -> [ emqx_client_connected , emqx_client_authenticate - , emqx_client_auth_anonymous + , emqx_client_auth_success_anonymous , emqx_client_check_acl , emqx_client_subscribe , emqx_client_unsubscribe diff --git a/apps/emqx_rule_engine/src/emqx_rule_date.erl b/apps/emqx_rule_engine/src/emqx_rule_date.erl new file mode 100644 index 000000000..fb9cad4c3 --- /dev/null +++ b/apps/emqx_rule_engine/src/emqx_rule_date.erl @@ -0,0 +1,248 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rule_date). + +-export([date/3, date/4, parse_date/4]). + +-export([ + is_int_char/1, + is_symbol_char/1, + is_m_char/1 +]). + +-record(result, { + %%year() + year = "1970" :: string(), + %%month() + month = "1" :: string(), + %%day() + day = "1" :: string(), + %%hour() + hour = "0" :: string(), + %%minute() %% epoch in millisecond precision + minute = "0" :: string(), + %%second() %% epoch in millisecond precision + second = "0" :: string(), + %%integer() %% zone maybe some value + zone = "+00:00" :: string() +}). + +%% -type time_unit() :: 'microsecond' +%% | 'millisecond' +%% | 'nanosecond' +%% | 'second'. +%% -type offset() :: [byte()] | (Time :: integer()). +date(TimeUnit, Offset, FormatString) -> + date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)). + +date(TimeUnit, Offset, FormatString, TimeEpoch) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + Res = lists:map( + fun(Expr) -> + eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr) + end, + R + ), + lists:concat(Res). + +parse_date(TimeUnit, Offset, FormatString, InputString) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + IsZ = fun(V) -> + case V of + {tag, $Z} -> true; + _ -> false + end + end, + R1 = lists:filter(IsZ, R), + IfFun = fun(Con, A, B) -> + case Con of + [] -> A; + _ -> B + end + end, + Res = parse_input(FormatString, InputString), + Str = + Res#result.year ++ "-" ++ + Res#result.month ++ "-" ++ + Res#result.day ++ "T" ++ + Res#result.hour ++ ":" ++ + Res#result.minute ++ ":" ++ + Res#result.second ++ + IfFun(R1, Offset, Res#result.zone), + calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]). + +mlist(R) -> + %% %H Shows hour in 24-hour format [15] + [ + {$H, R#result.hour}, + %% %M Displays minutes [00-59] + {$M, R#result.minute}, + %% %S Displays seconds [00-59] + {$S, R#result.second}, + %% %y Displays year YYYY [2021] + {$y, R#result.year}, + %% %m Displays the number of the month [01-12] + {$m, R#result.month}, + %% %d Displays the number of the month [01-12] + {$d, R#result.day}, + %% %Z Displays Time zone + {$Z, R#result.zone} + ]. + +rmap(Result) -> + maps:from_list(mlist(Result)). + +support_char() -> "HMSymdZ". + +create_tag(Head, []) -> + Head; +create_tag(Head, [Val1 | RVal]) -> + case Val1 of + [] -> + create_tag(Head ++ [{st, [$%]}], RVal); + [H | Other] -> + case lists:member(H, support_char()) of + true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal); + false -> create_tag(Head ++ [{st, [$% | Val1]}], RVal) + end + end. + +eval_tag(_, {st, Str}) -> + Str; +eval_tag(Map, {tag, Char}) -> + maps:get(Char, Map, "undefined"). + +%% make_time(TimeUnit, Offset) -> +%% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)). +make_time(TimeUnit, Offset, TimeEpoch) -> + Res = calendar:system_time_to_rfc3339(TimeEpoch, + [{unit, TimeUnit}, {offset, Offset}]), + [Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T, + H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = Res, + IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, + {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), + #result{ + year = [Y1, Y2, Y3, Y4] + , month = [Mon1, Mon2] + , day = [D1, D2] + , hour = [H1, H2] + , minute = [Min1, Min2] + , second = [S1, S2] ++ FractionStr + , zone = UtcOffset + }. + +is_int_char(C) -> + C >= $0 andalso C =< $9. +is_symbol_char(C) -> + C =:= $- orelse C =:= $+. +is_m_char(C) -> + C =:= $:. + +parse_char_with_fun(_, []) -> + error(null_input); +parse_char_with_fun(ValidFun, [C | Other]) -> + Res = + case erlang:is_function(ValidFun) of + true -> ValidFun(C); + false -> erlang:apply(emqx_rule_date, ValidFun, [C]) + end, + case Res of + true -> {C, Other}; + false -> error({unexpected, [C | Other]}) + end. +parse_string([], Input) -> + {[], Input}; +parse_string([C | Other], Input) -> + {C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input), + {Res, Input2} = parse_string(Other, Input1), + {[C1 | Res], Input2}. + +parse_times(0, _, Input) -> + {[], Input}; +parse_times(Times, Fun, Input) -> + {C1, Input1} = parse_char_with_fun(Fun, Input), + {Res, Input2} = parse_times((Times - 1), Fun, Input1), + {[C1 | Res], Input2}. + +parse_int_times(Times, Input) -> + parse_times(Times, is_int_char, Input). + +parse_fraction(Input) -> + IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, + lists:splitwith(IsFractionChar, Input). + +parse_second(Input) -> + {M, Input1} = parse_int_times(2, Input), + {M1, Input2} = parse_fraction(Input1), + {M ++ M1, Input2}. + +parse_zone(Input) -> + {S, Input1} = parse_char_with_fun(is_symbol_char, Input), + {M, Input2} = parse_int_times(2, Input1), + {C, Input3} = parse_char_with_fun(is_m_char, Input2), + {V, Input4} = parse_int_times(2, Input3), + {[S | M ++ [C | V]], Input4}. + +mlist1() -> + maps:from_list( + %% %H Shows hour in 24-hour format [15] + [ + {$H, fun(Input) -> parse_int_times(2, Input) end}, + %% %M Displays minutes [00-59] + {$M, fun(Input) -> parse_int_times(2, Input) end}, + %% %S Displays seconds [00-59] + {$S, fun(Input) -> parse_second(Input) end}, + %% %y Displays year YYYY [2021] + {$y, fun(Input) -> parse_int_times(4, Input) end}, + %% %m Displays the number of the month [01-12] + {$m, fun(Input) -> parse_int_times(2, Input) end}, + %% %d Displays the number of the month [01-12] + {$d, fun(Input) -> parse_int_times(2, Input) end}, + %% %Z Displays Time zone + {$Z, fun(Input) -> parse_zone(Input) end} + ] + ). + +update_result($H, Res, Str) -> Res#result{hour = Str}; +update_result($M, Res, Str) -> Res#result{minute = Str}; +update_result($S, Res, Str) -> Res#result{second = Str}; +update_result($y, Res, Str) -> Res#result{year = Str}; +update_result($m, Res, Str) -> Res#result{month = Str}; +update_result($d, Res, Str) -> Res#result{day = Str}; +update_result($Z, Res, Str) -> Res#result{zone = Str}. + +parse_tag(Res, {st, St}, InputString) -> + {_A, B} = parse_string(St, InputString), + {Res, B}; +parse_tag(Res, {tag, St}, InputString) -> + Fun = maps:get(St, mlist1()), + {A, B} = Fun(InputString), + NRes = update_result(St, Res, A), + {NRes, B}. + +parse_tags(Res, [], _) -> + Res; +parse_tags(Res, [Tag | Others], InputString) -> + {NRes, B} = parse_tag(Res, Tag, InputString), + parse_tags(NRes, Others, B). + +parse_input(FormatString, InputString) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + parse_tags(#result{}, R, InputString). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 93e38c44d..754eede15 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,17 +2,24 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.3", - [{load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_date}, + {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.4.2", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_date}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, @@ -27,6 +34,7 @@ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_date}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, @@ -34,6 +42,7 @@ {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {add_module,emqx_rule_date}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {update,emqx_rule_metrics,{advanced,["4.4.0"]}}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, @@ -42,17 +51,24 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.3", - [{load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {delete_module,emqx_rule_date}, + {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.4.2", - [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {delete_module,emqx_rule_date}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, @@ -67,6 +83,7 @@ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {delete_module,emqx_rule_date}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.0", [{load_module,emqx_rule_maps,brutal_purge,soft_purge,[]}, @@ -79,5 +96,6 @@ {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {delete_module,emqx_rule_date}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 477c5ad9a..312fc9e7d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -37,7 +37,9 @@ , test_resource/1 , start_resource/1 , get_resource_status/1 - , is_source_alive/1 + , is_resource_alive/1 + , is_resource_alive/2 + , is_resource_alive/3 , get_resource_params/1 , ensure_resource_deleted/1 , delete_resource/1 @@ -46,7 +48,7 @@ -export([ init_resource/4 , init_action/4 - , clear_resource/3 + , clear_resource/4 , clear_rule/1 , clear_actions/1 , clear_action/3 @@ -55,6 +57,13 @@ -export([ restore_action_metrics/2 ]). +-export([ fetch_resource_status/3 + ]). + +-ifdef(TEST). +-export([alarm_name_of_resource_down/2]). +-endif. + -type(rule() :: #rule{}). -type(action() :: #action{}). -type(resource() :: #resource{}). @@ -340,11 +349,11 @@ test_resource(#{type := Type} = Params) -> try case create_resource(maps:put(id, ResId, Params), no_retry) of {ok, _} -> - case is_source_alive(ResId) of + case is_resource_alive(ResId, #{fetch => true}) of true -> ok; false -> - %% in is_source_alive, the cluster-call RPC logs errors + %% in is_resource_alive, the cluster-call RPC logs errors %% so we do not log anything here {error, {resource_down, ResId}} end; @@ -362,18 +371,56 @@ test_resource(#{type := Type} = Params) -> {error, {resource_type_not_found, Type}} end. -is_source_alive(ResId) -> - case rpc:multicall(ekka_mnesia:running_nodes(), ?MODULE, get_resource_status, [ResId], 5000) of - {ResL, []} -> - is_source_alive_(ResL); - {_, _Errors} -> - false +is_resource_alive(ResId) -> + is_resource_alive(ResId, #{fetch => false}). + +is_resource_alive(ResId, Opts) -> + is_resource_alive(ekka_mnesia:running_nodes(), ResId, Opts). + +-spec(is_resource_alive(list(node()) | node(), resource_id(), #{fetch := boolean()}) -> boolean()). +is_resource_alive(Node, ResId, Opts) when is_atom(Node) -> + is_resource_alive([Node], ResId, Opts); +is_resource_alive(Nodes, ResId, _Opts = #{fetch := true}) -> + try + case emqx_rule_registry:find_resource(ResId) of + {ok, #resource{type = ResType}} -> + {ok, #resource_type{on_status = {Mod, OnStatus}}} + = emqx_rule_registry:find_resource_type(ResType), + case rpc:multicall(Nodes, + ?MODULE, fetch_resource_status, [Mod, OnStatus, ResId], 5000) of + {ResL, []} -> + is_resource_alive_(ResL); + {_, _Error} -> + false + end; + not_found -> + false + end + catch E:R:S -> + ?LOG(warning, "is_resource_alive failed, ~0p:~0p ~0p", [E, R, S]), + false + end; +is_resource_alive(Nodes, ResId, _Opts = #{fetch := false}) -> + try + case rpc:multicall(Nodes, ?MODULE, get_resource_status, [ResId], 5000) of + {ResL, []} -> + is_resource_alive_(ResL); + {_, _Errors} -> + false + end + catch E:R:S -> + ?LOG(warning, "is_resource_alive failed, ~0p:~0p ~0p", [E, R, S]), + false end. -is_source_alive_([]) -> true; -is_source_alive_([{ok, #{is_alive := true}} | ResL]) -> is_source_alive_(ResL); -is_source_alive_([{ok, #{is_alive := false}} | _ResL]) -> false; -is_source_alive_([_Error | _ResL]) -> false. +%% fetch_resource_status -> #{is_alive => boolean()} +%% get_resource_status -> {ok, #{is_alive => boolean()}} +is_resource_alive_([]) -> true; +is_resource_alive_([#{is_alive := true} | ResL]) -> is_resource_alive_(ResL); +is_resource_alive_([#{is_alive := false} | _ResL]) -> false; +is_resource_alive_([{ok, #{is_alive := true}} | ResL]) -> is_resource_alive_(ResL); +is_resource_alive_([{ok, #{is_alive := false}} | _ResL]) -> false; +is_resource_alive_([_Error | _ResL]) -> false. -spec(get_resource_status(resource_id()) -> {ok, resource_status()} | {error, Reason :: term()}). get_resource_status(ResId) -> @@ -407,7 +454,7 @@ delete_resource(ResId) -> try case emqx_rule_registry:remove_resource(ResId) of ok -> - _ = ?CLUSTER_CALL(clear_resource, [ModD, Destroy, ResId]), + _ = ?CLUSTER_CALL(clear_resource, [ModD, Destroy, ResId, ResType]), ok; {error, _} = R -> R end @@ -609,9 +656,13 @@ init_action(Module, OnCreate, ActionInstId, Params) -> #action_instance_params{id = ActionInstId, params = Params, apply = Apply}) end. -clear_resource(_Module, undefined, ResId) -> +clear_resource(_Module, undefined, Type, ResId) -> + Name = alarm_name_of_resource_down(Type, ResId), + _ = emqx_alarm:deactivate(Name), ok = emqx_rule_registry:remove_resource_params(ResId); -clear_resource(Module, Destroy, ResId) -> +clear_resource(Module, Destroy, Type, ResId) -> + Name = alarm_name_of_resource_down(Type, ResId), + _ = emqx_alarm:deactivate(Name), case emqx_rule_registry:find_resource_params(ResId) of {ok, #resource_params{params = Params}} -> ?RAISE(Module:Destroy(ResId, Params), diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 3430bc98d..6587f9c5f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -321,7 +321,7 @@ do_create_resource(Create, ParsedParams) -> list_resources(#{}, _Params) -> Data0 = lists:foldr(fun maybe_record_to_map/2, [], emqx_rule_registry:get_resources()), Data = lists:map(fun(Res = #{id := ResId}) -> - Status = emqx_rule_engine:is_source_alive(ResId), + Status = emqx_rule_engine:is_resource_alive(ResId), maps:put(status, Status, Res) end, Data0), return({ok, Data}). @@ -332,14 +332,14 @@ list_resources_by_type(#{type := Type}, _Params) -> show_resource(#{id := Id}, _Params) -> case emqx_rule_registry:find_resource(Id) of {ok, R} -> - Status = - lists:concat( - [ case rpc:call(Node, emqx_rule_engine, get_resource_status, [Id]) of - {badrpc, _} -> []; - {ok, St} -> [maps:put(node, Node, St)]; - {error, _} -> [maps:put(node, Node, #{is_alive => false})] - end - || Node <- ekka_mnesia:running_nodes()]), + StatusFun = + fun(Node) -> + #{ + node => Node, + is_alive => emqx_rule_engine:is_resource_alive(Node, Id, #{fetch => false}) + } + end, + Status = [StatusFun(Node) || Node <- ekka_mnesia:running_nodes()], return({ok, maps:put(status, Status, record_to_map(R))}); not_found -> return({error, 404, <<"Not Found">>}) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 6c2ff97e4..d829a3b14 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -224,6 +224,8 @@ eventmsg_disconnected(_ClientInfo = #{ ConnInfo = #{ peername := PeerName, sockname := SockName, + proto_name := ProtoName, + proto_ver := ProtoVer, disconnected_at := DisconnectedAt }, Reason) -> with_basic_columns('client.disconnected', @@ -232,6 +234,8 @@ eventmsg_disconnected(_ClientInfo = #{ username => Username, peername => ntoa(PeerName), sockname => ntoa(SockName), + proto_name => ProtoName, + proto_ver => ProtoVer, disconn_props => printable_maps(maps:get(disconn_props, ConnInfo, #{})), disconnected_at => DisconnectedAt }). @@ -686,6 +690,8 @@ columns_with_exam('client.disconnected') -> , {<<"username">>, <<"u_emqx">>} , {<<"peername">>, <<"192.168.0.10:56431">>} , {<<"sockname">>, <<"0.0.0.0:1883">>} + , {<<"proto_name">>, <<"MQTT">>} + , {<<"proto_ver">>, 5} , {<<"disconnected_at">>, erlang:system_time(millisecond)} , columns_example_props(disconn_props) , {<<"timestamp">>, erlang:system_time(millisecond)} diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 7bbc34378..33d09c16d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -96,6 +96,7 @@ , bool/1 , int/1 , float/1 + , float2str/2 , map/1 , bin2hexstr/1 , hexstr2bin/1 @@ -201,6 +202,9 @@ , now_rfc3339/1 , unix_ts_to_rfc3339/1 , unix_ts_to_rfc3339/2 + , format_date/3 + , format_date/4 + , date_to_unix_ts/4 , rfc3339_to_unix_ts/1 , rfc3339_to_unix_ts/2 , now_timestamp/0 @@ -538,6 +542,9 @@ int(Data) -> float(Data) -> emqx_rule_utils:float(Data). +float2str(Float, Precision) -> + emqx_rule_utils:float2str(Float, Precision). + map(Data) -> emqx_rule_utils:map(Data). @@ -924,6 +931,25 @@ time_unit(<<"millisecond">>) -> millisecond; time_unit(<<"microsecond">>) -> microsecond; time_unit(<<"nanosecond">>) -> nanosecond. +format_date(TimeUnit, Offset, FormatString) -> + emqx_rule_utils:bin( + emqx_rule_date:date(time_unit(TimeUnit), + emqx_rule_utils:str(Offset), + emqx_rule_utils:str(FormatString))). + +format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> + emqx_rule_utils:bin( + emqx_rule_date:date(time_unit(TimeUnit), + emqx_rule_utils:str(Offset), + emqx_rule_utils:str(FormatString), + TimeEpoch)). + +date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> + emqx_rule_date:parse_date(time_unit(TimeUnit), + emqx_rule_utils:str(Offset), + emqx_rule_utils:str(FormatString), + emqx_rule_utils:str(InputString)). + mongo_date() -> erlang:timestamp(). diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl index dc08ad0c3..d287f1ad0 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl @@ -32,6 +32,7 @@ %% type converting -export([ str/1 + , float2str/2 , bin/1 , bool/1 , int/1 @@ -265,6 +266,9 @@ str(List) when is_list(List) -> end; str(Data) -> error({invalid_str, Data}). +float2str(Float, Precision) when is_float(Float) and is_integer(Precision)-> + float_to_binary(Float, [{decimals, Precision}, compact]). + utf8_bin(Str) when is_binary(Str); is_list(Str) -> unicode:characters_to_binary(Str); utf8_bin(Str) -> diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 11f470e3b..6900f76aa 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -165,7 +165,7 @@ end_per_suite(_Config) -> on_resource_create(_id, _) -> #{}. on_resource_destroy(_id, _) -> ok. -on_get_resource_status(_id, _) -> #{}. +on_get_resource_status(_id, _) -> #{is_alive => true}. %%------------------------------------------------------------------------------ %% Group specific setup/teardown @@ -329,6 +329,24 @@ t_create_resource(_Config) -> emqx_rule_registry:remove_resource(ResId), ok. +t_clean_resource_alarms(_Config) -> + ok = emqx_rule_engine:load_providers(), + {ok, #resource{id = ResId}} = emqx_rule_engine:create_resource( + #{type => built_in, + config => #{}, + description => <<"debug resource">>}), + ?assert(true, is_binary(ResId)), + Name = emqx_rule_engine:alarm_name_of_resource_down(built_in, ResId), + _ = emqx_alarm:activate(Name, #{id => ResId, type => built_in}), + AlarmExist = fun(#{name := AName}) -> AName == Name end, + Len = length(lists:filter(AlarmExist, emqx_alarm:get_alarms())), + ?assert(Len == 1), + ok = emqx_rule_engine:unload_providers(), + emqx_rule_registry:remove_resource(ResId), + LenAfterRemove = length(lists:filter(AlarmExist, emqx_alarm:get_alarms())), + ?assert(LenAfterRemove == 0), + ok. + %%------------------------------------------------------------------------------ %% Test cases for rule actions %%------------------------------------------------------------------------------ diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 9a8c76e3f..98f6ad0b7 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -125,6 +125,13 @@ t_float(_) -> ?assertError({invalid_number, {a, v}}, emqx_rule_funcs:float({a, v})), ?assertError(_, emqx_rule_funcs:float("a")). + +t_float2str(_) -> + ?assertEqual(<<"20.2">>, emqx_rule_funcs:float2str(20.2, 1)), + ?assertEqual(<<"20.2">>, emqx_rule_funcs:float2str(20.2, 10)), + ?assertEqual(<<"20.199999999999999">>, emqx_rule_funcs:float2str(20.2, 15)), + ?assertEqual(<<"20.1999999999999993">>, emqx_rule_funcs:float2str(20.2, 16)). + t_map(_) -> ?assertEqual(#{ver => <<"1.0">>, name => "emqx"}, emqx_rule_funcs:map([{ver, <<"1.0">>}, {name, "emqx"}])), ?assertEqual(#{<<"a">> => 1}, emqx_rule_funcs:map(<<"{\"a\":1}">>)), @@ -159,8 +166,12 @@ t_term_encode(_) -> end, TestData). t_hexstr2bin(_) -> - ?assertEqual(<<1,2>>, emqx_rule_funcs:hexstr2bin(<<"0102">>)), - ?assertEqual(<<17,33>>, emqx_rule_funcs:hexstr2bin(<<"1121">>)). + ?assertEqual(<<6, 54, 79>>, emqx_rule_funcs:hexstr2bin(<<"6364f">>)), + ?assertEqual(<<10>>, emqx_rule_funcs:hexstr2bin(<<"a">>)), + ?assertEqual(<<15>>, emqx_rule_funcs:hexstr2bin(<<"f">>)), + ?assertEqual(<<5>>, emqx_rule_funcs:hexstr2bin(<<"5">>)), + ?assertEqual(<<1, 2>>, emqx_rule_funcs:hexstr2bin(<<"0102">>)), + ?assertEqual(<<17, 33>>, emqx_rule_funcs:hexstr2bin(<<"1121">>)). t_bin2hexstr(_) -> ?assertEqual(<<"0102">>, emqx_rule_funcs:bin2hexstr(<<1,2>>)), @@ -698,6 +709,25 @@ t_rfc3339_to_unix_ts(_) -> ?assertEqual(Epoch, emqx_rule_funcs:rfc3339_to_unix_ts(DateTime, BUnit)) end || Unit <- [second,millisecond,microsecond,nanosecond]]. +t_format_date_funcs(_) -> + ?PROPTEST(prop_format_date_fun). + +prop_format_date_fun() -> + Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>], + ?FORALL(S, erlang:system_time(second), + S == apply_func(date_to_unix_ts, + Args1 ++ [apply_func(format_date, + Args1 ++ [S])])), + Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>], + ?FORALL(S, erlang:system_time(millisecond), + S == apply_func(date_to_unix_ts, + Args2 ++ [apply_func(format_date, + Args2 ++ [S])])), + Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], + ?FORALL(S, erlang:system_time(second), + S == apply_func(date_to_unix_ts, + Args ++ [apply_func(format_date, + Args ++ [S])])). %%------------------------------------------------------------------------------ %% Utility functions %%------------------------------------------------------------------------------ diff --git a/apps/emqx_sn/src/emqx_sn.app.src b/apps/emqx_sn/src/emqx_sn.app.src index 319137fed..68e0340f8 100644 --- a/apps/emqx_sn/src/emqx_sn.app.src +++ b/apps/emqx_sn/src/emqx_sn.app.src @@ -1,6 +1,6 @@ {application, emqx_sn, [{description, "EMQ X MQTT-SN Plugin"}, - {vsn, "4.3.6"}, % strict semver, bump manually! + {vsn, "4.3.7"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,esockd]}, diff --git a/apps/emqx_sn/src/emqx_sn.appup.src b/apps/emqx_sn/src/emqx_sn.appup.src index 269605afa..bab8a2b7a 100644 --- a/apps/emqx_sn/src/emqx_sn.appup.src +++ b/apps/emqx_sn/src/emqx_sn.appup.src @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- {VSN, [ + {"4.3.6",[ + {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} + ]}, {"4.3.5",[ {load_module,emqx_sn_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sn_app,brutal_purge,soft_purge,[]}, @@ -26,6 +29,9 @@ {<<"4\\.3\\.[0-1]">>, [{restart_application,emqx_sn}]} ], [ + {"4.3.6",[ + {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} + ]}, {"4.3.5",[ {load_module,emqx_sn_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sn_app,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index a1b502d26..22fe51b6e 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -831,13 +831,15 @@ mqtt2sn(?CONNACK_PACKET(0, _SessPresent), _State) -> mqtt2sn(?CONNACK_PACKET(_ReturnCode, _SessPresent), _State) -> ?SN_CONNACK_MSG(?SN_RC_CONGESTION); -mqtt2sn(?PUBREC_PACKET(MsgId), _State) -> +mqtt2sn(?PUBACK_PACKET(MsgId, _ReasonCode), _State) -> + TopicIdFinal = get_topic_id(puback, MsgId), + ?SN_PUBACK_MSG(TopicIdFinal, MsgId, ?SN_RC_ACCEPTED); + +mqtt2sn(?PUBREC_PACKET(MsgId, _ReturnCode), _State) -> ?SN_PUBREC_MSG(?SN_PUBREC, MsgId); - -mqtt2sn(?PUBREL_PACKET(MsgId), _State) -> +mqtt2sn(?PUBREL_PACKET(MsgId, _ReturnCode), _State) -> ?SN_PUBREC_MSG(?SN_PUBREL, MsgId); - -mqtt2sn(?PUBCOMP_PACKET(MsgId), _State) -> +mqtt2sn(?PUBCOMP_PACKET(MsgId, _ReturnCode), _State) -> ?SN_PUBREC_MSG(?SN_PUBCOMP, MsgId); mqtt2sn(?UNSUBACK_PACKET(MsgId), _State)-> @@ -884,11 +886,7 @@ mqtt2sn(?SUBACK_PACKET(MsgId, ReturnCodes), _State)-> {?QOS_0, get_topic_id(suback, MsgId), ?SN_RC_NOT_SUPPORTED} end, Flags = #mqtt_sn_flags{qos = QoS}, - ?SN_SUBACK_MSG(Flags, TopicId, MsgId, NewReturnCode); - -mqtt2sn(?PUBACK_PACKET(MsgId, _ReasonCode), _State) -> - TopicIdFinal = get_topic_id(puback, MsgId), - ?SN_PUBACK_MSG(TopicIdFinal, MsgId, ?SN_RC_ACCEPTED). + ?SN_SUBACK_MSG(Flags, TopicId, MsgId, NewReturnCode). send_register(TopicName, TopicId, MsgId, State) -> send_message(?SN_REGISTER_MSG(TopicId, MsgId, TopicName), State). diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index a388c9004..e438a4677 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -36,7 +36,7 @@ -define(FLAG_RETAIN(X),X). -define(FLAG_SESSION(X),X). --define(LOG(Format, Args), ct:print("TEST: " ++ Format, Args)). +-define(LOG(Format, Args), ct:log("TEST: " ++ Format, Args)). -define(MAX_PRED_TOPIC_ID, 2). -define(PREDEF_TOPIC_ID1, 1). @@ -949,6 +949,42 @@ t_publish_qos2_case03(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). +t_publish_qos2_re_sent(_) -> + Dup = 0, + QoS = 2, + Retain = 0, + Will = 0, + CleanSession = 0, + MsgId = 7, + TopicId0 = 0, + {ok, Socket} = gen_udp:open(0, [binary]), + send_connect_msg(Socket, <<"test">>), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + send_subscribe_msg_normal_topic(Socket, QoS, <<"/#">>, MsgId), + ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket)), + + Payload1 = <<20, 21, 22, 23>>, + send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/a">>, Payload1), + ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC :2, <<"/a">>/binary, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + + %% re-sent qos2 PUBLISH + send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/a">>, Payload1), + %% still receive PUBREC normally + ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), + send_pubrel_msg(Socket, MsgId), + ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), + + timer:sleep(100), + + send_disconnect_msg(Socket, undefined), + %% note: receiving DISCONNECT packet here means that the qos2 is not duplicated + %% publish to broker + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + gen_udp:close(Socket). + t_delivery_qos1_register_invalid_topic_id(_) -> Dup = 0, QoS = 1, diff --git a/bin/emqx b/bin/emqx index 6a4f3fbb7..180b4eb77 100755 --- a/bin/emqx +++ b/bin/emqx @@ -530,6 +530,10 @@ fi NAME_TYPE="$(echo "$NAME_ARG" | awk '{print $1}')" NAME="$(echo "$NAME_ARG" | awk '{print $2}')" NODENAME="$(echo "$NAME" | awk -F'@' '{print $1}')" +if ! (echo "$NODENAME" | grep -q '^[0-9A-Za-z_\-]\+$'); then + echo "Invalid node name, should be of format '^[0-9A-Za-z_-]+$'." + exit 1 +fi export ESCRIPT_NAME="$NODENAME" PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}" diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 69cec9175..14d90156c 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -5,6 +5,7 @@ -define(TIMEOUT, 300000). -define(INFO(Fmt,Args), io:format(Fmt++"~n",Args)). +-define(SEMVER_RE, <<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">>). -mode(compile). @@ -54,6 +55,7 @@ unpack(_, Args) -> install({RelName, NameTypeArg, NodeName, Cookie}, Opts) -> TargetNode = start_distribution(NodeName, NameTypeArg, Cookie), Version = proplists:get_value(version, Opts), + validate_target_version(Version, TargetNode), case unpack_release(RelName, TargetNode, Version) of {ok, Vsn} -> ?INFO("Unpacked successfully: ~p.", [Vsn]), @@ -427,6 +429,29 @@ erts_vsn() -> [ErtsVsn, _] = string:tokens(binary_to_list(Str), " "), ErtsVsn. +validate_target_version(TargetVersion, TargetNode) -> + CurrentVersion = current_release_version(TargetNode), + case {get_major_minor_vsn(CurrentVersion), get_major_minor_vsn(TargetVersion)} of + {{Major, Minor}, {Major, Minor}} -> ok; + _ -> + ?INFO("Cannot upgrade/downgrade to ~s from ~s~n" + "We only support relup between patch versions", + [TargetVersion, CurrentVersion]), + error({relup_not_allowed, unsupported_target_version}) + end. + +get_major_minor_vsn(Version) -> + Parts = parse_semver(Version), + [Major | Rem0] = Parts, + [Minor | _Rem1] = Rem0, + {Major, Minor}. + +parse_semver(Version) -> + case re:run(Version, ?SEMVER_RE, [{capture, all_but_first, binary}]) of + {match, Parts} -> Parts; + nomatch -> error({invalid_semver, Version}) + end. + str(A) when is_atom(A) -> atom_to_list(A); str(A) when is_binary(A) -> diff --git a/etc/emqx_cloud/vm.args b/etc/emqx_cloud/vm.args index a199091c7..e0f4c70d7 100644 --- a/etc/emqx_cloud/vm.args +++ b/etc/emqx_cloud/vm.args @@ -115,4 +115,4 @@ -shutdown_time 30000 ## patches dir --pa {{ platform_data_dir }}/patches +-pa "{{ platform_data_dir }}/patches" diff --git a/etc/emqx_edge/vm.args b/etc/emqx_edge/vm.args index 2f61faa2d..a87449b79 100644 --- a/etc/emqx_edge/vm.args +++ b/etc/emqx_edge/vm.args @@ -113,4 +113,4 @@ -shutdown_time 10000 ## patches dir --pa {{ platform_data_dir }}/patches +-pa "{{ platform_data_dir }}/patches" diff --git a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema index d517a6e97..a2985429b 100644 --- a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema +++ b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema @@ -83,7 +83,7 @@ ]}. {mapping, "dashboard.listener.https.verify", "emqx_dashboard.listeners", [ - {datatype, string} + {datatype, atom} ]}. {mapping, "dashboard.listener.https.fail_if_no_peer_cert", "emqx_dashboard.listeners", [ @@ -149,4 +149,3 @@ end end, [http, https])) end}. - diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src b/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src index 8394836ef..74f8fa63e 100644 --- a/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src +++ b/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src @@ -1,6 +1,6 @@ {application, emqx_telemetry, [{description, "EMQ X Telemetry"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_telemetry_sup]}, {applications, [kernel,stdlib]}, diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src b/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src index 23de107d9..79e6d3d41 100644 --- a/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src +++ b/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src @@ -1,13 +1,13 @@ %% -*- mode: erlang -*- {VSN, [ - {<<"4\\.3\\.[0-1]">>, [ + {<<"4\\.3\\.[0-2]">>, [ {load_module, emqx_telemetry, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ - {<<"4\\.3\\.[0-1]">>, [ + {<<"4\\.3\\.[0-2]">>, [ {load_module, emqx_telemetry, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry.erl b/lib-ce/emqx_telemetry/src/emqx_telemetry.erl index 29edc6b9c..b87329076 100644 --- a/lib-ce/emqx_telemetry/src/emqx_telemetry.erl +++ b/lib-ce/emqx_telemetry/src/emqx_telemetry.erl @@ -367,7 +367,9 @@ report_telemetry(State = #state{url = URL}) -> end. httpc_request(Method, URL, Headers, Body) -> - httpc:request(Method, {URL, Headers, "application/json", Body}, [], []). + HTTPOptions = [{timeout, timer:seconds(10)}, {ssl, [{verify, verify_none}]}], + Options = [], + httpc:request(Method, {URL, Headers, "application/json", Body}, HTTPOptions, Options). ignore_lib_apps(Apps) -> LibApps = [kernel, stdlib, sasl, appmon, eldap, erts, diff --git a/rebar.config b/rebar.config index 913f3c994..7835a0e2f 100644 --- a/rebar.config +++ b/rebar.config @@ -45,7 +45,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.9"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.10"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 7a04551b2..147b37903 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,14 +2,27 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.3", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, {"4.4.2", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, @@ -18,6 +31,8 @@ {load_module,emqx_relup}]}, {"4.4.1", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -61,6 +76,7 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, @@ -72,14 +88,25 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.3", - [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, - {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_relup}]}, {"4.4.2", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, @@ -88,6 +115,7 @@ {load_module,emqx_relup}]}, {"4.4.1", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_listeners,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index 3ee367e9c..476fe91d4 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -34,11 +34,15 @@ -spec(authenticate(emqx_types:clientinfo()) -> {ok, result()} | {error, term()}). authenticate(ClientInfo = #{zone := Zone}) -> AuthResult = default_auth_result(Zone), - case emqx_zone:get_env(Zone, bypass_auth_plugins, false) of + case + begin ok = emqx_metrics:inc('client.authenticate'), + emqx_zone:get_env(Zone, bypass_auth_plugins, false) + end + of true -> return_auth_result(AuthResult); false -> - return_auth_result(run_hooks('client.authenticate', [ClientInfo], AuthResult)) + return_auth_result(emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult)) end. %% @doc Check ACL @@ -52,6 +56,10 @@ check_acl(ClientInfo, PubSub, Topic) -> inc_acl_metrics(Result), Result. +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +%% ACL check_acl_cache(ClientInfo, PubSub, Topic) -> case emqx_acl_cache:get_acl_cache(PubSub, Topic) of not_found -> @@ -66,23 +74,14 @@ check_acl_cache(ClientInfo, PubSub, Topic) -> do_check_acl(ClientInfo = #{zone := Zone}, PubSub, Topic) -> Default = emqx_zone:get_env(Zone, acl_nomatch, deny), - Result = case run_hooks('client.check_acl', [ClientInfo, PubSub, Topic], Default) of - allow -> allow; - _Other -> deny + ok = emqx_metrics:inc('client.check_acl'), + Result = case emqx_hooks:run_fold('client.check_acl', [ClientInfo, PubSub, Topic], Default) of + allow -> allow; + _Other -> deny end, emqx:run_hook('client.check_acl_complete', [ClientInfo, PubSub, Topic, Result, false]), Result. -default_auth_result(Zone) -> - case emqx_zone:get_env(Zone, allow_anonymous, false) of - true -> #{auth_result => success, anonymous => true}; - false -> #{auth_result => not_authorized, anonymous => false} - end. - --compile({inline, [run_hooks/3]}). -run_hooks(Name, Args, Acc) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc). - -compile({inline, [inc_acl_metrics/1]}). inc_acl_metrics(allow) -> emqx_metrics:inc('client.acl.allow'); @@ -91,8 +90,26 @@ inc_acl_metrics(deny) -> inc_acl_metrics(cache_hit) -> emqx_metrics:inc('client.acl.cache_hit'). +%% Auth +default_auth_result(Zone) -> + case emqx_zone:get_env(Zone, allow_anonymous, false) of + true -> #{auth_result => success, anonymous => true}; + false -> #{auth_result => not_authorized, anonymous => false} + end. + -compile({inline, [return_auth_result/1]}). -return_auth_result(Result = #{auth_result := success}) -> - {ok, Result}; -return_auth_result(Result) -> - {error, maps:get(auth_result, Result, unknown_error)}. +return_auth_result(AuthResult = #{auth_result := success}) -> + inc_auth_success_metrics(AuthResult), + {ok, AuthResult}; +return_auth_result(AuthResult) -> + emqx_metrics:inc('client.auth.failure'), + {error, maps:get(auth_result, AuthResult, unknown_error)}. + +-compile({inline, [inc_auth_success_metrics/1]}). +inc_auth_success_metrics(AuthResult) -> + is_anonymous(AuthResult) andalso + emqx_metrics:inc('client.auth.success.anonymous'), + emqx_metrics:inc('client.auth.success'). + +is_anonymous(#{anonymous := true}) -> true; +is_anonymous(_AuthResult) -> false. diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 074888939..5eaf88704 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1254,13 +1254,18 @@ check_connect(ConnPkt, #channel{clientinfo = #{zone := Zone}}) -> %% Enrich Client Info enrich_client(ConnPkt, Channel = #channel{clientinfo = ClientInfo}) -> - {ok, NConnPkt, NClientInfo} = pipeline([fun set_username/2, - fun set_bridge_mode/2, - fun maybe_username_as_clientid/2, - fun maybe_assign_clientid/2, - fun fix_mountpoint/2 - ], ConnPkt, ClientInfo), - {ok, NConnPkt, Channel#channel{clientinfo = NClientInfo}}. + Pipe = pipeline([fun set_username/2, + fun set_bridge_mode/2, + fun maybe_username_as_clientid/2, + fun maybe_assign_clientid/2, + fun fix_mountpoint/2 + ], ConnPkt, ClientInfo), + case Pipe of + {ok, NConnPkt, NClientInfo} -> + {ok, NConnPkt, Channel#channel{clientinfo = NClientInfo}}; + {error, ReasonCode, NClientInfo} -> + {error, ReasonCode, Channel#channel{clientinfo = NClientInfo}} + end. set_username(#mqtt_packet_connect{username = Username}, ClientInfo = #{username := undefined}) -> @@ -1275,7 +1280,8 @@ maybe_username_as_clientid(_ConnPkt, ClientInfo = #{username := undefined}) -> {ok, ClientInfo}; maybe_username_as_clientid(_ConnPkt, ClientInfo = #{zone := Zone, username := Username}) -> case emqx_zone:use_username_as_clientid(Zone) of - true -> {ok, ClientInfo#{clientid => Username}}; + true when Username =/= <<>> -> {ok, ClientInfo#{clientid => Username}}; + true -> {error, ?RC_CLIENT_IDENTIFIER_NOT_VALID, ClientInfo}; false -> ok end. @@ -1317,8 +1323,6 @@ auth_connect(#mqtt_packet_connect{password = Password}, username := Username} = ClientInfo, case emqx_access_control:authenticate(ClientInfo#{password => Password}) of {ok, AuthResult} -> - is_anonymous(AuthResult) andalso - emqx_metrics:inc('client.auth.anonymous'), NClientInfo = maps:merge(ClientInfo, AuthResult), {ok, Channel#channel{clientinfo = NClientInfo}}; {error, Reason} -> @@ -1327,9 +1331,6 @@ auth_connect(#mqtt_packet_connect{password = Password}, {error, emqx_reason_codes:connack_error(Reason)} end. -is_anonymous(#{anonymous := true}) -> true; -is_anonymous(_AuthResult) -> false. - %%-------------------------------------------------------------------- %% Enhanced Authentication diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl index 9bb19b30a..7bec83465 100644 --- a/src/emqx_metrics.erl +++ b/src/emqx_metrics.erl @@ -68,8 +68,10 @@ %% BACKW -export([%% v4.3.0 upgrade_retained_delayed_counter_type/0, - %% e4.4.0, e4.3.0-e4.3.6, v4.3.0-v4.3.11 - assign_acl_stats_from_ets_to_counter/0 + %% v4.3.0-v4.3.11, e4.3.0-e4.3.6; v4.4.0, e4.4.0 + assign_acl_stats_from_ets_to_counter/0, + %% v4.3.0-v4.3.14, e4.3.0-e4.3.9; v4.4.0-v4.4.3, e4.4.0-e4.4.3, + assign_auth_stats_from_ets_to_counter/0 ]). -export_type([metric_idx/0]). @@ -174,7 +176,6 @@ {counter, 'client.connack'}, {counter, 'client.connected'}, {counter, 'client.authenticate'}, - {counter, 'client.auth.anonymous'}, {counter, 'client.check_acl'}, {counter, 'client.subscribe'}, {counter, 'client.unsubscribe'}, @@ -189,8 +190,16 @@ {counter, 'session.discarded'}, {counter, 'session.terminated'} ]). -%% Statistic metrics for ACL checking --define(STASTS_ACL_METRICS, + +%% Statistic metrics for auth checking +-define(STATS_AUTH_METRICS, + [ {counter, 'client.auth.success'}, + {counter, 'client.auth.success.anonymous'}, + {counter, 'client.auth.failure'} + ]). + +%% Statistic metrics for ACL checking stats +-define(STATS_ACL_METRICS, [ {counter, 'client.acl.allow'}, {counter, 'client.acl.deny'}, {counter, 'client.acl.cache_hit'} @@ -228,6 +237,21 @@ assign_acl_stats_from_ets_to_counter() -> ok = counters:put(CRef, Idx, Val) end, Names). +%% BACKW: %% v4.3.0-v4.3.14, e4.3.0-e4.3.9; v4.4.0-v4.4.3, e4.4.0-e4.4.3, +assign_auth_stats_from_ets_to_counter() -> + CRef = persistent_term:get(?MODULE), + Names = ['client.auth.success', 'client.auth.success.anonymous', 'client.auth.failure'], + lists:foreach(fun(Name) -> + Val = case emqx_metrics:val(Name) of + undefined -> 0; + Val0 -> Val0 + end, + Idx = reserved_idx(Name), + Metric = #metric{name = Name, type = counter, idx = Idx}, + ok = gen_server:call(?SERVER, {set, Metric}), + ok = counters:put(CRef, Idx, Val) + end, Names). + %%-------------------------------------------------------------------- %% Metrics API %%-------------------------------------------------------------------- @@ -458,7 +482,8 @@ init([]) -> ?DELIVERY_METRICS, ?CLIENT_METRICS, ?SESSION_METRICS, - ?STASTS_ACL_METRICS + ?STATS_AUTH_METRICS, + ?STATS_ACL_METRICS ]), % Store reserved indices ok = lists:foreach(fun({Type, Name}) -> @@ -592,7 +617,6 @@ reserved_idx('client.connack') -> 201; reserved_idx('client.connected') -> 202; reserved_idx('client.authenticate') -> 203; reserved_idx('client.enhanced_authenticate') -> 204; -reserved_idx('client.auth.anonymous') -> 205; reserved_idx('client.check_acl') -> 206; reserved_idx('client.subscribe') -> 207; reserved_idx('client.unsubscribe') -> 208; @@ -604,9 +628,13 @@ reserved_idx('session.takeovered') -> 222; reserved_idx('session.discarded') -> 223; reserved_idx('session.terminated') -> 224; %% Stats metrics +%% ACL reserved_idx('client.acl.allow') -> 300; reserved_idx('client.acl.deny') -> 301; reserved_idx('client.acl.cache_hit') -> 302; +%% Auth +reserved_idx('client.auth.success') -> 310; +reserved_idx('client.auth.success.anonymous') -> 311; +reserved_idx('client.auth.failure') -> 312; reserved_idx(_) -> undefined. - diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index 76f6c3d50..c366b878d 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -312,7 +312,19 @@ int2hexchar(I, lower) -> I - 10 + $a. -spec(hexstr2bin(binary()) -> binary()). hexstr2bin(B) when is_binary(B) -> - << <<(hexchar2int(H)*16 + hexchar2int(L))>> || <> <= B>>. + hexstr2bin(B, erlang:bit_size(B)). + +hexstr2bin(B, Size) when is_binary(B) -> + case Size rem 16 of + 0 -> + make_binary(B); + 8 -> + make_binary(<<"0", B/binary>>); + _ -> + throw({unsupport_hex_string, B, Size}) + end. + +make_binary(B) -> <<<<(hexchar2int(H) * 16 + hexchar2int(L))>> || <> <= B>>. hexchar2int(I) when I >= $0 andalso I =< $9 -> I - $0; hexchar2int(I) when I >= $A andalso I =< $F -> I - $A + 10; diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 45ad18bbe..429275aff 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -491,10 +491,13 @@ deliver_msg(ClientInfo, Msg = #message{qos = QoS}, Session = end, {ok, Session1}; false -> + %% Note that we publish message without shared ack header + %% But add to inflight with ack headers + %% This ack header is required for redispatch-on-terminate feature to work Publish = {PacketId, maybe_ack(Msg)}, Msg2 = mark_begin_deliver(Msg), - Session1 = await(PacketId, Msg2, Session), - {ok, [Publish], next_pkt_id(Session1)} + Inflight1 = emqx_inflight:insert(PacketId, with_ts(Msg2), Inflight), + {ok, [Publish], next_pkt_id(Session#session{inflight = Inflight1})} end. -spec(enqueue(emqx_types:clientinfo(), list(emqx_types:deliver()) | emqx_types:message(), @@ -532,14 +535,10 @@ enrich_delivers({deliver, Topic, Msg}, Session = #session{subscriptions = Subs}) enrich_subopts(get_subopts(Topic, Subs), Msg, Session). maybe_ack(Msg) -> - case emqx_shared_sub:is_ack_required(Msg) of - true -> emqx_shared_sub:maybe_ack(Msg); - false -> Msg - end. + emqx_shared_sub:maybe_ack(Msg). maybe_nack(Msg) -> - emqx_shared_sub:is_ack_required(Msg) - andalso (ok == emqx_shared_sub:maybe_nack_dropped(Msg)). + emqx_shared_sub:maybe_nack_dropped(Msg). get_subopts(Topic, SubMap) -> case maps:find(Topic, SubMap) of @@ -572,14 +571,6 @@ enrich_subopts([{subid, SubId} | Opts], Msg, Session) -> Msg1 = emqx_message:set_header(properties, Props#{'Subscription-Identifier' => SubId}, Msg), enrich_subopts(Opts, Msg1, Session). -%%-------------------------------------------------------------------- -%% Awaiting ACK for QoS1/QoS2 Messages -%%-------------------------------------------------------------------- - -await(PacketId, Msg, Session = #session{inflight = Inflight}) -> - Inflight1 = emqx_inflight:insert(PacketId, with_ts(Msg), Inflight), - Session#session{inflight = Inflight1}. - %%-------------------------------------------------------------------- %% Retry Delivery %%-------------------------------------------------------------------- @@ -679,16 +670,43 @@ replay(ClientInfo, Session = #session{inflight = Inflight}) -> end. -spec(terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok). -terminate(ClientInfo, discarded, Session) -> - run_hook('session.discarded', [ClientInfo, info(Session)]); -terminate(ClientInfo, takeovered, Session) -> - run_hook('session.takeovered', [ClientInfo, info(Session)]); terminate(ClientInfo, Reason, Session) -> + run_terminate_hooks(ClientInfo, Reason, Session), + redispatch_shared_messages(Session), + ok. + +run_terminate_hooks(ClientInfo, discarded, Session) -> + run_hook('session.discarded', [ClientInfo, info(Session)]); +run_terminate_hooks(ClientInfo, takeovered, Session) -> + run_hook('session.takeovered', [ClientInfo, info(Session)]); +run_terminate_hooks(ClientInfo, Reason, Session) -> run_hook('session.terminated', [ClientInfo, Reason, info(Session)]). +redispatch_shared_messages(#session{inflight = Inflight}) -> + InflightList = emqx_inflight:to_list(Inflight), + lists:foreach(fun + %% Only QoS1 messages get redispatched, because QoS2 messages + %% must be sent to the same client, once they're in flight + ({_, {#message{qos = ?QOS_2} = Msg, _}}) -> + ?LOG(warning, "Not redispatching qos2 msg: ~s", [emqx_message:format(Msg)]); + ({_, {#message{topic = Topic, qos = ?QOS_1} = Msg, _}}) -> + case emqx_shared_sub:get_group(Msg) of + {ok, Group} -> + %% Note that dispatch is called with self() in failed subs + %% This is done to avoid dispatching back to caller + Delivery = #delivery{sender = self(), message = Msg}, + emqx_shared_sub:dispatch(Group, Topic, Delivery, [self()]); + _ -> + false + end; + (_) -> + ok + end, InflightList). + -compile({inline, [run_hook/2]}). run_hook(Name, Args) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run(Name, Args). + ok = emqx_metrics:inc(Name), + emqx_hooks:run(Name, Args). %%-------------------------------------------------------------------- %% Inc message/delivery expired counter diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index 12e64b59a..121d777ca 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -38,12 +38,15 @@ , unsubscribe/3 ]). --export([dispatch/3]). +-export([ dispatch/3 + , dispatch/4 + ]). -export([ maybe_ack/1 , maybe_nack_dropped/1 , nack_no_connection/1 , is_ack_required/1 + , get_group/1 ]). %% for testing @@ -131,7 +134,7 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> false -> {error, no_subscribers}; {Type, SubPid} -> - case do_dispatch(SubPid, Topic, Msg, Type) of + case do_dispatch(SubPid, Group, Topic, Msg, Type) of ok -> {ok, 1}; {error, _Reason} -> %% Failed to dispatch to this sub, try next. @@ -153,36 +156,33 @@ strategy(Group) -> ack_enabled() -> emqx:get_env(shared_dispatch_ack_enabled, false). -do_dispatch(SubPid, Topic, Msg, _Type) when SubPid =:= self() -> +do_dispatch(SubPid, _Group, Topic, Msg, _Type) when SubPid =:= self() -> %% Deadlock otherwise - _ = erlang:send(SubPid, {deliver, Topic, Msg}), + SubPid ! {deliver, Topic, Msg}, ok; -do_dispatch(SubPid, Topic, Msg, Type) -> - dispatch_per_qos(SubPid, Topic, Msg, Type). - %% return either 'ok' (when everything is fine) or 'error' -dispatch_per_qos(SubPid, Topic, #message{qos = ?QOS_0} = Msg, _Type) -> +do_dispatch(SubPid, _Group, Topic, #message{qos = ?QOS_0} = Msg, _Type) -> %% For QoS 0 message, send it as regular dispatch - _ = erlang:send(SubPid, {deliver, Topic, Msg}), + SubPid ! {deliver, Topic, Msg}, ok; -dispatch_per_qos(SubPid, Topic, Msg, retry) -> +do_dispatch(SubPid, _Group, Topic, Msg, retry) -> %% Retry implies all subscribers nack:ed, send again without ack - _ = erlang:send(SubPid, {deliver, Topic, Msg}), + SubPid ! {deliver, Topic, Msg}, ok; -dispatch_per_qos(SubPid, Topic, Msg, fresh) -> +do_dispatch(SubPid, Group, Topic, Msg, fresh) -> case ack_enabled() of true -> - dispatch_with_ack(SubPid, Topic, Msg); + dispatch_with_ack(SubPid, Group, Topic, Msg); false -> - _ = erlang:send(SubPid, {deliver, Topic, Msg}), + SubPid ! {deliver, Topic, Msg}, ok end. -dispatch_with_ack(SubPid, Topic, Msg) -> +dispatch_with_ack(SubPid, Group, Topic, Msg) -> %% For QoS 1/2 message, expect an ack Ref = erlang:monitor(process, SubPid), Sender = self(), - _ = erlang:send(SubPid, {deliver, Topic, with_ack_ref(Msg, {Sender, Ref})}), + SubPid ! {deliver, Topic, with_group_ack(Msg, Group, Sender, Ref)}, Timeout = case Msg#message.qos of ?QOS_1 -> timer:seconds(?SHARED_SUB_QOS1_DISPATCH_TIMEOUT_SECONDS); ?QOS_2 -> infinity @@ -204,24 +204,32 @@ dispatch_with_ack(SubPid, Topic, Msg) -> _ = erlang:demonitor(Ref, [flush]) end. -with_ack_ref(Msg, SenderRef) -> - emqx_message:set_headers(#{shared_dispatch_ack => SenderRef}, Msg). +with_group_ack(Msg, Group, Sender, Ref) -> + emqx_message:set_headers(#{shared_dispatch_ack => {Group, Sender, Ref}}, Msg). -without_ack_ref(Msg) -> +-spec(without_group_ack(emqx_types:message()) -> emqx_types:message()). +without_group_ack(Msg) -> emqx_message:set_headers(#{shared_dispatch_ack => ?NO_ACK}, Msg). -get_ack_ref(Msg) -> +get_group_ack(Msg) -> emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK). +-spec(get_group(emqx_types:message()) -> {ok, any()} | error). +get_group(Msg) -> + case get_group_ack(Msg) of + ?NO_ACK -> error; + {Group, _Sender, _Ref} -> {ok, Group} + end. + -spec(is_ack_required(emqx_types:message()) -> boolean()). -is_ack_required(Msg) -> ?NO_ACK =/= get_ack_ref(Msg). +is_ack_required(Msg) -> ?NO_ACK =/= get_group_ack(Msg). %% @doc Negative ack dropped message due to inflight window or message queue being full. --spec(maybe_nack_dropped(emqx_types:message()) -> ok). +-spec(maybe_nack_dropped(emqx_types:message()) -> boolean()). maybe_nack_dropped(Msg) -> - case get_ack_ref(Msg) of - ?NO_ACK -> ok; - {Sender, Ref} -> nack(Sender, Ref, dropped) + case get_group_ack(Msg) of + ?NO_ACK -> false; + {_Group, Sender, Ref} -> ok == nack(Sender, Ref, dropped) end. %% @doc Negative ack message due to connection down. @@ -229,22 +237,22 @@ maybe_nack_dropped(Msg) -> %% i.e is_ack_required returned true. -spec(nack_no_connection(emqx_types:message()) -> ok). nack_no_connection(Msg) -> - {Sender, Ref} = get_ack_ref(Msg), + {_Group, Sender, Ref} = get_group_ack(Msg), nack(Sender, Ref, no_connection). -spec(nack(pid(), reference(), dropped | no_connection) -> ok). nack(Sender, Ref, Reason) -> - erlang:send(Sender, {Ref, ?NACK(Reason)}), + Sender ! {Ref, ?NACK(Reason)}, ok. -spec(maybe_ack(emqx_types:message()) -> emqx_types:message()). maybe_ack(Msg) -> - case get_ack_ref(Msg) of + case get_group_ack(Msg) of ?NO_ACK -> Msg; - {Sender, Ref} -> - erlang:send(Sender, {Ref, ?ACK}), - without_ack_ref(Msg) + {_Group, Sender, Ref} -> + Sender ! {Ref, ?ACK}, + without_group_ack(Msg) end. pick(sticky, ClientId, SourceTopic, Group, Topic, FailedSubs) -> diff --git a/test/emqx_client_SUITE.erl b/test/emqx_client_SUITE.erl index 4d69d11b3..d45e7a8cf 100644 --- a/test/emqx_client_SUITE.erl +++ b/test/emqx_client_SUITE.erl @@ -278,9 +278,15 @@ t_username_as_clientid(_) -> {ok, C} = emqtt:start_link([{username, Username}]), {ok, _} = emqtt:connect(C), #{clientinfo := #{clientid := Username}} = emqx_cm:get_chan_info(Username), - emqtt:disconnect(C). - - + emqtt:disconnect(C), + erlang:process_flag(trap_exit, true), + {ok, C1} = emqtt:start_link([{username, <<>>}]), + ?assertEqual({error, {client_identifier_not_valid, undefined}}, emqtt:connect(C1)), + receive + {'EXIT', _, {shutdown, client_identifier_not_valid}} -> ok + after 100 -> + throw({error, "expect_client_identifier_not_valid"}) + end. t_certcn_as_clientid_default_config_tls(_) -> tls_certcn_as_clientid(default). diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index 0a6c47d60..5c2de94d0 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -47,20 +47,20 @@ t_is_ack_required(_) -> ?assertEqual(false, emqx_shared_sub:is_ack_required(#message{headers = #{}})). t_maybe_nack_dropped(_) -> - ?assertEqual(ok, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})), - Msg = #message{headers = #{shared_dispatch_ack => {self(), for_test}}}, - ?assertEqual(ok, emqx_shared_sub:maybe_nack_dropped(Msg)), + ?assertEqual(false, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})), + Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}}, + ?assertEqual(true, emqx_shared_sub:maybe_nack_dropped(Msg)), ?assertEqual(ok,receive {for_test, {shared_sub_nack, dropped}} -> ok after 100 -> timeout end). t_nack_no_connection(_) -> - Msg = #message{headers = #{shared_dispatch_ack => {self(), for_test}}}, + Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}}, ?assertEqual(ok, emqx_shared_sub:nack_no_connection(Msg)), ?assertEqual(ok,receive {for_test, {shared_sub_nack, no_connection}} -> ok after 100 -> timeout end). t_maybe_ack(_) -> ?assertEqual(#message{headers = #{}}, emqx_shared_sub:maybe_ack(#message{headers = #{}})), - Msg = #message{headers = #{shared_dispatch_ack => {self(), for_test}}}, + Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}}, ?assertEqual(#message{headers = #{shared_dispatch_ack => ?no_ack}}, emqx_shared_sub:maybe_ack(Msg)), ?assertEqual(ok,receive {for_test, ?ack} -> ok after 100 -> timeout end). @@ -284,11 +284,14 @@ test_two_messages(Strategy, Group) -> ok. last_message(ExpectedPayload, Pids) -> + last_message(ExpectedPayload, Pids, 100). + +last_message(ExpectedPayload, Pids, Timeout) -> receive {publish, #{client_pid := Pid, payload := ExpectedPayload}} -> ct:pal("last_message: ~p ====== ~p, payload=~p", [Pids, Pid, ExpectedPayload]), {true, Pid} - after 100 -> + after Timeout -> ct:pal("not yet"), <<"not yet?">> end. @@ -410,6 +413,39 @@ t_local_fallback(_) -> ?assertEqual(UsedSubPid1, UsedSubPid2), ok. +%% This one tests that broker tries to select another shared subscriber +%% If the first one doesn't return an ACK +t_redispatch(_) -> + ok = ensure_config(sticky, true), + application:set_env(emqx, shared_dispatch_ack_enabled, true), + + Group = <<"group1">>, + + Topic = <<"foo/bar">>, + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]), + {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, false}]), + {ok, _} = emqtt:connect(ConnPid1), + {ok, _} = emqtt:connect(ConnPid2), + + emqtt:subscribe(ConnPid1, {<<"$share/", Group/binary, "/foo/bar">>, 1}), + emqtt:subscribe(ConnPid2, {<<"$share/", Group/binary, "/foo/bar">>, 1}), + + Message = emqx_message:make(ClientId1, 1, Topic, <<"hello1">>), + + emqx:publish(Message), + + {true, UsedSubPid1} = last_message(<<"hello1">>, [ConnPid1, ConnPid2]), + ok = emqtt:stop(UsedSubPid1), + + Res = last_message(<<"hello1">>, [ConnPid1, ConnPid2], 6000), + ?assertMatch({true, Pid} when Pid =/= UsedSubPid1, Res), + + {true, UsedSubPid2} = Res, + emqtt:stop(UsedSubPid2), + ok. + %%-------------------------------------------------------------------- %% help functions %%-------------------------------------------------------------------- @@ -481,6 +517,7 @@ setup_node(Node, Port) -> opts => [{zone,internal}], proto => tcp}]), application:set_env(gen_rpc, port_discovery, manual), + application:set_env(gen_rpc, tcp_server_port, Port * 2), ok; (_) -> ok