From d535d44845b16c818839e92c287d692dd9c15b93 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Jan 2022 11:21:08 +0800 Subject: [PATCH 1/4] revert: revert #6797 due to the acl metrics refactor --- .../src/emqx_mod_acl_internal.erl | 30 +++---------------- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- .../emqx_modules/src/emqx_modules.appup.src | 25 ++++------------ 3 files changed, 11 insertions(+), 46 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_mod_acl_internal.erl b/lib-ce/emqx_modules/src/emqx_mod_acl_internal.erl index 0f999848e..8956229ea 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_acl_internal.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_acl_internal.erl @@ -38,26 +38,12 @@ -type(acl_rules() :: #{publish => [emqx_access_rule:rule()], subscribe => [emqx_access_rule:rule()]}). --record(acl_metrics, { - allow = 'client.acl.allow', - deny = 'client.acl.deny', - ignore = 'client.acl.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(ACL_METRICS, ?METRICS(acl_metrics)). --define(ACL_METRICS(K), ?METRICS(acl_metrics, K)). - - %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- load(Env) -> Rules = rules_from_file(proplists:get_value(acl_file, Env)), - register_metrics(), emqx_hooks:add('client.check_acl', {?MODULE, check_acl, [Rules]}, -1). unload(_Env) -> @@ -82,15 +68,9 @@ description() -> -> {ok, allow} | {ok, deny} | ok). check_acl(Client, PubSub, Topic, _AclResult, Rules) -> case match(Client, Topic, lookup(PubSub, Rules)) of - {matched, allow} -> - emqx_metrics:inc(?ACL_METRICS(allow)), - {ok, allow}; - {matched, deny} -> - emqx_metrics:inc(?ACL_METRICS(deny)), - {ok, deny}; - nomatch -> - emqx_metrics:inc(?ACL_METRICS(ignore)), - ok + {matched, allow} -> {ok, allow}; + {matched, deny} -> {ok, deny}; + nomatch -> ok end. %%-------------------------------------------------------------------- @@ -127,9 +107,6 @@ rules_from_file(AclFile) -> #{} end. -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS). - filter(_PubSub, {allow, all}) -> true; filter(_PubSub, {deny, all}) -> @@ -142,3 +119,4 @@ filter(subscribe, {_AllowDeny, _Who, subscribe, _Topics}) -> true; filter(_PubSub, {_AllowDeny, _Who, _, _Topics}) -> false. + diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index 47a3d8888..bcb05fe31 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.4"}, + {vsn, "4.3.3"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index 8ea2c5ebe..f52ba1a61 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,45 +1,32 @@ %% -*-: erlang -*- {VSN, [ - {"4.3.3", [ - {apply, {emqx_metrics, ensure, [['client.acl.allow', 'client.acl.ignore', 'client.acl.deny']]}}, - {load_module, emqx_mod_acl_internal, brutal_purge, soft_purge, []} - ]}, {"4.3.2", [ - {load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {apply, {emqx_metrics, ensure, [['client.acl.allow', 'client.acl.ignore', 'client.acl.deny']]}}, - {load_module, emqx_mod_acl_internal, brutal_purge, soft_purge, []} + {load_module, emqx_mod_presence, brutal_purge, soft_purge, []} ]}, {"4.3.1", [ {load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []}, - {apply, {emqx_metrics, ensure, [['client.acl.allow', 'client.acl.ignore', 'client.acl.deny']]}}, - {load_module, emqx_mod_acl_internal, brutal_purge, soft_purge, []} + {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {update, emqx_mod_delayed, {advanced, []}}, {load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []}, - {apply, {emqx_metrics, ensure, [['client.acl.allow', 'client.acl.ignore', 'client.acl.deny']]}}, - {load_module, emqx_mod_acl_internal, brutal_purge, soft_purge, []} + {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ {"4.3.2", [ - {load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_acl_internal, brutal_purge, soft_purge, []} + {load_module, emqx_mod_presence, brutal_purge, soft_purge, []} ]}, {"4.3.1", [ {load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_acl_internal, brutal_purge, soft_purge, []} + {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {update, emqx_mod_delayed, {advanced, []}}, {load_module, emqx_mod_presence, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []}, - {load_module, emqx_mod_acl_internal, brutal_purge, soft_purge, []} + {load_module, emqx_mod_api_topic_metrics, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ] From ad996afe473a0add9d366c14d10d8e5bcaacac68 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Jan 2022 11:14:06 +0800 Subject: [PATCH 2/4] chore: ensure connected_at field for connect/connack/disconnected event --- apps/emqx_web_hook/src/emqx_web_hook.erl | 3 +++ apps/emqx_web_hook/test/props/prop_webhook_hooks.erl | 5 ++++- lib-ce/emqx_modules/src/emqx_mod_presence.erl | 7 ++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook.erl b/apps/emqx_web_hook/src/emqx_web_hook.erl index b556c16f8..7da2e3c6b 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook.erl @@ -93,6 +93,7 @@ on_client_connect(ConnInfo = #{clientid := ClientId, username := Username, peern , ipaddress => iolist_to_binary(ntoa(Peerhost)) , keepalive => maps:get(keepalive, ConnInfo) , proto_ver => maps:get(proto_ver, ConnInfo) + , connected_at => maps:get(connected_at, ConnInfo) }, send_http_request(ClientId, Params). @@ -109,6 +110,7 @@ on_client_connack(ConnInfo = #{clientid := ClientId, username := Username, peern , ipaddress => iolist_to_binary(ntoa(Peerhost)) , keepalive => maps:get(keepalive, ConnInfo) , proto_ver => maps:get(proto_ver, ConnInfo) + , connected_at => maps:get(connected_at, ConnInfo) , conn_ack => Rc }, send_http_request(ClientId, Params). @@ -143,6 +145,7 @@ on_client_disconnected(#{clientid := ClientId, username := Username}, Reason, Co , clientid => ClientId , username => maybe(Username) , reason => stringfy(maybe(Reason)) + , connected_at => maps:get(connected_at, ConnInfo) , disconnected_at => maps:get(disconnected_at, ConnInfo, erlang:system_time(millisecond)) }, send_http_request(ClientId, Params). diff --git a/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl b/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl index 311585287..438d70d22 100644 --- a/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl +++ b/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl @@ -52,7 +52,8 @@ prop_client_connect() -> username => maybe(maps:get(username, ConnInfo)), ipaddress => peer2addr(maps:get(peername, ConnInfo)), keepalive => maps:get(keepalive, ConnInfo), - proto_ver => maps:get(proto_ver, ConnInfo) + proto_ver => maps:get(proto_ver, ConnInfo), + connected_at => maps:get(connected_at, ConnInfo) }), true end). @@ -71,6 +72,7 @@ prop_client_connack() -> ipaddress => peer2addr(maps:get(peername, ConnInfo)), keepalive => maps:get(keepalive, ConnInfo), proto_ver => maps:get(proto_ver, ConnInfo), + connected_at => maps:get(connected_at, ConnInfo), conn_ack => Rc }), true @@ -106,6 +108,7 @@ prop_client_disconnected() -> node => stringfy(node()), clientid => maps:get(clientid, ClientInfo), username => maybe(maps:get(username, ClientInfo)), + connected_at => maps:get(connected_at, ConnInfo), disconnected_at => maps:get(disconnected_at, ConnInfo), reason => stringfy(Reason) }), diff --git a/lib-ce/emqx_modules/src/emqx_mod_presence.erl b/lib-ce/emqx_modules/src/emqx_mod_presence.erl index 01a60218c..582a3280f 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_presence.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_presence.erl @@ -57,8 +57,7 @@ on_client_connected(ClientInfo = #{clientid := ClientId}, ConnInfo, Env) -> connack => 0, %% XXX: connack will be removed in 5.0 keepalive => maps:get(keepalive, ConnInfo, 0), clean_start => maps:get(clean_start, ConnInfo, true), - expiry_interval => maps:get(expiry_interval, ConnInfo, 0), - connected_at => maps:get(connected_at, ConnInfo) + expiry_interval => maps:get(expiry_interval, ConnInfo, 0) }, case emqx_json:safe_encode(NPresence) of {ok, Payload} -> @@ -95,7 +94,8 @@ common_infos( sockport := SockPort }, _ConnInfo = #{proto_name := ProtoName, - proto_ver := ProtoVer + proto_ver := ProtoVer, + connected_at := ConnectedAt }) -> #{clientid => ClientId, username => Username, @@ -103,6 +103,7 @@ common_infos( sockport => SockPort, proto_name => ProtoName, proto_ver => ProtoVer, + connected_at => ConnectedAt, ts => erlang:system_time(millisecond) }. From 77609ebd96fc3b0f562c4e828788e40f723fbd48 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Jan 2022 11:15:09 +0800 Subject: [PATCH 3/4] chore: bump vsn and update appup.src --- apps/emqx_web_hook/src/emqx_web_hook.app.src | 2 +- apps/emqx_web_hook/src/emqx_web_hook.appup.src | 6 ++++-- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- lib-ce/emqx_modules/src/emqx_modules.appup.src | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index 0a99a2e0a..acabcd954 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.8"}, % strict semver, bump manually! + {vsn, "4.3.9"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index c70a652c5..531ad308b 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -5,9 +5,10 @@ {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[3-7]">>, + {<<"4\\.3\\.[3-8]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{<<"4\\.3\\.[0-2]">>, @@ -15,8 +16,9 @@ {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[3-7]">>, + {<<"4\\.3\\.[3-8]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index bcb05fe31..47a3d8888 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.3"}, + {vsn, "4.3.4"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index f52ba1a61..1b9eeec84 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,7 +1,7 @@ %% -*-: erlang -*- {VSN, [ - {"4.3.2", [ + {<<"4\\.3\\.[2-3]">>, [ {load_module, emqx_mod_presence, brutal_purge, soft_purge, []} ]}, {"4.3.1", [ @@ -16,7 +16,7 @@ {<<".*">>, []} ], [ - {"4.3.2", [ + {<<"4\\.3\\.[2-3]">>, [ {load_module, emqx_mod_presence, brutal_purge, soft_purge, []} ]}, {"4.3.1", [ From 3b4ddd6f518e7cf8d1117febe00a2fe9091ef15a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Jan 2022 15:05:13 +0800 Subject: [PATCH 4/4] chore: bump emqx_modules vsn --- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index bcb05fe31..47a3d8888 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.3"}, + {vsn, "4.3.4"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}},