From 2a348c812c4eae355fd3b47a689e16e8fe7d33a8 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 19 May 2022 11:21:57 +0800 Subject: [PATCH 01/36] chore: show more error msg when check_erlang_start failed --- bin/emqx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/emqx b/bin/emqx index a3843ec7e..756d2f705 100755 --- a/bin/emqx +++ b/bin/emqx @@ -37,15 +37,15 @@ assert_node_alive() { fi } -check_eralng_start() { - "$BINDIR/$PROGNAME" -noshell -boot "$REL_DIR/start_clean" -s crypto start -s init stop +check_erlang_start() { + "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()" } -if ! check_eralng_start >/dev/null 2>&1; then +if ! check_erlang_start >/dev/null 2>&1; then BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")" ## failed to start, might be due to missing libs, try to be portable export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH" - if ! check_eralng_start; then + if ! check_erlang_start; then ## it's hopeless echoerr "FATAL: Unable to start Erlang." echoerr "Please make sure openssl-1.1.1 (libcrypto) and libncurses are installed." From c705f294a1c04ec648e7564d5fa38042ff4d125f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 19 May 2022 13:48:31 -0300 Subject: [PATCH 02/36] fix(dashboard): use atom for `verify` tls option in dashboard listener ``` 2022-05-19T10:47:34.262135-04:00 [error] Failed to start Ranch listener 'https:dashboard' in ranch_ssl:listen for reason {options,{verify,"verify_none"}} (unknown POSIX error) ``` --- lib-ce/emqx_dashboard/priv/emqx_dashboard.schema | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema index d517a6e97..a2985429b 100644 --- a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema +++ b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema @@ -83,7 +83,7 @@ ]}. {mapping, "dashboard.listener.https.verify", "emqx_dashboard.listeners", [ - {datatype, string} + {datatype, atom} ]}. {mapping, "dashboard.listener.https.fail_if_no_peer_cert", "emqx_dashboard.listeners", [ @@ -149,4 +149,3 @@ end end, [http, https])) end}. - From ae71125271d791e30cc55634adf293bd125f1452 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 20 May 2022 14:04:53 +0800 Subject: [PATCH 03/36] fix: update pgsql sdk for otp23 & otp24 --- apps/emqx_auth_pgsql/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_auth_pgsql/rebar.config b/apps/emqx_auth_pgsql/rebar.config index e1a1c752c..be395993b 100644 --- a/apps/emqx_auth_pgsql/rebar.config +++ b/apps/emqx_auth_pgsql/rebar.config @@ -1,5 +1,5 @@ {deps, - [{epgsql, {git, "https://github.com/epgsql/epgsql.git", {tag, "4.4.0"}}} + [{epgsql, {git, "https://github.com/epgsql/epgsql.git", {tag, "4.6.0"}}} ]}. {erl_opts, [warn_unused_vars, From c10c75ce002338780dd0a09c6201ae86d559c39a Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 20 May 2022 14:14:06 +0800 Subject: [PATCH 04/36] fix: appup for epgsql 4.4.0 -> 4.6.0 --- .../src/emqx_auth_pgsql.app.src | 2 +- .../src/emqx_auth_pgsql.appup.src | 41 +++++++++++++------ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src index 75175a814..35cae1622 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_pgsql, [{description, "EMQ X Authentication/ACL with PostgreSQL"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_pgsql_sup]}, {applications, [kernel,stdlib,epgsql,ecpool]}, diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src index 1c9a0c572..07c13cbff 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src @@ -1,16 +1,31 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.1", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, - {"4.3.0", - [{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}], - [{"4.3.1", []}, - {"4.3.0", - [{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}] + [ + {"4.3.2", [ + %% epgsql 4.4.0 -> 4.6.0. + %% epgsql has no appup ,so we can only restart it. + {restart_application, epgsql} + ]}, + {"4.3.1", [ + {restart_application, epgsql} + ]}, + {"4.3.0", [ + {restart_application, epgsql}, + {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]} + ]}, + {<<".*">>,[]}], + [ + {"4.3.2", [ + {restart_application, epgsql} + ]}, + {"4.3.1", [ + {restart_application, epgsql} + ]}, + {"4.3.0", [ + {restart_application, epgsql}, + {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]} + ]}, + {<<".*">>,[]}] }. From 35d091aa1be50b43cfebddef73d91fa26afbdfc5 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 20 May 2022 10:01:19 +0800 Subject: [PATCH 05/36] fix(test): flaky test cases of auth_ldap --- apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl | 4 +++- .../emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl index 5de24d913..8bf7d532b 100644 --- a/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl @@ -48,7 +48,9 @@ init_per_group(GrpName, Cfg) -> Cfg. end_per_group(_GrpName, _Cfg) -> - emqx_ct_helpers:stop_apps([emqx_auth_ldap]). + emqx_ct_helpers:stop_apps([emqx_auth_ldap]), + %% clear the application envs to avoid cross-suite testcase failure + application:unload(emqx_auth_ldap). %%-------------------------------------------------------------------- %% Cases diff --git a/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl index 4f56ab32a..99f733d78 100644 --- a/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl @@ -40,7 +40,9 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_auth_ldap]). + emqx_ct_helpers:stop_apps([emqx_auth_ldap]), + %% clear the application envs to avoid cross-suite testcase failure + application:unload(emqx_auth_ldap). check_auth(_) -> MqttUser1 = #{clientid => <<"mqttuser1">>, From 92c09fdd7b0a519419276a86ff4dfa3537a1782b Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 20 May 2022 15:40:59 +0800 Subject: [PATCH 06/36] fix: deny relup between minor versions --- bin/install_upgrade.escript | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 69cec9175..14d90156c 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -5,6 +5,7 @@ -define(TIMEOUT, 300000). -define(INFO(Fmt,Args), io:format(Fmt++"~n",Args)). +-define(SEMVER_RE, <<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">>). -mode(compile). @@ -54,6 +55,7 @@ unpack(_, Args) -> install({RelName, NameTypeArg, NodeName, Cookie}, Opts) -> TargetNode = start_distribution(NodeName, NameTypeArg, Cookie), Version = proplists:get_value(version, Opts), + validate_target_version(Version, TargetNode), case unpack_release(RelName, TargetNode, Version) of {ok, Vsn} -> ?INFO("Unpacked successfully: ~p.", [Vsn]), @@ -427,6 +429,29 @@ erts_vsn() -> [ErtsVsn, _] = string:tokens(binary_to_list(Str), " "), ErtsVsn. +validate_target_version(TargetVersion, TargetNode) -> + CurrentVersion = current_release_version(TargetNode), + case {get_major_minor_vsn(CurrentVersion), get_major_minor_vsn(TargetVersion)} of + {{Major, Minor}, {Major, Minor}} -> ok; + _ -> + ?INFO("Cannot upgrade/downgrade to ~s from ~s~n" + "We only support relup between patch versions", + [TargetVersion, CurrentVersion]), + error({relup_not_allowed, unsupported_target_version}) + end. + +get_major_minor_vsn(Version) -> + Parts = parse_semver(Version), + [Major | Rem0] = Parts, + [Minor | _Rem1] = Rem0, + {Major, Minor}. + +parse_semver(Version) -> + case re:run(Version, ?SEMVER_RE, [{capture, all_but_first, binary}]) of + {match, Parts} -> Parts; + nomatch -> error({invalid_semver, Version}) + end. + str(A) when is_atom(A) -> atom_to_list(A); str(A) when is_binary(A) -> From 9435fe0bb9fa2342f1cb77c1d331a4544530065a Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 20 May 2022 13:36:30 +0800 Subject: [PATCH 07/36] refactor(metrics): more understandable auth metrics `client.auth.success`/`client.auth.failure` `client.auth.success.anonymous` --- src/emqx_access_control.erl | 55 +++++++++++++++++++++++++------------ src/emqx_channel.erl | 5 ---- src/emqx_metrics.erl | 44 +++++++++++++++++++++++------ 3 files changed, 74 insertions(+), 30 deletions(-) diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index fe12ce7e6..93dde28c2 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -34,11 +34,15 @@ -spec(authenticate(emqx_types:clientinfo()) -> {ok, result()} | {error, term()}). authenticate(ClientInfo = #{zone := Zone}) -> AuthResult = default_auth_result(Zone), - case emqx_zone:get_env(Zone, bypass_auth_plugins, false) of + case + begin ok = emqx_metrics:inc('client.authenticate'), + emqx_zone:get_env(Zone, bypass_auth_plugins, false) + end + of true -> return_auth_result(AuthResult); false -> - return_auth_result(run_hooks('client.authenticate', [ClientInfo], AuthResult)) + return_auth_result(emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult)) end. %% @doc Check ACL @@ -51,6 +55,10 @@ check_acl(ClientInfo, PubSub, Topic) -> end, inc_acl_metrics(Result), Result. +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +%% ACL check_acl_cache(ClientInfo, PubSub, Topic) -> case emqx_acl_cache:get_acl_cache(PubSub, Topic) of not_found -> @@ -64,21 +72,16 @@ check_acl_cache(ClientInfo, PubSub, Topic) -> do_check_acl(ClientInfo = #{zone := Zone}, PubSub, Topic) -> Default = emqx_zone:get_env(Zone, acl_nomatch, deny), - case run_hooks('client.check_acl', [ClientInfo, PubSub, Topic], Default) of + case + begin + ok = emqx_metrics:inc('client.check_acl'), + emqx_hooks:run_fold('client.check_acl', [ClientInfo, PubSub, Topic], Default) + end + of allow -> allow; _Other -> deny end. -default_auth_result(Zone) -> - case emqx_zone:get_env(Zone, allow_anonymous, false) of - true -> #{auth_result => success, anonymous => true}; - false -> #{auth_result => not_authorized, anonymous => false} - end. - --compile({inline, [run_hooks/3]}). -run_hooks(Name, Args, Acc) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc). - -compile({inline, [inc_acl_metrics/1]}). inc_acl_metrics(allow) -> emqx_metrics:inc('client.acl.allow'); @@ -87,8 +90,26 @@ inc_acl_metrics(deny) -> inc_acl_metrics(cache_hit) -> emqx_metrics:inc('client.acl.cache_hit'). +%% Auth +default_auth_result(Zone) -> + case emqx_zone:get_env(Zone, allow_anonymous, false) of + true -> #{auth_result => success, anonymous => true}; + false -> #{auth_result => not_authorized, anonymous => false} + end. + -compile({inline, [return_auth_result/1]}). -return_auth_result(Result = #{auth_result := success}) -> - {ok, Result}; -return_auth_result(Result) -> - {error, maps:get(auth_result, Result, unknown_error)}. +return_auth_result(AuthResult = #{auth_result := success}) -> + inc_auth_success_metrics(AuthResult), + {ok, AuthResult}; +return_auth_result(AuthResult) -> + emqx_metrics:inc('client.auth.failure'), + {error, maps:get(auth_result, AuthResult, unknown_error)}. + +-compile({inline, [inc_auth_success_metrics/1]}). +inc_auth_success_metrics(AuthResult) -> + is_anonymous(AuthResult) andalso + emqx_metrics:inc('client.auth.success.anonymous'), + emqx_metrics:inc('client.auth.success'). + +is_anonymous(#{anonymous := true}) -> true; +is_anonymous(_AuthResult) -> false. diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 530b04250..63a67592f 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1285,8 +1285,6 @@ auth_connect(#mqtt_packet_connect{password = Password}, username := Username} = ClientInfo, case emqx_access_control:authenticate(ClientInfo#{password => Password}) of {ok, AuthResult} -> - is_anonymous(AuthResult) andalso - emqx_metrics:inc('client.auth.anonymous'), NClientInfo = maps:merge(ClientInfo, AuthResult), {ok, Channel#channel{clientinfo = NClientInfo}}; {error, Reason} -> @@ -1295,9 +1293,6 @@ auth_connect(#mqtt_packet_connect{password = Password}, {error, emqx_reason_codes:connack_error(Reason)} end. -is_anonymous(#{anonymous := true}) -> true; -is_anonymous(_AuthResult) -> false. - %%-------------------------------------------------------------------- %% Enhanced Authentication diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl index 9bb19b30a..7bec83465 100644 --- a/src/emqx_metrics.erl +++ b/src/emqx_metrics.erl @@ -68,8 +68,10 @@ %% BACKW -export([%% v4.3.0 upgrade_retained_delayed_counter_type/0, - %% e4.4.0, e4.3.0-e4.3.6, v4.3.0-v4.3.11 - assign_acl_stats_from_ets_to_counter/0 + %% v4.3.0-v4.3.11, e4.3.0-e4.3.6; v4.4.0, e4.4.0 + assign_acl_stats_from_ets_to_counter/0, + %% v4.3.0-v4.3.14, e4.3.0-e4.3.9; v4.4.0-v4.4.3, e4.4.0-e4.4.3, + assign_auth_stats_from_ets_to_counter/0 ]). -export_type([metric_idx/0]). @@ -174,7 +176,6 @@ {counter, 'client.connack'}, {counter, 'client.connected'}, {counter, 'client.authenticate'}, - {counter, 'client.auth.anonymous'}, {counter, 'client.check_acl'}, {counter, 'client.subscribe'}, {counter, 'client.unsubscribe'}, @@ -189,8 +190,16 @@ {counter, 'session.discarded'}, {counter, 'session.terminated'} ]). -%% Statistic metrics for ACL checking --define(STASTS_ACL_METRICS, + +%% Statistic metrics for auth checking +-define(STATS_AUTH_METRICS, + [ {counter, 'client.auth.success'}, + {counter, 'client.auth.success.anonymous'}, + {counter, 'client.auth.failure'} + ]). + +%% Statistic metrics for ACL checking stats +-define(STATS_ACL_METRICS, [ {counter, 'client.acl.allow'}, {counter, 'client.acl.deny'}, {counter, 'client.acl.cache_hit'} @@ -228,6 +237,21 @@ assign_acl_stats_from_ets_to_counter() -> ok = counters:put(CRef, Idx, Val) end, Names). +%% BACKW: %% v4.3.0-v4.3.14, e4.3.0-e4.3.9; v4.4.0-v4.4.3, e4.4.0-e4.4.3, +assign_auth_stats_from_ets_to_counter() -> + CRef = persistent_term:get(?MODULE), + Names = ['client.auth.success', 'client.auth.success.anonymous', 'client.auth.failure'], + lists:foreach(fun(Name) -> + Val = case emqx_metrics:val(Name) of + undefined -> 0; + Val0 -> Val0 + end, + Idx = reserved_idx(Name), + Metric = #metric{name = Name, type = counter, idx = Idx}, + ok = gen_server:call(?SERVER, {set, Metric}), + ok = counters:put(CRef, Idx, Val) + end, Names). + %%-------------------------------------------------------------------- %% Metrics API %%-------------------------------------------------------------------- @@ -458,7 +482,8 @@ init([]) -> ?DELIVERY_METRICS, ?CLIENT_METRICS, ?SESSION_METRICS, - ?STASTS_ACL_METRICS + ?STATS_AUTH_METRICS, + ?STATS_ACL_METRICS ]), % Store reserved indices ok = lists:foreach(fun({Type, Name}) -> @@ -592,7 +617,6 @@ reserved_idx('client.connack') -> 201; reserved_idx('client.connected') -> 202; reserved_idx('client.authenticate') -> 203; reserved_idx('client.enhanced_authenticate') -> 204; -reserved_idx('client.auth.anonymous') -> 205; reserved_idx('client.check_acl') -> 206; reserved_idx('client.subscribe') -> 207; reserved_idx('client.unsubscribe') -> 208; @@ -604,9 +628,13 @@ reserved_idx('session.takeovered') -> 222; reserved_idx('session.discarded') -> 223; reserved_idx('session.terminated') -> 224; %% Stats metrics +%% ACL reserved_idx('client.acl.allow') -> 300; reserved_idx('client.acl.deny') -> 301; reserved_idx('client.acl.cache_hit') -> 302; +%% Auth +reserved_idx('client.auth.success') -> 310; +reserved_idx('client.auth.success.anonymous') -> 311; +reserved_idx('client.auth.failure') -> 312; reserved_idx(_) -> undefined. - From e57e2f68d4cf6332079a94f516458b45442a74b4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 20 May 2022 16:14:23 -0300 Subject: [PATCH 08/36] fix(mgmt): allow binding to specific interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When one tries to define the management http listener as, for example, `172.25.0.2:8081`, ranch is given `{port, {"172.25.0.2",8081}}` and breaks. ```sh env EMQX_MANAGEMENT__LISTENER__HTTP=172.25.0.2:8081 make quickrun ``` ``` 2022-05-20T13:14:19.451272-03:00 [error] Supervisor: {<0.2485.0>,ranch_listener_sup}. Context: start_error. Reason: badarg. Offender: id=ranch_acceptors_sup,pid=undefined. 2022-05-20T13:14:19.451400-03:00 [error] crasher: initial call: supervisor:ranch_acceptors_sup/1, pid: <0.2487.0>, registered_name: [], exit: {badarg,[{inet_tcp,listen,2,[{file,"inet_tcp.erl"},{line,166}]},{ranch_acceptors_sup,init,1,[{file,"ranch_acceptors_sup.erl"},{line,39}]},{supervisor,init,1,[{file,"supervisor.erl"},{line,330}]},{gen_server,init_it,2,[{file,"gen_server.erl"},{line,423}]},{gen_server,init_it,6,[{file,"gen_server.erl"},{line,390}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [<0.2485.0>,ranch_sup,<0.2187.0>], message_queue_len: 0, messages: [], links: [<0.2485.0>], dictionary: [{logger,error_logger}], trap_exit: true, status: running, heap_size: 610, stack_size: 29, reductions: 612; neighbours: 2022-05-20T13:14:19.451754-03:00 [error] Minirest(Handler): Start http:management listener on {"172.25.0.2",8081} unsuccessfully: ``` So, it’s currently not possible to make the management listener bind to a specific interface. --- apps/emqx_management/priv/emqx_management.schema | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/priv/emqx_management.schema b/apps/emqx_management/priv/emqx_management.schema index 343a70de6..a30a20e4d 100644 --- a/apps/emqx_management/priv/emqx_management.schema +++ b/apps/emqx_management/priv/emqx_management.schema @@ -227,6 +227,13 @@ end}. Prefix = "management.listener." ++ atom_to_list(Proto), case cuttlefish:conf_get(Prefix, Conf, undefined) of undefined -> Acc; + {IPStr, Port} -> + {ok, IP} = inet:parse_address(IPStr), + [{Proto, Port, [{ip, IP}] ++ TcpOpts(Prefix) ++ Opts(Prefix) + ++ case Proto of + http -> []; + https -> SslOpts(Prefix) + end} | Acc]; Port -> [{Proto, Port, TcpOpts(Prefix) ++ Opts(Prefix) ++ case Proto of @@ -236,4 +243,3 @@ end}. end end, [], [http, https]) end}. - From bd23dae523299bf63f0bf5c9ee312b8863d13f99 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 20 May 2022 14:11:54 +0800 Subject: [PATCH 09/36] refactor(auth): auth apps use counter in emqx_metrics --- .../emqx_auth_http/include/emqx_auth_http.hrl | 13 -------- apps/emqx_auth_http/src/emqx_auth_http.erl | 12 ++------ .../emqx_auth_http/src/emqx_auth_http_app.erl | 2 -- apps/emqx_auth_jwt/src/emqx_auth_jwt.erl | 30 ++++--------------- apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl | 1 - .../emqx_auth_ldap/include/emqx_auth_ldap.hrl | 13 -------- apps/emqx_auth_ldap/src/emqx_auth_ldap.erl | 11 ++----- .../emqx_auth_ldap/src/emqx_auth_ldap_app.erl | 1 - .../include/emqx_auth_mnesia.hrl | 12 -------- .../emqx_auth_mnesia/src/emqx_auth_mnesia.erl | 8 ----- .../src/emqx_auth_mnesia_app.erl | 1 - .../include/emqx_auth_mongo.hrl | 13 -------- apps/emqx_auth_mongo/src/emqx_auth_mongo.erl | 12 ++------ .../src/emqx_auth_mongo_app.erl | 2 -- .../include/emqx_auth_mysql.hrl | 13 -------- apps/emqx_auth_mysql/src/emqx_auth_mysql.erl | 12 ++------ .../src/emqx_auth_mysql_app.erl | 1 - .../include/emqx_auth_pgsql.hrl | 12 -------- apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl | 12 ++------ .../src/emqx_auth_pgsql_app.erl | 2 -- .../include/emqx_auth_redis.hrl | 13 -------- apps/emqx_auth_redis/src/emqx_auth_redis.erl | 12 ++------ .../src/emqx_auth_redis_app.erl | 2 -- .../emqx_exproto/src/emqx_exproto_channel.erl | 5 ---- apps/emqx_prometheus/src/emqx_prometheus.erl | 6 ++-- 25 files changed, 20 insertions(+), 201 deletions(-) diff --git a/apps/emqx_auth_http/include/emqx_auth_http.hrl b/apps/emqx_auth_http/include/emqx_auth_http.hrl index b62f02bcd..0eaa59daf 100644 --- a/apps/emqx_auth_http/include/emqx_auth_http.hrl +++ b/apps/emqx_auth_http/include/emqx_auth_http.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_http). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_http/src/emqx_auth_http.erl b/apps/emqx_auth_http/src/emqx_auth_http.erl index f97276849..98a897a8c 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http.erl @@ -30,22 +30,16 @@ ]). %% Callbacks --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo, AuthResult, #{auth := AuthParms = #{path := Path}, super := SuperParams}) -> case authenticate(AuthParms, ClientInfo) of {ok, 200, <<"ignore">>} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; + ok; {ok, 200, Body} -> - emqx_metrics:inc(?AUTH_METRICS(success)), IsSuperuser = is_superuser(SuperParams, ClientInfo), {stop, AuthResult#{is_superuser => IsSuperuser, auth_result => success, @@ -54,12 +48,10 @@ check(ClientInfo, AuthResult, #{auth := AuthParms = #{path := Path}, {ok, Code, _Body} -> ?LOG(error, "Deny connection from path: ~s, response http code: ~p", [Path, Code]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => http_to_connack_error(Code), anonymous => false}}; {error, Error} -> ?LOG(error, "Request auth path: ~s, error: ~p", [Path, Error]), - emqx_metrics:inc(?AUTH_METRICS(failure)), %%FIXME later: server_unavailable is not right. {stop, AuthResult#{auth_result => server_unavailable, anonymous => false}} diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index fb23f9c0a..9e4b842d8 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -112,7 +112,6 @@ load_hooks() -> case application:get_env(?APP, auth_req) of undefined -> ok; {ok, AuthReq} -> - ok = emqx_auth_http:register_metrics(), PoolOpts = proplists:get_value(pool_opts, AuthReq), PoolName = proplists:get_value(pool_name, AuthReq), {ok, _} = ehttpc_sup:start_pool(PoolName, PoolOpts), @@ -160,4 +159,3 @@ path(#{path := ""}) -> "/"; path(#{path := Path}) -> Path. - diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl index b900f2bed..040f9b629 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl @@ -21,28 +21,11 @@ -logger_header("[JWT]"). --export([ register_metrics/0 - , check_auth/3 +-export([ check_auth/3 , check_acl/5 , description/0 ]). --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). - --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - %%-------------------------------------------------------------------- %% Authentication callbacks %%-------------------------------------------------------------------- @@ -50,17 +33,16 @@ register_metrics() -> check_auth(ClientInfo, AuthResult, #{from := From, checklists := Checklists}) -> case maps:find(From, ClientInfo) of error -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {ok, undefined} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {ok, Token} -> case emqx_auth_jwt_svr:verify(Token) of {error, not_found} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, not_token} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, Reason} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => Reason, anonymous => false}}; {ok, Claims} -> {stop, maps:merge(AuthResult, verify_claims(Checklists, Claims, ClientInfo))} @@ -121,10 +103,8 @@ verify_acl(ClientInfo, [AclTopic | AclTopics], Topic) -> verify_claims(Checklists, Claims, ClientInfo) -> case do_verify_claims(feedvar(Checklists, ClientInfo), Claims) of {error, Reason} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), #{auth_result => Reason, anonymous => false}; ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), #{auth_result => success, anonymous => false, jwt_claims => Claims} end. diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl index c94e8aa3c..d4e0e4960 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl @@ -32,7 +32,6 @@ start(_Type, _Args) -> {ok, Sup} = supervisor:start_link({local, ?MODULE}, ?MODULE, []), {ok, _} = start_auth_server(jwks_svr_options()), - ok = emqx_auth_jwt:register_metrics(), AuthEnv = auth_env(), _ = emqx:hook('client.authenticate', {emqx_auth_jwt, check_auth, [AuthEnv]}), diff --git a/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl b/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl index 97a7d0cfc..0c573e0e2 100644 --- a/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl +++ b/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_ldap). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl index f2b6140a2..da932c2fe 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl @@ -26,17 +26,12 @@ -import(emqx_auth_ldap_cli, [search/3]). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 , prepare_filter/4 , replace_vars/2 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{username := Username, password := Password}, AuthResult, State = #{password_attr := PasswdAttr, bind_as_user := BindAsUserRequired, pool := Pool}) -> CheckResult = @@ -63,12 +58,10 @@ check(ClientInfo = #{username := Username, password := Password}, AuthResult, end, case CheckResult of ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{auth_result => success, anonymous => false}}; {error, not_found} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, ResultCode} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), ?LOG(error, "[LDAP] Auth from ldap failed: ~p", [ResultCode]), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl index 63fde463b..14c31f1bd 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl @@ -49,7 +49,6 @@ stop(_State) -> ok. load_auth_hook(DeviceDn) -> - ok = emqx_auth_ldap:register_metrics(), Params = maps:from_list(DeviceDn), emqx:hook('client.authenticate', fun emqx_auth_ldap:check/3, [Params#{pool => ?APP}]). diff --git a/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl b/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl index 2d5a6eb6d..f8569ef8a 100644 --- a/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl +++ b/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl @@ -41,15 +41,3 @@ }). -type(acl_record() :: {acl_target(), emqx_topic:topic(), action(), access(), created_at()}). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl index 386adba4b..e14de3cf5 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl @@ -27,7 +27,6 @@ -define(TABLE, emqx_user). %% Auth callbacks -export([ init/1 - , register_metrics/0 , check/3 , description/0 ]). @@ -51,10 +50,6 @@ init(#{clientid_list := ClientidList, username_list := UsernameList}) -> ok = ekka_mnesia:copy_table(?TABLE, disc_copies). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - hash_type() -> application:get_env(emqx_auth_mnesia, password_hash, sha256). @@ -67,17 +62,14 @@ check(ClientInfo = #{ clientid := Clientid end), case ets:select(?TABLE, MatchSpec) of [] -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; List -> case match_password(NPassword, HashType, List) of false -> Info = maps:without([password], ClientInfo), ?LOG(info, "[Mnesia] Auth from mnesia failed: ~p", [Info]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{anonymous => false, auth_result => password_error}}; _ -> - emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{anonymous => false, auth_result => success}} end end. diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl index 467b6cf7c..ee38b7165 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl @@ -56,7 +56,6 @@ load_auth_hook() -> ClientidList = application:get_env(?APP, clientid_list, []), UsernameList = application:get_env(?APP, username_list, []), ok = emqx_auth_mnesia:init(#{clientid_list => ClientidList, username_list => UsernameList}), - ok = emqx_auth_mnesia:register_metrics(), Params = #{hash_type => emqx_auth_mnesia:hash_type()}, emqx:hook('client.authenticate', fun emqx_auth_mnesia:check/3, [Params]). diff --git a/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl b/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl index 69044e920..68a97ad33 100644 --- a/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl +++ b/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl @@ -1,4 +1,3 @@ - -define(APP, emqx_auth_mongo). -define(DEFAULT_SELECTORS, [{<<"username">>, <<"%u">>}]). @@ -14,15 +13,3 @@ -record(aclquery, {collection = <<"mqtt_acl">>, selector = {<<"username">>, <<"%u">>}}). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl index 0df342295..bfb911707 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl @@ -23,8 +23,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/types.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). @@ -39,20 +38,15 @@ , available/3 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{password := Password}, AuthResult, Env = #{authquery := AuthQuery, superquery := SuperQuery}) -> #authquery{collection = Collection, field = Fields, hash = HashType, selector = Selector} = AuthQuery, Pool = maps:get(pool, Env, ?APP), case query(Pool, Collection, maps:from_list(replvars(Selector, ClientInfo))) of - undefined -> emqx_metrics:inc(?AUTH_METRICS(ignore)); + undefined -> ok; {error, Reason} -> ?LOG(error, "[MongoDB] Can't connect to MongoDB server: ~0p", [Reason]), - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => not_authorized, anonymous => false}}; UserMap -> Result = case [maps:get(Field, UserMap, undefined) || Field <- Fields] of @@ -64,13 +58,11 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case Result of ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo), anonymous => false, auth_result => success}}; {error, Error} -> ?LOG(error, "[MongoDB] check auth fail: ~p", [Error]), - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => Error, anonymous => false}} end end. diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl index 92c025621..a63aa8193 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl @@ -68,7 +68,6 @@ safe_start() -> reg_authmod(AuthQuery) -> case emqx_auth_mongo:available(?APP, AuthQuery) of ok -> - emqx_auth_mongo:register_metrics(), HookFun = fun emqx_auth_mongo:check/3, HookOptions = #{authquery => AuthQuery, superquery => undefined, pool => ?APP}, case r(super_query, application:get_env(?APP, super_query, undefined)) of @@ -122,4 +121,3 @@ r(auth_query, Config) -> r(acl_query, Config) -> #aclquery{collection = list_to_binary(get_value(collection, Config, "mqtt_acl")), selector = get_value(selector, Config, [?DEFAULT_SELECTORS])}. - diff --git a/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl b/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl index 56da35401..b7c185fcf 100644 --- a/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl +++ b/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_mysql). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl b/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl index 17d618838..31d9a007f 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl @@ -22,17 +22,12 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/types.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). -define(EMPTY(Username), (Username =:= undefined orelse Username =:= <<>>)). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{password := Password}, AuthResult, #{auth_query := {AuthSql, AuthParams}, super_query := SuperQuery, @@ -51,15 +46,13 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case CheckPass of ok -> - emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo), anonymous => false, auth_result => success}}; {error, not_found} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; + ok; {error, ResultCode} -> ?LOG(error, "[MySQL] Auth from mysql failed: ~p", [ResultCode]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. @@ -88,4 +81,3 @@ check_pass(Password, HashType) -> end. description() -> "Authentication with MySQL". - diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl b/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl index 716cb7a7a..e58f62a90 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl @@ -50,7 +50,6 @@ stop(_State) -> ok. load_auth_hook(AuthQuery) -> - ok = emqx_auth_mysql:register_metrics(), SuperQuery = parse_query(application:get_env(?APP, super_query, undefined)), {ok, HashType} = application:get_env(?APP, password_hash), Params = #{auth_query => AuthQuery, diff --git a/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl b/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl index 92b971667..565aa4668 100644 --- a/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl +++ b/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl @@ -1,13 +1 @@ -define(APP, emqx_auth_pgsql). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl index f38552085..f673e07e4 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl @@ -21,15 +21,10 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - %%-------------------------------------------------------------------- %% Auth Module Callbacks %%-------------------------------------------------------------------- @@ -50,15 +45,13 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case CheckPass of ok -> - emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo), anonymous => false, auth_result => success}}; {error, not_found} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; + ok; {error, ResultCode} -> ?LOG(error, "[Postgres] Auth from pgsql failed: ~p", [ResultCode]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. @@ -88,4 +81,3 @@ check_pass(Password, HashType) -> end. description() -> "Authentication with PostgreSQL". - diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl index c658c4d11..dbfeb8423 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl @@ -42,7 +42,6 @@ start(_StartType, _StartArgs) -> super_query => SuperQuery, hash_type => HashType, pool => ?APP}, - ok = emqx_auth_pgsql:register_metrics(), ok = emqx:hook('client.authenticate', fun emqx_auth_pgsql:check/3, [AuthEnv]) end), if_enabled(acl_query, fun(AclQuery) -> @@ -59,4 +58,3 @@ if_enabled(Par, Fun) -> {ok, Query} -> Fun(parse_query(Par, Query)); undefined -> ok end. - diff --git a/apps/emqx_auth_redis/include/emqx_auth_redis.hrl b/apps/emqx_auth_redis/include/emqx_auth_redis.hrl index fe488c37f..075d649ec 100644 --- a/apps/emqx_auth_redis/include/emqx_auth_redis.hrl +++ b/apps/emqx_auth_redis/include/emqx_auth_redis.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_redis). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.erl b/apps/emqx_auth_redis/src/emqx_auth_redis.erl index 04d3542f0..d432e012b 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.erl @@ -21,15 +21,10 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{password := Password}, AuthResult, #{auth_cmd := AuthCmd, super_cmd := SuperCmd, @@ -52,15 +47,13 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case CheckPass of ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), IsSuperuser = is_superuser(Pool, Type, SuperCmd, ClientInfo, Timeout), {stop, AuthResult#{is_superuser => IsSuperuser, anonymous => false, auth_result => success}}; {error, not_found} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, ResultCode} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), ?LOG(error, "[Redis] Auth from redis failed: ~p", [ResultCode]), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. @@ -82,4 +75,3 @@ check_pass(Password, HashType) -> ok -> ok; {error, _Reason} -> {error, not_authorized} end. - diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl b/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl index c6de8b80a..cbea448f9 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl @@ -49,7 +49,6 @@ load_auth_hook(AuthCmd) -> timeout => Timeout, type => Type, pool => ?APP}, - ok = emqx_auth_redis:register_metrics(), emqx:hook('client.authenticate', fun emqx_auth_redis:check/3, [Config]). load_acl_hook(AclCmd) -> @@ -66,4 +65,3 @@ if_cmd_enabled(Par, Fun) -> {ok, Cmd} -> Fun(Cmd); undefined -> ok end. - diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 0e39b40a7..dce3b36e1 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -299,8 +299,6 @@ handle_call({auth, RequestedClientInfo, Password}, case emqx_access_control:authenticate(ClientInfo1#{password => Password}) of {ok, AuthResult} -> emqx_logger:set_metadata_clientid(ClientId), - is_anonymous(AuthResult) andalso - emqx_metrics:inc('client.auth.anonymous'), NClientInfo = maps:merge(ClientInfo1, AuthResult), NChannel = Channel1#channel{clientinfo = NClientInfo}, case emqx_cm:open_session(true, NClientInfo, NConnInfo) of @@ -424,9 +422,6 @@ terminate(Reason, Channel) -> Req = #{reason => stringfy(Reason)}, try_dispatch(on_socket_closed, wrap(Req), Channel). -is_anonymous(#{anonymous := true}) -> true; -is_anonymous(_AuthResult) -> false. - packet_to_message(Topic, Qos, Payload, #channel{ conninfo = #{proto_ver := ProtoVer}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 57c54446f..a7c450ba6 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -412,8 +412,8 @@ emqx_collect(emqx_client_connected, Stats) -> counter_metric(?C('client.connected', Stats)); emqx_collect(emqx_client_authenticate, Stats) -> counter_metric(?C('client.authenticate', Stats)); -emqx_collect(emqx_client_auth_anonymous, Stats) -> - counter_metric(?C('client.auth.anonymous', Stats)); +emqx_collect(emqx_client_auth_success_anonymous, Stats) -> + counter_metric(?C('client.auth.success.anonymous', Stats)); emqx_collect(emqx_client_check_acl, Stats) -> counter_metric(?C('client.check_acl', Stats)); emqx_collect(emqx_client_subscribe, Stats) -> @@ -566,7 +566,7 @@ emqx_metrics_delivery() -> emqx_metrics_client() -> [ emqx_client_connected , emqx_client_authenticate - , emqx_client_auth_anonymous + , emqx_client_auth_success_anonymous , emqx_client_check_acl , emqx_client_subscribe , emqx_client_unsubscribe From 69abca17f236106adc05577b3017436d2d31818f Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 20 May 2022 13:42:21 +0800 Subject: [PATCH 10/36] chore(appup): bump app version & update appup --- .../emqx_auth_http/src/emqx_auth_http.app.src | 2 +- .../src/emqx_auth_http.appup.src | 21 +++++-- .../emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 8 +-- .../emqx_auth_ldap/src/emqx_auth_ldap.app.src | 2 +- .../src/emqx_auth_ldap.appup.src | 63 ++++++++----------- .../src/emqx_auth_mnesia.appup.src | 12 ++-- .../src/emqx_auth_mongo.app.src | 2 +- .../src/emqx_auth_mongo.appup.src | 10 +-- .../src/emqx_auth_mysql.app.src | 2 +- .../src/emqx_auth_mysql.appup.src | 13 ++-- .../src/emqx_auth_pgsql.appup.src | 52 +++++++-------- .../src/emqx_auth_redis.app.src | 2 +- .../src/emqx_auth_redis.appup.src | 13 ++-- apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.appup.src | 21 +++---- .../src/emqx_prometheus.app.src | 2 +- .../src/emqx_prometheus.appup.src | 9 +++ src/emqx.appup.src | 28 +++++++++ 18 files changed, 144 insertions(+), 120 deletions(-) create mode 100644 apps/emqx_prometheus/src/emqx_prometheus.appup.src diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index fd8d1e046..fba56740f 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_http, [{description, "EMQ X Authentication/ACL with HTTP API"}, - {vsn, "4.3.5"}, % strict semver, bump manually! + {vsn, "4.3.6"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_http_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.appup.src b/apps/emqx_auth_http/src/emqx_auth_http.appup.src index 256bd7566..519604d24 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.appup.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.appup.src @@ -1,27 +1,38 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.4", - [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]} - ]}, + [{"4.3.5", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, + {"4.3.4", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{apply,{application,stop,[emqx_auth_http]}}, {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-1]">>, [{restart_application,emqx_auth_http}]}, {<<".*">>,[]}], - [{"4.3.4", - [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}]}, + [{"4.3.5", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, + {"4.3.4", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{apply,{application,stop,[emqx_auth_http]}}, {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-1]">>, diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index a159d2ce4..b94159225 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,13 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.2", - [{restart_application,emqx_auth_jwt}]}, - {<<"4\\.3\\.[0-1]">>, + [{<<"4\\.3\\.[0-2]">>, [{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [{"4.3.2", - [{restart_application,emqx_auth_jwt}]}, - {<<"4\\.3\\.[0-1]">>, + [{<<"4\\.3\\.[0-2]">>, [{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index a072234d7..6875fca30 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_ldap, [{description, "EMQ X Authentication/ACL with LDAP"}, - {vsn, "4.3.4"}, % strict semver, bump manually! + {vsn, "4.3.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_ldap_sup]}, {applications, [kernel,stdlib,eldap2,ecpool]}, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src index ea44d7a1a..fbb59a176 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src @@ -1,40 +1,29 @@ -%% -*-: erlang -*- +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ {"4.3.3", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, - {"4.3.0", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {"4.3.1", - [ {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, + [{<<"4\\.3\\.[3-4]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], - [ {"4.3.3", []}, - {"4.3.0", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {"4.3.1", - [ {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-1]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{<<"4\\.3\\.[3-4]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-1]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index a9e47eb65..a849002d6 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -2,9 +2,11 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{<<"4\\.3\\.[5-6]">>, - [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-3]">>, [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, @@ -27,7 +29,9 @@ {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{<<"4\\.3\\.[5-6]">>, - [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-3]">>, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src index 3090656f7..2b6fedbb5 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mongo, [{description, "EMQ X Authentication/ACL with MongoDB"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.3.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mongo_sup]}, {applications, [kernel,stdlib,mongodb,ecpool]}, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src index a12cb37b1..1907b7fa7 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src @@ -1,10 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.2", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, - {"4.3.1", + [{<<"4\\.3\\.[1-3]">>, [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, {"4.3.0", @@ -12,10 +9,7 @@ {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.2", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, - {"4.3.1", + [{<<"4\\.3\\.[1-3]">>, [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, {"4.3.0", diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src index 6df4545fd..a0ddc4dd4 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mysql, [{description, "EMQ X Authentication/ACL with MySQL"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mysql_sup]}, {applications, [kernel,stdlib,mysql,ecpool]}, diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src index 88ff1b38e..a116d7dbb 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src @@ -1,16 +1,19 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.1", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.1", []}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}] }. diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src index 07c13cbff..d1c10d6fa 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src @@ -1,31 +1,25 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {"4.3.2", [ - %% epgsql 4.4.0 -> 4.6.0. - %% epgsql has no appup ,so we can only restart it. - {restart_application, epgsql} - ]}, - {"4.3.1", [ - {restart_application, epgsql} - ]}, - {"4.3.0", [ - {restart_application, epgsql}, - {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]} - ]}, - {<<".*">>,[]}], - [ - {"4.3.2", [ - {restart_application, epgsql} - ]}, - {"4.3.1", [ - {restart_application, epgsql} - ]}, - {"4.3.0", [ - {restart_application, epgsql}, - {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]} - ]}, - {<<".*">>,[]}] -}. + [{<<"4\\.3\\.[1-2]">>, + %% epgsql 4.4.0 -> 4.6.0. + %% epgsql has no appup ,so we can only restart it. + [{restart_application,epgsql}, + {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}]}, + {"4.3.0", + [{restart_application,epgsql}, + {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{<<"4\\.3\\.[1-2]">>, + [{restart_application,epgsql}, + {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}]}, + {"4.3.0", + [{load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}, + {restart_application,epgsql}, + {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src index ea6aaefcf..e9e37a463 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_redis, [{description, "EMQ X Authentication/ACL with Redis"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_redis_sup]}, {applications, [kernel,stdlib,eredis,eredis_cluster,ecpool]}, diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src b/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src index 83f5d46be..9036d77a8 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src @@ -1,16 +1,19 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.1", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.1", []}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}] }. diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index fe566ae52..41c19f752 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.7"}, %% 4.3.3 is used by ee + {vsn, "4.3.8"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index 0da87e289..33bd4386c 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,14 +1,9 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {"4.3.6", - [ %% There are only changes to the schema file, so we don't need any - %% commands here - ]}, - {<<"4\\.3\\.[4-5]">>, - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, - {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]">>, + [{<<"4\\.3\\.[6-7]">>, + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[2-5]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, @@ -17,11 +12,9 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.6", []}, - {<<"4\\.3\\.[4-5]">>, - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, - {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]">>, + [{<<"4\\.3\\.[6-7]">>, + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[2-5]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index b96608edb..ded1bf46d 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -1,6 +1,6 @@ {application, emqx_prometheus, [{description, "Prometheus for EMQ X"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel,stdlib,prometheus]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.appup.src b/apps/emqx_prometheus/src/emqx_prometheus.appup.src new file mode 100644 index 000000000..a06d65fe1 --- /dev/null +++ b/apps/emqx_prometheus/src/emqx_prometheus.appup.src @@ -0,0 +1,9 @@ +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! +{VSN, + [{"4.3.0", + [{load_module,emqx_prometheus,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.3.0", + [{load_module,emqx_prometheus,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 09ad2d13a..b47e7b76c 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -8,6 +8,9 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -20,6 +23,9 @@ {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", [{load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -37,6 +43,9 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, @@ -56,6 +65,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -79,6 +89,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -107,6 +118,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -134,6 +146,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -166,6 +179,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -197,6 +211,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -229,6 +244,7 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -261,6 +277,7 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -293,6 +310,7 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -326,6 +344,7 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -360,6 +379,7 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -393,6 +413,7 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -430,6 +451,7 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -469,8 +491,10 @@ {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -478,6 +502,8 @@ {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -498,6 +524,8 @@ {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, From 4ed8f4d95307d4db71f76469c275a974d108fd74 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 20 May 2022 18:10:51 +0800 Subject: [PATCH 11/36] chore: update CHANGES.md --- CHANGES-4.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 68a0b4b0c..b74eae09e 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -23,6 +23,7 @@ File format: * Add proto_name and proto_ver fields for $event/client_disconnected event. * Mnesia auth/acl http api support multiple condition queries. * Inflight QoS1 Messages for shared topics are now redispatched to another alive subscribers upon chosen subscriber session termination. +* Make auth metrics name more understandable. ### Bug fixes * List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. From e73cd57b5a1e046d80b826625dd0572cd766837a Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 23 May 2022 14:35:23 +0800 Subject: [PATCH 12/36] fix: epgsql use emqx fork --- apps/emqx_auth_pgsql/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_auth_pgsql/rebar.config b/apps/emqx_auth_pgsql/rebar.config index be395993b..7a6aaf411 100644 --- a/apps/emqx_auth_pgsql/rebar.config +++ b/apps/emqx_auth_pgsql/rebar.config @@ -1,5 +1,5 @@ {deps, - [{epgsql, {git, "https://github.com/epgsql/epgsql.git", {tag, "4.6.0"}}} + [{epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} ]}. {erl_opts, [warn_unused_vars, From 6dcdb58482b5469899be290e4e899fa9d88d08b8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 23 May 2022 11:39:29 +0200 Subject: [PATCH 13/36] docs: add missing docs for changes made in bin/emqx and upgrade script --- CHANGES-4.3.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index b74eae09e..53ceb8c0c 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -14,6 +14,8 @@ File format: ### Enhancements +* Refactored `bin/emqx` help messages. +* Upgrade script refuses upgrade from incompatible versions. (e.g. hot upgrade from 4.3 to 4.4 will fail fast). * Made possible for EMQX to boot from a Linux directory which has white spaces in its path. * Add support for JWT authorization [#7596] Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists. From c93626e97f976a9d1a31a060c576984dc00a2441 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 23 May 2022 18:05:41 +0800 Subject: [PATCH 14/36] chore: update changes and appup.src for emqx_management --- CHANGES-4.3.md | 1 + apps/emqx_management/src/emqx_management.appup.src | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index b74eae09e..2a4715a7a 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -24,6 +24,7 @@ File format: * Mnesia auth/acl http api support multiple condition queries. * Inflight QoS1 Messages for shared topics are now redispatched to another alive subscribers upon chosen subscriber session termination. * Make auth metrics name more understandable. +* Allow emqx_management http listener binding to specific interface [#8005] ### Bug fixes * List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src index 9265913e6..15aef0463 100644 --- a/apps/emqx_management/src/emqx_management.appup.src +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -1,13 +1,13 @@ %% -*- mode: erlang -*- {VSN, - [ {<<"4\\.3\\.([0-9]|1[0-2])">>, + [ {<<"4\\.3\\.([0-9]|1[0-4])">>, [ {apply,{minirest,stop_http,['http:management']}}, {apply,{minirest,stop_http,['https:management']}}, {restart_application, emqx_management} ]}, {<<".*">>, []} ], - [ {<<"4\\.3\\.([0-9]|1[0-2])">>, + [ {<<"4\\.3\\.([0-9]|1[0-4])">>, [ {apply,{minirest,stop_http,['http:management']}}, {apply,{minirest,stop_http,['https:management']}}, {restart_application, emqx_management} From 2b81839126c92c0a54f930b6aa112992f849a376 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 23 May 2022 20:21:21 +0800 Subject: [PATCH 15/36] chore(pgsql): restart emqx_auth_pgsql due to the egpsql upgraded We have to restart the emqx_auth_pgsql because epgsql has upgraded 4.4.0 to 4.6.0 see: #8001 --- .../src/emqx_auth_pgsql.appup.src | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src index d1c10d6fa..494cc94c9 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src @@ -1,25 +1,11 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[1-2]">>, - %% epgsql 4.4.0 -> 4.6.0. - %% epgsql has no appup ,so we can only restart it. - [{restart_application,epgsql}, - {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{restart_application,epgsql}, - {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, + [{<<"4\\.3\\.[0-2]">>, + %% restart it due to epgsql upgraded from 4.4.0 to 4.6.0 + %% in emqx_auth_pgsql:v4.3.3 + [{restart_application,emqx_auth_pgsql}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[1-2]">>, - [{restart_application,epgsql}, - {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}, - {restart_application,epgsql}, - {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, + [{<<"4\\.3\\.[0-2]">>, + [{restart_application,emqx_auth_pgsql}]}, {<<".*">>,[]}]}. From 95d613e204331e5b88c69b74895bcc31e6dd0ac8 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 23 May 2022 21:48:26 +0800 Subject: [PATCH 16/36] chore: pretty change logs --- CHANGES-4.3.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 2a4715a7a..447593250 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -25,14 +25,13 @@ File format: * Inflight QoS1 Messages for shared topics are now redispatched to another alive subscribers upon chosen subscriber session termination. * Make auth metrics name more understandable. * Allow emqx_management http listener binding to specific interface [#8005] +* Add rule-engine function float2str/2, user can specify the float output precision [#7991] ### Bug fixes * List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. * SSL closed error bug fixed for redis client. * Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message * Rule-engine function hexstr2bin/1 support half byte [#7977] -* Add rule-engine function float2str/2, user can specify the float output precision [#7991] - * Improved resilience against autocluster partitioning during cluster startup. [#7876] [ekka-158](https://github.com/emqx/ekka/pull/158) From 41331c5c1ae24574d0639789286f198453a9ca35 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 23 May 2022 22:26:20 +0800 Subject: [PATCH 17/36] chore: release 4.3.15-rc.1 --- include/emqx_release.hrl | 2 +- scripts/get-dashboard.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 5145f7b5e..3945b3869 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.14"}). +-define(EMQX_RELEASE, {opensource, "4.3.15-rc.1"}). -else. diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index abe8e2415..62467420c 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -9,7 +9,7 @@ PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" case "${PKG_VSN}" in 4.3*) EMQX_CE_DASHBOARD_VERSION='v4.3.7' - EMQX_EE_DASHBOARD_VERSION='v4.3.18' + EMQX_EE_DASHBOARD_VERSION='v4.3.19' ;; *) echo "Unsupported version $PKG_VSN" >&2 From 24f0592e94f230cb4eb3f523cc54b98644140162 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 23 May 2022 16:18:56 -0300 Subject: [PATCH 18/36] fix(node_dump): define `RUNNER_ROOT_DIR` before sourcing vars Port of https://github.com/emqx/emqx/pull/7733 `emqx_vars` requires `RUNNER_ROOT_DIR` to be defined before being sourced. --- CHANGES-4.3.md | 1 + bin/node_dump | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index e387849e4..da2df758d 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -38,6 +38,7 @@ File format: startup. [#7876] [ekka-158](https://github.com/emqx/ekka/pull/158) * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979] +* Fix `node_dump` variable sourcing. [#8026] ## v4.3.14 diff --git a/bin/node_dump b/bin/node_dump index 83b41332b..79b469112 100755 --- a/bin/node_dump +++ b/bin/node_dump @@ -1,13 +1,13 @@ #!/bin/sh set -eu -ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" -echo "Running node dump in ${ROOT_DIR}" +RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" +echo "Running node dump in ${RUNNER_ROOT_DIR}" # shellcheck disable=SC1090 -. "$ROOT_DIR"/releases/emqx_vars +. "$RUNNER_ROOT_DIR"/releases/emqx_vars -cd "${ROOT_DIR}" +cd "${RUNNER_ROOT_DIR}" DUMP="$RUNNER_LOG_DIR/node_dump_$(date +"%Y%m%d_%H%M%S").tar.gz" CONF_DUMP="$RUNNER_LOG_DIR/conf.dump" From 6e7cbc1b9b1e235efff17164230b8d924ec84175 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 24 May 2022 09:33:17 +0800 Subject: [PATCH 19/36] chore: move epgsql deps to the top-level rebar.config --- apps/emqx_auth_pgsql/rebar.config | 2 +- rebar.config | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_auth_pgsql/rebar.config b/apps/emqx_auth_pgsql/rebar.config index 7a6aaf411..c351ed13e 100644 --- a/apps/emqx_auth_pgsql/rebar.config +++ b/apps/emqx_auth_pgsql/rebar.config @@ -1,5 +1,5 @@ {deps, - [{epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} + [ ]}. {erl_opts, [warn_unused_vars, diff --git a/rebar.config b/rebar.config index 8ed5933fc..6688e34a1 100644 --- a/rebar.config +++ b/rebar.config @@ -60,6 +60,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.2.1"}}} , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}} + , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} ]}. {xref_ignores, From 8fe933e88536c43104bede3188ee480e268f0fa7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 24 May 2022 11:51:10 +0800 Subject: [PATCH 20/36] fix: ensure auth_mnesia started first --- .../src/emqx_mgmt_data_backup.erl | 12 +++- ...x_auth_mnesia_data_export_import_SUITE.erl | 69 +++++++++++++++++++ .../ee435.json | 1 + 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 06dc0365a..bc9639e6a 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -662,6 +662,9 @@ do_import_data(Data, Version) -> import_blacklist(maps:get(<<"blacklist">>, Data, [])), import_applications(maps:get(<<"apps">>, Data, [])), import_users(maps:get(<<"users">>, Data, [])), + %% Import modules first to ensure the data of auth_mnesia module can be imported. + %% XXX: In opensource version, can't import if the emqx_auth_mnesia plug-in is not started?? + do_import_enterprise_modules(Data, Version), import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])), @@ -670,13 +673,20 @@ do_import_data(Data, Version) -> -ifdef(EMQX_ENTERPRISE). do_import_extra_data(Data, _Version) -> _ = import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), - _ = import_modules(maps:get(<<"modules">>, Data, [])), _ = import_schemas(maps:get(<<"schemas">>, Data, [])), ok. -else. do_import_extra_data(_Data, _Version) -> ok. -endif. +-ifdef(EMQX_ENTERPRISE). +do_import_enterprise_modules(Data, _Version) -> + _ = import_modules(maps:get(<<"modules">>, Data, [])), + ok. +-else. +do_import_enterprise_modules(_Data, _Version) -> ok. +-endif. + covert_empty_headers([]) -> #{}; covert_empty_headers(Other) -> Other. diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl new file mode 100644 index 000000000..abe496434 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_auth_mnesia_data_export_import_SUITE). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx_modules/include/emqx_modules.hrl"). +-compile([export_all, nowarn_export_all]). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Cfg) -> + _ = application:load(emqx_modules_spec), + emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_modules, + emqx_management, emqx_dashboard]), + Cfg. + +end_per_suite(Cfg) -> + emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management, + emqx_modules, emqx_rule_engine]), + Cfg. + +get_data_path() -> + emqx_ct_helpers:deps_path(emqx_management, "test/emqx_auth_mnesia_data_export_import_SUITE_data/"). + +import(FilePath, _Version) -> + dbg:tracer(),dbg:p(all,call), + dbg:tpl(emqx_mgmt_data_backup,import_auth_mnesia,x), + dbg:tp(emqx_modules_registry,get_modules,x), + dbg:tp(emqx_module_internal_acl,on_module_create,x), + ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), + [_] = lists:filter( + fun(#module{type = mnesia_authentication}) -> true; + (_) -> false + end, emqx_modules_registry:get_modules()), + ?assertNotEqual(0, ets:info(emqx_user, size)), + ?assertNotEqual(0, ets:info(emqx_acl, size)). + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- +-ifdef(EMQX_ENTERPRISE). + +t_importee430(_) -> + import("ee435.json", ee435), + {ok, _} = emqx_mgmt_data_backup:export(), + remove_all_users_and_acl(). + +-endif. + +remove_all_users_and_acl() -> + mnesia:delete_table(emqx_user), + mnesia:delete_table(emqx_acl). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json new file mode 100644 index 000000000..fd2f51ad6 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json @@ -0,0 +1 @@ +{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"02BzoSYaTxkscy2MDtU92EbX7b4=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=","created_at":1653360665243}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pub","access":"allow","created_at":1653360687955},{"type":"clientid","type_value":"clientida","topic":"t/a","action":"sub","access":"allow","created_at":1653360687955}],"modules":[{"id":"module:fcda7532","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653360656060,"description":""},{"id":"module:16f3f067","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:db849123","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:55987aaa","type":"presence","config":{"qos":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:78cae4f9","type":"recon","config":{},"enabled":true,"created_at":1653360591111,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2022-05-24 10:51:39"} \ No newline at end of file From 1531b34f8af2e81f4ff7235f3d3a13152958c27f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 24 May 2022 12:16:03 +0800 Subject: [PATCH 21/36] test: add cases for importing e427 --- .../test/emqx_auth_mnesia_data_export_import_SUITE.erl | 9 +++++---- .../ee427.json | 1 + .../ee435.json | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl index abe496434..e82e875e9 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl @@ -40,10 +40,6 @@ get_data_path() -> emqx_ct_helpers:deps_path(emqx_management, "test/emqx_auth_mnesia_data_export_import_SUITE_data/"). import(FilePath, _Version) -> - dbg:tracer(),dbg:p(all,call), - dbg:tpl(emqx_mgmt_data_backup,import_auth_mnesia,x), - dbg:tp(emqx_modules_registry,get_modules,x), - dbg:tp(emqx_module_internal_acl,on_module_create,x), ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), [_] = lists:filter( fun(#module{type = mnesia_authentication}) -> true; @@ -57,6 +53,11 @@ import(FilePath, _Version) -> %%-------------------------------------------------------------------- -ifdef(EMQX_ENTERPRISE). +t_importee427(_) -> + import("ee427.json", ee427), + {ok, _} = emqx_mgmt_data_backup:export(), + remove_all_users_and_acl(). + t_importee430(_) -> import("ee435.json", ee435), {ok, _} = emqx_mgmt_data_backup:export(), diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json new file mode 100644 index 000000000..763bf2c7b --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json @@ -0,0 +1 @@ +{"version":"4.2","date":"2022-05-24 12:09:56","modules":[{"id":"module:842a5c57","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653365372585,"description":""},{"id":"module:e3d38e5a","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1652436434273,"description":""},{"id":"module:4f911150","type":"presence","config":{"qos":0},"enabled":true,"created_at":1652436434273,"description":""},{"id":"module:db11d08f","type":"recon","config":{},"enabled":true,"created_at":1652436434273,"description":""}],"rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"oFu0ZiOAYJmB1DyOzoLwEervBK0=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"WVlGsjBiNGJkOWRkN2QyZmYyOWViYjI4MzRiZjQyMDgzNDhkYzJjZmZlOGVjMjUzOGU5NDkwYmYyYjY5N2Q3NjUyMDU=","created_at":1653365382492}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pubsub","access":"allow","created_at":1653365390351}],"schemas":[],"configs":[],"listeners_state":[]} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json index fd2f51ad6..b457ff6b8 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json @@ -1 +1 @@ -{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"02BzoSYaTxkscy2MDtU92EbX7b4=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=","created_at":1653360665243}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pub","access":"allow","created_at":1653360687955},{"type":"clientid","type_value":"clientida","topic":"t/a","action":"sub","access":"allow","created_at":1653360687955}],"modules":[{"id":"module:fcda7532","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653360656060,"description":""},{"id":"module:16f3f067","type":"internal_acl","config":{"acl_rule_file":"etc/acl.conf"},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:db849123","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:55987aaa","type":"presence","config":{"qos":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:78cae4f9","type":"recon","config":{},"enabled":true,"created_at":1653360591111,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2022-05-24 10:51:39"} \ No newline at end of file +{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"02BzoSYaTxkscy2MDtU92EbX7b4=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=","created_at":1653360665243}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pub","access":"allow","created_at":1653360687955},{"type":"clientid","type_value":"clientida","topic":"t/a","action":"sub","access":"allow","created_at":1653360687955}],"modules":[{"id":"module:fcda7532","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653360656060,"description":""},{"id":"module:db849123","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:55987aaa","type":"presence","config":{"qos":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:78cae4f9","type":"recon","config":{},"enabled":true,"created_at":1653360591111,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2022-05-24 10:51:39"} From 0eef2977471120cedcbec33863a86d8c8787ed62 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 24 May 2022 12:27:10 +0800 Subject: [PATCH 22/36] chore: fix dialyzer error --- .../emqx_auth_mnesia_data_export_import_SUITE.erl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl index e82e875e9..021cf735a 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl @@ -15,9 +15,13 @@ %%-------------------------------------------------------------------- -module(emqx_auth_mnesia_data_export_import_SUITE). + +-compile([export_all, nowarn_export_all]). + +-ifdef(EMQX_ENTERPRISE). + -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx_modules/include/emqx_modules.hrl"). --compile([export_all, nowarn_export_all]). %%-------------------------------------------------------------------- %% Setups @@ -51,7 +55,6 @@ import(FilePath, _Version) -> %%-------------------------------------------------------------------- %% Cases %%-------------------------------------------------------------------- --ifdef(EMQX_ENTERPRISE). t_importee427(_) -> import("ee427.json", ee427), @@ -63,8 +66,8 @@ t_importee430(_) -> {ok, _} = emqx_mgmt_data_backup:export(), remove_all_users_and_acl(). --endif. - remove_all_users_and_acl() -> mnesia:delete_table(emqx_user), mnesia:delete_table(emqx_acl). + +-endif. From 355f859a9b18714d37d7ed30c24bac7369d3ccb6 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 24 May 2022 13:50:27 +0800 Subject: [PATCH 23/36] test: format data-export files --- .../ee427.json | 89 ++++++++++++++++- .../ee435.json | 97 ++++++++++++++++++- 2 files changed, 184 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json index 763bf2c7b..43d58538a 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json @@ -1 +1,88 @@ -{"version":"4.2","date":"2022-05-24 12:09:56","modules":[{"id":"module:842a5c57","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653365372585,"description":""},{"id":"module:e3d38e5a","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1652436434273,"description":""},{"id":"module:4f911150","type":"presence","config":{"qos":0},"enabled":true,"created_at":1652436434273,"description":""},{"id":"module:db11d08f","type":"recon","config":{},"enabled":true,"created_at":1652436434273,"description":""}],"rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"oFu0ZiOAYJmB1DyOzoLwEervBK0=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"WVlGsjBiNGJkOWRkN2QyZmYyOWViYjI4MzRiZjQyMDgzNDhkYzJjZmZlOGVjMjUzOGU5NDkwYmYyYjY5N2Q3NjUyMDU=","created_at":1653365382492}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pubsub","access":"allow","created_at":1653365390351}],"schemas":[],"configs":[],"listeners_state":[]} +{ + "version": "4.2", + "date": "2022-05-24 12:09:56", + "modules": [ + { + "id": "module:842a5c57", + "type": "mnesia_authentication", + "config": { + "password_hash": "sha256" + }, + "enabled": true, + "created_at": 1653365372585, + "description": "" + }, + { + "id": "module:e3d38e5a", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1652436434273, + "description": "" + }, + { + "id": "module:4f911150", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1652436434273, + "description": "" + }, + { + "id": "module:db11d08f", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1652436434273, + "description": "" + } + ], + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "oFu0ZiOAYJmB1DyOzoLwEervBK0=", + "tags": "administrator" + } + ], + "auth_mnesia": [ + { + "login": "usera", + "type": "clientid", + "password": "WVlGsjBiNGJkOWRkN2QyZmYyOWViYjI4MzRiZjQyMDgzNDhkYzJjZmZlOGVjMjUzOGU5NDkwYmYyYjY5N2Q3NjUyMDU=", + "created_at": 1653365382492 + } + ], + "acl_mnesia": [ + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "pubsub", + "access": "allow", + "created_at": 1653365390351 + } + ], + "schemas": [], + "configs": [], + "listeners_state": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json index b457ff6b8..e6d9cbb3b 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json @@ -1 +1,96 @@ -{"version":"4.3","rules":[],"resources":[],"blacklist":[],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"02BzoSYaTxkscy2MDtU92EbX7b4=","tags":"administrator"}],"auth_mnesia":[{"login":"usera","type":"clientid","password":"joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=","created_at":1653360665243}],"acl_mnesia":[{"type":"clientid","type_value":"clientida","topic":"t/a","action":"pub","access":"allow","created_at":1653360687955},{"type":"clientid","type_value":"clientida","topic":"t/a","action":"sub","access":"allow","created_at":1653360687955}],"modules":[{"id":"module:fcda7532","type":"mnesia_authentication","config":{"password_hash":"sha256"},"enabled":true,"created_at":1653360656060,"description":""},{"id":"module:db849123","type":"retainer","config":{"storage_type":"ram","max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:55987aaa","type":"presence","config":{"qos":0},"enabled":true,"created_at":1653360591111,"description":""},{"id":"module:78cae4f9","type":"recon","config":{},"enabled":true,"created_at":1653360591111,"description":""}],"schemas":[],"configs":[],"listeners_state":[],"date":"2022-05-24 10:51:39"} +{ + "version": "4.3", + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "02BzoSYaTxkscy2MDtU92EbX7b4=", + "tags": "administrator" + } + ], + "auth_mnesia": [ + { + "login": "usera", + "type": "clientid", + "password": "joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=", + "created_at": 1653360665243 + } + ], + "acl_mnesia": [ + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "pub", + "access": "allow", + "created_at": 1653360687955 + }, + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "sub", + "access": "allow", + "created_at": 1653360687955 + } + ], + "modules": [ + { + "id": "module:fcda7532", + "type": "mnesia_authentication", + "config": { + "password_hash": "sha256" + }, + "enabled": true, + "created_at": 1653360656060, + "description": "" + }, + { + "id": "module:db849123", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1653360591111, + "description": "" + }, + { + "id": "module:55987aaa", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1653360591111, + "description": "" + }, + { + "id": "module:78cae4f9", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1653360591111, + "description": "" + } + ], + "schemas": [], + "configs": [], + "listeners_state": [], + "date": "2022-05-24 10:51:39" +} From d9252dc6727908ccdf4c0c5192ab2e31c4be4663 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 16 Apr 2022 01:17:27 +0800 Subject: [PATCH 24/36] fix(schema): init resources before restoring schema registry --- apps/emqx_management/src/emqx_mgmt_data_backup.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index bc9639e6a..cb94b7cf9 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -657,7 +657,6 @@ import(Filename, OverridesJson) -> -endif. do_import_data(Data, Version) -> - do_import_extra_data(Data, Version), import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), import_blacklist(maps:get(<<"blacklist">>, Data, [])), import_applications(maps:get(<<"apps">>, Data, [])), @@ -668,7 +667,10 @@ do_import_data(Data, Version) -> import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])), - import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])). + import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])), + %% always do extra import at last, to make sure resources are initiated before + %% creating the schemas + do_import_extra_data(Data, Version). -ifdef(EMQX_ENTERPRISE). do_import_extra_data(Data, _Version) -> From c741b9dfbe1e77876c6d2512890baa5b540bb70d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 24 May 2022 09:45:58 +0200 Subject: [PATCH 25/36] fix(bin/emqx): fix license option to allow space in path --- bin/emqx | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/bin/emqx b/bin/emqx index 756d2f705..7eed9ab3c 100755 --- a/bin/emqx +++ b/bin/emqx @@ -338,14 +338,12 @@ generate_config() { ## changing the config 'log.rotation.size' rm -rf "${RUNNER_LOG_DIR}"/*.siz - EMQX_LICENSE_CONF_OPTION="" - if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then - EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" - fi - set +e - # shellcheck disable=SC2086 - CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + if [ "${EMQX_LICENSE_CONF:-}" = "" ]; then + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + else + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -i "${EMQX_LICENSE_CONF}" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + fi # shellcheck disable=SC2181 RESULT=$? set -e From 17b9baf509ab1908249e846b02287b6342066ed6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 24 May 2022 18:56:28 +0200 Subject: [PATCH 26/36] fix(alarm): fix false sysmem usage alarm Use information from lc app to trigger sysmem alarm set/clear actions --- rebar.config | 2 +- src/emqx.appup.src | 78 ++++++++++------ src/emqx_alarm.erl | 20 +++++ src/emqx_alarm_handler.erl | 15 +--- src/emqx_os_mon.erl | 178 +++++++++++++++++++++++++++++-------- test/emqx_os_mon_SUITE.erl | 23 +++-- 6 files changed, 235 insertions(+), 81 deletions(-) diff --git a/rebar.config b/rebar.config index 6688e34a1..2307f3620 100644 --- a/rebar.config +++ b/rebar.config @@ -58,7 +58,7 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} - , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.2.1"}}} + , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.1"}}} , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}} , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} ]}. diff --git a/src/emqx.appup.src b/src/emqx.appup.src index b47e7b76c..0b5361758 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -11,6 +11,9 @@ {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -26,6 +29,9 @@ {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", [{load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -38,7 +44,6 @@ {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -48,6 +53,9 @@ {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, @@ -71,13 +79,15 @@ {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -99,14 +109,15 @@ {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -129,13 +140,14 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -163,12 +175,13 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -195,7 +208,7 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, @@ -229,7 +242,7 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, @@ -263,7 +276,7 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, @@ -297,7 +310,7 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, @@ -331,7 +344,7 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, @@ -366,7 +379,7 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, @@ -402,7 +415,7 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, @@ -439,7 +452,7 @@ {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, @@ -480,14 +493,16 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.15", - [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -495,9 +510,12 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, @@ -509,9 +527,12 @@ {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -532,7 +553,8 @@ {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -557,7 +579,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -584,7 +607,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -611,7 +635,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -642,7 +667,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_alarm.erl b/src/emqx_alarm.erl index 6573ec1b5..e3b45e295 100644 --- a/src/emqx_alarm.erl +++ b/src/emqx_alarm.erl @@ -39,6 +39,8 @@ , deactivate/1 , deactivate/2 , delete_all_deactivated_alarms/0 + , ensure_deactivated/1 + , ensure_deactivated/2 , get_alarms/0 , get_alarms/1 ]). @@ -132,6 +134,24 @@ activate(Name) -> activate(Name, Details) -> gen_server:call(?MODULE, {activate_alarm, Name, Details}). +-spec ensure_deactivated(binary() | atom()) -> ok. +ensure_deactivated(Name) -> + ensure_deactivated(Name, no_details). + +-spec ensure_deactivated(binary() | atom(), atom() | map()) -> ok. +ensure_deactivated(Name, Data) -> + %% this duplicates the dirty read in handle_call, + %% intention is to avoid making gen_server calls when there is no alarm + case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of + [] -> + ok; + _ -> + case deactivate(Name, Data) of + {error, not_found} -> ok; + Other -> Other + end + end. + deactivate(Name) -> gen_server:call(?MODULE, {deactivate_alarm, Name, no_details}). diff --git a/src/emqx_alarm_handler.erl b/src/emqx_alarm_handler.erl index b389d4175..1520a231b 100644 --- a/src/emqx_alarm_handler.erl +++ b/src/emqx_alarm_handler.erl @@ -56,21 +56,12 @@ init({_Args, {alarm_handler, _ExistingAlarms}}) -> init(_) -> {ok, []}. -handle_event({set_alarm, {system_memory_high_watermark, []}}, State) -> - emqx_alarm:activate(high_system_memory_usage, #{high_watermark => emqx_os_mon:get_sysmem_high_watermark()}), - {ok, State}; - -handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) -> +handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) -> emqx_alarm:activate(high_process_memory_usage, #{pid => list_to_binary(pid_to_list(Pid)), high_watermark => emqx_os_mon:get_procmem_high_watermark()}), {ok, State}; - -handle_event({clear_alarm, system_memory_high_watermark}, State) -> - emqx_alarm:deactivate(high_system_memory_usage), - {ok, State}; - -handle_event({clear_alarm, process_memory_high_watermark}, State) -> - emqx_alarm:deactivate(high_process_memory_usage), +handle_event({clear_alarm, process_memory_high_watermark}, State) -> + emqx_alarm:ensure_deactivate(high_process_memory_usage), {ok, State}; handle_event(_, State) -> diff --git a/src/emqx_os_mon.erl b/src/emqx_os_mon.erl index b23f3e65d..ab637645d 100644 --- a/src/emqx_os_mon.erl +++ b/src/emqx_os_mon.erl @@ -78,32 +78,30 @@ set_cpu_low_watermark(Float) -> call({set_cpu_low_watermark, Float}). get_mem_check_interval() -> - memsup:get_check_interval() div 1000. + call(?FUNCTION_NAME). -set_mem_check_interval(Seconds) when Seconds < 60 -> - memsup:set_check_interval(1); set_mem_check_interval(Seconds) -> - memsup:set_check_interval(Seconds div 60). + call({?FUNCTION_NAME, Seconds}). get_sysmem_high_watermark() -> - memsup:get_sysmem_high_watermark(). + call(?FUNCTION_NAME). -set_sysmem_high_watermark(Float) -> - V = Float/100, +set_sysmem_high_watermark(HW) -> + V = resolve_watermark(HW), case load_ctl:get_config() of #{ ?MEM_MON_F0 := true } = OldLC -> ok = load_ctl:put_config(OldLC#{ ?MEM_MON_F0 => true - , ?MEM_MON_F1 => V}); + , ?MEM_MON_F1 => V / 100}); _ -> skip end, - memsup:set_sysmem_high_watermark(V). + gen_server:call(?OS_MON, {?FUNCTION_NAME, V}, infinity). get_procmem_high_watermark() -> memsup:get_procmem_high_watermark(). -set_procmem_high_watermark(Float) -> - memsup:set_procmem_high_watermark(Float / 100). +set_procmem_high_watermark(HW) -> + memsup:set_procmem_high_watermark(resolve_watermark(HW) / 100). call(Req) -> gen_server:call(?OS_MON, Req, infinity). @@ -113,16 +111,38 @@ call(Req) -> %%-------------------------------------------------------------------- init([Opts]) -> - set_mem_check_interval(proplists:get_value(mem_check_interval, Opts)), - SysHW = proplists:get_value(sysmem_high_watermark, Opts), - set_sysmem_high_watermark(SysHW), + process_flag(trap_exit, true), + %% make sure memsup will not emit system memory alarms + memsup:set_sysmem_high_watermark(1), set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts)), - ensure_system_memory_alarm(SysHW), - {ok, ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts), + MemCehckInterval = do_resolve_mem_check_interval(proplists:get_value(mem_check_interval, Opts)), + SysHW = resolve_watermark(proplists:get_value(sysmem_high_watermark, Opts)), + St = ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts), cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts), cpu_check_interval => proplists:get_value(cpu_check_interval, Opts), - timer => undefined})}. + sysmem_high_watermark => SysHW, + mem_check_interval => MemCehckInterval, + timer => undefined}), + ok = do_set_mem_check_interval(MemCehckInterval), + %% update immediately after start/restart + ok = update_mem_alarm_status(SysHW), + {ok, ensure_mem_check_timer(St)}. +handle_call(get_sysmem_high_watermark, _From, State) -> + #{sysmem_high_watermark := SysHW} = State, + {reply, maybe_round(SysHW), State}; +handle_call(get_mem_check_interval, _From, State) -> + #{mem_check_interval := Interval} = State, + {reply, Interval, State}; +handle_call({set_sysmem_high_watermark, SysHW}, _From, State) -> + %% update immediately after start/restart + ok = update_mem_alarm_status(SysHW), + {reply, ok, State#{sysmem_high_watermark => SysHW}}; +handle_call({set_mem_check_interval, Seconds0}, _From, State) -> + Seconds = do_resolve_mem_check_interval(Seconds0), + ok = do_set_mem_check_interval(Seconds), + %% will start taking effect when the current timer expires + {reply, ok, State#{mem_check_interval => Seconds}}; handle_call(get_cpu_check_interval, _From, State) -> {reply, maps:get(cpu_check_interval, State, undefined), State}; @@ -145,6 +165,8 @@ handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, State}. +handle_cast(upgrade, State) -> + {noreply, State}; handle_cast(Msg, State) -> ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. @@ -168,16 +190,28 @@ handle_info({timeout, Timer, check}, State = #{timer := Timer, ensure_check_timer(State) end, {noreply, NState}; - +handle_info({timeout, Timer, check_mem}, #{mem_check_timer := Timer, + sysmem_high_watermark := SysHW + } = State) -> + ok = update_mem_alarm_status(SysHW), + NState = ensure_mem_check_timer(State#{mem_check_timer := undefined}), + {noreply, NState}; handle_info(Info, State) -> ?LOG(error, "unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #{timer := Timer}) -> +terminate(_Reason, #{timer := Timer} = St) -> + emqx_misc:cancel_timer(maps:get(mem_check_timer, St, undefined)), emqx_misc:cancel_timer(Timer). code_change(_OldVsn, State, _Extra) -> - {ok, State}. + %% NOTE: downgrade is not handled as the extra fields added to State + %% does not affect old version code. + %% The only thing which may slip through is that a started timer + %% will result in a "unexpected info" error log for the old version code + NewState = ensure_mem_check_timer(State), + SysHW = resolve_sysmem_high_watermark(State), + {ok, NewState#{sysmem_high_watermark => SysHW}}. %%-------------------------------------------------------------------- %% Internal functions @@ -189,19 +223,93 @@ ensure_check_timer(State = #{cpu_check_interval := Interval}) -> _ -> State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)} end. -%% At startup, memsup starts first and checks for memory alarms, -%% but emqx_alarm_handler is not yet used instead of alarm_handler, -%% so alarm_handler is used directly for notification (normally emqx_alarm_handler should be used). -%%The internal memsup will no longer trigger events that have been alerted, -%% and there is no exported function to remove the alerted flag, -%% so it can only be checked again at startup. -ensure_system_memory_alarm(HW) -> - case erlang:whereis(memsup) of - undefined -> ok; - _Pid -> - {Total, Allocated, _Worst} = memsup:get_memory_data(), - case Total =/= 0 andalso Allocated/Total * 100 > HW of - true -> emqx_alarm:activate(high_system_memory_usage, #{high_watermark => HW}); - false -> ok - end +ensure_mem_check_timer(#{mem_check_timer := Ref} = State) when is_reference(Ref) -> + %% timer already started + State; +ensure_mem_check_timer(State) -> + Interval = resolve_mem_check_interval(State), + case is_sysmem_check_supported() of + true -> + State#{mem_check_timer => emqx_misc:start_timer(timer:seconds(Interval), check_mem), + mem_check_interval => Interval + }; + false -> + State + end. + +resolve_mem_check_interval(#{mem_check_interval := Seconds}) when is_integer(Seconds) -> + Seconds; +resolve_mem_check_interval(_) -> + %% this only happens when hot-upgrade from older version (< 4.3.14, or < 4.4.4) + try + %% memsup has interval set API using minutes, but returns in milliseconds from get API + IntervalMs = memsup:get_check_interval(), + true = (IntervalMs > 1000), + IntervalMs div 1000 + catch + _ : _ -> + %% this is the memsup default + 60 + end. + +is_sysmem_check_supported() -> + %% sorry Mac and Windows, for now + {unix, linux} =:= os:type(). + +%% we still need to set memsup interval for process (not system) memory check +do_set_mem_check_interval(Seconds) -> + Minutes = Seconds div 60, + _ = memsup:set_check_interval(Minutes), + ok. + +%% keep the time unit alignment with memsup, minmum interval is 60 seconds. +do_resolve_mem_check_interval(Seconds) -> + case is_integer(Seconds) andalso Seconds >= 60 of + true -> Seconds; + false -> 60 + end. + +resolve_sysmem_high_watermark(#{sysmem_high_watermark := SysHW}) -> SysHW; +resolve_sysmem_high_watermark(_) -> + %% sysmem_high_watermark is not found in state map + %% get it from memsup + resolve_watermark(memsup:get_sysmem_high_watermark()). + +resolve_watermark(W) when W > 0 andalso W =< 1 -> + W * 100; +resolve_watermark(W) when W > 0 andalso W =< 100 -> + W. + +update_mem_alarm_status(SysHW) -> + Usage = current_sysmem_percent(), + case Usage > SysHW of + true -> + _ = emqx_alarm:activate( + high_system_memory_usage, + #{ + usage => Usage, + high_watermark => SysHW + } + ); + _ -> + ok = emqx_alarm:ensure_deactivated( + high_system_memory_usage, + #{ + usage => Usage, + high_watermark => SysHW + } + ) + end, + ok. + +current_sysmem_percent() -> + Ratio = load_ctl:get_memory_usage(), + erlang:floor(Ratio * 10000) / 100. + +maybe_round(X) when is_integer(X) -> X; +maybe_round(X) when is_float(X) -> + R = erlang:round(X), + case erlang:abs(X - R) > 1.0e-6 of + true -> X; + false -> R end. diff --git a/test/emqx_os_mon_SUITE.erl b/test/emqx_os_mon_SUITE.erl index 575c97225..70bf89b18 100644 --- a/test/emqx_os_mon_SUITE.erl +++ b/test/emqx_os_mon_SUITE.erl @@ -43,14 +43,23 @@ end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]), application:stop(os_mon). -% t_set_mem_check_interval(_) -> -% error('TODO'). +t_set_mem_check_interval(_) -> + emqx_os_mon:set_mem_check_interval(0), + ?assertEqual(60, emqx_os_mon:get_mem_check_interval()), + emqx_os_mon:set_mem_check_interval(61), + ?assertEqual(61, emqx_os_mon:get_mem_check_interval()), + ok. -% t_set_sysmem_high_watermark(_) -> -% error('TODO'). - -% t_set_procmem_high_watermark(_) -> -% error('TODO'). +t_set_sysmem_high_watermark(_) -> + emqx_os_mon:set_sysmem_high_watermark(10), + ?assertEqual(10, emqx_os_mon:get_sysmem_high_watermark()), + emqx_os_mon:set_sysmem_high_watermark(100), + ?assertEqual(100, emqx_os_mon:get_sysmem_high_watermark()), + emqx_os_mon:set_sysmem_high_watermark(0.9), + ?assertEqual(90, emqx_os_mon:get_sysmem_high_watermark()), + emqx_os_mon:set_sysmem_high_watermark(0.932), + ?assertEqual(93.2, emqx_os_mon:get_sysmem_high_watermark()), + ok. t_api(_) -> ?assertEqual(1, emqx_os_mon:get_cpu_check_interval()), From 0400638f71320938b527fed072a6f90b0a6fc129 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 25 May 2022 10:45:54 +0200 Subject: [PATCH 27/36] feat(update_appup.escript): allow skipped versions Say app current version is 1.1.2, and the compare baes is version 1.1.1, but there is a 1.1.2 in appup. We may skip updating appup instructions for 1.1.2 as it's never used, (at least not in EMQX) --- scripts/update_appup.escript | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 840f63509..fc6447f38 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -168,8 +168,8 @@ find_appup_actions(App, OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0), UpDiff = diff_app(up, App, CurrAppIdx, PrevAppIdx), DownDiff = diff_app(down, App, PrevAppIdx, CurrAppIdx), - Upgrade = merge_update_actions(App, UpDiff, OldUpgrade), - Downgrade = merge_update_actions(App, DownDiff, OldDowngrade), + Upgrade = merge_update_actions(App, UpDiff, OldUpgrade, PrevVersion), + Downgrade = merge_update_actions(App, DownDiff, OldDowngrade, PrevVersion), case OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade of true -> []; false -> [{App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}] @@ -258,14 +258,39 @@ find_base_appup_actions(App, PrevVersion) -> end, {ensure_version(PrevVersion, Upgrade), ensure_version(PrevVersion, Downgrade)}. -merge_update_actions(App, Changes, Vsns) -> +merge_update_actions(App, Changes, Vsns, PrevVersion) -> lists:map(fun(Ret = {<<".*">>, _}) -> Ret; ({Vsn, Actions}) -> - {Vsn, do_merge_update_actions(App, Changes, Actions)} + case is_skipped_version(App, Vsn, PrevVersion) of + true -> + log("WARN: ~p has version ~s skipped over?~n", [App, Vsn]), + {Vsn, Actions}; + false -> + {Vsn, do_merge_update_actions(App, Changes, Actions)} + end end, Vsns). +%% say current version is 1.1.2, and the compare baes is version 1.1.1, but there is a 1.1.2 in appup +%% we may skip merging instructions for 1.1.2 (it's never used) +is_skipped_version(App, Vsn, PrevVersion) when is_list(Vsn) andalso is_list(PrevVersion) -> + case is_app_external(App) andalso parse_version_number(Vsn) of + {ok, VsnTuple} -> + case parse_version_number(PrevVersion) of + {ok, PrevVsnTuple} -> + VsnTuple > PrevVsnTuple; + _ -> + false + end; + _ -> + false + end; +is_skipped_version(_App, _Vsn, _PrevVersion) -> + %% if app version is a regexp, we don't konw for sure + %% return 'false' to be on the safe side + false. + do_merge_update_actions(App, {New0, Changed0, Deleted0}, OldActions) -> AppSpecific = app_specific_actions(App) -- OldActions, AlreadyHandled = lists:flatten(lists:map(fun process_old_action/1, OldActions)), From 1bebbd6ee5fa6ee81d35b8a60aaaf804ee96ab2f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 25 May 2022 16:13:33 +0200 Subject: [PATCH 28/36] fix(emqx_os_mon): ensure alarm is deactivated on unsupported platforms --- scripts/update_appup.escript | 7 ++++--- src/emqx_os_mon.erl | 21 +++++++++++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index fc6447f38..dd8dd9ca2 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -272,8 +272,9 @@ merge_update_actions(App, Changes, Vsns, PrevVersion) -> end, Vsns). -%% say current version is 1.1.2, and the compare baes is version 1.1.1, but there is a 1.1.2 in appup -%% we may skip merging instructions for 1.1.2 (it's never used) +%% say current version is 1.1.3, and the compare base is version 1.1.1, +%% but there is a 1.1.2 in appup we may skip merging instructions for +%% 1.1.2 because it's not used and no way to know what has been changed is_skipped_version(App, Vsn, PrevVersion) when is_list(Vsn) andalso is_list(PrevVersion) -> case is_app_external(App) andalso parse_version_number(Vsn) of {ok, VsnTuple} -> @@ -287,7 +288,7 @@ is_skipped_version(App, Vsn, PrevVersion) when is_list(Vsn) andalso is_list(Prev false end; is_skipped_version(_App, _Vsn, _PrevVersion) -> - %% if app version is a regexp, we don't konw for sure + %% if app version is a regexp, we don't know for sure %% return 'false' to be on the safe side false. diff --git a/src/emqx_os_mon.erl b/src/emqx_os_mon.erl index ab637645d..c4aa04a46 100644 --- a/src/emqx_os_mon.erl +++ b/src/emqx_os_mon.erl @@ -115,15 +115,15 @@ init([Opts]) -> %% make sure memsup will not emit system memory alarms memsup:set_sysmem_high_watermark(1), set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts)), - MemCehckInterval = do_resolve_mem_check_interval(proplists:get_value(mem_check_interval, Opts)), + MemCheckInterval = do_resolve_mem_check_interval(proplists:get_value(mem_check_interval, Opts)), SysHW = resolve_watermark(proplists:get_value(sysmem_high_watermark, Opts)), St = ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts), cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts), cpu_check_interval => proplists:get_value(cpu_check_interval, Opts), sysmem_high_watermark => SysHW, - mem_check_interval => MemCehckInterval, + mem_check_interval => MemCheckInterval, timer => undefined}), - ok = do_set_mem_check_interval(MemCehckInterval), + ok = do_set_mem_check_interval(MemCheckInterval), %% update immediately after start/restart ok = update_mem_alarm_status(SysHW), {ok, ensure_mem_check_timer(St)}. @@ -165,8 +165,6 @@ handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, State}. -handle_cast(upgrade, State) -> - {noreply, State}; handle_cast(Msg, State) -> ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. @@ -234,7 +232,9 @@ ensure_mem_check_timer(State) -> mem_check_interval => Interval }; false -> - State + State#{mem_check_timer => undefined, + mem_check_interval => Interval + } end. resolve_mem_check_interval(#{mem_check_interval := Seconds}) when is_integer(Seconds) -> @@ -281,6 +281,15 @@ resolve_watermark(W) when W > 0 andalso W =< 100 -> W. update_mem_alarm_status(SysHW) -> + case is_sysmem_check_supported() of + true -> + do_update_mem_alarm_status(SysHW); + false -> + %% in case the old alarm is activated + ok = emqx_alarm:ensure_deactivated(high_system_memory_usage, #{reason => disabled}) + end. + +do_update_mem_alarm_status(SysHW) -> Usage = current_sysmem_percent(), case Usage > SysHW of true -> From 5a84b2c623007ff21fa4a224e0a1c67f62d1e963 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 25 May 2022 16:32:14 +0200 Subject: [PATCH 29/36] chore: only deal with 0-100 scale sysmem high watermark --- src/emqx_os_mon.erl | 16 +++++----------- test/emqx_os_mon_SUITE.erl | 4 ++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/emqx_os_mon.erl b/src/emqx_os_mon.erl index c4aa04a46..bbf4fd80e 100644 --- a/src/emqx_os_mon.erl +++ b/src/emqx_os_mon.erl @@ -87,21 +87,20 @@ get_sysmem_high_watermark() -> call(?FUNCTION_NAME). set_sysmem_high_watermark(HW) -> - V = resolve_watermark(HW), case load_ctl:get_config() of #{ ?MEM_MON_F0 := true } = OldLC -> ok = load_ctl:put_config(OldLC#{ ?MEM_MON_F0 => true - , ?MEM_MON_F1 => V / 100}); + , ?MEM_MON_F1 => HW / 100}); _ -> skip end, - gen_server:call(?OS_MON, {?FUNCTION_NAME, V}, infinity). + gen_server:call(?OS_MON, {?FUNCTION_NAME, HW}, infinity). get_procmem_high_watermark() -> memsup:get_procmem_high_watermark(). set_procmem_high_watermark(HW) -> - memsup:set_procmem_high_watermark(resolve_watermark(HW) / 100). + memsup:set_procmem_high_watermark(HW / 100). call(Req) -> gen_server:call(?OS_MON, Req, infinity). @@ -116,7 +115,7 @@ init([Opts]) -> memsup:set_sysmem_high_watermark(1), set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts)), MemCheckInterval = do_resolve_mem_check_interval(proplists:get_value(mem_check_interval, Opts)), - SysHW = resolve_watermark(proplists:get_value(sysmem_high_watermark, Opts)), + SysHW = proplists:get_value(sysmem_high_watermark, Opts), St = ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts), cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts), cpu_check_interval => proplists:get_value(cpu_check_interval, Opts), @@ -273,12 +272,7 @@ resolve_sysmem_high_watermark(#{sysmem_high_watermark := SysHW}) -> SysHW; resolve_sysmem_high_watermark(_) -> %% sysmem_high_watermark is not found in state map %% get it from memsup - resolve_watermark(memsup:get_sysmem_high_watermark()). - -resolve_watermark(W) when W > 0 andalso W =< 1 -> - W * 100; -resolve_watermark(W) when W > 0 andalso W =< 100 -> - W. + memsup:get_sysmem_high_watermark(). update_mem_alarm_status(SysHW) -> case is_sysmem_check_supported() of diff --git a/test/emqx_os_mon_SUITE.erl b/test/emqx_os_mon_SUITE.erl index 70bf89b18..d4b8a5d2a 100644 --- a/test/emqx_os_mon_SUITE.erl +++ b/test/emqx_os_mon_SUITE.erl @@ -55,9 +55,9 @@ t_set_sysmem_high_watermark(_) -> ?assertEqual(10, emqx_os_mon:get_sysmem_high_watermark()), emqx_os_mon:set_sysmem_high_watermark(100), ?assertEqual(100, emqx_os_mon:get_sysmem_high_watermark()), - emqx_os_mon:set_sysmem_high_watermark(0.9), + emqx_os_mon:set_sysmem_high_watermark(90), ?assertEqual(90, emqx_os_mon:get_sysmem_high_watermark()), - emqx_os_mon:set_sysmem_high_watermark(0.932), + emqx_os_mon:set_sysmem_high_watermark(93.2), ?assertEqual(93.2, emqx_os_mon:get_sysmem_high_watermark()), ok. From c9f45a692a6aa6474b6218278b877dfa2abf06f5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 25 May 2022 20:48:20 +0800 Subject: [PATCH 30/36] chore: bump to 4.3.15-rc.2 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 3945b3869..43d9dfcb7 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.15-rc.1"}). +-define(EMQX_RELEASE, {opensource, "4.3.15-rc.2"}). -else. From 4655d55192358defbba3c624a7bcc80ceb077e04 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 26 May 2022 11:34:23 +0800 Subject: [PATCH 31/36] build: fix 'fatal: unsafe repository ...' on ubuntu20.04 --- .github/workflows/apps_version_check.yaml | 2 ++ .github/workflows/build_slim_packages.yaml | 2 ++ .github/workflows/run_cts_tests.yaml | 10 ++++++++++ 3 files changed, 14 insertions(+) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 2ef18934f..59d6aebff 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -19,6 +19,8 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 # need full history + - name: fix-git-unsafe-repository + run: git config --global --add safe.directory /__w/emqx/emqx - name: Check relup (ce) if: endsWith(github.repository, 'emqx') run: ./scripts/update-appup.sh emqx --check diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index a10e72258..529faa559 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -29,6 +29,8 @@ jobs: else echo "EMQX_NAME=emqx" >> $GITHUB_ENV fi + - name: fix-git-unsafe-repository + run: git config --global --add safe.directory /__w/emqx/emqx - name: build zip packages run: make ${EMQX_NAME}-zip - name: build deb/rpm packages diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 84aa301c4..6b05a014e 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -45,6 +45,8 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi + - name: fix-git-unsafe-repository + run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx" - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -115,6 +117,8 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi + - name: fix-git-unsafe-repository + run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx" - name: run test cases run: | printenv | grep "^EMQX_" > .env @@ -197,6 +201,8 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi + - name: fix-git-unsafe-repository + run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx" - name: run test cases run: | printenv | grep "^EMQX_" > .env @@ -268,6 +274,8 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi + - name: fix-git-unsafe-repository + run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx" - name: run test cases run: | export EMQX_AUTH__PGSQL__USERNAME=root \ @@ -391,6 +399,8 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi + - name: fix-git-unsafe-repository + run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx" - name: run test cases run: | export EMQX_AUTH__REIDS__PASSWORD=public From 34ec0cc04b3d7f39fb484991e319b931c6ba2ac4 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 23 May 2022 14:33:40 +0800 Subject: [PATCH 32/36] fix: logger formatter --- etc/emqx.conf | 29 +++++- priv/emqx.schema | 100 ++++++++++++++++++++- src/emqx_logger_textfmt.erl | 173 +++++++++++++++++++++++++++++++++++- 3 files changed, 296 insertions(+), 6 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index bc45df1fc..fb7263f85 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -458,7 +458,34 @@ log.file = emqx.log ## Log formatter ## Value: text | json -#log.formatter = text +log.formatter = text + +## Text logger, date format. +## +## Value: rfc3339 | FormatString +## FormatString: +## %Y: year +## %m: month +## %d: day +## %H: hour +## %M: minute +## %S: second +## %6N: microseconds +## %z: timezone +## For example: +## log.formatter.text.date.format = emqx-server-date: %Y-%m-%dT%H:%M:%S:%6N %z +## Default: rfc3339 +# log.formatter.text.date.format = rfc3339 + +## Text logger, timezone. +## Only takes effect when log.formatter.text.date.format not rfc3339. +## +## Value: local | z | UTC TimeOffset +## TimeOffset: [+|-]HH:MM:SS, for example: +08:00 +## z: UTC zulu timezone. +## local is the system local time timezone. +## Default: local +# log.formatter.text.date.timezone = local ## Log to single line ## Value: Boolean diff --git a/priv/emqx.schema b/priv/emqx.schema index 4d7b1beda..7fde30c02 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -528,6 +528,18 @@ end}. {datatype, {enum, [text, json]}} ]}. +%% @doc format logs as text, date format part +{mapping, "log.formatter.text.date.format", "kernel.logger", [ + {default, rfc3339}, + {datatype, string} +]}. + +%% @doc format logs as text, date format part. local or any UTC offset +{mapping, "log.formatter.text.date.timezone", "kernel.logger", [ + {default, local}, + {datatype, string} +]}. + %% @doc format logs in a single line. {mapping, "log.single_line", "kernel.logger", [ {default, true}, @@ -629,9 +641,93 @@ end}. single_line => SingleLine }}; text -> + DateFormat = + case cuttlefish:conf_get("log.formatter.text.date.format", Conf) of + "rfc3339" -> + rfc3339; + DateStr -> + DateStrTrans = + fun + DST(<<>>, Formatter) -> lists:reverse(Formatter); + DST(<<"%Y", Tail/binary>>, Formatter) -> DST(Tail, [year | Formatter]); + DST(<<"%m", Tail/binary>>, Formatter) -> DST(Tail, [month | Formatter]); + DST(<<"%d", Tail/binary>>, Formatter) -> DST(Tail, [day | Formatter]); + DST(<<"%H", Tail/binary>>, Formatter) -> DST(Tail, [hour | Formatter]); + DST(<<"%M", Tail/binary>>, Formatter) -> DST(Tail, [minute | Formatter]); + DST(<<"%S", Tail/binary>>, Formatter) -> DST(Tail, [second | Formatter]); + DST(<<"%6N", Tail/binary>>, Formatter) -> DST(Tail, [nano_second | Formatter]); + DST(<<"%z", Tail/binary>>, Formatter) -> DST(Tail, [timezone | Formatter]); + DST(<>, [Str | Formatter]) when is_list(Str) -> + DST(Tail, [lists:append(Str, [Char]) | Formatter]); + DST(<>, Formatter) -> + DST(Tail, [[Char] | Formatter]) + end, + DateStrTrans(list_to_binary(DateStr), []) + end, + {DateOffsetStr, DateOffset} = + case cuttlefish:conf_get("log.formatter.text.date.timezone", Conf) of + "z" -> + {"z", 0}; + "Z" -> + {"Z", 0}; + "local" -> + UniversalTime = + calendar:system_time_to_universal_time( + erlang:system_time(second), + second), + LocalTime = erlang:universaltime_to_localtime(UniversalTime), + LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), + UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), + OffsetSecond = LocalSecs - UniversalSecs, + {Sign, PosOffsetSecond} = + case OffsetSecond >= 0 of + true -> {$+, OffsetSecond}; + false -> {$-, - OffsetSecond} + end, + {H, M, S} = calendar:seconds_to_time(PosOffsetSecond), + Str = + case S of + 0 -> + io_lib:format("~c~p:~p", [Sign, H, M]); + S -> + io_lib:format("~c~p:~p:~p", [Sign, H, M, S]) + end, + {Str, OffsetSecond}; + DateOStr -> + Sign = hd(DateOStr), + Signs = #{$+ => 1, $- => -1}, + case maps:get(Sign, Signs, undefined) of + undefined -> + error(bad_logger_offset); + PosNeg -> + try + [Sign | HM] = DateOStr, + {HourStr, MinuteStr, SecondStr} = + case string:tokens(HM, ":") of + [H, M] -> + {H, M, "0"}; + [H, M, S] -> + {H, M, S} + end, + Hour = erlang:list_to_integer(HourStr), + Minute = erlang:list_to_integer(MinuteStr), + Second = erlang:list_to_integer(SecondStr), + true = (Hour =< 23), + true = (Minute =< 59), + true = (Second =< 59), + {DateOStr, PosNeg * (Hour * 3600 + Minute * 60 + Second)} + catch _E : _R -> + error(bad_logger_offset) + end + end + end, {emqx_logger_textfmt, - #{template => - [time," [",level,"] ", + #{ + date_format => DateFormat, + timezone_offset => DateOffset, + timezone => DateOffsetStr, + template => + [" [",level,"] ", {clientid, [{peername, [clientid,"@",peername," "], diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index 98104f58c..df4118ba4 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -16,6 +16,16 @@ -module(emqx_logger_textfmt). +-define(SECONDS_PER_MINUTE, 60). +-define(SECONDS_PER_HOUR, 3600). +-define(SECONDS_PER_DAY, 86400). +-define(DAYS_PER_YEAR, 365). +-define(DAYS_PER_LEAP_YEAR, 366). +-define(DAYS_FROM_0_TO_1970, 719528). +-define(DAYS_FROM_0_TO_10000, 2932897). +-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY). +-define(SECONDS_FROM_0_TO_10000, ?DAYS_FROM_0_TO_10000 * ?SECONDS_PER_DAY). + -export([format/2]). -export([check_config/1]). @@ -28,11 +38,21 @@ , gl % not interesting ]). -check_config(X) -> logger_formatter:check_config(X). +check_config(Config0) -> + Config = maps:without([date_format, timezone_offset, timezone], Config0), + logger_formatter:check_config(Config). -format(#{msg := Msg0, meta := Meta} = Event, Config) -> +format(#{msg := Msg0, meta := Meta} = Event, + #{date_format := rfc3339, template := Template0} = Config) -> Msg = maybe_merge(Msg0, Meta), - logger_formatter:format(Event#{msg := Msg}, Config). + Template = [time | Template0], + logger_formatter:format(Event#{msg := Msg}, Config#{template => Template}); + +format(#{msg := Msg0, meta := Meta} = Event, + #{timezone_offset := TZO, timezone := TZ, date_format := DFS} = Config) -> + Msg = maybe_merge(Msg0, Meta), + Time = maps:get(time, Event, undefined), + [date_format(Time, TZO, TZ, DFS) | logger_formatter:format(Event#{msg := Msg}, Config)]. maybe_merge({report, Report}, Meta) when is_map(Report) -> {report, maps:merge(Report, filter(Meta))}; @@ -41,3 +61,150 @@ maybe_merge(Report, _Meta) -> filter(Meta) -> maps:without(?WITHOUT_MERGE, Meta). + +date_format(undefined, Offset, OffsetStr, Formatter) -> + Time = erlang:system_time(microsecond), + date_format(Time, Offset, OffsetStr, Formatter); +date_format(Time, Offset, OffsetStr, Formatter) -> + Adjustment = erlang:convert_time_unit(Offset, second, microsecond), + AdjustedTime = Time + Adjustment, + %% Factor 1000000 for microsecond. + Factor = 1000000, + Secs = AdjustedTime div Factor, + DateTime = system_time_to_datetime(Secs), + {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, + FractionStr = fraction_str(Factor, AdjustedTime), + Date = #{ + year => padding(Year, 4), + month => padding(Month, 2), + day => padding(Day, 2), + hour => padding(Hour, 2), + minute => padding(Min, 2), + second => padding(Sec, 2), + nano_second => FractionStr, + timezone => OffsetStr + }, + [maps:get(Key, Date, Key) || Key <- Formatter]. + +system_time_to_datetime(Seconds) -> + gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970). + +gregorian_seconds_to_datetime(Secs) when Secs >= 0 -> + Days = Secs div ?SECONDS_PER_DAY, + Rest = Secs rem ?SECONDS_PER_DAY, + {gregorian_days_to_date(Days), seconds_to_time(Rest)}. + +seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY -> + Secs0 = Secs rem ?SECONDS_PER_DAY, + Hour = Secs0 div ?SECONDS_PER_HOUR, + Secs1 = Secs0 rem ?SECONDS_PER_HOUR, + Minute = Secs1 div ?SECONDS_PER_MINUTE, + Second = Secs1 rem ?SECONDS_PER_MINUTE, + {Hour, Minute, Second}. + +gregorian_days_to_date(Days) -> + {Year, DayOfYear} = day_to_year(Days), + {Month, DayOfMonth} = year_day_to_date(Year, DayOfYear), + {Year, Month, DayOfMonth}. + +day_to_year(DayOfEpoch) when DayOfEpoch >= 0 -> + YMax = DayOfEpoch div ?DAYS_PER_YEAR, + YMin = DayOfEpoch div ?DAYS_PER_LEAP_YEAR, + {Y1, D1} = dty(YMin, YMax, DayOfEpoch, dy(YMin), dy(YMax)), + {Y1, DayOfEpoch - D1}. + +year_day_to_date(Year, DayOfYear) -> + ExtraDay = + case is_leap_year(Year) of + true -> + 1; + false -> + 0 + end, + {Month, Day} = year_day_to_date2(ExtraDay, DayOfYear), + {Month, Day + 1}. + +dty(Min, Max, _D1, DMin, _DMax) when Min == Max -> + {Min, DMin}; +dty(Min, Max, D1, DMin, DMax) -> + Diff = Max - Min, + Mid = Min + Diff * (D1 - DMin) div (DMax - DMin), + MidLength = + case is_leap_year(Mid) of + true -> + ?DAYS_PER_LEAP_YEAR; + false -> + ?DAYS_PER_YEAR + end, + case dy(Mid) of + D2 when D1 < D2 -> + NewMax = Mid - 1, + dty(Min, NewMax, D1, DMin, dy(NewMax)); + D2 when D1 - D2 >= MidLength -> + NewMin = Mid + 1, + dty(NewMin, Max, D1, dy(NewMin), DMax); + D2 -> + {Mid, D2} + end. + +dy(Y) when Y =< 0 -> + 0; +dy(Y) -> + X = Y - 1, + X div 4 - X div 100 + X div 400 + X * ?DAYS_PER_YEAR + ?DAYS_PER_LEAP_YEAR. + +is_leap_year(Y) when is_integer(Y), Y >= 0 -> + is_leap_year1(Y). + +is_leap_year1(Year) when Year rem 4 =:= 0, Year rem 100 > 0 -> + true; +is_leap_year1(Year) when Year rem 400 =:= 0 -> + true; +is_leap_year1(_) -> + false. + +year_day_to_date2(_, Day) when Day < 31 -> + {1, Day}; +year_day_to_date2(E, Day) when 31 =< Day, Day < 59 + E -> + {2, Day - 31}; +year_day_to_date2(E, Day) when 59 + E =< Day, Day < 90 + E -> + {3, Day - (59 + E)}; +year_day_to_date2(E, Day) when 90 + E =< Day, Day < 120 + E -> + {4, Day - (90 + E)}; +year_day_to_date2(E, Day) when 120 + E =< Day, Day < 151 + E -> + {5, Day - (120 + E)}; +year_day_to_date2(E, Day) when 151 + E =< Day, Day < 181 + E -> + {6, Day - (151 + E)}; +year_day_to_date2(E, Day) when 181 + E =< Day, Day < 212 + E -> + {7, Day - (181 + E)}; +year_day_to_date2(E, Day) when 212 + E =< Day, Day < 243 + E -> + {8, Day - (212 + E)}; +year_day_to_date2(E, Day) when 243 + E =< Day, Day < 273 + E -> + {9, Day - (243 + E)}; +year_day_to_date2(E, Day) when 273 + E =< Day, Day < 304 + E -> + {10, Day - (273 + E)}; +year_day_to_date2(E, Day) when 304 + E =< Day, Day < 334 + E -> + {11, Day - (304 + E)}; +year_day_to_date2(E, Day) when 334 + E =< Day -> + {12, Day - (334 + E)}. + +fraction_str(1, _Time) -> + ""; +fraction_str(Factor, Time) -> + Fraction = Time rem Factor, + S = integer_to_list(abs(Fraction)), + [padding(S, log10(Factor) - length(S))]. + +log10(1000) -> + 3; +log10(1000000) -> + 6; +log10(1000000000) -> + 9. + +padding(Data, Len) when is_integer(Data) -> + padding(integer_to_list(Data), Len); +padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> + [$0 | padding(Data, Len - 1)]; +padding(Data, _Len) -> + Data. From 664ebcc1927f420f4e880a54cfa169ca7c5df26f Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Wed, 25 May 2022 19:14:08 +0800 Subject: [PATCH 33/36] fix: log date format by emqx_calendar --- etc/emqx.conf | 11 +- priv/emqx.schema | 16 +- src/emqx_calendar.erl | 512 ++++++++++++++++++++++++++++++++++++ src/emqx_logger_textfmt.erl | 170 +----------- 4 files changed, 537 insertions(+), 172 deletions(-) create mode 100644 src/emqx_calendar.erl diff --git a/etc/emqx.conf b/etc/emqx.conf index fb7263f85..4cdd6e4d7 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -427,7 +427,7 @@ log.to = file ## this level will be logged. ## ## Default: warning -log.level = warning +log.level = debug ## The dir for log files. ## @@ -471,9 +471,10 @@ log.formatter = text ## %M: minute ## %S: second ## %6N: microseconds -## %z: timezone +## %3N: milliseconds +## %z: timezone, [+|-]HH:MM or [+|-]HH:MM:SS. Depends on `log.formatter.text.date.timezone` ## For example: -## log.formatter.text.date.format = emqx-server-date: %Y-%m-%dT%H:%M:%S:%6N %z +## log.formatter.text.date.format = emqx-server-date: %Y-%m-%dT%H:%M:%S.%6N %z ## Default: rfc3339 # log.formatter.text.date.format = rfc3339 @@ -481,9 +482,9 @@ log.formatter = text ## Only takes effect when log.formatter.text.date.format not rfc3339. ## ## Value: local | z | UTC TimeOffset -## TimeOffset: [+|-]HH:MM:SS, for example: +08:00 -## z: UTC zulu timezone. +## TimeOffset: [+|-]HH:MM or [+|-]HH:MM:SS, for example: +08:00. ## local is the system local time timezone. +## z: UTC zulu timezone. ## Default: local # log.formatter.text.date.timezone = local diff --git a/priv/emqx.schema b/priv/emqx.schema index 7fde30c02..dc15d87b3 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -530,13 +530,13 @@ end}. %% @doc format logs as text, date format part {mapping, "log.formatter.text.date.format", "kernel.logger", [ - {default, rfc3339}, + {default, "rfc3339"}, {datatype, string} ]}. %% @doc format logs as text, date format part. local or any UTC offset {mapping, "log.formatter.text.date.timezone", "kernel.logger", [ - {default, local}, + {default, "local"}, {datatype, string} ]}. @@ -642,7 +642,7 @@ end}. }}; text -> DateFormat = - case cuttlefish:conf_get("log.formatter.text.date.format", Conf) of + case cuttlefish:conf_get("log.formatter.text.date.format", Conf, "rfc3339") of "rfc3339" -> rfc3339; DateStr -> @@ -655,7 +655,9 @@ end}. DST(<<"%H", Tail/binary>>, Formatter) -> DST(Tail, [hour | Formatter]); DST(<<"%M", Tail/binary>>, Formatter) -> DST(Tail, [minute | Formatter]); DST(<<"%S", Tail/binary>>, Formatter) -> DST(Tail, [second | Formatter]); - DST(<<"%6N", Tail/binary>>, Formatter) -> DST(Tail, [nano_second | Formatter]); + DST(<<"%N", Tail/binary>>, Formatter) -> DST(Tail, [nanosecond | Formatter]); + DST(<<"%3N", Tail/binary>>, Formatter) -> DST(Tail, [millisecond | Formatter]); + DST(<<"%6N", Tail/binary>>, Formatter) -> DST(Tail, [microsecond | Formatter]); DST(<<"%z", Tail/binary>>, Formatter) -> DST(Tail, [timezone | Formatter]); DST(<>, [Str | Formatter]) when is_list(Str) -> DST(Tail, [lists:append(Str, [Char]) | Formatter]); @@ -665,7 +667,7 @@ end}. DateStrTrans(list_to_binary(DateStr), []) end, {DateOffsetStr, DateOffset} = - case cuttlefish:conf_get("log.formatter.text.date.timezone", Conf) of + case cuttlefish:conf_get("log.formatter.text.date.timezone", Conf, "local") of "z" -> {"z", 0}; "Z" -> @@ -688,9 +690,9 @@ end}. Str = case S of 0 -> - io_lib:format("~c~p:~p", [Sign, H, M]); + io_lib:format("~c~2.10.0B:~2.10.0B", [Sign, H, M]); S -> - io_lib:format("~c~p:~p:~p", [Sign, H, M, S]) + io_lib:format("~c~2.10.0B:~2.10.0B:~2.10.0B", [Sign, H, M, S]) end, {Str, OffsetSecond}; DateOStr -> diff --git a/src/emqx_calendar.erl b/src/emqx_calendar.erl new file mode 100644 index 000000000..ae9bae763 --- /dev/null +++ b/src/emqx_calendar.erl @@ -0,0 +1,512 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_calendar). + +-define(SECONDS_PER_MINUTE, 60). +-define(SECONDS_PER_HOUR, 3600). +-define(SECONDS_PER_DAY, 86400). +-define(DAYS_PER_YEAR, 365). +-define(DAYS_PER_LEAP_YEAR, 366). +-define(DAYS_FROM_0_TO_1970, 719528). +-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY). + +-export([ formatter/1 + , format/3 + , format/4 + , format/5 + , parse/3 + , parse/4 + , offset_second/1 + ]). + +-export([test/0]). + +-define(DATE_PART, + [ year + , month + , day + , hour + , minute + , second + , nanosecond + , millisecond + , microsecond + ]). + +-define(DATE_ZONE_NAME, + [ timezone + , timezone1 + , timezone2 + ]). + +formatter(FormatterStr) when is_list(FormatterStr) -> + formatter(list_to_binary(FormatterStr)); +formatter(FormatterBin) when is_binary(FormatterBin) -> + do_formatter(FormatterBin, []). + +offset_second(Offset) -> + do_offset_second(Offset). + +format(Time, Unit, Formatter) -> + format(Time, Unit, undefined, Formatter). + +format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> + format(Time, Unit, Offset, formatter(FormatterBin)); +format(Time, Unit, Offset, Formatter) -> + do_format(Time, time_unit(Unit), offset_second(Offset), Formatter). + +%% for logger format +format(Time, Unit, Offset, Zones, Formatter) -> + do_format(Time, time_unit(Unit), offset_second(Offset), Zones, Formatter). + +test() -> + Units = [second, millisecond, microsecond, nanosecond], + Formatters = [ + <<"%Y-%m-%d %H:%M:%S:%N%z">>, + <<"%Y-%m-%d %H:%M:%S:%N%:z">>, + <<"%Y-%m-%d %H:%M:%S:%N%::z">>, + <<"%Y-%m-%d %H:%M:%S:%N">>, + <<"%Y-%m-%d %H:%M:%S:%3N">>, + <<"%Y-%m-%d %H:%M:%S:%6N">> + ], + Offsets = [ + 0, + 8 * 3600, + -8 * 3600, + <<"+08:00">>, + <<"+08:00:01">>, + <<"+0802">>, + <<"-08:00">>, + <<"local">> + ], + [ + [ + [begin + Time = erlang:system_time(U), + FormatDate = erlang:list_to_binary(format(Time, U, O, F)), + PTime = emqx_calendar:parse(FormatDate, U, O, F), + io:format("~p ~p ~p ~p ~n~p~n~p~n", [U, O, F, FormatDate, Time, PTime]), + % Equal = (PTime == Time), + % F2 = + % fun() -> + % Delta = Time - PTime, + % DeltaFmt = erlang:list_to_binary(format(Delta, U, 0, <<"%Y-%m-%d %H:%M:%S:%N">>)), + % io:format("delta ~p~n ~p~n-----~n", [Delta, DeltaFmt]) + % end, + % Equal orelse F2() + ok + end||O <- Offsets] + || F <-Formatters] + || U <-Units], + ok. + +parse(DateStr, Unit, FormatterBin) -> + parse(DateStr, Unit, 0, FormatterBin). + +parse(DateStr, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> + parse(DateStr, Unit, Offset, formatter(FormatterBin)); +parse(DateStr, Unit, Offset, Formatter) -> + do_parse(DateStr, Unit, offset_second(Offset), Formatter). + +do_parse(DateStr, Unit, Offset, Formatter) -> + DateInfo = do_parse_date_str(DateStr, Formatter, #{}), + {Precise, PrecisionUnit} = precise(DateInfo), + Counter = + fun + (year, V, Res) -> + Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); + (month, V, Res) -> + Res + nano_sm(V); + (day, V, Res) -> + Res + (V * ?SECONDS_PER_DAY * Precise); + (hour, V, Res) -> + Res + (V * ?SECONDS_PER_HOUR * Precise); + (minute, V, Res) -> + Res + (V * ?SECONDS_PER_MINUTE * Precise); + (second, V, Res) -> + Res + V * Precise; + (millisecond, V, Res) -> + case PrecisionUnit of + millisecond -> + Res + V; + microsecond -> + Res + (V * 1000); + nanosecond -> + Res + (V * 1000000) + end; + (microsecond, V, Res) -> + case PrecisionUnit of + microsecond -> + Res + V; + nanosecond -> + Res + (V * 1000) + end; + (nanosecond, V, Res) -> + Res + V; + (parsed_offset, V, Res) -> + Res - V + end, + Counter1 = + fun(K,V,R) -> + Res = Counter(K,V,R), + % io:format("K, V, R = ~p, ~p, ~p, Res = ~p~n", [K, V, R, Res - R]), + Res + end, + Count0 = maps:fold(Counter1, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), + Count = + case maps:is_key(parsed_offset, DateInfo) of + true -> + Count0; + false -> + (Offset * Precise) + Count0 + end, + erlang:convert_time_unit(Count, PrecisionUnit, Unit). + +precise(#{nanosecond := _}) -> {1000_000_000, nanosecond}; +precise(#{microsecond := _}) -> {1000_000, microsecond}; +precise(#{millisecond := _}) -> {1000, millisecond}; +precise(#{second := _}) -> {1, second}; +precise(_) -> {1, second}. + +do_parse_date_str(<<>>, _, Result) -> Result; +do_parse_date_str(_, [], Result) -> Result; +do_parse_date_str(Date, [Key | Formatter], Result) -> + Size = date_size(Key), + <> = Date, + case lists:member(Key, ?DATE_PART) of + true -> + do_parse_date_str(Tail, Formatter, Result#{Key => erlang:binary_to_integer(DatePart)}); + false -> + case lists:member(Key, ?DATE_ZONE_NAME) of + true -> + io:format("DatePart -------------- ~p ~p~n", [DatePart, offset_second(DatePart)]), + do_parse_date_str(Tail, Formatter, Result#{parsed_offset => offset_second(DatePart)}); + false -> + do_parse_date_str(Tail, Formatter, Result) + end + end. + +date_size(Str) when is_list(Str) -> erlang:length(Str); +date_size(DateName) -> + Map = #{ + year => 4, + month => 2, + day => 2, + hour => 2, + minute => 2, + second => 2, + millisecond => 3, + microsecond => 6, + nanosecond => 9, + timezone => 5, + timezone1 => 6, + timezone2 => 9 + }, + maps:get(DateName, Map). + +nano_sm(Month) -> dm(Month) * ?SECONDS_PER_DAY * 1000_000. + +dm(Month) -> + MonthDays = #{ + 1 => 0, + 2 => 31, + 3 => 59, + 4 => 90, + 5 => 120, + 6 => 151, + 7 => 181, + 8 => 212, + 9 => 243, + 10 => 273, + 11 => 304, + 12 => 334 + }, + maps:get(Month, MonthDays). + +%% ------------------------------------------------------------------------------------------------- +%% internal +%% emqx_calendar:format(erlang:system_time(second), <<"second">>, <<"+8:00">> ,<<"%Y-%m-%d-%H:%M:%S%Z">>). + +time_unit(second) -> second; +time_unit(millisecond) -> millisecond; +time_unit(microsecond) -> microsecond; +time_unit(nanosecond) -> nanosecond; +time_unit("second") -> second; +time_unit("millisecond") -> millisecond; +time_unit("microsecond") -> microsecond; +time_unit("nanosecond") -> nanosecond; +time_unit(<<"second">>) -> second; +time_unit(<<"millisecond">>) -> millisecond; +time_unit(<<"microsecond">>) -> microsecond; +time_unit(<<"nanosecond">>) -> nanosecond. + +%% ------------------------------------------------------------------------------------------------- +%% internal: format part + +do_formatter(<<>>, Formatter) -> lists:reverse(Formatter); +do_formatter(<<"%Y", Tail/binary>>, Formatter) -> do_formatter(Tail, [year | Formatter]); +do_formatter(<<"%m", Tail/binary>>, Formatter) -> do_formatter(Tail, [month | Formatter]); +do_formatter(<<"%d", Tail/binary>>, Formatter) -> do_formatter(Tail, [day | Formatter]); +do_formatter(<<"%H", Tail/binary>>, Formatter) -> do_formatter(Tail, [hour | Formatter]); +do_formatter(<<"%M", Tail/binary>>, Formatter) -> do_formatter(Tail, [minute | Formatter]); +do_formatter(<<"%S", Tail/binary>>, Formatter) -> do_formatter(Tail, [second | Formatter]); +do_formatter(<<"%N", Tail/binary>>, Formatter) -> do_formatter(Tail, [nanosecond | Formatter]); +do_formatter(<<"%3N", Tail/binary>>, Formatter) -> do_formatter(Tail, [millisecond | Formatter]); +do_formatter(<<"%6N", Tail/binary>>, Formatter) -> do_formatter(Tail, [microsecond | Formatter]); +do_formatter(<<"%z", Tail/binary>>, Formatter) -> do_formatter(Tail, [timezone | Formatter]); +do_formatter(<<"%:z", Tail/binary>>, Formatter) -> do_formatter(Tail, [timezone1 | Formatter]); +do_formatter(<<"%::z", Tail/binary>>, Formatter) -> do_formatter(Tail, [timezone2 | Formatter]); +do_formatter(<>, [Str | Formatter]) when is_list(Str) -> + do_formatter(Tail, [lists:append(Str, [Char]) | Formatter]); +do_formatter(<>, Formatter) -> do_formatter(Tail, [[Char] | Formatter]). + +do_offset_second(OffsetSecond) when is_integer(OffsetSecond) -> OffsetSecond; +do_offset_second(undefined) -> 0; +do_offset_second("local") -> do_offset_second(local); +do_offset_second(<<"local">>) -> do_offset_second(local); +do_offset_second(local) -> + UniversalTime = calendar:system_time_to_universal_time(erlang:system_time(second), second), + LocalTime = erlang:universaltime_to_localtime(UniversalTime), + LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), + UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), + LocalSecs - UniversalSecs; +do_offset_second(Offset) when is_binary(Offset) -> + do_offset_second(erlang:binary_to_list(Offset)); +do_offset_second("Z") -> 0; +do_offset_second("z") -> 0; +do_offset_second(Offset) when is_list(Offset) -> + Sign = hd(Offset), + ((Sign == $+) orelse (Sign == $-)) + orelse error({bad_zone, Offset}), + Signs = #{$+ => 1, $- => -1}, + PosNeg = maps:get(Sign, Signs), + [Sign | HM] = Offset, + {HourStr, MinuteStr, SecondStr} = + case string:tokens(HM, ":") of + [H, M] -> + {H, M, "0"}; + [H, M, S] -> + {H, M, S}; + [HHMM] when erlang:length(HHMM) == 4 -> + {string:sub_string(HHMM, 1,2), string:sub_string(HHMM, 3,4), "0"}; + _ -> + error({bad_zone, Offset}) + end, + Hour = erlang:list_to_integer(HourStr), + Minute = erlang:list_to_integer(MinuteStr), + Second = erlang:list_to_integer(SecondStr), + (Hour =< 23) orelse error({bad_hour, Hour}), + (Minute =< 59) orelse error({bad_minute, Minute}), + (Second =< 59) orelse error({bad_second, Second}), + PosNeg * (Hour * 3600 + Minute * 60 + Second). + +do_format(Time, Unit, Offset, Formatter) -> + Timezones = formatter_timezones(Offset, Formatter, #{}), + do_format(Time, Unit, Offset, Timezones, Formatter). + +do_format(Time, Unit, Offset, Timezones, Formatter) -> + Adjustment = erlang:convert_time_unit(Offset, second, Unit), + AdjustedTime = Time + Adjustment, + Factor = factor(Unit), + Secs = AdjustedTime div Factor, + DateTime = system_time_to_datetime(Secs), + {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, + Date = #{ + year => padding(Year, 4), + month => padding(Month, 2), + day => padding(Day, 2), + hour => padding(Hour, 2), + minute => padding(Min, 2), + second => padding(Sec, 2), + millisecond => trans_x_second(Unit, millisecond, Time), + microsecond => trans_x_second(Unit, microsecond, Time), + nanosecond => trans_x_second(Unit, nanosecond, Time) + }, + DateWithZone = maps:merge(Date, Timezones), + [maps:get(Key, DateWithZone, Key) || Key <- Formatter]. + +formatter_timezones(_Offset, [], Zones) -> Zones; +formatter_timezones(Offset, [Timezone | Formatter], Zones) -> + case lists:member(Timezone, [timezone, timezone1, timezone2]) of + true -> + NZones = Zones#{Timezone => offset_to_timezone(Offset, Timezone)}, + formatter_timezones(Offset, Formatter, NZones); + false -> + formatter_timezones(Offset, Formatter, Zones) + end. + +offset_to_timezone(Offset, Timezone) -> + Sign = + case Offset >= 0 of + true -> + $+; + false -> + $- + end, + {H, M, S} = seconds_to_time(abs(Offset)), + %% TODO: Support zone define %:::z + %% Numeric time zone with ":" to necessary precision (e.g., -04, +05:30). + case Timezone of + timezone -> + %% +0800 + io_lib:format("~c~2.10.0B~2.10.0B", [Sign, H, M]); + timezone1 -> + %% +08:00 + io_lib:format("~c~2.10.0B:~2.10.0B", [Sign, H, M]); + timezone2 -> + %% +08:00:00 + io_lib:format("~c~2.10.0B:~2.10.0B:~2.10.0B", [Sign, H, M, S]) + end. + +factor(second) -> 1; +factor(millisecond) -> 1000; +factor(microsecond) -> 1000000; +factor(nanosecond) -> 1000000000. + +system_time_to_datetime(Seconds) -> + gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970). + +gregorian_seconds_to_datetime(Secs) when Secs >= 0 -> + Days = Secs div ?SECONDS_PER_DAY, + Rest = Secs rem ?SECONDS_PER_DAY, + {gregorian_days_to_date(Days), seconds_to_time(Rest)}. + +seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY -> + Secs0 = Secs rem ?SECONDS_PER_DAY, + Hour = Secs0 div ?SECONDS_PER_HOUR, + Secs1 = Secs0 rem ?SECONDS_PER_HOUR, + Minute = Secs1 div ?SECONDS_PER_MINUTE, + Second = Secs1 rem ?SECONDS_PER_MINUTE, + {Hour, Minute, Second}. + +gregorian_days_to_date(Days) -> + {Year, DayOfYear} = day_to_year(Days), + {Month, DayOfMonth} = year_day_to_date(Year, DayOfYear), + {Year, Month, DayOfMonth}. + +day_to_year(DayOfEpoch) when DayOfEpoch >= 0 -> + YMax = DayOfEpoch div ?DAYS_PER_YEAR, + YMin = DayOfEpoch div ?DAYS_PER_LEAP_YEAR, + {Y1, D1} = dty(YMin, YMax, DayOfEpoch, dy(YMin), dy(YMax)), + {Y1, DayOfEpoch - D1}. + +year_day_to_date(Year, DayOfYear) -> + ExtraDay = + case is_leap_year(Year) of + true -> + 1; + false -> + 0 + end, + {Month, Day} = year_day_to_date2(ExtraDay, DayOfYear), + {Month, Day + 1}. + +dty(Min, Max, _D1, DMin, _DMax) when Min == Max -> + {Min, DMin}; +dty(Min, Max, D1, DMin, DMax) -> + Diff = Max - Min, + Mid = Min + Diff * (D1 - DMin) div (DMax - DMin), + MidLength = + case is_leap_year(Mid) of + true -> + ?DAYS_PER_LEAP_YEAR; + false -> + ?DAYS_PER_YEAR + end, + case dy(Mid) of + D2 when D1 < D2 -> + NewMax = Mid - 1, + dty(Min, NewMax, D1, DMin, dy(NewMax)); + D2 when D1 - D2 >= MidLength -> + NewMin = Mid + 1, + dty(NewMin, Max, D1, dy(NewMin), DMax); + D2 -> + {Mid, D2} + end. + +dy(Y) when Y =< 0 -> + 0; +dy(Y) -> + X = Y - 1, + X div 4 - X div 100 + X div 400 + X * ?DAYS_PER_YEAR + ?DAYS_PER_LEAP_YEAR. + +is_leap_year(Y) when is_integer(Y), Y >= 0 -> + is_leap_year1(Y). + +is_leap_year1(Year) when Year rem 4 =:= 0, Year rem 100 > 0 -> + true; +is_leap_year1(Year) when Year rem 400 =:= 0 -> + true; +is_leap_year1(_) -> + false. + +year_day_to_date2(_, Day) when Day < 31 -> + {1, Day}; +year_day_to_date2(E, Day) when 31 =< Day, Day < 59 + E -> + {2, Day - 31}; +year_day_to_date2(E, Day) when 59 + E =< Day, Day < 90 + E -> + {3, Day - (59 + E)}; +year_day_to_date2(E, Day) when 90 + E =< Day, Day < 120 + E -> + {4, Day - (90 + E)}; +year_day_to_date2(E, Day) when 120 + E =< Day, Day < 151 + E -> + {5, Day - (120 + E)}; +year_day_to_date2(E, Day) when 151 + E =< Day, Day < 181 + E -> + {6, Day - (151 + E)}; +year_day_to_date2(E, Day) when 181 + E =< Day, Day < 212 + E -> + {7, Day - (181 + E)}; +year_day_to_date2(E, Day) when 212 + E =< Day, Day < 243 + E -> + {8, Day - (212 + E)}; +year_day_to_date2(E, Day) when 243 + E =< Day, Day < 273 + E -> + {9, Day - (243 + E)}; +year_day_to_date2(E, Day) when 273 + E =< Day, Day < 304 + E -> + {10, Day - (273 + E)}; +year_day_to_date2(E, Day) when 304 + E =< Day, Day < 334 + E -> + {11, Day - (304 + E)}; +year_day_to_date2(E, Day) when 334 + E =< Day -> + {12, Day - (334 + E)}. + +trans_x_second(FromUnit, ToUnit, Time) -> + XSecond = do_trans_x_second(FromUnit, ToUnit, Time), + Len = + case ToUnit of + millisecond -> 3; + microsecond -> 6; + nanosecond -> 9 + end, + padding(XSecond, Len). + +do_trans_x_second(second, second, Time) -> Time div 60; +do_trans_x_second(second, _, _Time) -> 0; + +do_trans_x_second(millisecond, millisecond, Time) -> Time rem 1000; +do_trans_x_second(millisecond, microsecond, _Time) -> 0; +do_trans_x_second(millisecond, nanosecond, _Time) -> 0; + +do_trans_x_second(microsecond, millisecond, Time) -> Time div 1000 rem 1000; +do_trans_x_second(microsecond, microsecond, Time) -> Time rem 1000000; +do_trans_x_second(microsecond, nanosecond, _Time) -> 0; + +do_trans_x_second(nanosecond, millisecond, Time) -> Time div 1000000 rem 1000; +do_trans_x_second(nanosecond, microsecond, Time) -> Time div 1000 rem 1000000; +do_trans_x_second(nanosecond, nanosecond, Time) -> Time rem 1000000000. + +padding(Data, Len) when is_integer(Data) -> + padding(integer_to_list(Data), Len); +padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> + [$0 | padding(Data, Len - 1)]; +padding(Data, _Len) -> + Data. diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index df4118ba4..b13871384 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -16,16 +16,6 @@ -module(emqx_logger_textfmt). --define(SECONDS_PER_MINUTE, 60). --define(SECONDS_PER_HOUR, 3600). --define(SECONDS_PER_DAY, 86400). --define(DAYS_PER_YEAR, 365). --define(DAYS_PER_LEAP_YEAR, 366). --define(DAYS_FROM_0_TO_1970, 719528). --define(DAYS_FROM_0_TO_10000, 2932897). --define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY). --define(SECONDS_FROM_0_TO_10000, ?DAYS_FROM_0_TO_10000 * ?SECONDS_PER_DAY). - -export([format/2]). -export([check_config/1]). @@ -39,7 +29,7 @@ ]). check_config(Config0) -> - Config = maps:without([date_format, timezone_offset, timezone], Config0), + Config = maps:without([timezone_offset, timezone, date_format], Config0), logger_formatter:check_config(Config). format(#{msg := Msg0, meta := Meta} = Event, @@ -51,8 +41,15 @@ format(#{msg := Msg0, meta := Meta} = Event, format(#{msg := Msg0, meta := Meta} = Event, #{timezone_offset := TZO, timezone := TZ, date_format := DFS} = Config) -> Msg = maybe_merge(Msg0, Meta), - Time = maps:get(time, Event, undefined), - [date_format(Time, TZO, TZ, DFS) | logger_formatter:format(Event#{msg := Msg}, Config)]. + Time = + case maps:get(time, Event, undefined) of + undefined -> + erlang:system_time(microsecond); + T -> + T + end, + Date = emqx_calendar:format(Time, microsecond, TZO, TZ, DFS), + [Date | logger_formatter:format(Event#{msg := Msg}, Config)]. maybe_merge({report, Report}, Meta) when is_map(Report) -> {report, maps:merge(Report, filter(Meta))}; @@ -61,150 +58,3 @@ maybe_merge(Report, _Meta) -> filter(Meta) -> maps:without(?WITHOUT_MERGE, Meta). - -date_format(undefined, Offset, OffsetStr, Formatter) -> - Time = erlang:system_time(microsecond), - date_format(Time, Offset, OffsetStr, Formatter); -date_format(Time, Offset, OffsetStr, Formatter) -> - Adjustment = erlang:convert_time_unit(Offset, second, microsecond), - AdjustedTime = Time + Adjustment, - %% Factor 1000000 for microsecond. - Factor = 1000000, - Secs = AdjustedTime div Factor, - DateTime = system_time_to_datetime(Secs), - {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, - FractionStr = fraction_str(Factor, AdjustedTime), - Date = #{ - year => padding(Year, 4), - month => padding(Month, 2), - day => padding(Day, 2), - hour => padding(Hour, 2), - minute => padding(Min, 2), - second => padding(Sec, 2), - nano_second => FractionStr, - timezone => OffsetStr - }, - [maps:get(Key, Date, Key) || Key <- Formatter]. - -system_time_to_datetime(Seconds) -> - gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970). - -gregorian_seconds_to_datetime(Secs) when Secs >= 0 -> - Days = Secs div ?SECONDS_PER_DAY, - Rest = Secs rem ?SECONDS_PER_DAY, - {gregorian_days_to_date(Days), seconds_to_time(Rest)}. - -seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY -> - Secs0 = Secs rem ?SECONDS_PER_DAY, - Hour = Secs0 div ?SECONDS_PER_HOUR, - Secs1 = Secs0 rem ?SECONDS_PER_HOUR, - Minute = Secs1 div ?SECONDS_PER_MINUTE, - Second = Secs1 rem ?SECONDS_PER_MINUTE, - {Hour, Minute, Second}. - -gregorian_days_to_date(Days) -> - {Year, DayOfYear} = day_to_year(Days), - {Month, DayOfMonth} = year_day_to_date(Year, DayOfYear), - {Year, Month, DayOfMonth}. - -day_to_year(DayOfEpoch) when DayOfEpoch >= 0 -> - YMax = DayOfEpoch div ?DAYS_PER_YEAR, - YMin = DayOfEpoch div ?DAYS_PER_LEAP_YEAR, - {Y1, D1} = dty(YMin, YMax, DayOfEpoch, dy(YMin), dy(YMax)), - {Y1, DayOfEpoch - D1}. - -year_day_to_date(Year, DayOfYear) -> - ExtraDay = - case is_leap_year(Year) of - true -> - 1; - false -> - 0 - end, - {Month, Day} = year_day_to_date2(ExtraDay, DayOfYear), - {Month, Day + 1}. - -dty(Min, Max, _D1, DMin, _DMax) when Min == Max -> - {Min, DMin}; -dty(Min, Max, D1, DMin, DMax) -> - Diff = Max - Min, - Mid = Min + Diff * (D1 - DMin) div (DMax - DMin), - MidLength = - case is_leap_year(Mid) of - true -> - ?DAYS_PER_LEAP_YEAR; - false -> - ?DAYS_PER_YEAR - end, - case dy(Mid) of - D2 when D1 < D2 -> - NewMax = Mid - 1, - dty(Min, NewMax, D1, DMin, dy(NewMax)); - D2 when D1 - D2 >= MidLength -> - NewMin = Mid + 1, - dty(NewMin, Max, D1, dy(NewMin), DMax); - D2 -> - {Mid, D2} - end. - -dy(Y) when Y =< 0 -> - 0; -dy(Y) -> - X = Y - 1, - X div 4 - X div 100 + X div 400 + X * ?DAYS_PER_YEAR + ?DAYS_PER_LEAP_YEAR. - -is_leap_year(Y) when is_integer(Y), Y >= 0 -> - is_leap_year1(Y). - -is_leap_year1(Year) when Year rem 4 =:= 0, Year rem 100 > 0 -> - true; -is_leap_year1(Year) when Year rem 400 =:= 0 -> - true; -is_leap_year1(_) -> - false. - -year_day_to_date2(_, Day) when Day < 31 -> - {1, Day}; -year_day_to_date2(E, Day) when 31 =< Day, Day < 59 + E -> - {2, Day - 31}; -year_day_to_date2(E, Day) when 59 + E =< Day, Day < 90 + E -> - {3, Day - (59 + E)}; -year_day_to_date2(E, Day) when 90 + E =< Day, Day < 120 + E -> - {4, Day - (90 + E)}; -year_day_to_date2(E, Day) when 120 + E =< Day, Day < 151 + E -> - {5, Day - (120 + E)}; -year_day_to_date2(E, Day) when 151 + E =< Day, Day < 181 + E -> - {6, Day - (151 + E)}; -year_day_to_date2(E, Day) when 181 + E =< Day, Day < 212 + E -> - {7, Day - (181 + E)}; -year_day_to_date2(E, Day) when 212 + E =< Day, Day < 243 + E -> - {8, Day - (212 + E)}; -year_day_to_date2(E, Day) when 243 + E =< Day, Day < 273 + E -> - {9, Day - (243 + E)}; -year_day_to_date2(E, Day) when 273 + E =< Day, Day < 304 + E -> - {10, Day - (273 + E)}; -year_day_to_date2(E, Day) when 304 + E =< Day, Day < 334 + E -> - {11, Day - (304 + E)}; -year_day_to_date2(E, Day) when 334 + E =< Day -> - {12, Day - (334 + E)}. - -fraction_str(1, _Time) -> - ""; -fraction_str(Factor, Time) -> - Fraction = Time rem Factor, - S = integer_to_list(abs(Fraction)), - [padding(S, log10(Factor) - length(S))]. - -log10(1000) -> - 3; -log10(1000000) -> - 6; -log10(1000000000) -> - 9. - -padding(Data, Len) when is_integer(Data) -> - padding(integer_to_list(Data), Len); -padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> - [$0 | padding(Data, Len - 1)]; -padding(Data, _Len) -> - Data. From 4fc5cb2817c4f1e8225c048bc04f4972148d0df4 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 26 May 2022 12:19:53 +0800 Subject: [PATCH 34/36] fix: rule engine date format fun & date to timestamp --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 39 ++- .../test/emqx_rule_funcs_SUITE.erl | 34 +- etc/emqx.conf | 11 +- priv/emqx.schema | 10 +- src/emqx_calendar.erl | 290 +++++++----------- src/emqx_logger_textfmt.erl | 4 +- 6 files changed, 174 insertions(+), 214 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 282c8929d..a8b347f5d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -201,6 +201,8 @@ , unix_ts_to_rfc3339/2 , format_date/3 , format_date/4 + , timezone_to_second/1 + , date_to_unix_ts/3 , date_to_unix_ts/4 , rfc3339_to_unix_ts/1 , rfc3339_to_unix_ts/2 @@ -923,26 +925,37 @@ now_timestamp(Unit) -> time_unit(<<"second">>) -> second; time_unit(<<"millisecond">>) -> millisecond; time_unit(<<"microsecond">>) -> microsecond; -time_unit(<<"nanosecond">>) -> nanosecond. +time_unit(<<"nanosecond">>) -> nanosecond; +time_unit(second) -> second; +time_unit(millisecond) -> millisecond; +time_unit(microsecond) -> microsecond; +time_unit(nanosecond) -> nanosecond. format_date(TimeUnit, Offset, FormatString) -> - emqx_rule_utils:bin( - emqx_rule_date:date(time_unit(TimeUnit), - emqx_rule_utils:str(Offset), - emqx_rule_utils:str(FormatString))). + Unit = time_unit(TimeUnit), + TimeEpoch = erlang:system_time(Unit), + format_date(Unit, Offset, FormatString, TimeEpoch). + +timezone_to_second(TimeZone) -> + emqx_calendar:offset_second(TimeZone). format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> + Unit = time_unit(TimeUnit), emqx_rule_utils:bin( - emqx_rule_date:date(time_unit(TimeUnit), - emqx_rule_utils:str(Offset), - emqx_rule_utils:str(FormatString), - TimeEpoch)). + lists:concat( + emqx_calendar:format(TimeEpoch, Unit, Offset, FormatString))). +%% date string has timezone information, calculate the offset. +date_to_unix_ts(TimeUnit, FormatString, InputString) -> + Unit = time_unit(TimeUnit), + emqx_calendar:parse(InputString, Unit, FormatString). + +%% date string has no timezone information, force add the offset. date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> - emqx_rule_date:parse_date(time_unit(TimeUnit), - emqx_rule_utils:str(Offset), - emqx_rule_utils:str(FormatString), - emqx_rule_utils:str(InputString)). + Unit = time_unit(TimeUnit), + OffsetSecond = emqx_calendar:offset_second(Offset), + OffsetDelta = erlang:convert_time_unit(OffsetSecond, second, Unit), + OffsetDelta + date_to_unix_ts(Unit, FormatString, InputString). mongo_date() -> erlang:timestamp(). diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 98f6ad0b7..632697a87 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -713,24 +713,32 @@ t_format_date_funcs(_) -> ?PROPTEST(prop_format_date_fun). prop_format_date_fun() -> - Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>], + Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%Y---%H:%M:%S%z">>], + Args1DTUS = [<<"second">>, 0, <<"%m--%d--%Y---%H:%M:%S%z">>], ?FORALL(S, erlang:system_time(second), S == apply_func(date_to_unix_ts, - Args1 ++ [apply_func(format_date, + Args1DTUS ++ [apply_func(format_date, Args1 ++ [S])])), - Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>], + Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], + Args2DTUS = [<<"millisecond">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], ?FORALL(S, erlang:system_time(millisecond), S == apply_func(date_to_unix_ts, - Args2 ++ [apply_func(format_date, + Args2DTUS ++ [apply_func(format_date, Args2 ++ [S])])), - Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], + Args = [<<"second">>, <<"+08:00">>, <<"%Y-%m-%d-%H:%M:%S%z">>], + ArgsDTUS = [<<"second">>, <<"%Y-%m-%d-%H:%M:%S%z">>], ?FORALL(S, erlang:system_time(second), S == apply_func(date_to_unix_ts, - Args ++ [apply_func(format_date, - Args ++ [S])])). -%%------------------------------------------------------------------------------ -%% Utility functions -%%------------------------------------------------------------------------------ + ArgsDTUS ++ [apply_func(format_date, + Args ++ [S])])), + % no offset in format string. force add offset + Second = erlang:system_time(second), + Args3 = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Second], + Formatters3 = apply_func(format_date, Args3), + % -04:00 remove offset + OffsetSeconds = emqx_rule_funcs:timezone_to_second(<<"-04:00">>), + Args3DTUS = [<<"second">>, OffsetSeconds, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], + Second == apply_func(date_to_unix_ts, Args3DTUS). apply_func(Name, Args) when is_atom(Name) -> erlang:apply(emqx_rule_funcs, Name, Args); @@ -880,9 +888,9 @@ message() -> %% CT functions %%------------------------------------------------------------------------------ -all() -> - IsTestCase = fun("t_" ++ _) -> true; (_) -> false end, - [F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))]. +all() -> [t_format_date_funcs]. + % IsTestCase = fun("t_" ++ _) -> true; (_) -> false end, + % [F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))]. suite() -> [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}]. diff --git a/etc/emqx.conf b/etc/emqx.conf index 4cdd6e4d7..fcc8557a2 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -427,7 +427,7 @@ log.to = file ## this level will be logged. ## ## Default: warning -log.level = debug +log.level = warning ## The dir for log files. ## @@ -472,9 +472,11 @@ log.formatter = text ## %S: second ## %6N: microseconds ## %3N: milliseconds -## %z: timezone, [+|-]HH:MM or [+|-]HH:MM:SS. Depends on `log.formatter.text.date.timezone` +## %z: timezone, [+-]HHMM +## %:z: timezone, [+-]HH:MM +## %::z: timezone, [+-]HH:MM:SS ## For example: -## log.formatter.text.date.format = emqx-server-date: %Y-%m-%dT%H:%M:%S.%6N %z +## emqx-server-date: %Y-%m-%dT%H:%M:%S.%6N %:z ## Default: rfc3339 # log.formatter.text.date.format = rfc3339 @@ -482,9 +484,10 @@ log.formatter = text ## Only takes effect when log.formatter.text.date.format not rfc3339. ## ## Value: local | z | UTC TimeOffset -## TimeOffset: [+|-]HH:MM or [+|-]HH:MM:SS, for example: +08:00. +## TimeOffset: [+-]HHMM or [+|-]HH:MM or [+|-]HH:MM:SS, for example: +08:00. ## local is the system local time timezone. ## z: UTC zulu timezone. +## Z: UTC zulu timezone. Same as z. ## Default: local # log.formatter.text.date.timezone = local diff --git a/priv/emqx.schema b/priv/emqx.schema index dc15d87b3..96d0005ad 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -659,6 +659,8 @@ end}. DST(<<"%3N", Tail/binary>>, Formatter) -> DST(Tail, [millisecond | Formatter]); DST(<<"%6N", Tail/binary>>, Formatter) -> DST(Tail, [microsecond | Formatter]); DST(<<"%z", Tail/binary>>, Formatter) -> DST(Tail, [timezone | Formatter]); + DST(<<"%:z", Tail/binary>>, Formatter) -> DST(Tail, [timezone1 | Formatter]); + DST(<<"%::z", Tail/binary>>, Formatter) -> DST(Tail, [timezone2 | Formatter]); DST(<>, [Str | Formatter]) when is_list(Str) -> DST(Tail, [lists:append(Str, [Char]) | Formatter]); DST(<>, Formatter) -> @@ -669,9 +671,9 @@ end}. {DateOffsetStr, DateOffset} = case cuttlefish:conf_get("log.formatter.text.date.timezone", Conf, "local") of "z" -> - {"z", 0}; + {"z", "z"}; "Z" -> - {"Z", 0}; + {"Z", "Z"}; "local" -> UniversalTime = calendar:system_time_to_universal_time( @@ -709,7 +711,9 @@ end}. [H, M] -> {H, M, "0"}; [H, M, S] -> - {H, M, S} + {H, M, S}; + [HHMM] when erlang:length(HHMM) == 4 -> + {string:sub_string(HHMM, 1,2), string:sub_string(HHMM, 3,4), "0"} end, Hour = erlang:list_to_integer(HourStr), Minute = erlang:list_to_integer(MinuteStr), diff --git a/src/emqx_calendar.erl b/src/emqx_calendar.erl index ae9bae763..c7e2d627f 100644 --- a/src/emqx_calendar.erl +++ b/src/emqx_calendar.erl @@ -27,14 +27,10 @@ -export([ formatter/1 , format/3 , format/4 - , format/5 , parse/3 - , parse/4 , offset_second/1 ]). --export([test/0]). - -define(DATE_PART, [ year , month @@ -69,174 +65,10 @@ format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> format(Time, Unit, Offset, Formatter) -> do_format(Time, time_unit(Unit), offset_second(Offset), Formatter). -%% for logger format -format(Time, Unit, Offset, Zones, Formatter) -> - do_format(Time, time_unit(Unit), offset_second(Offset), Zones, Formatter). - -test() -> - Units = [second, millisecond, microsecond, nanosecond], - Formatters = [ - <<"%Y-%m-%d %H:%M:%S:%N%z">>, - <<"%Y-%m-%d %H:%M:%S:%N%:z">>, - <<"%Y-%m-%d %H:%M:%S:%N%::z">>, - <<"%Y-%m-%d %H:%M:%S:%N">>, - <<"%Y-%m-%d %H:%M:%S:%3N">>, - <<"%Y-%m-%d %H:%M:%S:%6N">> - ], - Offsets = [ - 0, - 8 * 3600, - -8 * 3600, - <<"+08:00">>, - <<"+08:00:01">>, - <<"+0802">>, - <<"-08:00">>, - <<"local">> - ], - [ - [ - [begin - Time = erlang:system_time(U), - FormatDate = erlang:list_to_binary(format(Time, U, O, F)), - PTime = emqx_calendar:parse(FormatDate, U, O, F), - io:format("~p ~p ~p ~p ~n~p~n~p~n", [U, O, F, FormatDate, Time, PTime]), - % Equal = (PTime == Time), - % F2 = - % fun() -> - % Delta = Time - PTime, - % DeltaFmt = erlang:list_to_binary(format(Delta, U, 0, <<"%Y-%m-%d %H:%M:%S:%N">>)), - % io:format("delta ~p~n ~p~n-----~n", [Delta, DeltaFmt]) - % end, - % Equal orelse F2() - ok - end||O <- Offsets] - || F <-Formatters] - || U <-Units], - ok. - -parse(DateStr, Unit, FormatterBin) -> - parse(DateStr, Unit, 0, FormatterBin). - -parse(DateStr, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> - parse(DateStr, Unit, Offset, formatter(FormatterBin)); -parse(DateStr, Unit, Offset, Formatter) -> - do_parse(DateStr, Unit, offset_second(Offset), Formatter). - -do_parse(DateStr, Unit, Offset, Formatter) -> - DateInfo = do_parse_date_str(DateStr, Formatter, #{}), - {Precise, PrecisionUnit} = precise(DateInfo), - Counter = - fun - (year, V, Res) -> - Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); - (month, V, Res) -> - Res + nano_sm(V); - (day, V, Res) -> - Res + (V * ?SECONDS_PER_DAY * Precise); - (hour, V, Res) -> - Res + (V * ?SECONDS_PER_HOUR * Precise); - (minute, V, Res) -> - Res + (V * ?SECONDS_PER_MINUTE * Precise); - (second, V, Res) -> - Res + V * Precise; - (millisecond, V, Res) -> - case PrecisionUnit of - millisecond -> - Res + V; - microsecond -> - Res + (V * 1000); - nanosecond -> - Res + (V * 1000000) - end; - (microsecond, V, Res) -> - case PrecisionUnit of - microsecond -> - Res + V; - nanosecond -> - Res + (V * 1000) - end; - (nanosecond, V, Res) -> - Res + V; - (parsed_offset, V, Res) -> - Res - V - end, - Counter1 = - fun(K,V,R) -> - Res = Counter(K,V,R), - % io:format("K, V, R = ~p, ~p, ~p, Res = ~p~n", [K, V, R, Res - R]), - Res - end, - Count0 = maps:fold(Counter1, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), - Count = - case maps:is_key(parsed_offset, DateInfo) of - true -> - Count0; - false -> - (Offset * Precise) + Count0 - end, - erlang:convert_time_unit(Count, PrecisionUnit, Unit). - -precise(#{nanosecond := _}) -> {1000_000_000, nanosecond}; -precise(#{microsecond := _}) -> {1000_000, microsecond}; -precise(#{millisecond := _}) -> {1000, millisecond}; -precise(#{second := _}) -> {1, second}; -precise(_) -> {1, second}. - -do_parse_date_str(<<>>, _, Result) -> Result; -do_parse_date_str(_, [], Result) -> Result; -do_parse_date_str(Date, [Key | Formatter], Result) -> - Size = date_size(Key), - <> = Date, - case lists:member(Key, ?DATE_PART) of - true -> - do_parse_date_str(Tail, Formatter, Result#{Key => erlang:binary_to_integer(DatePart)}); - false -> - case lists:member(Key, ?DATE_ZONE_NAME) of - true -> - io:format("DatePart -------------- ~p ~p~n", [DatePart, offset_second(DatePart)]), - do_parse_date_str(Tail, Formatter, Result#{parsed_offset => offset_second(DatePart)}); - false -> - do_parse_date_str(Tail, Formatter, Result) - end - end. - -date_size(Str) when is_list(Str) -> erlang:length(Str); -date_size(DateName) -> - Map = #{ - year => 4, - month => 2, - day => 2, - hour => 2, - minute => 2, - second => 2, - millisecond => 3, - microsecond => 6, - nanosecond => 9, - timezone => 5, - timezone1 => 6, - timezone2 => 9 - }, - maps:get(DateName, Map). - -nano_sm(Month) -> dm(Month) * ?SECONDS_PER_DAY * 1000_000. - -dm(Month) -> - MonthDays = #{ - 1 => 0, - 2 => 31, - 3 => 59, - 4 => 90, - 5 => 120, - 6 => 151, - 7 => 181, - 8 => 212, - 9 => 243, - 10 => 273, - 11 => 304, - 12 => 334 - }, - maps:get(Month, MonthDays). - +parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) -> + parse(DateStr, Unit, formatter(FormatterBin)); +parse(DateStr, Unit, Formatter) -> + do_parse(DateStr, Unit, Formatter). %% ------------------------------------------------------------------------------------------------- %% internal %% emqx_calendar:format(erlang:system_time(second), <<"second">>, <<"+8:00">> ,<<"%Y-%m-%d-%H:%M:%S%Z">>). @@ -315,10 +147,6 @@ do_offset_second(Offset) when is_list(Offset) -> PosNeg * (Hour * 3600 + Minute * 60 + Second). do_format(Time, Unit, Offset, Formatter) -> - Timezones = formatter_timezones(Offset, Formatter, #{}), - do_format(Time, Unit, Offset, Timezones, Formatter). - -do_format(Time, Unit, Offset, Timezones, Formatter) -> Adjustment = erlang:convert_time_unit(Offset, second, Unit), AdjustedTime = Time + Adjustment, Factor = factor(Unit), @@ -336,6 +164,7 @@ do_format(Time, Unit, Offset, Timezones, Formatter) -> microsecond => trans_x_second(Unit, microsecond, Time), nanosecond => trans_x_second(Unit, nanosecond, Time) }, + Timezones = formatter_timezones(Offset, Formatter, #{}), DateWithZone = maps:merge(Date, Timezones), [maps:get(Key, DateWithZone, Key) || Key <- Formatter]. @@ -493,12 +322,12 @@ do_trans_x_second(second, second, Time) -> Time div 60; do_trans_x_second(second, _, _Time) -> 0; do_trans_x_second(millisecond, millisecond, Time) -> Time rem 1000; -do_trans_x_second(millisecond, microsecond, _Time) -> 0; -do_trans_x_second(millisecond, nanosecond, _Time) -> 0; +do_trans_x_second(millisecond, microsecond, Time) -> (Time rem 1000) * 1000; +do_trans_x_second(millisecond, nanosecond, Time) -> (Time rem 1000) * 1000_000; do_trans_x_second(microsecond, millisecond, Time) -> Time div 1000 rem 1000; do_trans_x_second(microsecond, microsecond, Time) -> Time rem 1000000; -do_trans_x_second(microsecond, nanosecond, _Time) -> 0; +do_trans_x_second(microsecond, nanosecond, Time) -> (Time rem 1000000) * 1000; do_trans_x_second(nanosecond, millisecond, Time) -> Time div 1000000 rem 1000; do_trans_x_second(nanosecond, microsecond, Time) -> Time div 1000 rem 1000000; @@ -510,3 +339,106 @@ padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> [$0 | padding(Data, Len - 1)]; padding(Data, _Len) -> Data. + +%% ------------------------------------------------------------------------------------------------- +%% internal +%% parse part + +do_parse(DateStr, Unit, Formatter) -> + DateInfo = do_parse_date_str(DateStr, Formatter, #{}), + {Precise, PrecisionUnit} = precise(DateInfo), + Counter = + fun + (year, V, Res) -> + Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); + (month, V, Res) -> + Res + dm(V) * ?SECONDS_PER_DAY * Precise; + (day, V, Res) -> + Res + (V * ?SECONDS_PER_DAY * Precise); + (hour, V, Res) -> + Res + (V * ?SECONDS_PER_HOUR * Precise); + (minute, V, Res) -> + Res + (V * ?SECONDS_PER_MINUTE * Precise); + (second, V, Res) -> + Res + V * Precise; + (millisecond, V, Res) -> + case PrecisionUnit of + millisecond -> + Res + V; + microsecond -> + Res + (V * 1000); + nanosecond -> + Res + (V * 1000000) + end; + (microsecond, V, Res) -> + case PrecisionUnit of + microsecond -> + Res + V; + nanosecond -> + Res + (V * 1000) + end; + (nanosecond, V, Res) -> + Res + V; + (parsed_offset, V, Res) -> + Res - V + end, + Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), + erlang:convert_time_unit(Count, PrecisionUnit, Unit). + +precise(#{nanosecond := _}) -> {1000_000_000, nanosecond}; +precise(#{microsecond := _}) -> {1000_000, microsecond}; +precise(#{millisecond := _}) -> {1000, millisecond}; +precise(#{second := _}) -> {1, second}; +precise(_) -> {1, second}. + +do_parse_date_str(<<>>, _, Result) -> Result; +do_parse_date_str(_, [], Result) -> Result; +do_parse_date_str(Date, [Key | Formatter], Result) -> + Size = date_size(Key), + <> = Date, + case lists:member(Key, ?DATE_PART) of + true -> + do_parse_date_str(Tail, Formatter, Result#{Key => erlang:binary_to_integer(DatePart)}); + false -> + case lists:member(Key, ?DATE_ZONE_NAME) of + true -> + do_parse_date_str(Tail, Formatter, Result#{parsed_offset => offset_second(DatePart)}); + false -> + do_parse_date_str(Tail, Formatter, Result) + end + end. + +date_size(Str) when is_list(Str) -> erlang:length(Str); +date_size(DateName) -> + Map = #{ + year => 4, + month => 2, + day => 2, + hour => 2, + minute => 2, + second => 2, + millisecond => 3, + microsecond => 6, + nanosecond => 9, + timezone => 5, + timezone1 => 6, + timezone2 => 9 + }, + maps:get(DateName, Map). + +dm(Month) -> + MonthDays = #{ + 1 => 0, + 2 => 31, + 3 => 59, + 4 => 90, + 5 => 120, + 6 => 151, + 7 => 181, + 8 => 212, + 9 => 243, + 10 => 273, + 11 => 304, + 12 => 334 + }, + maps:get(Month, MonthDays). diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index b13871384..16cc3a414 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -39,7 +39,7 @@ format(#{msg := Msg0, meta := Meta} = Event, logger_formatter:format(Event#{msg := Msg}, Config#{template => Template}); format(#{msg := Msg0, meta := Meta} = Event, - #{timezone_offset := TZO, timezone := TZ, date_format := DFS} = Config) -> + #{timezone_offset := TZO, date_format := DFS} = Config) -> Msg = maybe_merge(Msg0, Meta), Time = case maps:get(time, Event, undefined) of @@ -48,7 +48,7 @@ format(#{msg := Msg0, meta := Meta} = Event, T -> T end, - Date = emqx_calendar:format(Time, microsecond, TZO, TZ, DFS), + Date = emqx_calendar:format(Time, microsecond, TZO, DFS), [Date | logger_formatter:format(Event#{msg := Msg}, Config)]. maybe_merge({report, Report}, Meta) when is_map(Report) -> From e1ad8aab465cc4723a7f18e02b7df029abe8d450 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 26 May 2022 12:50:16 +0800 Subject: [PATCH 35/36] fix: date format funs & logger conf & appup & change log --- CHANGES-4.3.md | 2 +- .../test/emqx_rule_funcs_SUITE.erl | 9 +- etc/emqx.conf | 25 ++-- priv/emqx.schema | 67 --------- src/emqx.appup.src | 129 +++++++++++++----- src/emqx_calendar.erl | 1 - src/emqx_logger_textfmt.erl | 4 +- 7 files changed, 112 insertions(+), 125 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index da2df758d..dd75320b7 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -21,7 +21,7 @@ File format: Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists. * Better randomisation of app screts (changed from timestamp seeded sha hash (uuid) to crypto:strong_rand_bytes) * Return a client_identifier_not_valid error when username is empty and username_as_clientid is set to true [#7862] -* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/4 [#7894] +* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/3, date_to_unix_ts/4 [#7894] * Add proto_name and proto_ver fields for $event/client_disconnected event. * Mnesia auth/acl http api support multiple condition queries. * Inflight QoS1 Messages for shared topics are now redispatched to another alive subscribers upon chosen subscriber session termination. diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 632697a87..fe9cf688b 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -736,8 +736,7 @@ prop_format_date_fun() -> Args3 = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Second], Formatters3 = apply_func(format_date, Args3), % -04:00 remove offset - OffsetSeconds = emqx_rule_funcs:timezone_to_second(<<"-04:00">>), - Args3DTUS = [<<"second">>, OffsetSeconds, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], + Args3DTUS = [<<"second">>, <<"-04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], Second == apply_func(date_to_unix_ts, Args3DTUS). apply_func(Name, Args) when is_atom(Name) -> @@ -888,9 +887,9 @@ message() -> %% CT functions %%------------------------------------------------------------------------------ -all() -> [t_format_date_funcs]. - % IsTestCase = fun("t_" ++ _) -> true; (_) -> false end, - % [F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))]. +all() -> + IsTestCase = fun("t_" ++ _) -> true; (_) -> false end, + [F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))]. suite() -> [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}]. diff --git a/etc/emqx.conf b/etc/emqx.conf index fcc8557a2..161f9fc2b 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -460,37 +460,28 @@ log.file = emqx.log ## Value: text | json log.formatter = text -## Text logger, date format. +## Format of the text logger. ## -## Value: rfc3339 | FormatString -## FormatString: +## Value: rfc3339 | FORMAT +## Where FORMAT is the format string of the timestamp. Supported specifiers: ## %Y: year ## %m: month ## %d: day ## %H: hour ## %M: minute ## %S: second -## %6N: microseconds -## %3N: milliseconds +## %N: nanoseconds (000000000 - 999999999) +## %6N: microseconds (00000 - 999999) +## %3N: milliseconds (000 - 999) ## %z: timezone, [+-]HHMM ## %:z: timezone, [+-]HH:MM ## %::z: timezone, [+-]HH:MM:SS +## ## For example: -## emqx-server-date: %Y-%m-%dT%H:%M:%S.%6N %:z +## log.formatter.text.date.format = %Y-%m-%dT%H:%M:%S.%6N %:z ## Default: rfc3339 # log.formatter.text.date.format = rfc3339 -## Text logger, timezone. -## Only takes effect when log.formatter.text.date.format not rfc3339. -## -## Value: local | z | UTC TimeOffset -## TimeOffset: [+-]HHMM or [+|-]HH:MM or [+|-]HH:MM:SS, for example: +08:00. -## local is the system local time timezone. -## z: UTC zulu timezone. -## Z: UTC zulu timezone. Same as z. -## Default: local -# log.formatter.text.date.timezone = local - ## Log to single line ## Value: Boolean #log.single_line = true diff --git a/priv/emqx.schema b/priv/emqx.schema index 96d0005ad..304fc0634 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -534,12 +534,6 @@ end}. {datatype, string} ]}. -%% @doc format logs as text, date format part. local or any UTC offset -{mapping, "log.formatter.text.date.timezone", "kernel.logger", [ - {default, "local"}, - {datatype, string} -]}. - %% @doc format logs in a single line. {mapping, "log.single_line", "kernel.logger", [ {default, true}, @@ -668,70 +662,9 @@ end}. end, DateStrTrans(list_to_binary(DateStr), []) end, - {DateOffsetStr, DateOffset} = - case cuttlefish:conf_get("log.formatter.text.date.timezone", Conf, "local") of - "z" -> - {"z", "z"}; - "Z" -> - {"Z", "Z"}; - "local" -> - UniversalTime = - calendar:system_time_to_universal_time( - erlang:system_time(second), - second), - LocalTime = erlang:universaltime_to_localtime(UniversalTime), - LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), - UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), - OffsetSecond = LocalSecs - UniversalSecs, - {Sign, PosOffsetSecond} = - case OffsetSecond >= 0 of - true -> {$+, OffsetSecond}; - false -> {$-, - OffsetSecond} - end, - {H, M, S} = calendar:seconds_to_time(PosOffsetSecond), - Str = - case S of - 0 -> - io_lib:format("~c~2.10.0B:~2.10.0B", [Sign, H, M]); - S -> - io_lib:format("~c~2.10.0B:~2.10.0B:~2.10.0B", [Sign, H, M, S]) - end, - {Str, OffsetSecond}; - DateOStr -> - Sign = hd(DateOStr), - Signs = #{$+ => 1, $- => -1}, - case maps:get(Sign, Signs, undefined) of - undefined -> - error(bad_logger_offset); - PosNeg -> - try - [Sign | HM] = DateOStr, - {HourStr, MinuteStr, SecondStr} = - case string:tokens(HM, ":") of - [H, M] -> - {H, M, "0"}; - [H, M, S] -> - {H, M, S}; - [HHMM] when erlang:length(HHMM) == 4 -> - {string:sub_string(HHMM, 1,2), string:sub_string(HHMM, 3,4), "0"} - end, - Hour = erlang:list_to_integer(HourStr), - Minute = erlang:list_to_integer(MinuteStr), - Second = erlang:list_to_integer(SecondStr), - true = (Hour =< 23), - true = (Minute =< 59), - true = (Second =< 59), - {DateOStr, PosNeg * (Hour * 3600 + Minute * 60 + Second)} - catch _E : _R -> - error(bad_logger_offset) - end - end - end, {emqx_logger_textfmt, #{ date_format => DateFormat, - timezone_offset => DateOffset, - timezone => DateOffsetStr, template => [" [",level,"] ", {clientid, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 0b5361758..1f1443f37 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,9 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.15", - [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, @@ -16,7 +18,9 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, @@ -34,7 +38,9 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -58,7 +64,9 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -86,7 +94,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -116,7 +126,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -146,7 +158,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -180,7 +194,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -214,7 +230,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -248,7 +266,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -282,7 +302,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -316,7 +338,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -350,7 +374,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -385,7 +411,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -420,7 +448,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -458,7 +487,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -500,7 +530,9 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.15", - [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -513,7 +545,9 @@ {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -530,7 +564,9 @@ {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, @@ -553,7 +589,10 @@ {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -579,7 +618,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -607,7 +649,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -635,7 +680,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -667,7 +715,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -699,7 +750,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -731,7 +784,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -763,7 +818,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -795,7 +852,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -827,7 +886,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -860,7 +921,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -893,7 +956,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -929,7 +993,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_calendar.erl b/src/emqx_calendar.erl index c7e2d627f..7f9faf3be 100644 --- a/src/emqx_calendar.erl +++ b/src/emqx_calendar.erl @@ -71,7 +71,6 @@ parse(DateStr, Unit, Formatter) -> do_parse(DateStr, Unit, Formatter). %% ------------------------------------------------------------------------------------------------- %% internal -%% emqx_calendar:format(erlang:system_time(second), <<"second">>, <<"+8:00">> ,<<"%Y-%m-%d-%H:%M:%S%Z">>). time_unit(second) -> second; time_unit(millisecond) -> millisecond; diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index 16cc3a414..ba13c0cb7 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -39,7 +39,7 @@ format(#{msg := Msg0, meta := Meta} = Event, logger_formatter:format(Event#{msg := Msg}, Config#{template => Template}); format(#{msg := Msg0, meta := Meta} = Event, - #{timezone_offset := TZO, date_format := DFS} = Config) -> + #{date_format := DFS} = Config) -> Msg = maybe_merge(Msg0, Meta), Time = case maps:get(time, Event, undefined) of @@ -48,7 +48,7 @@ format(#{msg := Msg0, meta := Meta} = Event, T -> T end, - Date = emqx_calendar:format(Time, microsecond, TZO, DFS), + Date = emqx_calendar:format(Time, microsecond, local, DFS), [Date | logger_formatter:format(Event#{msg := Msg}, Config)]. maybe_merge({report, Report}, Meta) when is_map(Report) -> From 64a455bf8ee1180e81aed3c92a5033532b8ddc79 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 26 May 2022 14:14:18 +0800 Subject: [PATCH 36/36] fix: old date format style example & some code format --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 2 +- .../test/emqx_rule_funcs_SUITE.erl | 6 +- etc/emqx.conf | 4 + src/emqx.appup.src | 5 - src/emqx_calendar.erl | 101 ++++++++---------- src/emqx_logger_textfmt.erl | 3 +- 6 files changed, 55 insertions(+), 66 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index a8b347f5d..18e61967e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -955,7 +955,7 @@ date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> Unit = time_unit(TimeUnit), OffsetSecond = emqx_calendar:offset_second(Offset), OffsetDelta = erlang:convert_time_unit(OffsetSecond, second, Unit), - OffsetDelta + date_to_unix_ts(Unit, FormatString, InputString). + date_to_unix_ts(Unit, FormatString, InputString) - OffsetDelta. mongo_date() -> erlang:timestamp(). diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index fe9cf688b..aa2781e8c 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -714,10 +714,9 @@ t_format_date_funcs(_) -> prop_format_date_fun() -> Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%Y---%H:%M:%S%z">>], - Args1DTUS = [<<"second">>, 0, <<"%m--%d--%Y---%H:%M:%S%z">>], ?FORALL(S, erlang:system_time(second), S == apply_func(date_to_unix_ts, - Args1DTUS ++ [apply_func(format_date, + Args1 ++ [apply_func(format_date, Args1 ++ [S])])), Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], Args2DTUS = [<<"millisecond">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], @@ -735,8 +734,7 @@ prop_format_date_fun() -> Second = erlang:system_time(second), Args3 = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Second], Formatters3 = apply_func(format_date, Args3), - % -04:00 remove offset - Args3DTUS = [<<"second">>, <<"-04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], + Args3DTUS = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], Second == apply_func(date_to_unix_ts, Args3DTUS). apply_func(Name, Args) when is_atom(Name) -> diff --git a/etc/emqx.conf b/etc/emqx.conf index 161f9fc2b..0f8f93a91 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -479,6 +479,10 @@ log.formatter = text ## ## For example: ## log.formatter.text.date.format = %Y-%m-%dT%H:%M:%S.%6N %:z +## +## Before 4.2, the default date format was: +## log.formatter.text.date.format = %Y-%m-%d %H:%M:%S.%3N +## ## Default: rfc3339 # log.formatter.text.date.format = rfc3339 diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 1f1443f37..9dbf47f6f 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -591,7 +591,6 @@ {"4.3.12", [{delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -641,7 +640,6 @@ {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -673,7 +671,6 @@ {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -708,7 +705,6 @@ {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -743,7 +739,6 @@ {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_calendar.erl b/src/emqx_calendar.erl index 7f9faf3be..0c094df81 100644 --- a/src/emqx_calendar.erl +++ b/src/emqx_calendar.erl @@ -55,7 +55,7 @@ formatter(FormatterBin) when is_binary(FormatterBin) -> do_formatter(FormatterBin, []). offset_second(Offset) -> - do_offset_second(Offset). + offset_second_(Offset). format(Time, Unit, Formatter) -> format(Time, Unit, undefined, Formatter). @@ -105,24 +105,24 @@ do_formatter(<>, [Str | Formatter]) when is_list(Str) -> do_formatter(Tail, [lists:append(Str, [Char]) | Formatter]); do_formatter(<>, Formatter) -> do_formatter(Tail, [[Char] | Formatter]). -do_offset_second(OffsetSecond) when is_integer(OffsetSecond) -> OffsetSecond; -do_offset_second(undefined) -> 0; -do_offset_second("local") -> do_offset_second(local); -do_offset_second(<<"local">>) -> do_offset_second(local); -do_offset_second(local) -> +offset_second_(OffsetSecond) when is_integer(OffsetSecond) -> OffsetSecond; +offset_second_(undefined) -> 0; +offset_second_("local") -> offset_second_(local); +offset_second_(<<"local">>) -> offset_second_(local); +offset_second_(local) -> UniversalTime = calendar:system_time_to_universal_time(erlang:system_time(second), second), LocalTime = erlang:universaltime_to_localtime(UniversalTime), LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), LocalSecs - UniversalSecs; -do_offset_second(Offset) when is_binary(Offset) -> - do_offset_second(erlang:binary_to_list(Offset)); -do_offset_second("Z") -> 0; -do_offset_second("z") -> 0; -do_offset_second(Offset) when is_list(Offset) -> +offset_second_(Offset) when is_binary(Offset) -> + offset_second_(erlang:binary_to_list(Offset)); +offset_second_("Z") -> 0; +offset_second_("z") -> 0; +offset_second_(Offset) when is_list(Offset) -> Sign = hd(Offset), ((Sign == $+) orelse (Sign == $-)) - orelse error({bad_zone, Offset}), + orelse error({bad_time_offset, Offset}), Signs = #{$+ => 1, $- => -1}, PosNeg = maps:get(Sign, Signs), [Sign | HM] = Offset, @@ -135,14 +135,14 @@ do_offset_second(Offset) when is_list(Offset) -> [HHMM] when erlang:length(HHMM) == 4 -> {string:sub_string(HHMM, 1,2), string:sub_string(HHMM, 3,4), "0"}; _ -> - error({bad_zone, Offset}) + error({bad_time_offset, Offset}) end, Hour = erlang:list_to_integer(HourStr), Minute = erlang:list_to_integer(MinuteStr), Second = erlang:list_to_integer(SecondStr), - (Hour =< 23) orelse error({bad_hour, Hour}), - (Minute =< 59) orelse error({bad_minute, Minute}), - (Second =< 59) orelse error({bad_second, Second}), + (Hour =< 23) orelse error({bad_time_offset_hour, Hour}), + (Minute =< 59) orelse error({bad_time_offset_minute, Minute}), + (Second =< 59) orelse error({bad_time_offset_second, Second}), PosNeg * (Hour * 3600 + Minute * 60 + Second). do_format(Time, Unit, Offset, Formatter) -> @@ -345,7 +345,7 @@ padding(Data, _Len) -> do_parse(DateStr, Unit, Formatter) -> DateInfo = do_parse_date_str(DateStr, Formatter, #{}), - {Precise, PrecisionUnit} = precise(DateInfo), + {Precise, PrecisionUnit} = precision(DateInfo), Counter = fun (year, V, Res) -> @@ -384,11 +384,11 @@ do_parse(DateStr, Unit, Formatter) -> Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), erlang:convert_time_unit(Count, PrecisionUnit, Unit). -precise(#{nanosecond := _}) -> {1000_000_000, nanosecond}; -precise(#{microsecond := _}) -> {1000_000, microsecond}; -precise(#{millisecond := _}) -> {1000, millisecond}; -precise(#{second := _}) -> {1, second}; -precise(_) -> {1, second}. +precision(#{nanosecond := _}) -> {1000_000_000, nanosecond}; +precision(#{microsecond := _}) -> {1000_000, microsecond}; +precision(#{millisecond := _}) -> {1000, millisecond}; +precision(#{second := _}) -> {1, second}; +precision(_) -> {1, second}. do_parse_date_str(<<>>, _, Result) -> Result; do_parse_date_str(_, [], Result) -> Result; @@ -408,36 +408,29 @@ do_parse_date_str(Date, [Key | Formatter], Result) -> end. date_size(Str) when is_list(Str) -> erlang:length(Str); -date_size(DateName) -> - Map = #{ - year => 4, - month => 2, - day => 2, - hour => 2, - minute => 2, - second => 2, - millisecond => 3, - microsecond => 6, - nanosecond => 9, - timezone => 5, - timezone1 => 6, - timezone2 => 9 - }, - maps:get(DateName, Map). +date_size(year) -> 4; +date_size(month) -> 2; +date_size(day) -> 2; +date_size(hour) -> 2; +date_size(minute) -> 2; +date_size(second) -> 2; +date_size(millisecond) -> 3; +date_size(microsecond) -> 6; +date_size(nanosecond) -> 9; +date_size(timezone) -> 5; +date_size(timezone1) -> 6; +date_size(timezone2) -> 9. + +dm(1) -> 0; +dm(2) -> 31; +dm(3) -> 59; +dm(4) -> 90; +dm(5) -> 120; +dm(6) -> 151; +dm(7) -> 181; +dm(8) -> 212; +dm(9) -> 243; +dm(10) -> 273; +dm(11) -> 304; +dm(12) -> 334. -dm(Month) -> - MonthDays = #{ - 1 => 0, - 2 => 31, - 3 => 59, - 4 => 90, - 5 => 120, - 6 => 151, - 7 => 181, - 8 => 212, - 9 => 243, - 10 => 273, - 11 => 304, - 12 => 334 - }, - maps:get(Month, MonthDays). diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index ba13c0cb7..1d1bb8dcd 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -29,7 +29,7 @@ ]). check_config(Config0) -> - Config = maps:without([timezone_offset, timezone, date_format], Config0), + Config = maps:without([date_format], Config0), logger_formatter:check_config(Config). format(#{msg := Msg0, meta := Meta} = Event, @@ -37,7 +37,6 @@ format(#{msg := Msg0, meta := Meta} = Event, Msg = maybe_merge(Msg0, Meta), Template = [time | Template0], logger_formatter:format(Event#{msg := Msg}, Config#{template => Template}); - format(#{msg := Msg0, meta := Meta} = Event, #{date_format := DFS} = Config) -> Msg = maybe_merge(Msg0, Meta),