From 31d4c92a175ced11f4df78a15965653c41cfd224 Mon Sep 17 00:00:00 2001 From: Georgy Sychev Date: Wed, 18 May 2022 14:33:09 +0400 Subject: [PATCH 01/10] fix(shared): retry dispatch when inflights are full --- src/emqx_session.erl | 7 +- src/emqx_shared_sub.erl | 132 ++++++++++++++++++++++++--------- test/emqx_shared_sub_SUITE.erl | 57 ++++++++++++-- 3 files changed, 151 insertions(+), 45 deletions(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index d01f33e6a..20a61b4f6 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -449,8 +449,8 @@ deliver_msg(ClientInfo, Msg = #message{qos = QoS}, Session = case emqx_inflight:is_full(Inflight) of true -> Session1 = case maybe_nack(Msg) of - true -> Session; - false -> enqueue(ClientInfo, Msg, Session) + drop -> Session; + store -> enqueue(ClientInfo, Msg, Session) end, {ok, Session1}; false -> @@ -650,7 +650,7 @@ redispatch_shared_messages(#session{inflight = Inflight}) -> %% Note that dispatch is called with self() in failed subs %% This is done to avoid dispatching back to caller Delivery = #delivery{sender = self(), message = Msg}, - emqx_shared_sub:dispatch(Group, Topic, Delivery, [self()]); + emqx_shared_sub:dispatch_to_non_self(Group, Topic, Delivery); _ -> false end; @@ -716,4 +716,3 @@ age(Now, Ts) -> Now - Ts. set_field(Name, Value, Session) -> Pos = emqx_misc:index_of(Name, record_info(fields, session)), setelement(Pos+1, Session, Value). - diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index 121d777ca..c7fb1a3cb 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -39,8 +39,8 @@ ]). -export([ dispatch/3 - , dispatch/4 - ]). + , dispatch_to_non_self/3 + ]). -export([ maybe_ack/1 , maybe_nack_dropped/1 @@ -123,22 +123,28 @@ unsubscribe(Group, Topic, SubPid) when is_pid(SubPid) -> record(Group, Topic, SubPid) -> #emqx_shared_subscription{group = Group, topic = Topic, subpid = SubPid}. +-spec(dispatch_to_non_self(emqx_topic:group(), emqx_topic:topic(), emqx_types:delivery()) + -> emqx_types:deliver_result()). +dispatch_to_non_self(Group, Topic, Delivery) -> + Strategy = strategy(Group), + dispatch(Strategy, Group, Topic, Delivery, _FailedSubs = #{self() => sender}). + -spec(dispatch(emqx_topic:group(), emqx_topic:topic(), emqx_types:delivery()) -> emqx_types:deliver_result()). dispatch(Group, Topic, Delivery) -> - dispatch(Group, Topic, Delivery, _FailedSubs = []). + Strategy = strategy(Group), + dispatch(Strategy, Group, Topic, Delivery, _FailedSubs = #{}). -dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> +dispatch(Strategy, Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> #message{from = ClientId, topic = SourceTopic} = Msg, - case pick(strategy(Group), ClientId, SourceTopic, Group, Topic, FailedSubs) of - false -> - {error, no_subscribers}; + case pick(Strategy, ClientId, SourceTopic, Group, Topic, FailedSubs) of + false -> {error, no_subscribers}; {Type, SubPid} -> case do_dispatch(SubPid, Group, Topic, Msg, Type) of ok -> {ok, 1}; - {error, _Reason} -> + {error, Reason} -> %% Failed to dispatch to this sub, try next. - dispatch(Group, Topic, Delivery, [SubPid | FailedSubs]) + dispatch(Strategy, Group, Topic, Delivery, FailedSubs#{SubPid => Reason}) end end. @@ -165,33 +171,33 @@ do_dispatch(SubPid, _Group, Topic, #message{qos = ?QOS_0} = Msg, _Type) -> %% For QoS 0 message, send it as regular dispatch SubPid ! {deliver, Topic, Msg}, ok; -do_dispatch(SubPid, _Group, Topic, Msg, retry) -> - %% Retry implies all subscribers nack:ed, send again without ack - SubPid ! {deliver, Topic, Msg}, - ok; -do_dispatch(SubPid, Group, Topic, Msg, fresh) -> +do_dispatch(SubPid, Group, Topic, Msg, Type) -> case ack_enabled() of true -> - dispatch_with_ack(SubPid, Group, Topic, Msg); + dispatch_with_ack(SubPid, Group, Topic, Msg, Type); false -> SubPid ! {deliver, Topic, Msg}, ok end. -dispatch_with_ack(SubPid, Group, Topic, Msg) -> +dispatch_with_ack(SubPid, Group, Topic, Msg, Type) -> %% For QoS 1/2 message, expect an ack Ref = erlang:monitor(process, SubPid), Sender = self(), - SubPid ! {deliver, Topic, with_group_ack(Msg, Group, Sender, Ref)}, + SubPid ! {deliver, Topic, with_group_ack(Msg, Group, Type, Sender, Ref)}, Timeout = case Msg#message.qos of ?QOS_1 -> timer:seconds(?SHARED_SUB_QOS1_DISPATCH_TIMEOUT_SECONDS); ?QOS_2 -> infinity end, + + %% This OpaqueRef is a forward compatibilty workaround. Pre 4.3.15 versions + %% pass Ref from `{Sender, Ref}` ack header back as it is. + OpaqueRef = old_ref(Type, Group, Ref), try receive - {Ref, ?ACK} -> + {ReceivedRef, ?ACK} when ReceivedRef =:= Ref; ReceivedRef =:= OpaqueRef -> ok; - {Ref, ?NACK(Reason)} -> + {ReceivedRef, ?NACK(Reason)} when ReceivedRef =:= Ref; ReceivedRef =:= OpaqueRef -> %% the receive session may nack this message when its queue is full {error, Reason}; {'DOWN', Ref, process, SubPid, Reason} -> @@ -204,8 +210,11 @@ dispatch_with_ack(SubPid, Group, Topic, Msg) -> _ = erlang:demonitor(Ref, [flush]) end. -with_group_ack(Msg, Group, Sender, Ref) -> - emqx_message:set_headers(#{shared_dispatch_ack => {Group, Sender, Ref}}, Msg). +with_group_ack(Msg, Group, Type, Sender, Ref) -> + emqx_message:set_headers(#{shared_dispatch_ack => {Sender, old_ref(Type, Group, Ref)}}, Msg). + +old_ref(Type, Group, Ref) -> + {Type, Group, Ref}. -spec(without_group_ack(emqx_types:message()) -> emqx_types:message()). without_group_ack(Msg) -> @@ -217,19 +226,32 @@ get_group_ack(Msg) -> -spec(get_group(emqx_types:message()) -> {ok, any()} | error). get_group(Msg) -> case get_group_ack(Msg) of - ?NO_ACK -> error; - {Group, _Sender, _Ref} -> {ok, Group} + {_Sender, {_Type, Group, _Ref}} -> {ok, Group}; + _ -> error end. -spec(is_ack_required(emqx_types:message()) -> boolean()). is_ack_required(Msg) -> ?NO_ACK =/= get_group_ack(Msg). %% @doc Negative ack dropped message due to inflight window or message queue being full. --spec(maybe_nack_dropped(emqx_types:message()) -> boolean()). +-spec(maybe_nack_dropped(emqx_types:message()) -> store | drop). maybe_nack_dropped(Msg) -> case get_group_ack(Msg) of - ?NO_ACK -> false; - {_Group, Sender, Ref} -> ok == nack(Sender, Ref, dropped) + %% No ack header is present, put it into mqueue + ?NO_ACK -> store; + + %% For fresh Ref we send a nack and return true, to note that the inflight is full + {Sender, {fresh, _Group, Ref}} -> nack(Sender, Ref, dropped), drop; + + %% For retry Ref we can't reject a message if inflight is full, so we mark it as + %% acknowledged and put it into mqueue + {_Sender, {retry, _Group, _Ref}} -> maybe_ack(Msg), store; + + %% This clause is for backward compatibility + Ack -> + {Sender, Ref} = fetch_sender_ref(Ack), + nack(Sender, Ref, dropped), + drop end. %% @doc Negative ack message due to connection down. @@ -237,7 +259,7 @@ maybe_nack_dropped(Msg) -> %% i.e is_ack_required returned true. -spec(nack_no_connection(emqx_types:message()) -> ok). nack_no_connection(Msg) -> - {_Group, Sender, Ref} = get_group_ack(Msg), + {Sender, Ref} = fetch_sender_ref(get_group_ack(Msg)), nack(Sender, Ref, no_connection). -spec(nack(pid(), reference(), dropped | no_connection) -> ok). @@ -250,11 +272,16 @@ maybe_ack(Msg) -> case get_group_ack(Msg) of ?NO_ACK -> Msg; - {_Group, Sender, Ref} -> + Ack -> + {Sender, Ref} = fetch_sender_ref(Ack), Sender ! {Ref, ?ACK}, without_group_ack(Msg) end. +fetch_sender_ref({Sender, {_Type, _Group, Ref}}) -> {Sender, Ref}; +%% These clauses are for backward compatibility +fetch_sender_ref({Sender, Ref}) -> {Sender, Ref}. + pick(sticky, ClientId, SourceTopic, Group, Topic, FailedSubs) -> Sub0 = erlang:get({shared_sub_sticky, Group, Topic}), case is_active_sub(Sub0, FailedSubs) of @@ -264,23 +291,37 @@ pick(sticky, ClientId, SourceTopic, Group, Topic, FailedSubs) -> {fresh, Sub0}; false -> %% randomly pick one for the first message - {Type, Sub} = do_pick(random, ClientId, SourceTopic, Group, Topic, [Sub0 | FailedSubs]), - %% stick to whatever pick result - erlang:put({shared_sub_sticky, Group, Topic}, Sub), - {Type, Sub} + FailedSubs1 = maps_put_new(FailedSubs, Sub0, inactive), + case do_pick(random, ClientId, SourceTopic, Group, Topic, FailedSubs1) of + false -> false; + {Type, Sub} -> + %% stick to whatever pick result + erlang:put({shared_sub_sticky, Group, Topic}, Sub), + {Type, Sub} + end end; pick(Strategy, ClientId, SourceTopic, Group, Topic, FailedSubs) -> do_pick(Strategy, ClientId, SourceTopic, Group, Topic, FailedSubs). do_pick(Strategy, ClientId, SourceTopic, Group, Topic, FailedSubs) -> All = subscribers(Group, Topic), - case All -- FailedSubs of + case lists:filter(fun(Sub) -> not maps:is_key(Sub, FailedSubs) end, All) of [] when All =:= [] -> %% Genuinely no subscriber false; [] -> - %% All offline? pick one anyway - {retry, pick_subscriber(Group, Topic, Strategy, ClientId, SourceTopic, All)}; + %% We try redispatch to subs who dropped the message because inflight was full. + Found = maps_find_by(FailedSubs, fun({SubPid, FailReason}) -> + FailReason == dropped andalso is_alive_sub(SubPid) + end), + case Found of + {ok, Dropped, dropped} -> + %% Found dropped client + {retry, pick_subscriber(Group, Topic, Strategy, ClientId, SourceTopic, [Dropped])}; + error -> + %% All offline? pick one anyway + {retry, pick_subscriber(Group, Topic, Strategy, ClientId, SourceTopic, All)} + end; Subs -> %% More than one available {fresh, pick_subscriber(Group, Topic, Strategy, ClientId, SourceTopic, Subs)} @@ -414,7 +455,7 @@ update_stats(State) -> %% Return 'true' if the subscriber process is alive AND not in the failed list is_active_sub(Pid, FailedSubs) -> - is_alive_sub(Pid) andalso not lists:member(Pid, FailedSubs). + not maps:is_key(Pid, FailedSubs) andalso is_alive_sub(Pid). %% erlang:is_process_alive/1 does not work with remote pid. is_alive_sub(Pid) when ?IS_LOCAL_PID(Pid) -> @@ -427,3 +468,22 @@ delete_route_if_needed({Group, Topic}) -> true -> ok; false -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) end. + +maps_find_by(Map, Predicate) when is_map(Map) -> + maps_find_by(maps:iterator(Map), Predicate); + +maps_find_by(Iterator, Predicate) -> + case maps:next(Iterator) of + none -> error; + {Key, Value, NewIterator} -> + case Predicate(Key, Value) of + true -> {ok, Key, Value}; + false -> maps_find_by(NewIterator, Predicate) + end + end. + +maps_put_new(Map, Key, Value) -> + case Map of + #{Key := _} -> Map; + _ -> Map#{Key => Value} + end. diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index 5c2de94d0..30bd96a66 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -47,20 +47,20 @@ t_is_ack_required(_) -> ?assertEqual(false, emqx_shared_sub:is_ack_required(#message{headers = #{}})). t_maybe_nack_dropped(_) -> - ?assertEqual(false, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})), - Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}}, - ?assertEqual(true, emqx_shared_sub:maybe_nack_dropped(Msg)), + ?assertEqual(store, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})), + Msg = #message{headers = #{shared_dispatch_ack => {self(), {fresh, <<"group">>, for_test}}}}, + ?assertEqual(drop, emqx_shared_sub:maybe_nack_dropped(Msg)), ?assertEqual(ok,receive {for_test, {shared_sub_nack, dropped}} -> ok after 100 -> timeout end). t_nack_no_connection(_) -> - Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}}, + Msg = #message{headers = #{shared_dispatch_ack => {self(), {fresh, <<"group">>, for_test}}}}, ?assertEqual(ok, emqx_shared_sub:nack_no_connection(Msg)), ?assertEqual(ok,receive {for_test, {shared_sub_nack, no_connection}} -> ok after 100 -> timeout end). t_maybe_ack(_) -> ?assertEqual(#message{headers = #{}}, emqx_shared_sub:maybe_ack(#message{headers = #{}})), - Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}}, + Msg = #message{headers = #{shared_dispatch_ack => {self(), {fresh, <<"group">>, for_test}}}}, ?assertEqual(#message{headers = #{shared_dispatch_ack => ?no_ack}}, emqx_shared_sub:maybe_ack(Msg)), ?assertEqual(ok,receive {for_test, ?ack} -> ok after 100 -> timeout end). @@ -446,6 +446,53 @@ t_redispatch(_) -> emqtt:stop(UsedSubPid2), ok. +t_dispatch_when_inflights_are_full(_) -> + ok = ensure_config(round_robin, true), + Topic = <<"foo/bar">>, + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + + %% Note that max_inflight is 1 + {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {max_inflight, 1}]), + {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {max_inflight, 1}]), + {ok, _} = emqtt:connect(ConnPid1), + {ok, _} = emqtt:connect(ConnPid2), + + emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar">>, 2}), + emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar">>, 2}), + + Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>), + Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>), + Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>), + Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>), + ct:sleep(100), + + sys:suspend(ConnPid1), + sys:suspend(ConnPid2), + + %% Fill in the inflight for first client + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message1)), + + %% Fill in the inflight for second client + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message2)), + + %% Now kill any client + erlang:exit(ConnPid1, normal), + ct:sleep(100), + + %% And try to send the message + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message3)), + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message4)), + + %% And see that it gets dispatched to the client which is alive, even if it's inflight is full + sys:resume(ConnPid2), + ct:sleep(100), + ?assertMatch({true, ConnPid2}, last_message(<<"hello3">>, [ConnPid1, ConnPid2])), + ?assertMatch({true, ConnPid2}, last_message(<<"hello4">>, [ConnPid1, ConnPid2])), + + emqtt:stop(ConnPid2), + ok. + %%-------------------------------------------------------------------- %% help functions %%-------------------------------------------------------------------- From 34ec0cc04b3d7f39fb484991e319b931c6ba2ac4 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 23 May 2022 14:33:40 +0800 Subject: [PATCH 02/10] fix: logger formatter --- etc/emqx.conf | 29 +++++- priv/emqx.schema | 100 ++++++++++++++++++++- src/emqx_logger_textfmt.erl | 173 +++++++++++++++++++++++++++++++++++- 3 files changed, 296 insertions(+), 6 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index bc45df1fc..fb7263f85 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -458,7 +458,34 @@ log.file = emqx.log ## Log formatter ## Value: text | json -#log.formatter = text +log.formatter = text + +## Text logger, date format. +## +## Value: rfc3339 | FormatString +## FormatString: +## %Y: year +## %m: month +## %d: day +## %H: hour +## %M: minute +## %S: second +## %6N: microseconds +## %z: timezone +## For example: +## log.formatter.text.date.format = emqx-server-date: %Y-%m-%dT%H:%M:%S:%6N %z +## Default: rfc3339 +# log.formatter.text.date.format = rfc3339 + +## Text logger, timezone. +## Only takes effect when log.formatter.text.date.format not rfc3339. +## +## Value: local | z | UTC TimeOffset +## TimeOffset: [+|-]HH:MM:SS, for example: +08:00 +## z: UTC zulu timezone. +## local is the system local time timezone. +## Default: local +# log.formatter.text.date.timezone = local ## Log to single line ## Value: Boolean diff --git a/priv/emqx.schema b/priv/emqx.schema index 4d7b1beda..7fde30c02 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -528,6 +528,18 @@ end}. {datatype, {enum, [text, json]}} ]}. +%% @doc format logs as text, date format part +{mapping, "log.formatter.text.date.format", "kernel.logger", [ + {default, rfc3339}, + {datatype, string} +]}. + +%% @doc format logs as text, date format part. local or any UTC offset +{mapping, "log.formatter.text.date.timezone", "kernel.logger", [ + {default, local}, + {datatype, string} +]}. + %% @doc format logs in a single line. {mapping, "log.single_line", "kernel.logger", [ {default, true}, @@ -629,9 +641,93 @@ end}. single_line => SingleLine }}; text -> + DateFormat = + case cuttlefish:conf_get("log.formatter.text.date.format", Conf) of + "rfc3339" -> + rfc3339; + DateStr -> + DateStrTrans = + fun + DST(<<>>, Formatter) -> lists:reverse(Formatter); + DST(<<"%Y", Tail/binary>>, Formatter) -> DST(Tail, [year | Formatter]); + DST(<<"%m", Tail/binary>>, Formatter) -> DST(Tail, [month | Formatter]); + DST(<<"%d", Tail/binary>>, Formatter) -> DST(Tail, [day | Formatter]); + DST(<<"%H", Tail/binary>>, Formatter) -> DST(Tail, [hour | Formatter]); + DST(<<"%M", Tail/binary>>, Formatter) -> DST(Tail, [minute | Formatter]); + DST(<<"%S", Tail/binary>>, Formatter) -> DST(Tail, [second | Formatter]); + DST(<<"%6N", Tail/binary>>, Formatter) -> DST(Tail, [nano_second | Formatter]); + DST(<<"%z", Tail/binary>>, Formatter) -> DST(Tail, [timezone | Formatter]); + DST(<>, [Str | Formatter]) when is_list(Str) -> + DST(Tail, [lists:append(Str, [Char]) | Formatter]); + DST(<>, Formatter) -> + DST(Tail, [[Char] | Formatter]) + end, + DateStrTrans(list_to_binary(DateStr), []) + end, + {DateOffsetStr, DateOffset} = + case cuttlefish:conf_get("log.formatter.text.date.timezone", Conf) of + "z" -> + {"z", 0}; + "Z" -> + {"Z", 0}; + "local" -> + UniversalTime = + calendar:system_time_to_universal_time( + erlang:system_time(second), + second), + LocalTime = erlang:universaltime_to_localtime(UniversalTime), + LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), + UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), + OffsetSecond = LocalSecs - UniversalSecs, + {Sign, PosOffsetSecond} = + case OffsetSecond >= 0 of + true -> {$+, OffsetSecond}; + false -> {$-, - OffsetSecond} + end, + {H, M, S} = calendar:seconds_to_time(PosOffsetSecond), + Str = + case S of + 0 -> + io_lib:format("~c~p:~p", [Sign, H, M]); + S -> + io_lib:format("~c~p:~p:~p", [Sign, H, M, S]) + end, + {Str, OffsetSecond}; + DateOStr -> + Sign = hd(DateOStr), + Signs = #{$+ => 1, $- => -1}, + case maps:get(Sign, Signs, undefined) of + undefined -> + error(bad_logger_offset); + PosNeg -> + try + [Sign | HM] = DateOStr, + {HourStr, MinuteStr, SecondStr} = + case string:tokens(HM, ":") of + [H, M] -> + {H, M, "0"}; + [H, M, S] -> + {H, M, S} + end, + Hour = erlang:list_to_integer(HourStr), + Minute = erlang:list_to_integer(MinuteStr), + Second = erlang:list_to_integer(SecondStr), + true = (Hour =< 23), + true = (Minute =< 59), + true = (Second =< 59), + {DateOStr, PosNeg * (Hour * 3600 + Minute * 60 + Second)} + catch _E : _R -> + error(bad_logger_offset) + end + end + end, {emqx_logger_textfmt, - #{template => - [time," [",level,"] ", + #{ + date_format => DateFormat, + timezone_offset => DateOffset, + timezone => DateOffsetStr, + template => + [" [",level,"] ", {clientid, [{peername, [clientid,"@",peername," "], diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index 98104f58c..df4118ba4 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -16,6 +16,16 @@ -module(emqx_logger_textfmt). +-define(SECONDS_PER_MINUTE, 60). +-define(SECONDS_PER_HOUR, 3600). +-define(SECONDS_PER_DAY, 86400). +-define(DAYS_PER_YEAR, 365). +-define(DAYS_PER_LEAP_YEAR, 366). +-define(DAYS_FROM_0_TO_1970, 719528). +-define(DAYS_FROM_0_TO_10000, 2932897). +-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY). +-define(SECONDS_FROM_0_TO_10000, ?DAYS_FROM_0_TO_10000 * ?SECONDS_PER_DAY). + -export([format/2]). -export([check_config/1]). @@ -28,11 +38,21 @@ , gl % not interesting ]). -check_config(X) -> logger_formatter:check_config(X). +check_config(Config0) -> + Config = maps:without([date_format, timezone_offset, timezone], Config0), + logger_formatter:check_config(Config). -format(#{msg := Msg0, meta := Meta} = Event, Config) -> +format(#{msg := Msg0, meta := Meta} = Event, + #{date_format := rfc3339, template := Template0} = Config) -> Msg = maybe_merge(Msg0, Meta), - logger_formatter:format(Event#{msg := Msg}, Config). + Template = [time | Template0], + logger_formatter:format(Event#{msg := Msg}, Config#{template => Template}); + +format(#{msg := Msg0, meta := Meta} = Event, + #{timezone_offset := TZO, timezone := TZ, date_format := DFS} = Config) -> + Msg = maybe_merge(Msg0, Meta), + Time = maps:get(time, Event, undefined), + [date_format(Time, TZO, TZ, DFS) | logger_formatter:format(Event#{msg := Msg}, Config)]. maybe_merge({report, Report}, Meta) when is_map(Report) -> {report, maps:merge(Report, filter(Meta))}; @@ -41,3 +61,150 @@ maybe_merge(Report, _Meta) -> filter(Meta) -> maps:without(?WITHOUT_MERGE, Meta). + +date_format(undefined, Offset, OffsetStr, Formatter) -> + Time = erlang:system_time(microsecond), + date_format(Time, Offset, OffsetStr, Formatter); +date_format(Time, Offset, OffsetStr, Formatter) -> + Adjustment = erlang:convert_time_unit(Offset, second, microsecond), + AdjustedTime = Time + Adjustment, + %% Factor 1000000 for microsecond. + Factor = 1000000, + Secs = AdjustedTime div Factor, + DateTime = system_time_to_datetime(Secs), + {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, + FractionStr = fraction_str(Factor, AdjustedTime), + Date = #{ + year => padding(Year, 4), + month => padding(Month, 2), + day => padding(Day, 2), + hour => padding(Hour, 2), + minute => padding(Min, 2), + second => padding(Sec, 2), + nano_second => FractionStr, + timezone => OffsetStr + }, + [maps:get(Key, Date, Key) || Key <- Formatter]. + +system_time_to_datetime(Seconds) -> + gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970). + +gregorian_seconds_to_datetime(Secs) when Secs >= 0 -> + Days = Secs div ?SECONDS_PER_DAY, + Rest = Secs rem ?SECONDS_PER_DAY, + {gregorian_days_to_date(Days), seconds_to_time(Rest)}. + +seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY -> + Secs0 = Secs rem ?SECONDS_PER_DAY, + Hour = Secs0 div ?SECONDS_PER_HOUR, + Secs1 = Secs0 rem ?SECONDS_PER_HOUR, + Minute = Secs1 div ?SECONDS_PER_MINUTE, + Second = Secs1 rem ?SECONDS_PER_MINUTE, + {Hour, Minute, Second}. + +gregorian_days_to_date(Days) -> + {Year, DayOfYear} = day_to_year(Days), + {Month, DayOfMonth} = year_day_to_date(Year, DayOfYear), + {Year, Month, DayOfMonth}. + +day_to_year(DayOfEpoch) when DayOfEpoch >= 0 -> + YMax = DayOfEpoch div ?DAYS_PER_YEAR, + YMin = DayOfEpoch div ?DAYS_PER_LEAP_YEAR, + {Y1, D1} = dty(YMin, YMax, DayOfEpoch, dy(YMin), dy(YMax)), + {Y1, DayOfEpoch - D1}. + +year_day_to_date(Year, DayOfYear) -> + ExtraDay = + case is_leap_year(Year) of + true -> + 1; + false -> + 0 + end, + {Month, Day} = year_day_to_date2(ExtraDay, DayOfYear), + {Month, Day + 1}. + +dty(Min, Max, _D1, DMin, _DMax) when Min == Max -> + {Min, DMin}; +dty(Min, Max, D1, DMin, DMax) -> + Diff = Max - Min, + Mid = Min + Diff * (D1 - DMin) div (DMax - DMin), + MidLength = + case is_leap_year(Mid) of + true -> + ?DAYS_PER_LEAP_YEAR; + false -> + ?DAYS_PER_YEAR + end, + case dy(Mid) of + D2 when D1 < D2 -> + NewMax = Mid - 1, + dty(Min, NewMax, D1, DMin, dy(NewMax)); + D2 when D1 - D2 >= MidLength -> + NewMin = Mid + 1, + dty(NewMin, Max, D1, dy(NewMin), DMax); + D2 -> + {Mid, D2} + end. + +dy(Y) when Y =< 0 -> + 0; +dy(Y) -> + X = Y - 1, + X div 4 - X div 100 + X div 400 + X * ?DAYS_PER_YEAR + ?DAYS_PER_LEAP_YEAR. + +is_leap_year(Y) when is_integer(Y), Y >= 0 -> + is_leap_year1(Y). + +is_leap_year1(Year) when Year rem 4 =:= 0, Year rem 100 > 0 -> + true; +is_leap_year1(Year) when Year rem 400 =:= 0 -> + true; +is_leap_year1(_) -> + false. + +year_day_to_date2(_, Day) when Day < 31 -> + {1, Day}; +year_day_to_date2(E, Day) when 31 =< Day, Day < 59 + E -> + {2, Day - 31}; +year_day_to_date2(E, Day) when 59 + E =< Day, Day < 90 + E -> + {3, Day - (59 + E)}; +year_day_to_date2(E, Day) when 90 + E =< Day, Day < 120 + E -> + {4, Day - (90 + E)}; +year_day_to_date2(E, Day) when 120 + E =< Day, Day < 151 + E -> + {5, Day - (120 + E)}; +year_day_to_date2(E, Day) when 151 + E =< Day, Day < 181 + E -> + {6, Day - (151 + E)}; +year_day_to_date2(E, Day) when 181 + E =< Day, Day < 212 + E -> + {7, Day - (181 + E)}; +year_day_to_date2(E, Day) when 212 + E =< Day, Day < 243 + E -> + {8, Day - (212 + E)}; +year_day_to_date2(E, Day) when 243 + E =< Day, Day < 273 + E -> + {9, Day - (243 + E)}; +year_day_to_date2(E, Day) when 273 + E =< Day, Day < 304 + E -> + {10, Day - (273 + E)}; +year_day_to_date2(E, Day) when 304 + E =< Day, Day < 334 + E -> + {11, Day - (304 + E)}; +year_day_to_date2(E, Day) when 334 + E =< Day -> + {12, Day - (334 + E)}. + +fraction_str(1, _Time) -> + ""; +fraction_str(Factor, Time) -> + Fraction = Time rem Factor, + S = integer_to_list(abs(Fraction)), + [padding(S, log10(Factor) - length(S))]. + +log10(1000) -> + 3; +log10(1000000) -> + 6; +log10(1000000000) -> + 9. + +padding(Data, Len) when is_integer(Data) -> + padding(integer_to_list(Data), Len); +padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> + [$0 | padding(Data, Len - 1)]; +padding(Data, _Len) -> + Data. From 664ebcc1927f420f4e880a54cfa169ca7c5df26f Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Wed, 25 May 2022 19:14:08 +0800 Subject: [PATCH 03/10] fix: log date format by emqx_calendar --- etc/emqx.conf | 11 +- priv/emqx.schema | 16 +- src/emqx_calendar.erl | 512 ++++++++++++++++++++++++++++++++++++ src/emqx_logger_textfmt.erl | 170 +----------- 4 files changed, 537 insertions(+), 172 deletions(-) create mode 100644 src/emqx_calendar.erl diff --git a/etc/emqx.conf b/etc/emqx.conf index fb7263f85..4cdd6e4d7 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -427,7 +427,7 @@ log.to = file ## this level will be logged. ## ## Default: warning -log.level = warning +log.level = debug ## The dir for log files. ## @@ -471,9 +471,10 @@ log.formatter = text ## %M: minute ## %S: second ## %6N: microseconds -## %z: timezone +## %3N: milliseconds +## %z: timezone, [+|-]HH:MM or [+|-]HH:MM:SS. Depends on `log.formatter.text.date.timezone` ## For example: -## log.formatter.text.date.format = emqx-server-date: %Y-%m-%dT%H:%M:%S:%6N %z +## log.formatter.text.date.format = emqx-server-date: %Y-%m-%dT%H:%M:%S.%6N %z ## Default: rfc3339 # log.formatter.text.date.format = rfc3339 @@ -481,9 +482,9 @@ log.formatter = text ## Only takes effect when log.formatter.text.date.format not rfc3339. ## ## Value: local | z | UTC TimeOffset -## TimeOffset: [+|-]HH:MM:SS, for example: +08:00 -## z: UTC zulu timezone. +## TimeOffset: [+|-]HH:MM or [+|-]HH:MM:SS, for example: +08:00. ## local is the system local time timezone. +## z: UTC zulu timezone. ## Default: local # log.formatter.text.date.timezone = local diff --git a/priv/emqx.schema b/priv/emqx.schema index 7fde30c02..dc15d87b3 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -530,13 +530,13 @@ end}. %% @doc format logs as text, date format part {mapping, "log.formatter.text.date.format", "kernel.logger", [ - {default, rfc3339}, + {default, "rfc3339"}, {datatype, string} ]}. %% @doc format logs as text, date format part. local or any UTC offset {mapping, "log.formatter.text.date.timezone", "kernel.logger", [ - {default, local}, + {default, "local"}, {datatype, string} ]}. @@ -642,7 +642,7 @@ end}. }}; text -> DateFormat = - case cuttlefish:conf_get("log.formatter.text.date.format", Conf) of + case cuttlefish:conf_get("log.formatter.text.date.format", Conf, "rfc3339") of "rfc3339" -> rfc3339; DateStr -> @@ -655,7 +655,9 @@ end}. DST(<<"%H", Tail/binary>>, Formatter) -> DST(Tail, [hour | Formatter]); DST(<<"%M", Tail/binary>>, Formatter) -> DST(Tail, [minute | Formatter]); DST(<<"%S", Tail/binary>>, Formatter) -> DST(Tail, [second | Formatter]); - DST(<<"%6N", Tail/binary>>, Formatter) -> DST(Tail, [nano_second | Formatter]); + DST(<<"%N", Tail/binary>>, Formatter) -> DST(Tail, [nanosecond | Formatter]); + DST(<<"%3N", Tail/binary>>, Formatter) -> DST(Tail, [millisecond | Formatter]); + DST(<<"%6N", Tail/binary>>, Formatter) -> DST(Tail, [microsecond | Formatter]); DST(<<"%z", Tail/binary>>, Formatter) -> DST(Tail, [timezone | Formatter]); DST(<>, [Str | Formatter]) when is_list(Str) -> DST(Tail, [lists:append(Str, [Char]) | Formatter]); @@ -665,7 +667,7 @@ end}. DateStrTrans(list_to_binary(DateStr), []) end, {DateOffsetStr, DateOffset} = - case cuttlefish:conf_get("log.formatter.text.date.timezone", Conf) of + case cuttlefish:conf_get("log.formatter.text.date.timezone", Conf, "local") of "z" -> {"z", 0}; "Z" -> @@ -688,9 +690,9 @@ end}. Str = case S of 0 -> - io_lib:format("~c~p:~p", [Sign, H, M]); + io_lib:format("~c~2.10.0B:~2.10.0B", [Sign, H, M]); S -> - io_lib:format("~c~p:~p:~p", [Sign, H, M, S]) + io_lib:format("~c~2.10.0B:~2.10.0B:~2.10.0B", [Sign, H, M, S]) end, {Str, OffsetSecond}; DateOStr -> diff --git a/src/emqx_calendar.erl b/src/emqx_calendar.erl new file mode 100644 index 000000000..ae9bae763 --- /dev/null +++ b/src/emqx_calendar.erl @@ -0,0 +1,512 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_calendar). + +-define(SECONDS_PER_MINUTE, 60). +-define(SECONDS_PER_HOUR, 3600). +-define(SECONDS_PER_DAY, 86400). +-define(DAYS_PER_YEAR, 365). +-define(DAYS_PER_LEAP_YEAR, 366). +-define(DAYS_FROM_0_TO_1970, 719528). +-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY). + +-export([ formatter/1 + , format/3 + , format/4 + , format/5 + , parse/3 + , parse/4 + , offset_second/1 + ]). + +-export([test/0]). + +-define(DATE_PART, + [ year + , month + , day + , hour + , minute + , second + , nanosecond + , millisecond + , microsecond + ]). + +-define(DATE_ZONE_NAME, + [ timezone + , timezone1 + , timezone2 + ]). + +formatter(FormatterStr) when is_list(FormatterStr) -> + formatter(list_to_binary(FormatterStr)); +formatter(FormatterBin) when is_binary(FormatterBin) -> + do_formatter(FormatterBin, []). + +offset_second(Offset) -> + do_offset_second(Offset). + +format(Time, Unit, Formatter) -> + format(Time, Unit, undefined, Formatter). + +format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> + format(Time, Unit, Offset, formatter(FormatterBin)); +format(Time, Unit, Offset, Formatter) -> + do_format(Time, time_unit(Unit), offset_second(Offset), Formatter). + +%% for logger format +format(Time, Unit, Offset, Zones, Formatter) -> + do_format(Time, time_unit(Unit), offset_second(Offset), Zones, Formatter). + +test() -> + Units = [second, millisecond, microsecond, nanosecond], + Formatters = [ + <<"%Y-%m-%d %H:%M:%S:%N%z">>, + <<"%Y-%m-%d %H:%M:%S:%N%:z">>, + <<"%Y-%m-%d %H:%M:%S:%N%::z">>, + <<"%Y-%m-%d %H:%M:%S:%N">>, + <<"%Y-%m-%d %H:%M:%S:%3N">>, + <<"%Y-%m-%d %H:%M:%S:%6N">> + ], + Offsets = [ + 0, + 8 * 3600, + -8 * 3600, + <<"+08:00">>, + <<"+08:00:01">>, + <<"+0802">>, + <<"-08:00">>, + <<"local">> + ], + [ + [ + [begin + Time = erlang:system_time(U), + FormatDate = erlang:list_to_binary(format(Time, U, O, F)), + PTime = emqx_calendar:parse(FormatDate, U, O, F), + io:format("~p ~p ~p ~p ~n~p~n~p~n", [U, O, F, FormatDate, Time, PTime]), + % Equal = (PTime == Time), + % F2 = + % fun() -> + % Delta = Time - PTime, + % DeltaFmt = erlang:list_to_binary(format(Delta, U, 0, <<"%Y-%m-%d %H:%M:%S:%N">>)), + % io:format("delta ~p~n ~p~n-----~n", [Delta, DeltaFmt]) + % end, + % Equal orelse F2() + ok + end||O <- Offsets] + || F <-Formatters] + || U <-Units], + ok. + +parse(DateStr, Unit, FormatterBin) -> + parse(DateStr, Unit, 0, FormatterBin). + +parse(DateStr, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> + parse(DateStr, Unit, Offset, formatter(FormatterBin)); +parse(DateStr, Unit, Offset, Formatter) -> + do_parse(DateStr, Unit, offset_second(Offset), Formatter). + +do_parse(DateStr, Unit, Offset, Formatter) -> + DateInfo = do_parse_date_str(DateStr, Formatter, #{}), + {Precise, PrecisionUnit} = precise(DateInfo), + Counter = + fun + (year, V, Res) -> + Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); + (month, V, Res) -> + Res + nano_sm(V); + (day, V, Res) -> + Res + (V * ?SECONDS_PER_DAY * Precise); + (hour, V, Res) -> + Res + (V * ?SECONDS_PER_HOUR * Precise); + (minute, V, Res) -> + Res + (V * ?SECONDS_PER_MINUTE * Precise); + (second, V, Res) -> + Res + V * Precise; + (millisecond, V, Res) -> + case PrecisionUnit of + millisecond -> + Res + V; + microsecond -> + Res + (V * 1000); + nanosecond -> + Res + (V * 1000000) + end; + (microsecond, V, Res) -> + case PrecisionUnit of + microsecond -> + Res + V; + nanosecond -> + Res + (V * 1000) + end; + (nanosecond, V, Res) -> + Res + V; + (parsed_offset, V, Res) -> + Res - V + end, + Counter1 = + fun(K,V,R) -> + Res = Counter(K,V,R), + % io:format("K, V, R = ~p, ~p, ~p, Res = ~p~n", [K, V, R, Res - R]), + Res + end, + Count0 = maps:fold(Counter1, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), + Count = + case maps:is_key(parsed_offset, DateInfo) of + true -> + Count0; + false -> + (Offset * Precise) + Count0 + end, + erlang:convert_time_unit(Count, PrecisionUnit, Unit). + +precise(#{nanosecond := _}) -> {1000_000_000, nanosecond}; +precise(#{microsecond := _}) -> {1000_000, microsecond}; +precise(#{millisecond := _}) -> {1000, millisecond}; +precise(#{second := _}) -> {1, second}; +precise(_) -> {1, second}. + +do_parse_date_str(<<>>, _, Result) -> Result; +do_parse_date_str(_, [], Result) -> Result; +do_parse_date_str(Date, [Key | Formatter], Result) -> + Size = date_size(Key), + <> = Date, + case lists:member(Key, ?DATE_PART) of + true -> + do_parse_date_str(Tail, Formatter, Result#{Key => erlang:binary_to_integer(DatePart)}); + false -> + case lists:member(Key, ?DATE_ZONE_NAME) of + true -> + io:format("DatePart -------------- ~p ~p~n", [DatePart, offset_second(DatePart)]), + do_parse_date_str(Tail, Formatter, Result#{parsed_offset => offset_second(DatePart)}); + false -> + do_parse_date_str(Tail, Formatter, Result) + end + end. + +date_size(Str) when is_list(Str) -> erlang:length(Str); +date_size(DateName) -> + Map = #{ + year => 4, + month => 2, + day => 2, + hour => 2, + minute => 2, + second => 2, + millisecond => 3, + microsecond => 6, + nanosecond => 9, + timezone => 5, + timezone1 => 6, + timezone2 => 9 + }, + maps:get(DateName, Map). + +nano_sm(Month) -> dm(Month) * ?SECONDS_PER_DAY * 1000_000. + +dm(Month) -> + MonthDays = #{ + 1 => 0, + 2 => 31, + 3 => 59, + 4 => 90, + 5 => 120, + 6 => 151, + 7 => 181, + 8 => 212, + 9 => 243, + 10 => 273, + 11 => 304, + 12 => 334 + }, + maps:get(Month, MonthDays). + +%% ------------------------------------------------------------------------------------------------- +%% internal +%% emqx_calendar:format(erlang:system_time(second), <<"second">>, <<"+8:00">> ,<<"%Y-%m-%d-%H:%M:%S%Z">>). + +time_unit(second) -> second; +time_unit(millisecond) -> millisecond; +time_unit(microsecond) -> microsecond; +time_unit(nanosecond) -> nanosecond; +time_unit("second") -> second; +time_unit("millisecond") -> millisecond; +time_unit("microsecond") -> microsecond; +time_unit("nanosecond") -> nanosecond; +time_unit(<<"second">>) -> second; +time_unit(<<"millisecond">>) -> millisecond; +time_unit(<<"microsecond">>) -> microsecond; +time_unit(<<"nanosecond">>) -> nanosecond. + +%% ------------------------------------------------------------------------------------------------- +%% internal: format part + +do_formatter(<<>>, Formatter) -> lists:reverse(Formatter); +do_formatter(<<"%Y", Tail/binary>>, Formatter) -> do_formatter(Tail, [year | Formatter]); +do_formatter(<<"%m", Tail/binary>>, Formatter) -> do_formatter(Tail, [month | Formatter]); +do_formatter(<<"%d", Tail/binary>>, Formatter) -> do_formatter(Tail, [day | Formatter]); +do_formatter(<<"%H", Tail/binary>>, Formatter) -> do_formatter(Tail, [hour | Formatter]); +do_formatter(<<"%M", Tail/binary>>, Formatter) -> do_formatter(Tail, [minute | Formatter]); +do_formatter(<<"%S", Tail/binary>>, Formatter) -> do_formatter(Tail, [second | Formatter]); +do_formatter(<<"%N", Tail/binary>>, Formatter) -> do_formatter(Tail, [nanosecond | Formatter]); +do_formatter(<<"%3N", Tail/binary>>, Formatter) -> do_formatter(Tail, [millisecond | Formatter]); +do_formatter(<<"%6N", Tail/binary>>, Formatter) -> do_formatter(Tail, [microsecond | Formatter]); +do_formatter(<<"%z", Tail/binary>>, Formatter) -> do_formatter(Tail, [timezone | Formatter]); +do_formatter(<<"%:z", Tail/binary>>, Formatter) -> do_formatter(Tail, [timezone1 | Formatter]); +do_formatter(<<"%::z", Tail/binary>>, Formatter) -> do_formatter(Tail, [timezone2 | Formatter]); +do_formatter(<>, [Str | Formatter]) when is_list(Str) -> + do_formatter(Tail, [lists:append(Str, [Char]) | Formatter]); +do_formatter(<>, Formatter) -> do_formatter(Tail, [[Char] | Formatter]). + +do_offset_second(OffsetSecond) when is_integer(OffsetSecond) -> OffsetSecond; +do_offset_second(undefined) -> 0; +do_offset_second("local") -> do_offset_second(local); +do_offset_second(<<"local">>) -> do_offset_second(local); +do_offset_second(local) -> + UniversalTime = calendar:system_time_to_universal_time(erlang:system_time(second), second), + LocalTime = erlang:universaltime_to_localtime(UniversalTime), + LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), + UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), + LocalSecs - UniversalSecs; +do_offset_second(Offset) when is_binary(Offset) -> + do_offset_second(erlang:binary_to_list(Offset)); +do_offset_second("Z") -> 0; +do_offset_second("z") -> 0; +do_offset_second(Offset) when is_list(Offset) -> + Sign = hd(Offset), + ((Sign == $+) orelse (Sign == $-)) + orelse error({bad_zone, Offset}), + Signs = #{$+ => 1, $- => -1}, + PosNeg = maps:get(Sign, Signs), + [Sign | HM] = Offset, + {HourStr, MinuteStr, SecondStr} = + case string:tokens(HM, ":") of + [H, M] -> + {H, M, "0"}; + [H, M, S] -> + {H, M, S}; + [HHMM] when erlang:length(HHMM) == 4 -> + {string:sub_string(HHMM, 1,2), string:sub_string(HHMM, 3,4), "0"}; + _ -> + error({bad_zone, Offset}) + end, + Hour = erlang:list_to_integer(HourStr), + Minute = erlang:list_to_integer(MinuteStr), + Second = erlang:list_to_integer(SecondStr), + (Hour =< 23) orelse error({bad_hour, Hour}), + (Minute =< 59) orelse error({bad_minute, Minute}), + (Second =< 59) orelse error({bad_second, Second}), + PosNeg * (Hour * 3600 + Minute * 60 + Second). + +do_format(Time, Unit, Offset, Formatter) -> + Timezones = formatter_timezones(Offset, Formatter, #{}), + do_format(Time, Unit, Offset, Timezones, Formatter). + +do_format(Time, Unit, Offset, Timezones, Formatter) -> + Adjustment = erlang:convert_time_unit(Offset, second, Unit), + AdjustedTime = Time + Adjustment, + Factor = factor(Unit), + Secs = AdjustedTime div Factor, + DateTime = system_time_to_datetime(Secs), + {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, + Date = #{ + year => padding(Year, 4), + month => padding(Month, 2), + day => padding(Day, 2), + hour => padding(Hour, 2), + minute => padding(Min, 2), + second => padding(Sec, 2), + millisecond => trans_x_second(Unit, millisecond, Time), + microsecond => trans_x_second(Unit, microsecond, Time), + nanosecond => trans_x_second(Unit, nanosecond, Time) + }, + DateWithZone = maps:merge(Date, Timezones), + [maps:get(Key, DateWithZone, Key) || Key <- Formatter]. + +formatter_timezones(_Offset, [], Zones) -> Zones; +formatter_timezones(Offset, [Timezone | Formatter], Zones) -> + case lists:member(Timezone, [timezone, timezone1, timezone2]) of + true -> + NZones = Zones#{Timezone => offset_to_timezone(Offset, Timezone)}, + formatter_timezones(Offset, Formatter, NZones); + false -> + formatter_timezones(Offset, Formatter, Zones) + end. + +offset_to_timezone(Offset, Timezone) -> + Sign = + case Offset >= 0 of + true -> + $+; + false -> + $- + end, + {H, M, S} = seconds_to_time(abs(Offset)), + %% TODO: Support zone define %:::z + %% Numeric time zone with ":" to necessary precision (e.g., -04, +05:30). + case Timezone of + timezone -> + %% +0800 + io_lib:format("~c~2.10.0B~2.10.0B", [Sign, H, M]); + timezone1 -> + %% +08:00 + io_lib:format("~c~2.10.0B:~2.10.0B", [Sign, H, M]); + timezone2 -> + %% +08:00:00 + io_lib:format("~c~2.10.0B:~2.10.0B:~2.10.0B", [Sign, H, M, S]) + end. + +factor(second) -> 1; +factor(millisecond) -> 1000; +factor(microsecond) -> 1000000; +factor(nanosecond) -> 1000000000. + +system_time_to_datetime(Seconds) -> + gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970). + +gregorian_seconds_to_datetime(Secs) when Secs >= 0 -> + Days = Secs div ?SECONDS_PER_DAY, + Rest = Secs rem ?SECONDS_PER_DAY, + {gregorian_days_to_date(Days), seconds_to_time(Rest)}. + +seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY -> + Secs0 = Secs rem ?SECONDS_PER_DAY, + Hour = Secs0 div ?SECONDS_PER_HOUR, + Secs1 = Secs0 rem ?SECONDS_PER_HOUR, + Minute = Secs1 div ?SECONDS_PER_MINUTE, + Second = Secs1 rem ?SECONDS_PER_MINUTE, + {Hour, Minute, Second}. + +gregorian_days_to_date(Days) -> + {Year, DayOfYear} = day_to_year(Days), + {Month, DayOfMonth} = year_day_to_date(Year, DayOfYear), + {Year, Month, DayOfMonth}. + +day_to_year(DayOfEpoch) when DayOfEpoch >= 0 -> + YMax = DayOfEpoch div ?DAYS_PER_YEAR, + YMin = DayOfEpoch div ?DAYS_PER_LEAP_YEAR, + {Y1, D1} = dty(YMin, YMax, DayOfEpoch, dy(YMin), dy(YMax)), + {Y1, DayOfEpoch - D1}. + +year_day_to_date(Year, DayOfYear) -> + ExtraDay = + case is_leap_year(Year) of + true -> + 1; + false -> + 0 + end, + {Month, Day} = year_day_to_date2(ExtraDay, DayOfYear), + {Month, Day + 1}. + +dty(Min, Max, _D1, DMin, _DMax) when Min == Max -> + {Min, DMin}; +dty(Min, Max, D1, DMin, DMax) -> + Diff = Max - Min, + Mid = Min + Diff * (D1 - DMin) div (DMax - DMin), + MidLength = + case is_leap_year(Mid) of + true -> + ?DAYS_PER_LEAP_YEAR; + false -> + ?DAYS_PER_YEAR + end, + case dy(Mid) of + D2 when D1 < D2 -> + NewMax = Mid - 1, + dty(Min, NewMax, D1, DMin, dy(NewMax)); + D2 when D1 - D2 >= MidLength -> + NewMin = Mid + 1, + dty(NewMin, Max, D1, dy(NewMin), DMax); + D2 -> + {Mid, D2} + end. + +dy(Y) when Y =< 0 -> + 0; +dy(Y) -> + X = Y - 1, + X div 4 - X div 100 + X div 400 + X * ?DAYS_PER_YEAR + ?DAYS_PER_LEAP_YEAR. + +is_leap_year(Y) when is_integer(Y), Y >= 0 -> + is_leap_year1(Y). + +is_leap_year1(Year) when Year rem 4 =:= 0, Year rem 100 > 0 -> + true; +is_leap_year1(Year) when Year rem 400 =:= 0 -> + true; +is_leap_year1(_) -> + false. + +year_day_to_date2(_, Day) when Day < 31 -> + {1, Day}; +year_day_to_date2(E, Day) when 31 =< Day, Day < 59 + E -> + {2, Day - 31}; +year_day_to_date2(E, Day) when 59 + E =< Day, Day < 90 + E -> + {3, Day - (59 + E)}; +year_day_to_date2(E, Day) when 90 + E =< Day, Day < 120 + E -> + {4, Day - (90 + E)}; +year_day_to_date2(E, Day) when 120 + E =< Day, Day < 151 + E -> + {5, Day - (120 + E)}; +year_day_to_date2(E, Day) when 151 + E =< Day, Day < 181 + E -> + {6, Day - (151 + E)}; +year_day_to_date2(E, Day) when 181 + E =< Day, Day < 212 + E -> + {7, Day - (181 + E)}; +year_day_to_date2(E, Day) when 212 + E =< Day, Day < 243 + E -> + {8, Day - (212 + E)}; +year_day_to_date2(E, Day) when 243 + E =< Day, Day < 273 + E -> + {9, Day - (243 + E)}; +year_day_to_date2(E, Day) when 273 + E =< Day, Day < 304 + E -> + {10, Day - (273 + E)}; +year_day_to_date2(E, Day) when 304 + E =< Day, Day < 334 + E -> + {11, Day - (304 + E)}; +year_day_to_date2(E, Day) when 334 + E =< Day -> + {12, Day - (334 + E)}. + +trans_x_second(FromUnit, ToUnit, Time) -> + XSecond = do_trans_x_second(FromUnit, ToUnit, Time), + Len = + case ToUnit of + millisecond -> 3; + microsecond -> 6; + nanosecond -> 9 + end, + padding(XSecond, Len). + +do_trans_x_second(second, second, Time) -> Time div 60; +do_trans_x_second(second, _, _Time) -> 0; + +do_trans_x_second(millisecond, millisecond, Time) -> Time rem 1000; +do_trans_x_second(millisecond, microsecond, _Time) -> 0; +do_trans_x_second(millisecond, nanosecond, _Time) -> 0; + +do_trans_x_second(microsecond, millisecond, Time) -> Time div 1000 rem 1000; +do_trans_x_second(microsecond, microsecond, Time) -> Time rem 1000000; +do_trans_x_second(microsecond, nanosecond, _Time) -> 0; + +do_trans_x_second(nanosecond, millisecond, Time) -> Time div 1000000 rem 1000; +do_trans_x_second(nanosecond, microsecond, Time) -> Time div 1000 rem 1000000; +do_trans_x_second(nanosecond, nanosecond, Time) -> Time rem 1000000000. + +padding(Data, Len) when is_integer(Data) -> + padding(integer_to_list(Data), Len); +padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> + [$0 | padding(Data, Len - 1)]; +padding(Data, _Len) -> + Data. diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index df4118ba4..b13871384 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -16,16 +16,6 @@ -module(emqx_logger_textfmt). --define(SECONDS_PER_MINUTE, 60). --define(SECONDS_PER_HOUR, 3600). --define(SECONDS_PER_DAY, 86400). --define(DAYS_PER_YEAR, 365). --define(DAYS_PER_LEAP_YEAR, 366). --define(DAYS_FROM_0_TO_1970, 719528). --define(DAYS_FROM_0_TO_10000, 2932897). --define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY). --define(SECONDS_FROM_0_TO_10000, ?DAYS_FROM_0_TO_10000 * ?SECONDS_PER_DAY). - -export([format/2]). -export([check_config/1]). @@ -39,7 +29,7 @@ ]). check_config(Config0) -> - Config = maps:without([date_format, timezone_offset, timezone], Config0), + Config = maps:without([timezone_offset, timezone, date_format], Config0), logger_formatter:check_config(Config). format(#{msg := Msg0, meta := Meta} = Event, @@ -51,8 +41,15 @@ format(#{msg := Msg0, meta := Meta} = Event, format(#{msg := Msg0, meta := Meta} = Event, #{timezone_offset := TZO, timezone := TZ, date_format := DFS} = Config) -> Msg = maybe_merge(Msg0, Meta), - Time = maps:get(time, Event, undefined), - [date_format(Time, TZO, TZ, DFS) | logger_formatter:format(Event#{msg := Msg}, Config)]. + Time = + case maps:get(time, Event, undefined) of + undefined -> + erlang:system_time(microsecond); + T -> + T + end, + Date = emqx_calendar:format(Time, microsecond, TZO, TZ, DFS), + [Date | logger_formatter:format(Event#{msg := Msg}, Config)]. maybe_merge({report, Report}, Meta) when is_map(Report) -> {report, maps:merge(Report, filter(Meta))}; @@ -61,150 +58,3 @@ maybe_merge(Report, _Meta) -> filter(Meta) -> maps:without(?WITHOUT_MERGE, Meta). - -date_format(undefined, Offset, OffsetStr, Formatter) -> - Time = erlang:system_time(microsecond), - date_format(Time, Offset, OffsetStr, Formatter); -date_format(Time, Offset, OffsetStr, Formatter) -> - Adjustment = erlang:convert_time_unit(Offset, second, microsecond), - AdjustedTime = Time + Adjustment, - %% Factor 1000000 for microsecond. - Factor = 1000000, - Secs = AdjustedTime div Factor, - DateTime = system_time_to_datetime(Secs), - {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, - FractionStr = fraction_str(Factor, AdjustedTime), - Date = #{ - year => padding(Year, 4), - month => padding(Month, 2), - day => padding(Day, 2), - hour => padding(Hour, 2), - minute => padding(Min, 2), - second => padding(Sec, 2), - nano_second => FractionStr, - timezone => OffsetStr - }, - [maps:get(Key, Date, Key) || Key <- Formatter]. - -system_time_to_datetime(Seconds) -> - gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970). - -gregorian_seconds_to_datetime(Secs) when Secs >= 0 -> - Days = Secs div ?SECONDS_PER_DAY, - Rest = Secs rem ?SECONDS_PER_DAY, - {gregorian_days_to_date(Days), seconds_to_time(Rest)}. - -seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY -> - Secs0 = Secs rem ?SECONDS_PER_DAY, - Hour = Secs0 div ?SECONDS_PER_HOUR, - Secs1 = Secs0 rem ?SECONDS_PER_HOUR, - Minute = Secs1 div ?SECONDS_PER_MINUTE, - Second = Secs1 rem ?SECONDS_PER_MINUTE, - {Hour, Minute, Second}. - -gregorian_days_to_date(Days) -> - {Year, DayOfYear} = day_to_year(Days), - {Month, DayOfMonth} = year_day_to_date(Year, DayOfYear), - {Year, Month, DayOfMonth}. - -day_to_year(DayOfEpoch) when DayOfEpoch >= 0 -> - YMax = DayOfEpoch div ?DAYS_PER_YEAR, - YMin = DayOfEpoch div ?DAYS_PER_LEAP_YEAR, - {Y1, D1} = dty(YMin, YMax, DayOfEpoch, dy(YMin), dy(YMax)), - {Y1, DayOfEpoch - D1}. - -year_day_to_date(Year, DayOfYear) -> - ExtraDay = - case is_leap_year(Year) of - true -> - 1; - false -> - 0 - end, - {Month, Day} = year_day_to_date2(ExtraDay, DayOfYear), - {Month, Day + 1}. - -dty(Min, Max, _D1, DMin, _DMax) when Min == Max -> - {Min, DMin}; -dty(Min, Max, D1, DMin, DMax) -> - Diff = Max - Min, - Mid = Min + Diff * (D1 - DMin) div (DMax - DMin), - MidLength = - case is_leap_year(Mid) of - true -> - ?DAYS_PER_LEAP_YEAR; - false -> - ?DAYS_PER_YEAR - end, - case dy(Mid) of - D2 when D1 < D2 -> - NewMax = Mid - 1, - dty(Min, NewMax, D1, DMin, dy(NewMax)); - D2 when D1 - D2 >= MidLength -> - NewMin = Mid + 1, - dty(NewMin, Max, D1, dy(NewMin), DMax); - D2 -> - {Mid, D2} - end. - -dy(Y) when Y =< 0 -> - 0; -dy(Y) -> - X = Y - 1, - X div 4 - X div 100 + X div 400 + X * ?DAYS_PER_YEAR + ?DAYS_PER_LEAP_YEAR. - -is_leap_year(Y) when is_integer(Y), Y >= 0 -> - is_leap_year1(Y). - -is_leap_year1(Year) when Year rem 4 =:= 0, Year rem 100 > 0 -> - true; -is_leap_year1(Year) when Year rem 400 =:= 0 -> - true; -is_leap_year1(_) -> - false. - -year_day_to_date2(_, Day) when Day < 31 -> - {1, Day}; -year_day_to_date2(E, Day) when 31 =< Day, Day < 59 + E -> - {2, Day - 31}; -year_day_to_date2(E, Day) when 59 + E =< Day, Day < 90 + E -> - {3, Day - (59 + E)}; -year_day_to_date2(E, Day) when 90 + E =< Day, Day < 120 + E -> - {4, Day - (90 + E)}; -year_day_to_date2(E, Day) when 120 + E =< Day, Day < 151 + E -> - {5, Day - (120 + E)}; -year_day_to_date2(E, Day) when 151 + E =< Day, Day < 181 + E -> - {6, Day - (151 + E)}; -year_day_to_date2(E, Day) when 181 + E =< Day, Day < 212 + E -> - {7, Day - (181 + E)}; -year_day_to_date2(E, Day) when 212 + E =< Day, Day < 243 + E -> - {8, Day - (212 + E)}; -year_day_to_date2(E, Day) when 243 + E =< Day, Day < 273 + E -> - {9, Day - (243 + E)}; -year_day_to_date2(E, Day) when 273 + E =< Day, Day < 304 + E -> - {10, Day - (273 + E)}; -year_day_to_date2(E, Day) when 304 + E =< Day, Day < 334 + E -> - {11, Day - (304 + E)}; -year_day_to_date2(E, Day) when 334 + E =< Day -> - {12, Day - (334 + E)}. - -fraction_str(1, _Time) -> - ""; -fraction_str(Factor, Time) -> - Fraction = Time rem Factor, - S = integer_to_list(abs(Fraction)), - [padding(S, log10(Factor) - length(S))]. - -log10(1000) -> - 3; -log10(1000000) -> - 6; -log10(1000000000) -> - 9. - -padding(Data, Len) when is_integer(Data) -> - padding(integer_to_list(Data), Len); -padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> - [$0 | padding(Data, Len - 1)]; -padding(Data, _Len) -> - Data. From 4fc5cb2817c4f1e8225c048bc04f4972148d0df4 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 26 May 2022 12:19:53 +0800 Subject: [PATCH 04/10] fix: rule engine date format fun & date to timestamp --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 39 ++- .../test/emqx_rule_funcs_SUITE.erl | 34 +- etc/emqx.conf | 11 +- priv/emqx.schema | 10 +- src/emqx_calendar.erl | 290 +++++++----------- src/emqx_logger_textfmt.erl | 4 +- 6 files changed, 174 insertions(+), 214 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 282c8929d..a8b347f5d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -201,6 +201,8 @@ , unix_ts_to_rfc3339/2 , format_date/3 , format_date/4 + , timezone_to_second/1 + , date_to_unix_ts/3 , date_to_unix_ts/4 , rfc3339_to_unix_ts/1 , rfc3339_to_unix_ts/2 @@ -923,26 +925,37 @@ now_timestamp(Unit) -> time_unit(<<"second">>) -> second; time_unit(<<"millisecond">>) -> millisecond; time_unit(<<"microsecond">>) -> microsecond; -time_unit(<<"nanosecond">>) -> nanosecond. +time_unit(<<"nanosecond">>) -> nanosecond; +time_unit(second) -> second; +time_unit(millisecond) -> millisecond; +time_unit(microsecond) -> microsecond; +time_unit(nanosecond) -> nanosecond. format_date(TimeUnit, Offset, FormatString) -> - emqx_rule_utils:bin( - emqx_rule_date:date(time_unit(TimeUnit), - emqx_rule_utils:str(Offset), - emqx_rule_utils:str(FormatString))). + Unit = time_unit(TimeUnit), + TimeEpoch = erlang:system_time(Unit), + format_date(Unit, Offset, FormatString, TimeEpoch). + +timezone_to_second(TimeZone) -> + emqx_calendar:offset_second(TimeZone). format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> + Unit = time_unit(TimeUnit), emqx_rule_utils:bin( - emqx_rule_date:date(time_unit(TimeUnit), - emqx_rule_utils:str(Offset), - emqx_rule_utils:str(FormatString), - TimeEpoch)). + lists:concat( + emqx_calendar:format(TimeEpoch, Unit, Offset, FormatString))). +%% date string has timezone information, calculate the offset. +date_to_unix_ts(TimeUnit, FormatString, InputString) -> + Unit = time_unit(TimeUnit), + emqx_calendar:parse(InputString, Unit, FormatString). + +%% date string has no timezone information, force add the offset. date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> - emqx_rule_date:parse_date(time_unit(TimeUnit), - emqx_rule_utils:str(Offset), - emqx_rule_utils:str(FormatString), - emqx_rule_utils:str(InputString)). + Unit = time_unit(TimeUnit), + OffsetSecond = emqx_calendar:offset_second(Offset), + OffsetDelta = erlang:convert_time_unit(OffsetSecond, second, Unit), + OffsetDelta + date_to_unix_ts(Unit, FormatString, InputString). mongo_date() -> erlang:timestamp(). diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 98f6ad0b7..632697a87 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -713,24 +713,32 @@ t_format_date_funcs(_) -> ?PROPTEST(prop_format_date_fun). prop_format_date_fun() -> - Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>], + Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%Y---%H:%M:%S%z">>], + Args1DTUS = [<<"second">>, 0, <<"%m--%d--%Y---%H:%M:%S%z">>], ?FORALL(S, erlang:system_time(second), S == apply_func(date_to_unix_ts, - Args1 ++ [apply_func(format_date, + Args1DTUS ++ [apply_func(format_date, Args1 ++ [S])])), - Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>], + Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], + Args2DTUS = [<<"millisecond">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], ?FORALL(S, erlang:system_time(millisecond), S == apply_func(date_to_unix_ts, - Args2 ++ [apply_func(format_date, + Args2DTUS ++ [apply_func(format_date, Args2 ++ [S])])), - Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], + Args = [<<"second">>, <<"+08:00">>, <<"%Y-%m-%d-%H:%M:%S%z">>], + ArgsDTUS = [<<"second">>, <<"%Y-%m-%d-%H:%M:%S%z">>], ?FORALL(S, erlang:system_time(second), S == apply_func(date_to_unix_ts, - Args ++ [apply_func(format_date, - Args ++ [S])])). -%%------------------------------------------------------------------------------ -%% Utility functions -%%------------------------------------------------------------------------------ + ArgsDTUS ++ [apply_func(format_date, + Args ++ [S])])), + % no offset in format string. force add offset + Second = erlang:system_time(second), + Args3 = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Second], + Formatters3 = apply_func(format_date, Args3), + % -04:00 remove offset + OffsetSeconds = emqx_rule_funcs:timezone_to_second(<<"-04:00">>), + Args3DTUS = [<<"second">>, OffsetSeconds, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], + Second == apply_func(date_to_unix_ts, Args3DTUS). apply_func(Name, Args) when is_atom(Name) -> erlang:apply(emqx_rule_funcs, Name, Args); @@ -880,9 +888,9 @@ message() -> %% CT functions %%------------------------------------------------------------------------------ -all() -> - IsTestCase = fun("t_" ++ _) -> true; (_) -> false end, - [F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))]. +all() -> [t_format_date_funcs]. + % IsTestCase = fun("t_" ++ _) -> true; (_) -> false end, + % [F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))]. suite() -> [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}]. diff --git a/etc/emqx.conf b/etc/emqx.conf index 4cdd6e4d7..fcc8557a2 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -427,7 +427,7 @@ log.to = file ## this level will be logged. ## ## Default: warning -log.level = debug +log.level = warning ## The dir for log files. ## @@ -472,9 +472,11 @@ log.formatter = text ## %S: second ## %6N: microseconds ## %3N: milliseconds -## %z: timezone, [+|-]HH:MM or [+|-]HH:MM:SS. Depends on `log.formatter.text.date.timezone` +## %z: timezone, [+-]HHMM +## %:z: timezone, [+-]HH:MM +## %::z: timezone, [+-]HH:MM:SS ## For example: -## log.formatter.text.date.format = emqx-server-date: %Y-%m-%dT%H:%M:%S.%6N %z +## emqx-server-date: %Y-%m-%dT%H:%M:%S.%6N %:z ## Default: rfc3339 # log.formatter.text.date.format = rfc3339 @@ -482,9 +484,10 @@ log.formatter = text ## Only takes effect when log.formatter.text.date.format not rfc3339. ## ## Value: local | z | UTC TimeOffset -## TimeOffset: [+|-]HH:MM or [+|-]HH:MM:SS, for example: +08:00. +## TimeOffset: [+-]HHMM or [+|-]HH:MM or [+|-]HH:MM:SS, for example: +08:00. ## local is the system local time timezone. ## z: UTC zulu timezone. +## Z: UTC zulu timezone. Same as z. ## Default: local # log.formatter.text.date.timezone = local diff --git a/priv/emqx.schema b/priv/emqx.schema index dc15d87b3..96d0005ad 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -659,6 +659,8 @@ end}. DST(<<"%3N", Tail/binary>>, Formatter) -> DST(Tail, [millisecond | Formatter]); DST(<<"%6N", Tail/binary>>, Formatter) -> DST(Tail, [microsecond | Formatter]); DST(<<"%z", Tail/binary>>, Formatter) -> DST(Tail, [timezone | Formatter]); + DST(<<"%:z", Tail/binary>>, Formatter) -> DST(Tail, [timezone1 | Formatter]); + DST(<<"%::z", Tail/binary>>, Formatter) -> DST(Tail, [timezone2 | Formatter]); DST(<>, [Str | Formatter]) when is_list(Str) -> DST(Tail, [lists:append(Str, [Char]) | Formatter]); DST(<>, Formatter) -> @@ -669,9 +671,9 @@ end}. {DateOffsetStr, DateOffset} = case cuttlefish:conf_get("log.formatter.text.date.timezone", Conf, "local") of "z" -> - {"z", 0}; + {"z", "z"}; "Z" -> - {"Z", 0}; + {"Z", "Z"}; "local" -> UniversalTime = calendar:system_time_to_universal_time( @@ -709,7 +711,9 @@ end}. [H, M] -> {H, M, "0"}; [H, M, S] -> - {H, M, S} + {H, M, S}; + [HHMM] when erlang:length(HHMM) == 4 -> + {string:sub_string(HHMM, 1,2), string:sub_string(HHMM, 3,4), "0"} end, Hour = erlang:list_to_integer(HourStr), Minute = erlang:list_to_integer(MinuteStr), diff --git a/src/emqx_calendar.erl b/src/emqx_calendar.erl index ae9bae763..c7e2d627f 100644 --- a/src/emqx_calendar.erl +++ b/src/emqx_calendar.erl @@ -27,14 +27,10 @@ -export([ formatter/1 , format/3 , format/4 - , format/5 , parse/3 - , parse/4 , offset_second/1 ]). --export([test/0]). - -define(DATE_PART, [ year , month @@ -69,174 +65,10 @@ format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> format(Time, Unit, Offset, Formatter) -> do_format(Time, time_unit(Unit), offset_second(Offset), Formatter). -%% for logger format -format(Time, Unit, Offset, Zones, Formatter) -> - do_format(Time, time_unit(Unit), offset_second(Offset), Zones, Formatter). - -test() -> - Units = [second, millisecond, microsecond, nanosecond], - Formatters = [ - <<"%Y-%m-%d %H:%M:%S:%N%z">>, - <<"%Y-%m-%d %H:%M:%S:%N%:z">>, - <<"%Y-%m-%d %H:%M:%S:%N%::z">>, - <<"%Y-%m-%d %H:%M:%S:%N">>, - <<"%Y-%m-%d %H:%M:%S:%3N">>, - <<"%Y-%m-%d %H:%M:%S:%6N">> - ], - Offsets = [ - 0, - 8 * 3600, - -8 * 3600, - <<"+08:00">>, - <<"+08:00:01">>, - <<"+0802">>, - <<"-08:00">>, - <<"local">> - ], - [ - [ - [begin - Time = erlang:system_time(U), - FormatDate = erlang:list_to_binary(format(Time, U, O, F)), - PTime = emqx_calendar:parse(FormatDate, U, O, F), - io:format("~p ~p ~p ~p ~n~p~n~p~n", [U, O, F, FormatDate, Time, PTime]), - % Equal = (PTime == Time), - % F2 = - % fun() -> - % Delta = Time - PTime, - % DeltaFmt = erlang:list_to_binary(format(Delta, U, 0, <<"%Y-%m-%d %H:%M:%S:%N">>)), - % io:format("delta ~p~n ~p~n-----~n", [Delta, DeltaFmt]) - % end, - % Equal orelse F2() - ok - end||O <- Offsets] - || F <-Formatters] - || U <-Units], - ok. - -parse(DateStr, Unit, FormatterBin) -> - parse(DateStr, Unit, 0, FormatterBin). - -parse(DateStr, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> - parse(DateStr, Unit, Offset, formatter(FormatterBin)); -parse(DateStr, Unit, Offset, Formatter) -> - do_parse(DateStr, Unit, offset_second(Offset), Formatter). - -do_parse(DateStr, Unit, Offset, Formatter) -> - DateInfo = do_parse_date_str(DateStr, Formatter, #{}), - {Precise, PrecisionUnit} = precise(DateInfo), - Counter = - fun - (year, V, Res) -> - Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); - (month, V, Res) -> - Res + nano_sm(V); - (day, V, Res) -> - Res + (V * ?SECONDS_PER_DAY * Precise); - (hour, V, Res) -> - Res + (V * ?SECONDS_PER_HOUR * Precise); - (minute, V, Res) -> - Res + (V * ?SECONDS_PER_MINUTE * Precise); - (second, V, Res) -> - Res + V * Precise; - (millisecond, V, Res) -> - case PrecisionUnit of - millisecond -> - Res + V; - microsecond -> - Res + (V * 1000); - nanosecond -> - Res + (V * 1000000) - end; - (microsecond, V, Res) -> - case PrecisionUnit of - microsecond -> - Res + V; - nanosecond -> - Res + (V * 1000) - end; - (nanosecond, V, Res) -> - Res + V; - (parsed_offset, V, Res) -> - Res - V - end, - Counter1 = - fun(K,V,R) -> - Res = Counter(K,V,R), - % io:format("K, V, R = ~p, ~p, ~p, Res = ~p~n", [K, V, R, Res - R]), - Res - end, - Count0 = maps:fold(Counter1, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), - Count = - case maps:is_key(parsed_offset, DateInfo) of - true -> - Count0; - false -> - (Offset * Precise) + Count0 - end, - erlang:convert_time_unit(Count, PrecisionUnit, Unit). - -precise(#{nanosecond := _}) -> {1000_000_000, nanosecond}; -precise(#{microsecond := _}) -> {1000_000, microsecond}; -precise(#{millisecond := _}) -> {1000, millisecond}; -precise(#{second := _}) -> {1, second}; -precise(_) -> {1, second}. - -do_parse_date_str(<<>>, _, Result) -> Result; -do_parse_date_str(_, [], Result) -> Result; -do_parse_date_str(Date, [Key | Formatter], Result) -> - Size = date_size(Key), - <> = Date, - case lists:member(Key, ?DATE_PART) of - true -> - do_parse_date_str(Tail, Formatter, Result#{Key => erlang:binary_to_integer(DatePart)}); - false -> - case lists:member(Key, ?DATE_ZONE_NAME) of - true -> - io:format("DatePart -------------- ~p ~p~n", [DatePart, offset_second(DatePart)]), - do_parse_date_str(Tail, Formatter, Result#{parsed_offset => offset_second(DatePart)}); - false -> - do_parse_date_str(Tail, Formatter, Result) - end - end. - -date_size(Str) when is_list(Str) -> erlang:length(Str); -date_size(DateName) -> - Map = #{ - year => 4, - month => 2, - day => 2, - hour => 2, - minute => 2, - second => 2, - millisecond => 3, - microsecond => 6, - nanosecond => 9, - timezone => 5, - timezone1 => 6, - timezone2 => 9 - }, - maps:get(DateName, Map). - -nano_sm(Month) -> dm(Month) * ?SECONDS_PER_DAY * 1000_000. - -dm(Month) -> - MonthDays = #{ - 1 => 0, - 2 => 31, - 3 => 59, - 4 => 90, - 5 => 120, - 6 => 151, - 7 => 181, - 8 => 212, - 9 => 243, - 10 => 273, - 11 => 304, - 12 => 334 - }, - maps:get(Month, MonthDays). - +parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) -> + parse(DateStr, Unit, formatter(FormatterBin)); +parse(DateStr, Unit, Formatter) -> + do_parse(DateStr, Unit, Formatter). %% ------------------------------------------------------------------------------------------------- %% internal %% emqx_calendar:format(erlang:system_time(second), <<"second">>, <<"+8:00">> ,<<"%Y-%m-%d-%H:%M:%S%Z">>). @@ -315,10 +147,6 @@ do_offset_second(Offset) when is_list(Offset) -> PosNeg * (Hour * 3600 + Minute * 60 + Second). do_format(Time, Unit, Offset, Formatter) -> - Timezones = formatter_timezones(Offset, Formatter, #{}), - do_format(Time, Unit, Offset, Timezones, Formatter). - -do_format(Time, Unit, Offset, Timezones, Formatter) -> Adjustment = erlang:convert_time_unit(Offset, second, Unit), AdjustedTime = Time + Adjustment, Factor = factor(Unit), @@ -336,6 +164,7 @@ do_format(Time, Unit, Offset, Timezones, Formatter) -> microsecond => trans_x_second(Unit, microsecond, Time), nanosecond => trans_x_second(Unit, nanosecond, Time) }, + Timezones = formatter_timezones(Offset, Formatter, #{}), DateWithZone = maps:merge(Date, Timezones), [maps:get(Key, DateWithZone, Key) || Key <- Formatter]. @@ -493,12 +322,12 @@ do_trans_x_second(second, second, Time) -> Time div 60; do_trans_x_second(second, _, _Time) -> 0; do_trans_x_second(millisecond, millisecond, Time) -> Time rem 1000; -do_trans_x_second(millisecond, microsecond, _Time) -> 0; -do_trans_x_second(millisecond, nanosecond, _Time) -> 0; +do_trans_x_second(millisecond, microsecond, Time) -> (Time rem 1000) * 1000; +do_trans_x_second(millisecond, nanosecond, Time) -> (Time rem 1000) * 1000_000; do_trans_x_second(microsecond, millisecond, Time) -> Time div 1000 rem 1000; do_trans_x_second(microsecond, microsecond, Time) -> Time rem 1000000; -do_trans_x_second(microsecond, nanosecond, _Time) -> 0; +do_trans_x_second(microsecond, nanosecond, Time) -> (Time rem 1000000) * 1000; do_trans_x_second(nanosecond, millisecond, Time) -> Time div 1000000 rem 1000; do_trans_x_second(nanosecond, microsecond, Time) -> Time div 1000 rem 1000000; @@ -510,3 +339,106 @@ padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> [$0 | padding(Data, Len - 1)]; padding(Data, _Len) -> Data. + +%% ------------------------------------------------------------------------------------------------- +%% internal +%% parse part + +do_parse(DateStr, Unit, Formatter) -> + DateInfo = do_parse_date_str(DateStr, Formatter, #{}), + {Precise, PrecisionUnit} = precise(DateInfo), + Counter = + fun + (year, V, Res) -> + Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); + (month, V, Res) -> + Res + dm(V) * ?SECONDS_PER_DAY * Precise; + (day, V, Res) -> + Res + (V * ?SECONDS_PER_DAY * Precise); + (hour, V, Res) -> + Res + (V * ?SECONDS_PER_HOUR * Precise); + (minute, V, Res) -> + Res + (V * ?SECONDS_PER_MINUTE * Precise); + (second, V, Res) -> + Res + V * Precise; + (millisecond, V, Res) -> + case PrecisionUnit of + millisecond -> + Res + V; + microsecond -> + Res + (V * 1000); + nanosecond -> + Res + (V * 1000000) + end; + (microsecond, V, Res) -> + case PrecisionUnit of + microsecond -> + Res + V; + nanosecond -> + Res + (V * 1000) + end; + (nanosecond, V, Res) -> + Res + V; + (parsed_offset, V, Res) -> + Res - V + end, + Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), + erlang:convert_time_unit(Count, PrecisionUnit, Unit). + +precise(#{nanosecond := _}) -> {1000_000_000, nanosecond}; +precise(#{microsecond := _}) -> {1000_000, microsecond}; +precise(#{millisecond := _}) -> {1000, millisecond}; +precise(#{second := _}) -> {1, second}; +precise(_) -> {1, second}. + +do_parse_date_str(<<>>, _, Result) -> Result; +do_parse_date_str(_, [], Result) -> Result; +do_parse_date_str(Date, [Key | Formatter], Result) -> + Size = date_size(Key), + <> = Date, + case lists:member(Key, ?DATE_PART) of + true -> + do_parse_date_str(Tail, Formatter, Result#{Key => erlang:binary_to_integer(DatePart)}); + false -> + case lists:member(Key, ?DATE_ZONE_NAME) of + true -> + do_parse_date_str(Tail, Formatter, Result#{parsed_offset => offset_second(DatePart)}); + false -> + do_parse_date_str(Tail, Formatter, Result) + end + end. + +date_size(Str) when is_list(Str) -> erlang:length(Str); +date_size(DateName) -> + Map = #{ + year => 4, + month => 2, + day => 2, + hour => 2, + minute => 2, + second => 2, + millisecond => 3, + microsecond => 6, + nanosecond => 9, + timezone => 5, + timezone1 => 6, + timezone2 => 9 + }, + maps:get(DateName, Map). + +dm(Month) -> + MonthDays = #{ + 1 => 0, + 2 => 31, + 3 => 59, + 4 => 90, + 5 => 120, + 6 => 151, + 7 => 181, + 8 => 212, + 9 => 243, + 10 => 273, + 11 => 304, + 12 => 334 + }, + maps:get(Month, MonthDays). diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index b13871384..16cc3a414 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -39,7 +39,7 @@ format(#{msg := Msg0, meta := Meta} = Event, logger_formatter:format(Event#{msg := Msg}, Config#{template => Template}); format(#{msg := Msg0, meta := Meta} = Event, - #{timezone_offset := TZO, timezone := TZ, date_format := DFS} = Config) -> + #{timezone_offset := TZO, date_format := DFS} = Config) -> Msg = maybe_merge(Msg0, Meta), Time = case maps:get(time, Event, undefined) of @@ -48,7 +48,7 @@ format(#{msg := Msg0, meta := Meta} = Event, T -> T end, - Date = emqx_calendar:format(Time, microsecond, TZO, TZ, DFS), + Date = emqx_calendar:format(Time, microsecond, TZO, DFS), [Date | logger_formatter:format(Event#{msg := Msg}, Config)]. maybe_merge({report, Report}, Meta) when is_map(Report) -> From e1ad8aab465cc4723a7f18e02b7df029abe8d450 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 26 May 2022 12:50:16 +0800 Subject: [PATCH 05/10] fix: date format funs & logger conf & appup & change log --- CHANGES-4.3.md | 2 +- .../test/emqx_rule_funcs_SUITE.erl | 9 +- etc/emqx.conf | 25 ++-- priv/emqx.schema | 67 --------- src/emqx.appup.src | 129 +++++++++++++----- src/emqx_calendar.erl | 1 - src/emqx_logger_textfmt.erl | 4 +- 7 files changed, 112 insertions(+), 125 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index da2df758d..dd75320b7 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -21,7 +21,7 @@ File format: Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists. * Better randomisation of app screts (changed from timestamp seeded sha hash (uuid) to crypto:strong_rand_bytes) * Return a client_identifier_not_valid error when username is empty and username_as_clientid is set to true [#7862] -* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/4 [#7894] +* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/3, date_to_unix_ts/4 [#7894] * Add proto_name and proto_ver fields for $event/client_disconnected event. * Mnesia auth/acl http api support multiple condition queries. * Inflight QoS1 Messages for shared topics are now redispatched to another alive subscribers upon chosen subscriber session termination. diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 632697a87..fe9cf688b 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -736,8 +736,7 @@ prop_format_date_fun() -> Args3 = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Second], Formatters3 = apply_func(format_date, Args3), % -04:00 remove offset - OffsetSeconds = emqx_rule_funcs:timezone_to_second(<<"-04:00">>), - Args3DTUS = [<<"second">>, OffsetSeconds, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], + Args3DTUS = [<<"second">>, <<"-04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], Second == apply_func(date_to_unix_ts, Args3DTUS). apply_func(Name, Args) when is_atom(Name) -> @@ -888,9 +887,9 @@ message() -> %% CT functions %%------------------------------------------------------------------------------ -all() -> [t_format_date_funcs]. - % IsTestCase = fun("t_" ++ _) -> true; (_) -> false end, - % [F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))]. +all() -> + IsTestCase = fun("t_" ++ _) -> true; (_) -> false end, + [F || {F, _A} <- module_info(exports), IsTestCase(atom_to_list(F))]. suite() -> [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}]. diff --git a/etc/emqx.conf b/etc/emqx.conf index fcc8557a2..161f9fc2b 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -460,37 +460,28 @@ log.file = emqx.log ## Value: text | json log.formatter = text -## Text logger, date format. +## Format of the text logger. ## -## Value: rfc3339 | FormatString -## FormatString: +## Value: rfc3339 | FORMAT +## Where FORMAT is the format string of the timestamp. Supported specifiers: ## %Y: year ## %m: month ## %d: day ## %H: hour ## %M: minute ## %S: second -## %6N: microseconds -## %3N: milliseconds +## %N: nanoseconds (000000000 - 999999999) +## %6N: microseconds (00000 - 999999) +## %3N: milliseconds (000 - 999) ## %z: timezone, [+-]HHMM ## %:z: timezone, [+-]HH:MM ## %::z: timezone, [+-]HH:MM:SS +## ## For example: -## emqx-server-date: %Y-%m-%dT%H:%M:%S.%6N %:z +## log.formatter.text.date.format = %Y-%m-%dT%H:%M:%S.%6N %:z ## Default: rfc3339 # log.formatter.text.date.format = rfc3339 -## Text logger, timezone. -## Only takes effect when log.formatter.text.date.format not rfc3339. -## -## Value: local | z | UTC TimeOffset -## TimeOffset: [+-]HHMM or [+|-]HH:MM or [+|-]HH:MM:SS, for example: +08:00. -## local is the system local time timezone. -## z: UTC zulu timezone. -## Z: UTC zulu timezone. Same as z. -## Default: local -# log.formatter.text.date.timezone = local - ## Log to single line ## Value: Boolean #log.single_line = true diff --git a/priv/emqx.schema b/priv/emqx.schema index 96d0005ad..304fc0634 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -534,12 +534,6 @@ end}. {datatype, string} ]}. -%% @doc format logs as text, date format part. local or any UTC offset -{mapping, "log.formatter.text.date.timezone", "kernel.logger", [ - {default, "local"}, - {datatype, string} -]}. - %% @doc format logs in a single line. {mapping, "log.single_line", "kernel.logger", [ {default, true}, @@ -668,70 +662,9 @@ end}. end, DateStrTrans(list_to_binary(DateStr), []) end, - {DateOffsetStr, DateOffset} = - case cuttlefish:conf_get("log.formatter.text.date.timezone", Conf, "local") of - "z" -> - {"z", "z"}; - "Z" -> - {"Z", "Z"}; - "local" -> - UniversalTime = - calendar:system_time_to_universal_time( - erlang:system_time(second), - second), - LocalTime = erlang:universaltime_to_localtime(UniversalTime), - LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), - UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), - OffsetSecond = LocalSecs - UniversalSecs, - {Sign, PosOffsetSecond} = - case OffsetSecond >= 0 of - true -> {$+, OffsetSecond}; - false -> {$-, - OffsetSecond} - end, - {H, M, S} = calendar:seconds_to_time(PosOffsetSecond), - Str = - case S of - 0 -> - io_lib:format("~c~2.10.0B:~2.10.0B", [Sign, H, M]); - S -> - io_lib:format("~c~2.10.0B:~2.10.0B:~2.10.0B", [Sign, H, M, S]) - end, - {Str, OffsetSecond}; - DateOStr -> - Sign = hd(DateOStr), - Signs = #{$+ => 1, $- => -1}, - case maps:get(Sign, Signs, undefined) of - undefined -> - error(bad_logger_offset); - PosNeg -> - try - [Sign | HM] = DateOStr, - {HourStr, MinuteStr, SecondStr} = - case string:tokens(HM, ":") of - [H, M] -> - {H, M, "0"}; - [H, M, S] -> - {H, M, S}; - [HHMM] when erlang:length(HHMM) == 4 -> - {string:sub_string(HHMM, 1,2), string:sub_string(HHMM, 3,4), "0"} - end, - Hour = erlang:list_to_integer(HourStr), - Minute = erlang:list_to_integer(MinuteStr), - Second = erlang:list_to_integer(SecondStr), - true = (Hour =< 23), - true = (Minute =< 59), - true = (Second =< 59), - {DateOStr, PosNeg * (Hour * 3600 + Minute * 60 + Second)} - catch _E : _R -> - error(bad_logger_offset) - end - end - end, {emqx_logger_textfmt, #{ date_format => DateFormat, - timezone_offset => DateOffset, - timezone => DateOffsetStr, template => [" [",level,"] ", {clientid, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 0b5361758..1f1443f37 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,9 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.15", - [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, @@ -16,7 +18,9 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, @@ -34,7 +38,9 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -58,7 +64,9 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -86,7 +94,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -116,7 +126,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -146,7 +158,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -180,7 +194,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -214,7 +230,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -248,7 +266,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -282,7 +302,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -316,7 +338,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -350,7 +374,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -385,7 +411,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -420,7 +448,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -458,7 +487,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -500,7 +530,9 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.15", - [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -513,7 +545,9 @@ {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -530,7 +564,9 @@ {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, @@ -553,7 +589,10 @@ {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -579,7 +618,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -607,7 +649,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -635,7 +680,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -667,7 +715,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -699,7 +750,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -731,7 +784,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -763,7 +818,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -795,7 +852,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -827,7 +886,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -860,7 +921,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -893,7 +956,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -929,7 +993,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_calendar.erl b/src/emqx_calendar.erl index c7e2d627f..7f9faf3be 100644 --- a/src/emqx_calendar.erl +++ b/src/emqx_calendar.erl @@ -71,7 +71,6 @@ parse(DateStr, Unit, Formatter) -> do_parse(DateStr, Unit, Formatter). %% ------------------------------------------------------------------------------------------------- %% internal -%% emqx_calendar:format(erlang:system_time(second), <<"second">>, <<"+8:00">> ,<<"%Y-%m-%d-%H:%M:%S%Z">>). time_unit(second) -> second; time_unit(millisecond) -> millisecond; diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index 16cc3a414..ba13c0cb7 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -39,7 +39,7 @@ format(#{msg := Msg0, meta := Meta} = Event, logger_formatter:format(Event#{msg := Msg}, Config#{template => Template}); format(#{msg := Msg0, meta := Meta} = Event, - #{timezone_offset := TZO, date_format := DFS} = Config) -> + #{date_format := DFS} = Config) -> Msg = maybe_merge(Msg0, Meta), Time = case maps:get(time, Event, undefined) of @@ -48,7 +48,7 @@ format(#{msg := Msg0, meta := Meta} = Event, T -> T end, - Date = emqx_calendar:format(Time, microsecond, TZO, DFS), + Date = emqx_calendar:format(Time, microsecond, local, DFS), [Date | logger_formatter:format(Event#{msg := Msg}, Config)]. maybe_merge({report, Report}, Meta) when is_map(Report) -> From 64a455bf8ee1180e81aed3c92a5033532b8ddc79 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 26 May 2022 14:14:18 +0800 Subject: [PATCH 06/10] fix: old date format style example & some code format --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 2 +- .../test/emqx_rule_funcs_SUITE.erl | 6 +- etc/emqx.conf | 4 + src/emqx.appup.src | 5 - src/emqx_calendar.erl | 101 ++++++++---------- src/emqx_logger_textfmt.erl | 3 +- 6 files changed, 55 insertions(+), 66 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index a8b347f5d..18e61967e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -955,7 +955,7 @@ date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> Unit = time_unit(TimeUnit), OffsetSecond = emqx_calendar:offset_second(Offset), OffsetDelta = erlang:convert_time_unit(OffsetSecond, second, Unit), - OffsetDelta + date_to_unix_ts(Unit, FormatString, InputString). + date_to_unix_ts(Unit, FormatString, InputString) - OffsetDelta. mongo_date() -> erlang:timestamp(). diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index fe9cf688b..aa2781e8c 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -714,10 +714,9 @@ t_format_date_funcs(_) -> prop_format_date_fun() -> Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%Y---%H:%M:%S%z">>], - Args1DTUS = [<<"second">>, 0, <<"%m--%d--%Y---%H:%M:%S%z">>], ?FORALL(S, erlang:system_time(second), S == apply_func(date_to_unix_ts, - Args1DTUS ++ [apply_func(format_date, + Args1 ++ [apply_func(format_date, Args1 ++ [S])])), Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], Args2DTUS = [<<"millisecond">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], @@ -735,8 +734,7 @@ prop_format_date_fun() -> Second = erlang:system_time(second), Args3 = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Second], Formatters3 = apply_func(format_date, Args3), - % -04:00 remove offset - Args3DTUS = [<<"second">>, <<"-04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], + Args3DTUS = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], Second == apply_func(date_to_unix_ts, Args3DTUS). apply_func(Name, Args) when is_atom(Name) -> diff --git a/etc/emqx.conf b/etc/emqx.conf index 161f9fc2b..0f8f93a91 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -479,6 +479,10 @@ log.formatter = text ## ## For example: ## log.formatter.text.date.format = %Y-%m-%dT%H:%M:%S.%6N %:z +## +## Before 4.2, the default date format was: +## log.formatter.text.date.format = %Y-%m-%d %H:%M:%S.%3N +## ## Default: rfc3339 # log.formatter.text.date.format = rfc3339 diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 1f1443f37..9dbf47f6f 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -591,7 +591,6 @@ {"4.3.12", [{delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -641,7 +640,6 @@ {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -673,7 +671,6 @@ {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -708,7 +705,6 @@ {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -743,7 +739,6 @@ {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_calendar.erl b/src/emqx_calendar.erl index 7f9faf3be..0c094df81 100644 --- a/src/emqx_calendar.erl +++ b/src/emqx_calendar.erl @@ -55,7 +55,7 @@ formatter(FormatterBin) when is_binary(FormatterBin) -> do_formatter(FormatterBin, []). offset_second(Offset) -> - do_offset_second(Offset). + offset_second_(Offset). format(Time, Unit, Formatter) -> format(Time, Unit, undefined, Formatter). @@ -105,24 +105,24 @@ do_formatter(<>, [Str | Formatter]) when is_list(Str) -> do_formatter(Tail, [lists:append(Str, [Char]) | Formatter]); do_formatter(<>, Formatter) -> do_formatter(Tail, [[Char] | Formatter]). -do_offset_second(OffsetSecond) when is_integer(OffsetSecond) -> OffsetSecond; -do_offset_second(undefined) -> 0; -do_offset_second("local") -> do_offset_second(local); -do_offset_second(<<"local">>) -> do_offset_second(local); -do_offset_second(local) -> +offset_second_(OffsetSecond) when is_integer(OffsetSecond) -> OffsetSecond; +offset_second_(undefined) -> 0; +offset_second_("local") -> offset_second_(local); +offset_second_(<<"local">>) -> offset_second_(local); +offset_second_(local) -> UniversalTime = calendar:system_time_to_universal_time(erlang:system_time(second), second), LocalTime = erlang:universaltime_to_localtime(UniversalTime), LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), LocalSecs - UniversalSecs; -do_offset_second(Offset) when is_binary(Offset) -> - do_offset_second(erlang:binary_to_list(Offset)); -do_offset_second("Z") -> 0; -do_offset_second("z") -> 0; -do_offset_second(Offset) when is_list(Offset) -> +offset_second_(Offset) when is_binary(Offset) -> + offset_second_(erlang:binary_to_list(Offset)); +offset_second_("Z") -> 0; +offset_second_("z") -> 0; +offset_second_(Offset) when is_list(Offset) -> Sign = hd(Offset), ((Sign == $+) orelse (Sign == $-)) - orelse error({bad_zone, Offset}), + orelse error({bad_time_offset, Offset}), Signs = #{$+ => 1, $- => -1}, PosNeg = maps:get(Sign, Signs), [Sign | HM] = Offset, @@ -135,14 +135,14 @@ do_offset_second(Offset) when is_list(Offset) -> [HHMM] when erlang:length(HHMM) == 4 -> {string:sub_string(HHMM, 1,2), string:sub_string(HHMM, 3,4), "0"}; _ -> - error({bad_zone, Offset}) + error({bad_time_offset, Offset}) end, Hour = erlang:list_to_integer(HourStr), Minute = erlang:list_to_integer(MinuteStr), Second = erlang:list_to_integer(SecondStr), - (Hour =< 23) orelse error({bad_hour, Hour}), - (Minute =< 59) orelse error({bad_minute, Minute}), - (Second =< 59) orelse error({bad_second, Second}), + (Hour =< 23) orelse error({bad_time_offset_hour, Hour}), + (Minute =< 59) orelse error({bad_time_offset_minute, Minute}), + (Second =< 59) orelse error({bad_time_offset_second, Second}), PosNeg * (Hour * 3600 + Minute * 60 + Second). do_format(Time, Unit, Offset, Formatter) -> @@ -345,7 +345,7 @@ padding(Data, _Len) -> do_parse(DateStr, Unit, Formatter) -> DateInfo = do_parse_date_str(DateStr, Formatter, #{}), - {Precise, PrecisionUnit} = precise(DateInfo), + {Precise, PrecisionUnit} = precision(DateInfo), Counter = fun (year, V, Res) -> @@ -384,11 +384,11 @@ do_parse(DateStr, Unit, Formatter) -> Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), erlang:convert_time_unit(Count, PrecisionUnit, Unit). -precise(#{nanosecond := _}) -> {1000_000_000, nanosecond}; -precise(#{microsecond := _}) -> {1000_000, microsecond}; -precise(#{millisecond := _}) -> {1000, millisecond}; -precise(#{second := _}) -> {1, second}; -precise(_) -> {1, second}. +precision(#{nanosecond := _}) -> {1000_000_000, nanosecond}; +precision(#{microsecond := _}) -> {1000_000, microsecond}; +precision(#{millisecond := _}) -> {1000, millisecond}; +precision(#{second := _}) -> {1, second}; +precision(_) -> {1, second}. do_parse_date_str(<<>>, _, Result) -> Result; do_parse_date_str(_, [], Result) -> Result; @@ -408,36 +408,29 @@ do_parse_date_str(Date, [Key | Formatter], Result) -> end. date_size(Str) when is_list(Str) -> erlang:length(Str); -date_size(DateName) -> - Map = #{ - year => 4, - month => 2, - day => 2, - hour => 2, - minute => 2, - second => 2, - millisecond => 3, - microsecond => 6, - nanosecond => 9, - timezone => 5, - timezone1 => 6, - timezone2 => 9 - }, - maps:get(DateName, Map). +date_size(year) -> 4; +date_size(month) -> 2; +date_size(day) -> 2; +date_size(hour) -> 2; +date_size(minute) -> 2; +date_size(second) -> 2; +date_size(millisecond) -> 3; +date_size(microsecond) -> 6; +date_size(nanosecond) -> 9; +date_size(timezone) -> 5; +date_size(timezone1) -> 6; +date_size(timezone2) -> 9. + +dm(1) -> 0; +dm(2) -> 31; +dm(3) -> 59; +dm(4) -> 90; +dm(5) -> 120; +dm(6) -> 151; +dm(7) -> 181; +dm(8) -> 212; +dm(9) -> 243; +dm(10) -> 273; +dm(11) -> 304; +dm(12) -> 334. -dm(Month) -> - MonthDays = #{ - 1 => 0, - 2 => 31, - 3 => 59, - 4 => 90, - 5 => 120, - 6 => 151, - 7 => 181, - 8 => 212, - 9 => 243, - 10 => 273, - 11 => 304, - 12 => 334 - }, - maps:get(Month, MonthDays). diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index ba13c0cb7..1d1bb8dcd 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -29,7 +29,7 @@ ]). check_config(Config0) -> - Config = maps:without([timezone_offset, timezone, date_format], Config0), + Config = maps:without([date_format], Config0), logger_formatter:check_config(Config). format(#{msg := Msg0, meta := Meta} = Event, @@ -37,7 +37,6 @@ format(#{msg := Msg0, meta := Meta} = Event, Msg = maybe_merge(Msg0, Meta), Template = [time | Template0], logger_formatter:format(Event#{msg := Msg}, Config#{template => Template}); - format(#{msg := Msg0, meta := Meta} = Event, #{date_format := DFS} = Config) -> Msg = maybe_merge(Msg0, Meta), From aabfb718043dd2d5a90ee0a13ad522f91b405c28 Mon Sep 17 00:00:00 2001 From: Georgy Sychev Date: Thu, 26 May 2022 12:32:15 +0400 Subject: [PATCH 07/10] chore: changes.md --- CHANGES-4.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 68a0b4b0c..5f904913f 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -30,6 +30,7 @@ File format: * Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message * Rule-engine function hexstr2bin/1 support half byte [#7977] * Add rule-engine function float2str/2, user can specify the float output precision [#7991] +* Shared message delivery when all alive shared subs have full inflight * Improved resilience against autocluster partitioning during cluster startup. [#7876] From 9274899ce96347465c490152385fa8a02777e28c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 26 May 2022 16:53:56 +0200 Subject: [PATCH 08/10] ci: delete raspbian10 from build list --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index fc2a23b99..309d56ee5 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -227,7 +227,7 @@ jobs: - centos8 - centos7 - centos6 - - raspbian10 + # - raspbian10 #armv6l is too slow to emulate # - raspbian9 exclude: - os: centos6 From 29622558c586d0d19cd8b37cc833f5d2d7bfb3e0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 25 May 2022 11:31:50 +0200 Subject: [PATCH 09/10] ci: pin emqx-buider 4.4-12 Centos/Rockylinux had troulbe booting because OpenSSL version was too new (1.1.1n). builder 4.4-12 has * Centos7: only one version (1.1.1k) OpenSSL installed * Centos8: go back to use default OpenSSL (1.1.1k) --- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/apps_version_check.yaml | 2 +- .github/workflows/build_packages.yaml | 13 +++++++------ .github/workflows/build_slim_packages.yaml | 5 +++-- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 2 +- Makefile | 2 +- deploy/docker/Dockerfile | 2 +- scripts/buildx.sh | 6 +++--- 12 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 819522dbd..a95acc772 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 16dc5c218..9ab6fa5ad 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -13,7 +13,7 @@ jobs: os: - ubuntu20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 09e9aaa6a..5c861a88f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -34,6 +34,7 @@ jobs: shell: bash working-directory: source run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" if make emqx-ee --dry-run > /dev/null 2>&1; then echo "::set-output name=profiles::[\"emqx-ee\"]" else @@ -217,8 +218,8 @@ jobs: - debian11 - debian10 - debian9 - - rockylinux8 - - centos7 + - el8 + - el7 - raspbian10 exclude: - package: pkg @@ -265,7 +266,7 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --builder "ghcr.io/emqx/emqx-builder/4.4-8:${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/4.4-12:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: @@ -342,7 +343,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -358,7 +359,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.otp }}-alpine3.15.1 + BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.otp }}-alpine3.15.1 RUN_FROM=alpine:3.15.1 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index dab02d1ec..2ac23d19c 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -21,14 +21,15 @@ jobs: - 24.1.5-3 os: - ubuntu20.04 - - rockylinux8 + - el8 - container: ghcr.io/emqx/emqx-builder/4.4-8:${{ matrix.erl_otp }}-${{ matrix.os }} + container: ghcr.io/emqx/emqx-builder/4.4-12:${{ matrix.erl_otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 - name: prepare run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" if make emqx-ee --dry-run > /dev/null 2>&1; then echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 2b49d4d77..e50738914 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ffcb69247..b03ac2077 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index 3b11a9368..b491ef47b 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index dbc79a8f5..db1a351c4 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -224,7 +224,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -275,7 +275,7 @@ jobs: otp: - 24.1.5-3 runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index efca499ea..d58486cda 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,7 +10,7 @@ on: jobs: run_proper_test: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 0ba09b68c..417423f4e 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-alpine3.15.1 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-alpine3.15.1 export EMQX_DEFAULT_RUNNER = alpine:3.15.1 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index d3a58ce74..63702e2c4 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-alpine3.15.1 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-alpine3.15.1 ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 6efc6bcc0..749ca565e 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-debian10 --arch arm64 +## ./scripts/buildx.sh --profile emqx --pkgtype zip --builder ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-debian10 --arch arm64 set -euo pipefail @@ -20,7 +20,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-8:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-12:24.1.5-3-debian10" } while [ "$#" -gt 0 ]; do @@ -77,4 +77,4 @@ docker run -i --rm \ --workdir /emqx \ --platform="linux/$ARCH" \ "$BUILDER" \ - bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" + bash -euc "git config --global --add safe.directory /emqx && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" From 0402903d710336a254c901870331a9ecdccf6bf5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 26 May 2022 23:49:52 +0200 Subject: [PATCH 10/10] chore: update to version 4.3.15-rc.3 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 43d9dfcb7..72e52a4f1 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.15-rc.2"}). +-define(EMQX_RELEASE, {opensource, "4.3.15-rc.3"}). -else.