diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 735025e2b..3f2bb99a0 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -1181,7 +1181,7 @@ format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> date_to_unix_ts(TimeUnit, FormatString, InputString) -> Unit = time_unit(TimeUnit), - emqx_utils_calendar:parse(InputString, Unit, FormatString). + emqx_utils_calendar:formatted_datetime_to_system_time(InputString, Unit, FormatString). date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> Unit = time_unit(TimeUnit), diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 5bcb48417..d74055a66 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -1143,6 +1143,50 @@ timezone_to_offset_seconds_helper(FunctionName) -> apply_func(FunctionName, [local]), ok. +t_date_to_unix_ts(_) -> + TestTab = [ + {{"2024-03-01T10:30:38+08:00", second}, [ + <<"second">>, <<"+08:00">>, <<"%Y-%m-%d %H-%M-%S">>, <<"2024-03-01 10:30:38">> + ]}, + {{"2024-03-01T10:30:38.333+08:00", second}, [ + <<"second">>, <<"+08:00">>, <<"%Y-%m-%d %H-%M-%S.%3N">>, <<"2024-03-01 10:30:38.333">> + ]}, + {{"2024-03-01T10:30:38.333+08:00", millisecond}, [ + <<"millisecond">>, + <<"+08:00">>, + <<"%Y-%m-%d %H-%M-%S.%3N">>, + <<"2024-03-01 10:30:38.333">> + ]}, + {{"2024-03-01T10:30:38.333+08:00", microsecond}, [ + <<"microsecond">>, + <<"+08:00">>, + <<"%Y-%m-%d %H-%M-%S.%3N">>, + <<"2024-03-01 10:30:38.333">> + ]}, + {{"2024-03-01T10:30:38.333+08:00", nanosecond}, [ + <<"nanosecond">>, + <<"+08:00">>, + <<"%Y-%m-%d %H-%M-%S.%3N">>, + <<"2024-03-01 10:30:38.333">> + ]}, + {{"2024-03-01T10:30:38.333444+08:00", microsecond}, [ + <<"microsecond">>, + <<"+08:00">>, + <<"%Y-%m-%d %H-%M-%S.%6N">>, + <<"2024-03-01 10:30:38.333444">> + ]} + ], + lists:foreach( + fun({{DateTime3339, Unit}, DateToTsArgs}) -> + ?assertEqual( + calendar:rfc3339_to_system_time(DateTime3339, [{unit, Unit}]), + apply_func(date_to_unix_ts, DateToTsArgs), + "Failed on test: " ++ DateTime3339 ++ "/" ++ atom_to_list(Unit) + ) + end, + TestTab + ). + t_parse_date_errors(_) -> ?assertError( bad_formatter_or_date, @@ -1154,6 +1198,37 @@ t_parse_date_errors(_) -> bad_formatter_or_date, emqx_rule_funcs:date_to_unix_ts(second, <<"%y-%m-%d %H:%M:%S">>, <<"2022-05-26 10:40:12">>) ), + %% invalid formats + ?assertThrow( + {missing_date_part, month}, + emqx_rule_funcs:date_to_unix_ts( + second, <<"%Y-%d %H:%M:%S">>, <<"2022-32 10:40:12">> + ) + ), + ?assertThrow( + {missing_date_part, year}, + emqx_rule_funcs:date_to_unix_ts( + second, <<"%H:%M:%S">>, <<"10:40:12">> + ) + ), + ?assertError( + _, + emqx_rule_funcs:date_to_unix_ts( + second, <<"%Y-%m-%d %H:%M:%S">>, <<"2022-05-32 10:40:12">> + ) + ), + ?assertError( + _, + emqx_rule_funcs:date_to_unix_ts( + second, <<"%Y-%m-%d %H:%M:%S">>, <<"2023-02-29 10:40:12">> + ) + ), + ?assertError( + _, + emqx_rule_funcs:date_to_unix_ts( + second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-30 10:40:12">> + ) + ), %% Compatibility test %% UTC+0 @@ -1173,25 +1248,42 @@ t_parse_date_errors(_) -> emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2022-05-26 10-40-12">>) ), - %% UTC+0 - UnixTsLeap0 = 1582986700, + %% leap year checks ?assertEqual( - UnixTsLeap0, - emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2020-02-29 14:31:40">>) + %% UTC+0 + 1709217100, + emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-29 14:31:40">>) ), - - %% UTC+0 - UnixTsLeap1 = 1709297071, ?assertEqual( - UnixTsLeap1, + %% UTC+0 + 1709297071, emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-01 12:44:31">>) ), - - %% UTC+0 - UnixTsLeap2 = 1709535387, ?assertEqual( - UnixTsLeap2, - emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-04 06:56:27">>) + %% UTC+0 + 4107588271, + emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2100-03-01 12:44:31">>) + ), + ?assertEqual( + %% UTC+8 + 1709188300, + emqx_rule_funcs:date_to_unix_ts( + second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-29 14:31:40">> + ) + ), + ?assertEqual( + %% UTC+8 + 1709268271, + emqx_rule_funcs:date_to_unix_ts( + second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-01 12:44:31">> + ) + ), + ?assertEqual( + %% UTC+8 + 4107559471, + emqx_rule_funcs:date_to_unix_ts( + second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2100-03-01 12:44:31">> + ) ), %% None zero zone shift with millisecond level precision diff --git a/apps/emqx_utils/src/emqx_utils_calendar.erl b/apps/emqx_utils/src/emqx_utils_calendar.erl index a3c1450cd..3c98c2828 100644 --- a/apps/emqx_utils/src/emqx_utils_calendar.erl +++ b/apps/emqx_utils/src/emqx_utils_calendar.erl @@ -22,7 +22,7 @@ formatter/1, format/3, format/4, - parse/3, + formatted_datetime_to_system_time/3, offset_second/1 ]). @@ -48,8 +48,9 @@ -define(DAYS_PER_YEAR, 365). -define(DAYS_PER_LEAP_YEAR, 366). -define(DAYS_FROM_0_TO_1970, 719528). --define(SECONDS_FROM_0_TO_1970, (?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY)). - +-define(DAYS_FROM_0_TO_10000, 2932897). +-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY). +-define(SECONDS_FROM_0_TO_10000, (?DAYS_FROM_0_TO_10000 * ?SECONDS_PER_DAY)). %% the maximum value is the SECONDS_FROM_0_TO_10000 in the calendar.erl, %% here minus SECONDS_PER_DAY to tolerate timezone time offset, %% so the maximum date can reach 9999-12-31 which is ample. @@ -171,10 +172,10 @@ format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> format(Time, Unit, Offset, Formatter) -> do_format(Time, time_unit(Unit), offset_second(Offset), Formatter). -parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) -> - parse(DateStr, Unit, formatter(FormatterBin)); -parse(DateStr, Unit, Formatter) -> - do_parse(DateStr, Unit, Formatter). +formatted_datetime_to_system_time(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) -> + formatted_datetime_to_system_time(DateStr, Unit, formatter(FormatterBin)); +formatted_datetime_to_system_time(DateStr, Unit, Formatter) -> + do_formatted_datetime_to_system_time(DateStr, Unit, Formatter). %%-------------------------------------------------------------------- %% Time unit @@ -467,56 +468,51 @@ padding(Data, _Len) -> Data. %%-------------------------------------------------------------------- -%% internal: parse part +%% internal: formatted_datetime_to_system_time part %%-------------------------------------------------------------------- -do_parse(DateStr, Unit, Formatter) -> +do_formatted_datetime_to_system_time(DateStr, Unit, Formatter) -> DateInfo = do_parse_date_str(DateStr, Formatter, #{}), - {Precise, PrecisionUnit} = precision(DateInfo), - Counter = - fun - (year, V, Res) -> - Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); - (month, V, Res) -> - Dm = dym(maps:get(year, DateInfo, 0), V), - Res + Dm * ?SECONDS_PER_DAY * Precise; - (day, V, Res) -> - Res + (V * ?SECONDS_PER_DAY * Precise); - (hour, V, Res) -> - Res + (V * ?SECONDS_PER_HOUR * Precise); - (minute, V, Res) -> - Res + (V * ?SECONDS_PER_MINUTE * Precise); - (second, V, Res) -> - Res + V * Precise; - (millisecond, V, Res) -> - case PrecisionUnit of - millisecond -> - Res + V; - microsecond -> - Res + (V * 1000); - nanosecond -> - Res + (V * 1000000) - end; - (microsecond, V, Res) -> - case PrecisionUnit of - microsecond -> - Res + V; - nanosecond -> - Res + (V * 1000) - end; - (nanosecond, V, Res) -> - Res + V; - (parsed_offset, V, Res) -> - Res - V * Precise - end, - Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), - erlang:convert_time_unit(Count, PrecisionUnit, Unit). + PrecisionUnit = precision(DateInfo), + ToPrecisionUnit = fun(Time, FromUnit) -> + erlang:convert_time_unit(Time, FromUnit, PrecisionUnit) + end, + GetRequiredPart = fun(Key) -> + case maps:get(Key, DateInfo, undefined) of + undefined -> throw({missing_date_part, Key}); + Value -> Value + end + end, + GetOptionalPart = fun(Key) -> maps:get(Key, DateInfo, 0) end, + Year = GetRequiredPart(year), + Month = GetRequiredPart(month), + Day = GetRequiredPart(day), + Hour = GetRequiredPart(hour), + Min = GetRequiredPart(minute), + Sec = GetRequiredPart(second), + DateTime = {{Year, Month, Day}, {Hour, Min, Sec}}, + TotalSecs = datetime_to_system_time(DateTime) - GetOptionalPart(parsed_offset), + check(TotalSecs, DateStr, Unit), + TotalTime = + ToPrecisionUnit(TotalSecs, second) + + ToPrecisionUnit(GetOptionalPart(millisecond), millisecond) + + ToPrecisionUnit(GetOptionalPart(microsecond), microsecond) + + ToPrecisionUnit(GetOptionalPart(nanosecond), nanosecond), + erlang:convert_time_unit(TotalTime, PrecisionUnit, Unit). -precision(#{nanosecond := _}) -> {1000_000_000, nanosecond}; -precision(#{microsecond := _}) -> {1000_000, microsecond}; -precision(#{millisecond := _}) -> {1000, millisecond}; -precision(#{second := _}) -> {1, second}; -precision(_) -> {1, second}. +check(Secs, _, _) when Secs >= -?SECONDS_FROM_0_TO_1970, Secs < ?SECONDS_FROM_0_TO_10000 -> + ok; +check(_Secs, DateStr, Unit) -> + throw({bad_format, #{date_string => DateStr, to_unit => Unit}}). + +datetime_to_system_time(DateTime) -> + calendar:datetime_to_gregorian_seconds(DateTime) - ?SECONDS_FROM_0_TO_1970. + +precision(#{nanosecond := _}) -> nanosecond; +precision(#{microsecond := _}) -> microsecond; +precision(#{millisecond := _}) -> millisecond; +precision(#{second := _}) -> second; +precision(_) -> second. do_parse_date_str(<<>>, _, Result) -> Result; diff --git a/changes/ce/fix-12668.en.md b/changes/ce/fix-12668.en.md new file mode 100644 index 000000000..c8ff95f9e --- /dev/null +++ b/changes/ce/fix-12668.en.md @@ -0,0 +1,2 @@ +Refactor the SQL function: `date_to_unix_ts()` by using `calendar:datetime_to_gregorian_seconds/1`. +This change also added validation for the input date format.