diff --git a/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl b/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl index 0bbe24ae4..b5beb9ae7 100644 --- a/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl +++ b/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl @@ -516,7 +516,7 @@ do_t_session_expiration(_Config, Opts) -> t_session_gc(Config) -> GCInterval = ?config(gc_interval, Config), - [Node1, Node2, Node3] = Nodes = ?config(nodes, Config), + [Node1, Node2, _Node3] = Nodes = ?config(nodes, Config), CoreNodes = [Node1, Node2], [ Port1, diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl index ad6afa9c4..ffbfaebb3 100644 --- a/apps/emqx_audit/src/emqx_audit.erl +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -119,7 +119,7 @@ log_to_db(Log) -> Audit0 = to_audit(Log), Audit = Audit0#?AUDIT{ node = node(), - created_at = erlang:system_time(millisecond) + created_at = erlang:system_time(microsecond) }, mria:dirty_write(?AUDIT, Audit). diff --git a/apps/emqx_audit/src/emqx_audit_api.erl b/apps/emqx_audit/src/emqx_audit_api.erl index 303ba044b..c6cf2737c 100644 --- a/apps/emqx_audit/src/emqx_audit_api.erl +++ b/apps/emqx_audit/src/emqx_audit_api.erl @@ -32,7 +32,7 @@ {<<"http_method">>, atom}, {<<"gte_created_at">>, timestamp}, {<<"lte_created_at">>, timestamp}, - {<<"gte_duration_ms">>, timestamp}, + {<<"gte_duration_ms">>, integer}, {<<"lte_duration_ms">>, integer} ]). -define(DISABLE_MSG, <<"Audit is disabled">>). @@ -130,14 +130,14 @@ schema("/audit") -> desc => ?DESC(filter_lte_duration_ms) })}, {gte_created_at, - ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{ + ?HOCON(emqx_utils_calendar:epoch_microsecond(), #{ in => query, required => false, example => <<"2023-10-15T00:00:00.820384+08:00">>, desc => ?DESC(filter_gte_created_at) })}, {lte_created_at, - ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{ + ?HOCON(emqx_utils_calendar:epoch_microsecond(), #{ in => query, example => <<"2023-10-16T00:00:00.820384+08:00">>, required => false, @@ -170,7 +170,7 @@ fields(audit) -> [ {created_at, ?HOCON( - emqx_utils_calendar:epoch_millisecond(), + emqx_utils_calendar:epoch_microsecond(), #{ desc => "The time when the log is created" } diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index 954292f9b..2f401e7a8 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -140,9 +140,9 @@ t_disabled(_) -> t_cli(_Config) -> Size = mnesia:table_info(emqx_audit, size), - TimeInt = erlang:system_time(millisecond) - 10, + TimeInt = erlang:system_time(microsecond) - 1000, Time = integer_to_list(TimeInt), - DateStr = calendar:system_time_to_rfc3339(TimeInt, [{unit, millisecond}]), + DateStr = calendar:system_time_to_rfc3339(TimeInt, [{unit, microsecond}]), Date = emqx_http_lib:uri_encode(DateStr), ok = emqx_ctl:run_command(["conf", "show", "log"]), AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), @@ -164,7 +164,11 @@ t_cli(_Config) -> ], Data ), - + %% check create at is valid + [#{<<"created_at">> := CreateAtRaw}] = Data, + CreateAt = calendar:rfc3339_to_system_time(binary_to_list(CreateAtRaw), [{unit, microsecond}]), + ?assert(CreateAt > TimeInt, CreateAtRaw), + ?assert(CreateAt < TimeInt + 5000000, CreateAtRaw), %% check cli filter {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader), #{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]), @@ -174,25 +178,41 @@ t_cli(_Config) -> ), ?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res2, [return_maps])), - %% check created_at filter + %% check created_at filter microsecond {ok, Res3} = emqx_mgmt_api_test_util:request_api( get, AuditPath, "gte_created_at=" ++ Time, AuthHeader ), #{<<"data">> := Data3} = emqx_utils_json:decode(Res3, [return_maps]), ?assertEqual(1, erlang:length(Data3)), + %% check created_at filter rfc3339 {ok, Res31} = emqx_mgmt_api_test_util:request_api( get, AuditPath, "gte_created_at=" ++ Date, AuthHeader ), ?assertEqual(Res3, Res31), + %% check created_at filter millisecond + TimeMs = integer_to_list(TimeInt div 1000), + {ok, Res32} = emqx_mgmt_api_test_util:request_api( + get, AuditPath, "gte_created_at=" ++ TimeMs, AuthHeader + ), + ?assertEqual(Res3, Res32), + + %% check created_at filter microsecond {ok, Res4} = emqx_mgmt_api_test_util:request_api( get, AuditPath, "lte_created_at=" ++ Time, AuthHeader ), #{<<"data">> := Data4} = emqx_utils_json:decode(Res4, [return_maps]), ?assertEqual(Size, erlang:length(Data4)), + + %% check created_at filter rfc3339 {ok, Res41} = emqx_mgmt_api_test_util:request_api( get, AuditPath, "lte_created_at=" ++ Date, AuthHeader ), ?assertEqual(Res4, Res41), + %% check created_at filter millisecond + {ok, Res42} = emqx_mgmt_api_test_util:request_api( + get, AuditPath, "lte_created_at=" ++ TimeMs, AuthHeader + ), + ?assertEqual(Res4, Res42), %% check duration_ms filter {ok, Res5} = emqx_mgmt_api_test_util:request_api( @@ -224,7 +244,7 @@ t_max_size(_Config) -> fun(_) -> ok = emqx_ctl:run_command(["conf", "show", "log"]) end, - lists:duplicate(110, 1) + lists:duplicate(100, 1) ), _ = mnesia:dump_log(), LogCount = wait_for_dirty_write_log_done(1500), diff --git a/apps/emqx_conf/src/emqx_conf_schema_types.erl b/apps/emqx_conf/src/emqx_conf_schema_types.erl index dc3af77b2..239624a81 100644 --- a/apps/emqx_conf/src/emqx_conf_schema_types.erl +++ b/apps/emqx_conf/src/emqx_conf_schema_types.erl @@ -143,6 +143,24 @@ readable("epoch_millisecond()") -> ] } }; +readable("epoch_microsecond()") -> + %% only for swagger + #{ + swagger => #{ + <<"oneOf">> => [ + #{ + type => integer, + example => 1640995200000000, + description => <<"epoch-microsecond">> + }, + #{ + type => string, + example => <<"2022-01-01T00:00:00.000000Z">>, + format => <<"date-time">> + } + ] + } + }; readable("duration()") -> #{ swagger => #{type => string, example => <<"12m">>}, diff --git a/apps/emqx_utils/src/emqx_utils_calendar.erl b/apps/emqx_utils/src/emqx_utils_calendar.erl index a42b8d0ca..8f1e406bf 100644 --- a/apps/emqx_utils/src/emqx_utils_calendar.erl +++ b/apps/emqx_utils/src/emqx_utils_calendar.erl @@ -29,6 +29,7 @@ %% API -export([ to_epoch_millisecond/1, + to_epoch_microsecond/1, to_epoch_second/1, human_readable_duration_string/1 ]). @@ -54,6 +55,7 @@ %% so the maximum date can reach 9999-12-31 which is ample. -define(MAXIMUM_EPOCH, 253402214400). -define(MAXIMUM_EPOCH_MILLI, 253402214400_000). +-define(MAXIMUM_EPOCH_MICROS, 253402214400_000_000). -define(DATE_PART, [ year, @@ -75,13 +77,16 @@ -reflect_type([ epoch_millisecond/0, - epoch_second/0 + epoch_second/0, + epoch_microsecond/0 ]). -type epoch_second() :: non_neg_integer(). -type epoch_millisecond() :: non_neg_integer(). +-type epoch_microsecond() :: non_neg_integer(). -typerefl_from_string({epoch_second/0, ?MODULE, to_epoch_second}). -typerefl_from_string({epoch_millisecond/0, ?MODULE, to_epoch_millisecond}). +-typerefl_from_string({epoch_microsecond/0, ?MODULE, to_epoch_microsecond}). %%-------------------------------------------------------------------- %% Epoch <-> RFC 3339 @@ -93,6 +98,9 @@ to_epoch_second(DateTime) -> to_epoch_millisecond(DateTime) -> to_epoch(DateTime, millisecond). +to_epoch_microsecond(DateTime) -> + to_epoch(DateTime, microsecond). + to_epoch(DateTime, Unit) -> try case string:to_integer(DateTime) of @@ -131,6 +139,14 @@ validate_epoch(Epoch, second) when Epoch =< ?MAXIMUM_EPOCH -> {ok, Epoch}; validate_epoch(Epoch, millisecond) when Epoch =< ?MAXIMUM_EPOCH_MILLI -> {ok, Epoch}; +%% http api use millisecond but we should transform to microsecond +validate_epoch(Epoch, microsecond) when + Epoch >= ?MAXIMUM_EPOCH andalso + Epoch =< ?MAXIMUM_EPOCH_MILLI +-> + {ok, Epoch * 1000}; +validate_epoch(Epoch, microsecond) when Epoch =< ?MAXIMUM_EPOCH_MICROS -> + {ok, Epoch}; validate_epoch(_Epoch, _Unit) -> {error, bad_epoch}.