From 3ed031db706e1eb19be6b61a74194f188f8dc029 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 7 Aug 2023 16:10:30 +0200 Subject: [PATCH 1/3] fix: rule SQL mongo_date function should return a string in test mode The rule SQL mongo_date function should return a string with the format ISODate(*), where * is an ISO date string when running the rule in test mode. Fixes: https://emqx.atlassian.net/browse/EMQX-10727 --- .../src/emqx_rule_engine.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 19 +++++++++++++-- .../src/emqx_rule_sqltester.erl | 23 ++++++++++++++++++- .../emqx_rule_engine_api_rule_test_SUITE.erl | 21 +++++++++++++++++ 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 09d57a4f9..e6d00bcae 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.22"}, + {vsn, "5.0.23"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 64522ee60..0c55f92b4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -1185,16 +1185,31 @@ function_literal(Fun, Args) -> {invalid_func, {Fun, Args}}. mongo_date() -> - erlang:timestamp(). + maybe_isodate_format(erlang:timestamp()). mongo_date(MillisecondsTimestamp) -> - convert_timestamp(MillisecondsTimestamp). + maybe_isodate_format(convert_timestamp(MillisecondsTimestamp)). mongo_date(Timestamp, Unit) -> InsertedTimeUnit = time_unit(Unit), ScaledEpoch = erlang:convert_time_unit(Timestamp, InsertedTimeUnit, millisecond), convert_timestamp(ScaledEpoch). +maybe_isodate_format(ErlTimestamp) -> + case emqx_rule_sqltester:is_test_runtime_env() of + false -> + ErlTimestamp; + true -> + %% if this is called from sqltest, we need to convert it to the ISODate() format, + %% so that it can be correctly converted into a JSON string. + isodate_format(ErlTimestamp) + end. + +isodate_format({MegaSecs, Secs, MicroSecs}) -> + SystemTimeMs = (MegaSecs * 1000_000_000_000 + Secs * 1000_000 + MicroSecs) div 1000, + Ts3339Str = calendar:system_time_to_rfc3339(SystemTimeMs, [{unit, millisecond}, {offset, "Z"}]), + iolist_to_binary(["ISODate(", Ts3339Str, ")"]). + convert_timestamp(MillisecondsTimestamp) -> MicroTimestamp = MillisecondsTimestamp * 1000, MegaSecs = MicroTimestamp div 1000_000_000_000, diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index f3b4e2790..867fffcc1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -18,7 +18,9 @@ -export([ test/1, - get_selected_data/3 + get_selected_data/3, + %% Some SQL functions return different results in the test environment + is_test_runtime_env/0 ]). -spec test(#{sql := binary(), context := map()}) -> {ok, map() | list()} | {error, term()}. @@ -63,12 +65,14 @@ test_rule(Sql, Select, Context, EventTopics) -> created_at => erlang:system_time(millisecond) }, FullContext = fill_default_values(hd(EventTopics), emqx_rule_maps:atom_key_map(Context)), + set_is_test_runtime_env(), try emqx_rule_runtime:apply_rule(Rule, FullContext, #{}) of {ok, Data} -> {ok, flatten(Data)}; {error, Reason} -> {error, Reason} after + unset_is_test_runtime_env(), ok = emqx_rule_engine:clear_metrics_for_rule(RuleId) end. @@ -97,3 +101,20 @@ envs_examp(EventTopic) -> emqx_rule_events:columns_with_exam(EventName) ) ). + +is_test_runtime_env_atom() -> + 'emqx_rule_sqltester:is_test_runtime_env'. + +set_is_test_runtime_env() -> + erlang:put(is_test_runtime_env_atom(), true), + ok. + +unset_is_test_runtime_env() -> + erlang:erase(is_test_runtime_env_atom()), + ok. + +is_test_runtime_env() -> + case erlang:get(is_test_runtime_env_atom()) of + true -> true; + _ -> false + end. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl index 575d35238..0c772958e 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl @@ -214,6 +214,27 @@ t_ctx_delivery_dropped(_) -> Expected = check_result([from_clientid, from_username, reason, qos, topic], [], Context), do_test(SQL, Context, Expected). +t_mongo_date_function_should_return_string_in_test_env(_) -> + SQL = + <<"SELECT mongo_date() as mongo_date FROM \"t/1\"">>, + Context = + #{ + action => <<"publish">>, + clientid => <<"c_emqx">>, + event_type => client_check_authz_complete, + result => <<"allow">>, + topic => <<"t/1">>, + username => <<"u_emqx">> + }, + CheckFunction = fun(Result) -> + MongoDate = maps:get(mongo_date, Result), + %% Use regex to match the expected string + MatchResult = re:run(MongoDate, <<"ISODate\\([0-9]{4}-[0-9]{2}-[0-9]{2}T.*\\)">>), + ?assertMatch({match, _}, MatchResult), + ok + end, + do_test(SQL, Context, CheckFunction). + do_test(SQL, Context, Expected0) -> Res = emqx_rule_engine_api:'/rule_test'( post, From 19d091eef1e833f86fe3a27ecc56bb175b5ab468 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 7 Aug 2023 16:28:39 +0200 Subject: [PATCH 2/3] docs: add changelog entry --- changes/ee/fix-11401.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/fix-11401.en.md diff --git a/changes/ee/fix-11401.en.md b/changes/ee/fix-11401.en.md new file mode 100644 index 000000000..2bce7170a --- /dev/null +++ b/changes/ee/fix-11401.en.md @@ -0,0 +1 @@ +When running one of the rule engine SQL `mongo_date` functions in the EMQX dashboard test interface, the resulting date is formatted as `ISODate(*)`, where * is the date in ISO date format instead of only the ISO date string. This is the format used by MongoDB to store dates. From b38461e50af081f1557ca7a196cf68a0bfd06879 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 8 Aug 2023 14:47:07 +0200 Subject: [PATCH 3/3] fix: mongo_date/2 shall give user friendly value in the test environment --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 0c55f92b4..e498dc642 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -1193,7 +1193,7 @@ mongo_date(MillisecondsTimestamp) -> mongo_date(Timestamp, Unit) -> InsertedTimeUnit = time_unit(Unit), ScaledEpoch = erlang:convert_time_unit(Timestamp, InsertedTimeUnit, millisecond), - convert_timestamp(ScaledEpoch). + mongo_date(ScaledEpoch). maybe_isodate_format(ErlTimestamp) -> case emqx_rule_sqltester:is_test_runtime_env() of