From 48d8f7e7a90bde1599565ef6a39d2529acdf2839 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 10 Oct 2022 16:03:31 +0800 Subject: [PATCH 01/12] fix(channel): add warning log if the acl check of a subscribed topic failed --- src/emqx_channel.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index b8f3c5b2c..3e20c7fef 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1501,12 +1501,15 @@ check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, check_sub_acls(TopicFilters, Channel) -> check_sub_acls(TopicFilters, Channel, []). -check_sub_acls([ TopicFilter = {Topic, _} | More] , Channel, Acc) -> +check_sub_acls([ TopicFilter = {Topic, SubOpts} | More] , Channel, Acc) -> case check_sub_acl(Topic, Channel) of allow -> check_sub_acls(More, Channel, [ {TopicFilter, 0} | Acc]); deny -> - check_sub_acls(More, Channel, [ {TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) + ReasonCode = ?RC_NOT_AUTHORIZED, + ?LOG(warning, "Cannot subscribe ~s with options ~p due to ~s.", + [Topic, SubOpts, emqx_reason_codes:text(ReasonCode)]), + check_sub_acls(More, Channel, [ {TopicFilter, ReasonCode} | Acc]) end; check_sub_acls([], _Channel, Acc) -> lists:reverse(Acc). From ef04310c2746a160e14a6273bd225a07c07da28d Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 10 Oct 2022 16:04:57 +0800 Subject: [PATCH 02/12] chore: update CHANGES.md --- CHANGES-4.3.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 673dbef8a..f136b7835 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -39,6 +39,8 @@ File format: - Added a test to prevent a last will testament message to be published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894) +- Add warning log if the acl check of a subscribed topic failed. [#9124](https://github.com/emqx/emqx/pull/9124) + ### Bug fixes - Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) From fd7230353c459f3d67c5232aa602defb36a7d393 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Sep 2022 14:29:59 +0800 Subject: [PATCH 03/12] refactor(test): move rule_engine sql test cases into a separate file --- apps/emqx_rule_engine/rebar.config | 10 + .../test/emqx_rule_engine_SUITE.erl | 1443 +---------------- apps/emqx_rule_engine/test/emqx_rule_test.hrl | 29 + .../test/emqx_rule_test_lib.erl | 141 ++ .../test/emqx_rulesql_SUITE.erl | 1378 ++++++++++++++++ 5 files changed, 1564 insertions(+), 1437 deletions(-) create mode 100644 apps/emqx_rule_engine/test/emqx_rule_test.hrl create mode 100644 apps/emqx_rule_engine/test/emqx_rule_test_lib.erl create mode 100644 apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 097c18a3d..f9ad1b283 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1,5 +1,7 @@ +%% -*- mode: erlang -*- {deps, []}. +%% Comple Opts {erl_opts, [warn_unused_vars, warn_shadow_vars, warn_unused_import, @@ -18,6 +20,14 @@ warnings_as_errors, deprecated_functions ]}. +%% {erl_opts, [...]}, but for CT runs +%% NOT WORKING!!! +%% %% == Common Test == +%% {ct_compile_opts, [ export_all +%% , nowarn_export_all +%% ]}. +%% {ct_opts, []}. + {cover_enabled, true}. {cover_opts, [verbose]}. {cover_export_enabled, true}. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index dc95c0368..42816e869 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -25,6 +25,9 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include("emqx_rule_test.hrl"). +-import(emqx_rule_test_lib, [make_simple_resource_type/1]). + %%-define(PROPTEST(M,F), true = proper:quickcheck(M:F())). all() -> @@ -92,47 +95,7 @@ groups() -> t_resource_types ]}, {runtime, [], - [t_match_atom_and_binary, - t_sqlselect_0, - t_sqlselect_00, - t_sqlselect_01, - t_sqlselect_02, - t_sqlselect_1, - t_sqlselect_2, - t_sqlselect_2_1, - t_sqlselect_2_2, - t_sqlselect_2_3, - t_sqlselect_3, - t_sqlparse_event_1, - t_sqlparse_event_2, - t_sqlparse_event_3, - t_sqlparse_foreach_1, - t_sqlparse_foreach_2, - t_sqlparse_foreach_3, - t_sqlparse_foreach_4, - t_sqlparse_foreach_5, - t_sqlparse_foreach_6, - t_sqlparse_foreach_7, - t_sqlparse_foreach_8, - t_sqlparse_case_when_1, - t_sqlparse_case_when_2, - t_sqlparse_case_when_3, - t_sqlparse_array_index_1, - t_sqlparse_array_index_2, - t_sqlparse_array_index_3, - t_sqlparse_array_index_4, - t_sqlparse_array_index_5, - t_sqlparse_select_matadata_1, - t_sqlparse_array_range_1, - t_sqlparse_array_range_2, - t_sqlparse_true_false, - t_sqlparse_compare_undefined, - t_sqlparse_compare_null_null, - t_sqlparse_compare_null_notnull, - t_sqlparse_compare_notnull_null, - t_sqlparse_compare, - t_sqlparse_new_map, - t_sqlparse_invalid_json + [t_match_atom_and_binary ]}, {rule_metrics, [], [t_metrics, @@ -169,10 +132,6 @@ end_per_suite(_Config) -> stop_apps(), ok. -on_resource_create(_id, _) -> #{}. -on_resource_destroy(_id, _) -> ok. -on_get_resource_status(_id, _) -> #{is_alive => true}. - %%------------------------------------------------------------------------------ %% Group specific setup/teardown %%------------------------------------------------------------------------------ @@ -269,15 +228,7 @@ init_per_testcase(Test, Config) | Config]; init_per_testcase(_TestCase, Config) -> ok = emqx_rule_registry:register_resource_types( - [#resource_type{ - name = built_in, - provider = ?APP, - params_spec = #{}, - on_create = {?MODULE, on_resource_create}, - on_destroy = {?MODULE, on_resource_destroy}, - on_status = {?MODULE, on_get_resource_status}, - title = #{en => <<"Built-In Resource Type (debug)">>}, - description = #{en => <<"The built in resource type for debug purpose">>}}]), + [make_simple_debug_resource_type()]), %ct:pal("============ ~p", [ets:tab2list(emqx_resource_type)]), Config. @@ -1280,324 +1231,6 @@ t_match_atom_and_binary(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_0(_Config) -> - %% Verify SELECT with and without 'AS' - Sql = "select * " - "from \"t/#\" " - "where payload.cmd.info = 'tt'", - ?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "select payload.cmd as cmd " - "from \"t/#\" " - "where cmd.info = 'tt'", - ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "select payload.cmd as cmd, cmd.info as info " - "from \"t/#\" " - "where cmd.info = 'tt' and info = 'tt'", - ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, - <<"info">> := <<"tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), - %% cascaded as - Sql4 = "select payload.cmd as cmd, cmd.info as meta.info " - "from \"t/#\" " - "where cmd.info = 'tt' and meta.info = 'tt'", - ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, - <<"meta">> := #{<<"info">> := <<"tt">>}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlselect_00(_Config) -> - %% Verify plus/subtract and unary_add_or_subtract - Sql = "select 1-1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 0}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - Sql1 = "select -1 + 1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 0}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "select 1 + 1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 2}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "select +1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 1}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlselect_01(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule1 = create_simple_repub_rule( - <<"t2">>, - "SELECT json_decode(payload) as p, payload " - "FROM \"t3/#\", \"t1\" " - "WHERE p.x = 1"), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), - ct:sleep(100), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"{\"x\":1}">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, - - emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), - receive {publish, #{topic := T3, payload := Payload3}} -> - ?assertEqual(<<"t2">>, T3), - ?assertEqual(<<"{\"x\":1}">>, Payload3) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule1). - -t_sqlselect_02(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule1 = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"t3/#\", \"t1\" " - "WHERE payload.x = 1"), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), - ct:sleep(100), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"{\"x\":1}">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := Payload0}} -> - ct:fail({unexpected_t2, Payload0}) - after 1000 -> - ok - end, - - emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), - receive {publish, #{topic := T3, payload := Payload3}} -> - ?assertEqual(<<"t2">>, T3), - ?assertEqual(<<"{\"x\":1}">>, Payload3) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule1). - -t_sqlselect_1(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT json_decode(payload) as p, payload " - "FROM \"t1\" " - "WHERE p.x = 1 and p.y = 2"), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ct:sleep(200), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":2}">>, 0), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"{\"x\":1,\"y\":2}">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2 - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"t2\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_1(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg dropped - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_dropped\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_nothing = Fun(), - - %% it should not keep republishing "t2" - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_2(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg acked - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_acked\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_3(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg delivered - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_delivered\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_3(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% republish the client.connected msg - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/client_connected\" " - "WHERE username = 'emqx1'", - <<"clientid=${clientid}">>), - {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ct:sleep(200), - {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), - {ok, _} = emqtt:connect(Client1), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - t_metrics(_Config) -> ok = emqx_rule_engine:load_providers(), TopicRule = create_simple_repub_rule( @@ -1899,964 +1532,6 @@ t_sqlselect_multi_actoins_4(Config) -> emqx_rule_registry:remove_rule(Rule). -t_sqlparse_event_1(_Config) -> - Sql = "select topic as tp " - "from \"$events/session_subscribed\" ", - ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). - -t_sqlparse_event_2(_Config) -> - Sql = "select clientid " - "from \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>}})). - -t_sqlparse_event_3(_Config) -> - Sql = "select clientid, topic as tp " - "from \"t/tt\", \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})). - -t_sqlparse_foreach_1(_Config) -> - %% Verify foreach with and without 'AS' - Sql = "foreach payload.sensors as s " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"s">> := 1}, #{<<"s">> := 2}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "from \"t/#\" ", - ?assertMatch({ok,[#{item := 1}, #{item := 2}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "foreach payload.sensors " - "from \"t/#\" ", - ?assertMatch({ok,[#{item := #{<<"cmd">> := <<"1">>}, clientid := <<"c_a">>}, - #{item := #{ <<"cmd">> := <<"2">> - , <<"name">> := <<"ct">> - } - , clientid := <<"c_a">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{ <<"payload">> => <<"{\"sensors\": [{\"cmd\":\"1\"}, " - "{\"cmd\":\"2\",\"name\":\"ct\"}]}">> - , <<"clientid">> => <<"c_a">> - , <<"topic">> => <<"t/a">> - }})), - Sql4 = "foreach payload.sensors " - "from \"t/#\" ", - {ok,[#{metadata := #{rule_id := TRuleId}}, - #{metadata := #{rule_id := TRuleId}}]} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{ - <<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}}), - ?assert(is_binary(TRuleId)). - -t_sqlparse_foreach_2(_Config) -> - %% Verify foreach-do with and without 'AS' - Sql = "foreach payload.sensors as s " - "do s.cmd as msg_type " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "do item.cmd as msg_type " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "foreach payload.sensors " - "do item as item " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"item">> := 1},#{<<"item">> := 2}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_3(_Config) -> - %% Verify foreach-incase with and without 'AS' - Sql = "foreach payload.sensors as s " - "incase s.cmd != 1 " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"s">> := #{<<"cmd">> := 2}}, - #{<<"s">> := #{<<"cmd">> := 3}} - ]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "incase item.cmd != 1 " - "from \"t/#\" ", - ?assertMatch({ok,[#{item := #{<<"cmd">> := 2}}, - #{item := #{<<"cmd">> := 3}} - ]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_4(_Config) -> - %% Verify foreach-do-incase - Sql = "foreach payload.sensors as s " - "do s.cmd as msg_type, s.name as name " - "incase is_not_null(s.cmd) " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ ok - , [ #{<<"msg_type">> := <<"1">>, <<"name">> := <<"n1">>} - , #{<<"msg_type">> := <<"2">>} - ] - }, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\", \"name\":\"n1\"}, " - "{\"cmd\":\"2\"}, {\"name\":\"n3\"}]}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok,[]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_5(_Config) -> - %% Verify foreach on a empty-list or non-list variable - Sql = "foreach payload.sensors as s " - "do s.cmd as msg_type, s.name as name " - "from \"t/#\" ", - ?assertMatch({ok,[]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok,[]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": []}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "from \"t/#\" ", - ?assertMatch({ok,[]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": 1}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_6(_Config) -> - %% Verify foreach on a empty-list or non-list variable - Sql = "foreach json_decode(payload) " - "do item.id as zid, timestamp as t " - "from \"t/#\" ", - {ok, Res} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"[{\"id\": 5},{\"id\": 15}]">>, - <<"topic">> => <<"t/a">>}}), - [#{<<"t">> := Ts1, <<"zid">> := Zid1}, - #{<<"t">> := Ts2, <<"zid">> := Zid2}] = Res, - ?assertEqual(true, is_integer(Ts1)), - ?assertEqual(true, is_integer(Ts2)), - ?assert(Zid1 == 5 orelse Zid1 == 15), - ?assert(Zid2 == 5 orelse Zid2 == 15). - -t_sqlparse_foreach_7(_Config) -> - %% Verify foreach-do-incase and cascaded AS - Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " - "incase is_not_null(info.cmd) " - "from \"t/#\" " - "where s.page = '2' ", - Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " - "{\"info\":[{\"name\":\"cmd1\", \"cmd\":\"1\"}, {\"cmd\":\"2\"}]} } }">>, - ?assertMatch({ ok - , [ #{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>} - , #{<<"msg_type">> := <<"2">>} - ] - }, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " - "incase is_not_null(info.cmd) " - "from \"t/#\" " - "where s.page = '3' ", - ?assertMatch({error, nomatch}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_8(_Config) -> - %% Verify foreach-do-incase and cascaded AS - Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " - "incase is_map(info) " - "from \"t/#\" " - "where s.page = '2' ", - Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " - "{\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }">>, - ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})), - - Sql3 = "foreach json_decode(payload) as p, p.sensors as s," - " s.collection as c, sublist(2,1,c.info) as info " - "do info.cmd as msg_type, info.name as name " - "from \"t/#\" " - "where s.page = '2' ", - [?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => SqlN, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})) - || SqlN <- [Sql3]]. - -t_sqlparse_case_when_1(_Config) -> - %% case-when-else clause - Sql = "select " - " case when payload.x < 0 then 0 " - " when payload.x > 7 then 7 " - " else payload.x " - " end as y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"y">> := 1}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})), - ok. - -t_sqlparse_case_when_2(_Config) -> - % switch clause - Sql = "select " - " case payload.x when 1 then 2 " - " when 2 then 3 " - " else 4 " - " end as y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"y">> := 2}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 2}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 4}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_case_when_3(_Config) -> - %% case-when clause - Sql = "select " - " case when payload.x < 0 then 0 " - " when payload.x > 7 then 7 " - " end as y " - "from \"t/#\" ", - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 5}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})), - ok. - -t_sqlparse_array_index_1(_Config) -> - %% index get - Sql = "select " - " json_decode(payload) as p, " - " p[1] as a " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"a">> := #{<<"x">> := 1}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"[{\"x\": 1}]">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - %% index get without 'as' - Sql2 = "select " - " payload.x[2] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [3]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,3,4]}, - <<"topic">> => <<"t/a">>}})), - %% index get without 'as' again - Sql3 = "select " - " payload.x[2].y " - "from \"t/#\" ", - ?assertMatch( {ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 3}]}}} - , emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}}) - ), - - %% index get with 'as' - Sql4 = "select " - " payload.x[2].y as b " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_index_2(_Config) -> - %% array get with negative index - Sql1 = "select " - " payload.x[-2].y as b " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}})), - %% array append to head or tail of a list: - Sql2 = "select " - " payload.x as b, " - " 1 as c[-0], " - " 2 as c[-0], " - " b as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := 0, <<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => 0}, - <<"topic">> => <<"t/a">>}})), - %% construct an empty list: - Sql3 = "select " - " [] as c, " - " 1 as c[-0], " - " 2 as c[-0], " - " 0 as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - %% construct a list: - Sql4 = "select " - " [payload.a, \"topic\", 'c'] as c, " - " 1 as c[-0], " - " 2 as c[-0], " - " 0 as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,11,<<"t/a">>,<<"c">>,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":11}">>, - <<"topic">> => <<"t/a">> - }})). - -t_sqlparse_array_index_3(_Config) -> - %% array with json string payload: - Sql0 = "select " - "payload," - "payload.x[2].y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := [1,2]}, 3]}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), - %% same as above but don't select payload: - Sql1 = "select " - "payload.x[2].y as b " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := [1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), - %% same as above but add 'as' clause: - Sql2 = "select " - "payload.x[2].y as b.c " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := #{<<"c">> := [1,2]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_index_4(_Config) -> - %% array with json string payload: - Sql0 = "select " - "0 as payload.x[2].y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 0}]}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), - %% array with json string payload, and also select payload.x: - Sql1 = "select " - "payload.x, " - "0 as payload.x[2].y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := 0}, 3]}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_index_5(_Config) -> - Sql00 = "select " - " [1,2,3,4] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assert(lists:any(fun({_K, V}) -> - V =:= [1,2,3,4] - end, maps:to_list(Res00))). - -t_sqlparse_select_matadata_1(_Config) -> - %% array with json string payload: - Sql0 = "select " - "payload " - "from \"t/#\" ", - ?assertNotMatch({ok, #{<<"payload">> := <<"abc">>, metadata := _}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"abc">>, - <<"topic">> => <<"t/a">>}})), - Sql1 = "select " - "payload, metadata " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := <<"abc">>, <<"metadata">> := _}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"abc">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_range_1(_Config) -> - %% get a range of list - Sql0 = "select " - " payload.a[1..4] as c " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,1,2,3]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})), - %% get a range from non-list data - Sql02 = "select " - " payload.a[1..4] as c " - "from \"t/#\" ", - ?assertMatch({error, {select_and_transform_error, {error,{range_get,non_list_data},_}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => - #{<<"payload">> => <<"{\"x\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})), - - %% construct a range: - Sql1 = "select " - " [1..4] as c, " - " 5 as c[-0], " - " 6 as c[-0], " - " 0 as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,1,2,3,4,5,6]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_range_2(_Config) -> - %% construct a range without 'as' - Sql00 = "select " - " [1..4] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assert(lists:any(fun({_K, V}) -> - V =:= [1,2,3,4] - end, maps:to_list(Res00))), - %% construct a range without 'as' - Sql01 = "select " - " a[2..4] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"a">> := [2,3,4]}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql01, - <<"ctx">> => #{<<"a">> => [1,2,3,4,5], - <<"topic">> => <<"t/a">>}})), - %% get a range of list without 'as' - Sql02 = "select " - " payload.a[1..4] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"a">> := [0,1,2,3]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_true_false(_Config) -> - %% construct a range without 'as' - Sql00 = "select " - " true as a, false as b, " - " false as x.y, true as c[-0] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assertMatch(#{<<"a">> := true, <<"b">> := false, - <<"x">> := #{<<"y">> := false}, - <<"c">> := [true] - }, Res00). - --define(TEST_SQL(SQL), - emqx_rule_sqltester:test( - #{<<"rawsql">> => SQL, - <<"ctx">> => #{<<"payload">> => <<"{}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_compare_undefined(_Config) -> - Sql00 = "select " - " * " - "from \"t/#\" " - "where dev != undefined ", - %% no match - ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00)), - - Sql00_1 = "select " - " * " - "from \"t/#\" " - "where dev <> undefined ", - %% no match - ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00_1)), - - Sql01 = "select " - " 'd' as dev " - "from \"t/#\" " - "where dev != undefined ", - {ok, Res01} = ?TEST_SQL(Sql01), - %% pass - ?assertMatch(#{}, Res01), - - Sql01_1 = "select " - " 'd' as dev " - "from \"t/#\" " - "where dev <> undefined ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - %% pass - ?assertMatch(#{}, Res01_1), - - Sql02 = "select " - " * " - "from \"t/#\" " - "where dev != 'undefined' ", - {ok, Res02} = ?TEST_SQL(Sql02), - %% pass - ?assertMatch(#{}, Res02), - - Sql03 = "select " - " * " - "from \"t/#\" " - "where dev =~ 'undefined' ", - Res03 = ?TEST_SQL(Sql03), - %% no match - ?assertMatch({error, nomatch}, Res03). - -t_sqlparse_compare_null_null(_Config) -> - %% test undefined == undefined - Sql00 = "select " - " a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := true - }, Res00), - - %% test undefined != undefined - Sql01 = "select " - " a != b as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := false - }, Res01), - - %% test undefined <> undefined - Sql01_1 = "select " - " a <> b as c " - "from \"t/#\" ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - ?assertMatch(#{<<"c">> := false - }, Res01_1), - - %% test undefined > undefined - Sql02 = "select " - " a > b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := false - }, Res02), - - %% test undefined < undefined - Sql03 = "select " - " a < b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - %% test undefined <= undefined - Sql04 = "select " - " a <= b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := true - }, Res04), - - %% test undefined >= undefined - Sql05 = "select " - " a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := true - }, Res05), - - %% test undefined =~ undefined - Sql06 = "select " - " a =~ b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := true - }, Res06). - -t_sqlparse_compare_null_notnull(_Config) -> - %% test undefined == 'b' - Sql00 = "select " - " 'b' as b, a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := false - }, Res00), - - %% test undefined != 'b' - Sql01 = "select " - " 'b' as b, a != b as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := true - }, Res01), - - %% test undefined <> 'b' - Sql01_1 = "select " - " 'b' as b, a <> b as c " - "from \"t/#\" ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - ?assertMatch(#{<<"c">> := true - }, Res01_1), - - %% test undefined > 'b' - Sql02 = "select " - " 'b' as b, a > b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := false - }, Res02), - - %% test undefined < 'b' - Sql03 = "select " - " 'b' as b, a < b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - %% test undefined <= 'b' - Sql04 = "select " - " 'b' as b, a <= b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := false - }, Res04), - - %% test undefined >= 'b' - Sql05 = "select " - " 'b' as b, a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := false - }, Res05), - - %% test undefined =~ 'b' - Sql06 = "select " - " 'b' as b, a =~ b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := false - }, Res06). - -t_sqlparse_compare_notnull_null(_Config) -> - %% test 'a' == undefined - Sql00 = "select " - " 'a' as a, a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := false - }, Res00), - - %% test 'a' != undefined - Sql01 = "select " - " 'a' as a, a != b as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := true - }, Res01), - - %% test 'a' <> undefined - Sql01_1 = "select " - " 'a' as a, a <> b as c " - "from \"t/#\" ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - ?assertMatch(#{<<"c">> := true - }, Res01_1), - - %% test 'a' > undefined - Sql02 = "select " - " 'a' as a, a > b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := false - }, Res02), - - %% test 'a' < undefined - Sql03 = "select " - " 'a' as a, a < b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - %% test 'a' <= undefined - Sql04 = "select " - " 'a' as a, a <= b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := false - }, Res04), - - %% test 'a' >= undefined - Sql05 = "select " - " 'a' as a, a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := false - }, Res05), - - %% test 'a' =~ undefined - Sql06 = "select " - " 'a' as a, a =~ b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := false - }, Res06). - -t_sqlparse_compare(_Config) -> - Sql00 = "select " - " 'a' as a, 'a' as b, a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := true - }, Res00), - - Sql00_1 = "select " - " 'true' as a, true as b, a = b as c " - "from \"t/#\" ", - {ok, Res00_1} = ?TEST_SQL(Sql00_1), - ?assertMatch(#{<<"c">> := true - }, Res00_1), - - Sql01 = "select " - " is_null(a) as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := true - }, Res01), - - Sql02 = "select " - " 1 as a, 2 as b, a < b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := true - }, Res02), - - Sql03 = "select " - " 1 as a, 2 as b, a > b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - Sql04 = "select " - " 1 as a, 2 as b, a = b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := false - }, Res04), - - Sql04_0 = "select " - " 1 as a, 1 as b, a = b as c " - "from \"t/#\" ", - {ok, Res04_0} = ?TEST_SQL(Sql04_0), - ?assertMatch(#{<<"c">> := true - }, Res04_0), - - Sql04_1 = "select " - " 1 as a, '1' as b, a = b as c " - "from \"t/#\" ", - {ok, Res04_1} = ?TEST_SQL(Sql04_1), - ?assertMatch(#{<<"c">> := true - }, Res04_1), - - %% test 1 >= 2 - Sql05 = "select " - " 1 as a, 2 as b, a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := false - }, Res05), - - %% test 1 <= 2 - Sql06 = "select " - " 1 as a, 2 as b, a <= b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := true - }, Res06), - - %% test 1 != 2 - Sql07 = "select " - " 1 as a, 2 as b, a != b as c " - "from \"t/#\" ", - {ok, Res07} = ?TEST_SQL(Sql07), - ?assertMatch(#{<<"c">> := true - }, Res07), - - %% test 1 <> 2 - Sql07_1 = "select " - " 1 as a, 2 as b, a <> b as c " - "from \"t/#\" ", - {ok, Res07_1} = ?TEST_SQL(Sql07_1), - ?assertMatch(#{<<"c">> := true - }, Res07_1), - - %% test 't' =~ 't' - Sql08 = "select " - " 't' as a, 't' as b, a =~ b as c " - "from \"t/#\" ", - {ok, Res08} = ?TEST_SQL(Sql08), - ?assertMatch(#{<<"c">> := true - }, Res08). - -t_sqlparse_new_map(_Config) -> - %% construct a range without 'as' - Sql00 = "select " - " map_new() as a, map_new() as b, " - " map_new() as x.y, map_new() as c[-0] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assertMatch(#{<<"a">> := #{}, <<"b">> := #{}, - <<"x">> := #{<<"y">> := #{}}, - <<"c">> := [#{}] - }, Res00). - t_sqlparse_payload_as(_Config) -> %% https://github.com/emqx/emqx/issues/3866 Sql00 = "SELECT " @@ -2908,29 +1583,6 @@ t_sqlparse_nested_get(_Config) -> <<"payload">> => <<"{\"a\": {\"b\": 0}}">> }})). -t_sqlparse_invalid_json(_Config) -> - Sql02 = "select " - " payload.a[1..4] as c " - "from \"t/#\" ", - ?assertMatch({error, {select_and_transform_error, {error,{decode_json_failed,_},_}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => - #{<<"payload">> => <<"{\"x\":[0,1,2,3,}">>, - <<"topic">> => <<"t/a">>}})), - - - Sql2 = "foreach payload.sensors " - "do item.cmd as msg_type " - "from \"t/#\" ", - ?assertMatch({error, {select_and_collect_error, {error,{decode_json_failed,_},_}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"} {\"cmd\":}]}">>, - <<"topic">> => <<"t/a">>}})). - %%------------------------------------------------------------------------------ %% Internal helpers %%------------------------------------------------------------------------------ @@ -2966,20 +1618,6 @@ make_simple_rule(RuleId, SQL, ForTopics) when is_binary(RuleId) -> actions = [{'inspect', #{}}], description = <<"simple rule">>}. -create_simple_repub_rule(TargetTopic, SQL) -> - create_simple_repub_rule(TargetTopic, SQL, <<"${payload}">>). - -create_simple_repub_rule(TargetTopic, SQL, Template) -> - {ok, Rule} = emqx_rule_engine:create_rule( - #{rawsql => SQL, - actions => [#{name => 'republish', - args => #{<<"target_topic">> => TargetTopic, - <<"target_qos">> => -1, - <<"payload_tmpl">> => Template} - }], - description => <<"simple repub rule">>}), - Rule. - make_simple_action(ActionName) when is_atom(ActionName) -> #action{name = ActionName, app = ?APP, module = ?MODULE, on_create = simple_action_inspect, params_spec = #{}, @@ -3002,19 +1640,6 @@ make_simple_resource(ResId) -> config = #{}, description = <<"Simple Resource">>}. -make_simple_resource_type(ResTypeName) -> - #resource_type{name = ResTypeName, provider = ?APP, - params_spec = #{}, - on_create = {?MODULE, on_simple_resource_type_create}, - on_destroy = {?MODULE, on_simple_resource_type_destroy}, - on_status = {?MODULE, on_simple_resource_type_status}, - title = #{en => <<"Simple Resource Type">>}, - description = #{en => <<"Simple Resource Type">>}}. - -on_simple_resource_type_create(_Id, #{}) -> #{}. -on_simple_resource_type_destroy(_Id, #{}) -> ok. -on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}. - hook_metrics_action(_Id, _Params) -> fun(Data = #{event := EventName}, _Envs) -> ct:pal("applying hook_metrics_action: ~p", [Data]), @@ -3305,66 +1930,10 @@ verify_peername(PeerName) -> verify_ipaddr(IPAddrS) -> ?assertMatch({ok, _}, inet:parse_address(binary_to_list(IPAddrS))). -init_events_counters() -> - ets:new(events_record_tab, [named_table, bag, public]). - %%------------------------------------------------------------------------------ -%% Start Apps +%% Mock funcs %%------------------------------------------------------------------------------ -stop_apps() -> - stopped = mnesia:stop(), - [application:stop(App) || App <- [emqx_rule_engine, emqx]]. - -start_apps() -> - [start_apps(App, SchemaFile, ConfigFile) || - {App, SchemaFile, ConfigFile} - <- [{emqx, deps_path(emqx, "priv/emqx.schema"), - deps_path(emqx, "etc/emqx.conf")}, - {emqx_rule_engine, local_path("priv/emqx_rule_engine.schema"), - local_path("etc/emqx_rule_engine.conf")}]]. - -start_apps(App, SchemaFile, ConfigFile) -> - read_schema_configs(App, SchemaFile, ConfigFile), - set_special_configs(App), - {ok, _} = application:ensure_all_started(App). - -read_schema_configs(App, SchemaFile, ConfigFile) -> - ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]), - Schema = cuttlefish_schema:files([SchemaFile]), - Conf = conf_parse:file(ConfigFile), - NewConfig = cuttlefish_generator:map(Schema, Conf), - Vals = proplists:get_value(App, NewConfig, []), - [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. - -deps_path(App, RelativePath) -> - %% Note: not lib_dir because etc dir is not sym-link-ed to _build dir - %% but priv dir is - Path0 = code:priv_dir(App), - Path = case file:read_link(Path0) of - {ok, Resolved} -> Resolved; - {error, _} -> Path0 - end, - filename:join([Path, "..", RelativePath]). - -local_path(RelativePath) -> - deps_path(emqx_rule_engine, RelativePath). - -set_special_configs(emqx_rule_engine) -> - application:set_env(emqx_rule_engine, ignore_sys_message, true), - application:set_env(emqx_rule_engine, events, - [{'client.connected',on,1}, - {'client.disconnected',on,1}, - {'session.subscribed',on,1}, - {'session.unsubscribed',on,1}, - {'message.acked',on,1}, - {'message.dropped',on,1}, - {'message.delivered',on,1} - ]), - ok; -set_special_configs(_App) -> - ok. - mock_print() -> catch meck:unload(emqx_ctl), meck:new(emqx_ctl, [non_strict, passthrough]), diff --git a/apps/emqx_rule_engine/test/emqx_rule_test.hrl b/apps/emqx_rule_engine/test/emqx_rule_test.hrl new file mode 100644 index 000000000..748cc2e9a --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rule_test.hrl @@ -0,0 +1,29 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Test Suite funcs +-import(emqx_rule_test_lib, + [ stop_apps/0 + , start_apps/0 + ]). + +%% RULE helper funcs +-import(emqx_rule_test_lib, + [ create_simple_repub_rule/2 + , create_simple_repub_rule/3 + , make_simple_debug_resource_type/0 + , init_events_counters/0 + ]). diff --git a/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl b/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl new file mode 100644 index 000000000..24550fbc7 --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl @@ -0,0 +1,141 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rule_test_lib). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx_rule_engine/include/rule_engine.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +%%------------------------------------------------------------------------------ +%% Start Apps +%%------------------------------------------------------------------------------ + +stop_apps() -> + stopped = mnesia:stop(), + [application:stop(App) || App <- [emqx_rule_engine, emqx]]. + +start_apps() -> + [start_apps(App, SchemaFile, ConfigFile) || + {App, SchemaFile, ConfigFile} + <- [{emqx, deps_path(emqx, "priv/emqx.schema"), + deps_path(emqx, "etc/emqx.conf")}, + {emqx_rule_engine, local_path("priv/emqx_rule_engine.schema"), + local_path("etc/emqx_rule_engine.conf")}]]. + +%%-------------------------------------- +%% start apps helper funcs + +start_apps(App, SchemaFile, ConfigFile) -> + read_schema_configs(App, SchemaFile, ConfigFile), + set_special_configs(App), + {ok, _} = application:ensure_all_started(App). + +read_schema_configs(App, SchemaFile, ConfigFile) -> + ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]), + Schema = cuttlefish_schema:files([SchemaFile]), + Conf = conf_parse:file(ConfigFile), + NewConfig = cuttlefish_generator:map(Schema, Conf), + Vals = proplists:get_value(App, NewConfig, []), + [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. + +deps_path(App, RelativePath) -> + %% Note: not lib_dir because etc dir is not sym-link-ed to _build dir + %% but priv dir is + Path0 = code:priv_dir(App), + Path = case file:read_link(Path0) of + {ok, Resolved} -> Resolved; + {error, _} -> Path0 + end, + filename:join([Path, "..", RelativePath]). + +local_path(RelativePath) -> + deps_path(emqx_rule_engine, RelativePath). + +set_special_configs(emqx_rule_engine) -> + application:set_env(emqx_rule_engine, ignore_sys_message, true), + application:set_env(emqx_rule_engine, events, + [{'client.connected',on,1}, + {'client.disconnected',on,1}, + {'session.subscribed',on,1}, + {'session.unsubscribed',on,1}, + {'message.acked',on,1}, + {'message.dropped',on,1}, + {'message.delivered',on,1} + ]), + ok; +set_special_configs(_App) -> + ok. + +%%------------------------------------------------------------------------------ +%% rule test helper funcs +%%------------------------------------------------------------------------------ + +create_simple_repub_rule(TargetTopic, SQL) -> + create_simple_repub_rule(TargetTopic, SQL, <<"${payload}">>). + +create_simple_repub_rule(TargetTopic, SQL, Template) -> + {ok, Rule} = emqx_rule_engine:create_rule( + #{rawsql => SQL, + actions => [#{name => 'republish', + args => #{<<"target_topic">> => TargetTopic, + <<"target_qos">> => -1, + <<"payload_tmpl">> => Template} + }], + description => <<"simple repub rule">>}), + Rule. + +make_simple_debug_resource_type() -> + #resource_type{ + name = built_in, + provider = ?APP, + params_spec = #{}, + on_create = {?MODULE, on_resource_create}, + on_destroy = {?MODULE, on_resource_destroy}, + on_status = {?MODULE, on_get_resource_status}, + title = #{en => <<"Built-In Resource Type (debug)">>}, + description = #{en => <<"The built in resource type for debug purpose">>}}. + +make_simple_resource_type(ResTypeName) -> + #resource_type{ + name = ResTypeName, + provider = ?APP, + params_spec = #{}, + on_create = {?MODULE, on_simple_resource_type_create}, + on_destroy = {?MODULE, on_simple_resource_type_destroy}, + on_status = {?MODULE, on_simple_resource_type_status}, + title = #{en => <<"Simple Resource Type">>}, + description = #{en => <<"Simple Resource Type">>}}. + +init_events_counters() -> + ets:new(events_record_tab, [named_table, bag, public]). + +%%------------------------------------------------------------------------------ +%% Internal helper funcs +%%------------------------------------------------------------------------------ + +on_resource_create(_id, _) -> #{}. +on_resource_destroy(_id, _) -> ok. +on_get_resource_status(_id, _) -> #{is_alive => true}. + +on_simple_resource_type_create(_Id, #{}) -> #{}. +on_simple_resource_type_destroy(_Id, #{}) -> ok. +on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}. diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl new file mode 100644 index 000000000..d93b747c0 --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -0,0 +1,1378 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rulesql_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx_rule_engine/include/rule_engine.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-include("emqx_rule_test.hrl"). + +all() -> + [ {group, rulesql_select} + , {group, rulesql_select_events} + , {group, rulesql_select_metadata} + , {group, rulesql_foreach} + , {group, rulesql_case_when} + , {group, rulesql_array_index} + , {group, rulesql_array_range} + , {group, rulesql_compare} + , {group, rulesql_boolean} + , {group, rulesql_others} + ]. + +suite() -> + [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}]. + +groups() -> + [{rulesql_select, [], + [ t_sqlselect_0 + , t_sqlselect_00 + , t_sqlselect_01 + , t_sqlselect_02 + , t_sqlselect_1 + , t_sqlselect_2 + , t_sqlselect_2_1 + , t_sqlselect_2_2 + , t_sqlselect_2_3 + , t_sqlselect_3 + ]}, + {rulesql_select_events, [], + [ t_sqlparse_event_1 + , t_sqlparse_event_2 + , t_sqlparse_event_3 + ]}, + {rulesql_select_metadata, [], + [ t_sqlparse_select_matadata_1 + ]}, + {rulesql_foreach, [], + [ t_sqlparse_foreach_1 + , t_sqlparse_foreach_2 + , t_sqlparse_foreach_3 + , t_sqlparse_foreach_4 + , t_sqlparse_foreach_5 + , t_sqlparse_foreach_6 + , t_sqlparse_foreach_7 + , t_sqlparse_foreach_8 + ]}, + {rulesql_case_when, [], + [ t_sqlparse_case_when_1 + , t_sqlparse_case_when_2 + , t_sqlparse_case_when_3 + ]}, + {rulesql_array_index, [], + [ t_sqlparse_array_index_1 + , t_sqlparse_array_index_2 + , t_sqlparse_array_index_3 + , t_sqlparse_array_index_4 + , t_sqlparse_array_index_5 + ]}, + {rulesql_array_range, [], + [ t_sqlparse_array_range_1 + , t_sqlparse_array_range_2 + ]}, + {rulesql_compare, [], + [ t_sqlparse_compare_undefined + , t_sqlparse_compare_null_null + , t_sqlparse_compare_null_notnull + , t_sqlparse_compare_notnull_null + , t_sqlparse_compare + ]}, + {rulesql_boolean, [], + [ t_sqlparse_true_false + ]}, + {rulesql_others, [], + [ t_sqlparse_new_map + , t_sqlparse_invalid_json + ]} + ]. + +%%------------------------------------------------------------------------------ +%% Overall setup/teardown +%%------------------------------------------------------------------------------ + +init_per_suite(Config) -> + ok = ekka_mnesia:start(), + ok = emqx_rule_registry:mnesia(boot), + start_apps(), + Config. + +end_per_suite(_Config) -> + stop_apps(), + ok. + +on_resource_create(_id, _) -> #{}. +on_resource_destroy(_id, _) -> ok. +on_get_resource_status(_id, _) -> #{is_alive => true}. + +%%------------------------------------------------------------------------------ +%% Group specific setup/teardown +%%------------------------------------------------------------------------------ + +group(_Groupname) -> + []. + +init_per_group(registry, Config) -> + Config; +init_per_group(_Groupname, Config) -> + Config. + +end_per_group(_Groupname, _Config) -> + ok. + +%%------------------------------------------------------------------------------ +%% Testcase specific setup/teardown +%%------------------------------------------------------------------------------ + +init_per_testcase(_TestCase, Config) -> + init_events_counters(), + ok = emqx_rule_registry:register_resource_types( + [make_simple_debug_resource_type()]), + %ct:pal("============ ~p", [ets:tab2list(emqx_resource_type)]), + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%%------------------------------------------------------------------------------ +%% Test cases for `select` +%%------------------------------------------------------------------------------ + +t_sqlselect_0(_Config) -> + %% Verify SELECT with and without 'AS' + Sql = "select * " + "from \"t/#\" " + "where payload.cmd.info = 'tt'", + ?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "select payload.cmd as cmd " + "from \"t/#\" " + "where cmd.info = 'tt'", + ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "select payload.cmd as cmd, cmd.info as info " + "from \"t/#\" " + "where cmd.info = 'tt' and info = 'tt'", + ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, + <<"info">> := <<"tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})), + %% cascaded as + Sql4 = "select payload.cmd as cmd, cmd.info as meta.info " + "from \"t/#\" " + "where cmd.info = 'tt' and meta.info = 'tt'", + ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, + <<"meta">> := #{<<"info">> := <<"tt">>}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlselect_00(_Config) -> + %% Verify plus/subtract and unary_add_or_subtract + Sql = "select 1-1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 0}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + Sql1 = "select -1 + 1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 0}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "select 1 + 1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 2}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "select +1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 1}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlselect_01(_Config) -> + ok = emqx_rule_engine:load_providers(), + TopicRule1 = create_simple_repub_rule( + <<"t2">>, + "SELECT json_decode(payload) as p, payload " + "FROM \"t3/#\", \"t1\" " + "WHERE p.x = 1"), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), + ct:sleep(100), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"{\"x\":1}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, + + emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), + receive {publish, #{topic := T3, payload := Payload3}} -> + ?assertEqual(<<"t2">>, T3), + ?assertEqual(<<"{\"x\":1}">>, Payload3) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule1). + +t_sqlselect_02(_Config) -> + ok = emqx_rule_engine:load_providers(), + TopicRule1 = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"t3/#\", \"t1\" " + "WHERE payload.x = 1"), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), + ct:sleep(100), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"{\"x\":1}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := Payload0}} -> + ct:fail({unexpected_t2, Payload0}) + after 1000 -> + ok + end, + + emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), + receive {publish, #{topic := T3, payload := Payload3}} -> + ?assertEqual(<<"t2">>, T3), + ?assertEqual(<<"{\"x\":1}">>, Payload3) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule1). + + + +t_sqlselect_1(_Config) -> + ok = emqx_rule_engine:load_providers(), + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT json_decode(payload) as p, payload " + "FROM \"t1\" " + "WHERE p.x = 1 and p.y = 2"), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + ct:sleep(200), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":2}">>, 0), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"{\"x\":1,\"y\":2}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2 + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"t2\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_t2 = Fun(), + received_t2 = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2_1(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2, if the msg dropped + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/message_dropped\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_nothing = Fun(), + + %% it should not keep republishing "t2" + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2_2(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2, if the msg acked + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/message_acked\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1), + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_t2 = Fun(), + received_t2 = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2_3(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2, if the msg delivered + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/message_delivered\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_t2 = Fun(), + received_t2 = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_3(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% republish the client.connected msg + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/client_connected\" " + "WHERE username = 'emqx1'", + <<"clientid=${clientid}">>), + {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + ct:sleep(200), + {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), + {ok, _} = emqtt:connect(Client1), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"clientid=c_emqx1">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +%%------------------------------------------------------------------------------ +%% Test cases for events +%%------------------------------------------------------------------------------ + +t_sqlparse_event_1(_Config) -> + Sql = "select topic as tp " + "from \"$events/session_subscribed\" ", + ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). + +t_sqlparse_event_2(_Config) -> + Sql = "select clientid " + "from \"$events/client_connected\" ", + ?assertMatch({ok,#{<<"clientid">> := <<"abc">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"clientid">> => <<"abc">>}})). + +t_sqlparse_event_3(_Config) -> + Sql = "select clientid, topic as tp " + "from \"t/tt\", \"$events/client_connected\" ", + ?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})). + +%%------------------------------------------------------------------------------ +%% Test cases for `foreach` +%%------------------------------------------------------------------------------ + +t_sqlparse_foreach_1(_Config) -> + %% Verify foreach with and without 'AS' + Sql = "foreach payload.sensors as s " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"s">> := 1}, #{<<"s">> := 2}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "from \"t/#\" ", + ?assertMatch({ok,[#{item := 1}, #{item := 2}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "foreach payload.sensors " + "from \"t/#\" ", + ?assertMatch({ok,[#{item := #{<<"cmd">> := <<"1">>}, clientid := <<"c_a">>}, + #{item := #{ <<"cmd">> := <<"2">> + , <<"name">> := <<"ct">> + } + , clientid := <<"c_a">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{ <<"payload">> => <<"{\"sensors\": [{\"cmd\":\"1\"}, " + "{\"cmd\":\"2\",\"name\":\"ct\"}]}">> + , <<"clientid">> => <<"c_a">> + , <<"topic">> => <<"t/a">> + }})), + Sql4 = "foreach payload.sensors " + "from \"t/#\" ", + {ok,[#{metadata := #{rule_id := TRuleId}}, + #{metadata := #{rule_id := TRuleId}}]} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => #{ + <<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}}), + ?assert(is_binary(TRuleId)). + +t_sqlparse_foreach_2(_Config) -> + %% Verify foreach-do with and without 'AS' + Sql = "foreach payload.sensors as s " + "do s.cmd as msg_type " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "do item.cmd as msg_type " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "foreach payload.sensors " + "do item as item " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"item">> := 1},#{<<"item">> := 2}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_3(_Config) -> + %% Verify foreach-incase with and without 'AS' + Sql = "foreach payload.sensors as s " + "incase s.cmd != 1 " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"s">> := #{<<"cmd">> := 2}}, + #{<<"s">> := #{<<"cmd">> := 3}} + ]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "incase item.cmd != 1 " + "from \"t/#\" ", + ?assertMatch({ok,[#{item := #{<<"cmd">> := 2}}, + #{item := #{<<"cmd">> := 3}} + ]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_4(_Config) -> + %% Verify foreach-do-incase + Sql = "foreach payload.sensors as s " + "do s.cmd as msg_type, s.name as name " + "incase is_not_null(s.cmd) " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ ok + , [ #{<<"msg_type">> := <<"1">>, <<"name">> := <<"n1">>} + , #{<<"msg_type">> := <<"2">>} + ] + }, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\", \"name\":\"n1\"}, " + "{\"cmd\":\"2\"}, {\"name\":\"n3\"}]}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok,[]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_5(_Config) -> + %% Verify foreach on a empty-list or non-list variable + Sql = "foreach payload.sensors as s " + "do s.cmd as msg_type, s.name as name " + "from \"t/#\" ", + ?assertMatch({ok,[]}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok,[]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": []}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "from \"t/#\" ", + ?assertMatch({ok,[]}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": 1}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_6(_Config) -> + %% Verify foreach on a empty-list or non-list variable + Sql = "foreach json_decode(payload) " + "do item.id as zid, timestamp as t " + "from \"t/#\" ", + {ok, Res} = emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"[{\"id\": 5},{\"id\": 15}]">>, + <<"topic">> => <<"t/a">>}}), + [#{<<"t">> := Ts1, <<"zid">> := Zid1}, + #{<<"t">> := Ts2, <<"zid">> := Zid2}] = Res, + ?assertEqual(true, is_integer(Ts1)), + ?assertEqual(true, is_integer(Ts2)), + ?assert(Zid1 == 5 orelse Zid1 == 15), + ?assert(Zid2 == 5 orelse Zid2 == 15). + +t_sqlparse_foreach_7(_Config) -> + %% Verify foreach-do-incase and cascaded AS + Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " + "do info.cmd as msg_type, info.name as name " + "incase is_not_null(info.cmd) " + "from \"t/#\" " + "where s.page = '2' ", + Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " + "{\"info\":[{\"name\":\"cmd1\", \"cmd\":\"1\"}, {\"cmd\":\"2\"}]} } }">>, + ?assertMatch({ ok + , [ #{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>} + , #{<<"msg_type">> := <<"2">>} + ] + }, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " + "do info.cmd as msg_type, info.name as name " + "incase is_not_null(info.cmd) " + "from \"t/#\" " + "where s.page = '3' ", + ?assertMatch({error, nomatch}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_8(_Config) -> + %% Verify foreach-do-incase and cascaded AS + Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " + "do info.cmd as msg_type, info.name as name " + "incase is_map(info) " + "from \"t/#\" " + "where s.page = '2' ", + Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " + "{\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }">>, + ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})), + + Sql3 = "foreach json_decode(payload) as p, p.sensors as s," + " s.collection as c, sublist(2,1,c.info) as info " + "do info.cmd as msg_type, info.name as name " + "from \"t/#\" " + "where s.page = '2' ", + [?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => SqlN, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})) + || SqlN <- [Sql3]]. + +%%------------------------------------------------------------------------------ +%% Test cases for `case..when..` +%%------------------------------------------------------------------------------ + +t_sqlparse_case_when_1(_Config) -> + %% case-when-else clause + Sql = "select " + " case when payload.x < 0 then 0 " + " when payload.x > 7 then 7 " + " else payload.x " + " end as y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"y">> := 1}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, + <<"topic">> => <<"t/a">>}})), + ok. + +t_sqlparse_case_when_2(_Config) -> + % switch clause + Sql = "select " + " case payload.x when 1 then 2 " + " when 2 then 3 " + " else 4 " + " end as y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"y">> := 2}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 3}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 2}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 4}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_case_when_3(_Config) -> + %% case-when clause + Sql = "select " + " case when payload.x < 0 then 0 " + " when payload.x > 7 then 7 " + " end as y " + "from \"t/#\" ", + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 5}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, + <<"topic">> => <<"t/a">>}})), + ok. + +%%------------------------------------------------------------------------------ +%% Test cases for array index +%%------------------------------------------------------------------------------ + +t_sqlparse_array_index_1(_Config) -> + %% index get + Sql = "select " + " json_decode(payload) as p, " + " p[1] as a " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"a">> := #{<<"x">> := 1}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"[{\"x\": 1}]">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + %% index get without 'as' + Sql2 = "select " + " payload.x[2] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [3]}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,3,4]}, + <<"topic">> => <<"t/a">>}})), + %% index get without 'as' again + Sql3 = "select " + " payload.x[2].y " + "from \"t/#\" ", + ?assertMatch( {ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 3}]}}} + , emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, + <<"topic">> => <<"t/a">>}}) + ), + + %% index get with 'as' + Sql4 = "select " + " payload.x[2].y as b " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_index_2(_Config) -> + %% array get with negative index + Sql1 = "select " + " payload.x[-2].y as b " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, + <<"topic">> => <<"t/a">>}})), + %% array append to head or tail of a list: + Sql2 = "select " + " payload.x as b, " + " 1 as c[-0], " + " 2 as c[-0], " + " b as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := 0, <<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => 0}, + <<"topic">> => <<"t/a">>}})), + %% construct an empty list: + Sql3 = "select " + " [] as c, " + " 1 as c[-0], " + " 2 as c[-0], " + " 0 as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + %% construct a list: + Sql4 = "select " + " [payload.a, \"topic\", 'c'] as c, " + " 1 as c[-0], " + " 2 as c[-0], " + " 0 as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,11,<<"t/a">>,<<"c">>,1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => #{<<"payload">> => <<"{\"a\":11}">>, + <<"topic">> => <<"t/a">> + }})). + +t_sqlparse_array_index_3(_Config) -> + %% array with json string payload: + Sql0 = "select " + "payload," + "payload.x[2].y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := [1,2]}, 3]}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})), + %% same as above but don't select payload: + Sql1 = "select " + "payload.x[2].y as b " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := [1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})), + %% same as above but add 'as' clause: + Sql2 = "select " + "payload.x[2].y as b.c " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := #{<<"c">> := [1,2]}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_index_4(_Config) -> + %% array with json string payload: + Sql0 = "select " + "0 as payload.x[2].y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 0}]}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})), + %% array with json string payload, and also select payload.x: + Sql1 = "select " + "payload.x, " + "0 as payload.x[2].y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := 0}, 3]}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_index_5(_Config) -> + Sql00 = "select " + " [1,2,3,4] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assert(lists:any(fun({_K, V}) -> + V =:= [1,2,3,4] + end, maps:to_list(Res00))). + +%%------------------------------------------------------------------------------ +%% Test cases for rule metadata +%%------------------------------------------------------------------------------ + +t_sqlparse_select_matadata_1(_Config) -> + %% array with json string payload: + Sql0 = "select " + "payload " + "from \"t/#\" ", + ?assertNotMatch({ok, #{<<"payload">> := <<"abc">>, metadata := _}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"abc">>, + <<"topic">> => <<"t/a">>}})), + Sql1 = "select " + "payload, metadata " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := <<"abc">>, <<"metadata">> := _}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"abc">>, + <<"topic">> => <<"t/a">>}})). + +%%------------------------------------------------------------------------------ +%% Test cases for array range +%%------------------------------------------------------------------------------ + +t_sqlparse_array_range_1(_Config) -> + %% get a range of list + Sql0 = "select " + " payload.a[1..4] as c " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,1,2,3]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, + <<"topic">> => <<"t/a">>}})), + %% get a range from non-list data + Sql02 = "select " + " payload.a[1..4] as c " + "from \"t/#\" ", + ?assertMatch({error, {select_and_transform_error, {error,{range_get,non_list_data},_}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql02, + <<"ctx">> => + #{<<"payload">> => <<"{\"x\":[0,1,2,3,4,5]}">>, + <<"topic">> => <<"t/a">>}})), + + %% construct a range: + Sql1 = "select " + " [1..4] as c, " + " 5 as c[-0], " + " 6 as c[-0], " + " 0 as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,1,2,3,4,5,6]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_range_2(_Config) -> + %% construct a range without 'as' + Sql00 = "select " + " [1..4] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assert(lists:any(fun({_K, V}) -> + V =:= [1,2,3,4] + end, maps:to_list(Res00))), + %% construct a range without 'as' + Sql01 = "select " + " a[2..4] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"a">> := [2,3,4]}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql01, + <<"ctx">> => #{<<"a">> => [1,2,3,4,5], + <<"topic">> => <<"t/a">>}})), + %% get a range of list without 'as' + Sql02 = "select " + " payload.a[1..4] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"a">> := [0,1,2,3]}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql02, + <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, + <<"topic">> => <<"t/a">>}})). + +%%------------------------------------------------------------------------------ +%% Test cases for boolean +%%------------------------------------------------------------------------------ + +t_sqlparse_true_false(_Config) -> + %% construct a range without 'as' + Sql00 = "select " + " true as a, false as b, " + " false as x.y, true as c[-0] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assertMatch(#{<<"a">> := true, <<"b">> := false, + <<"x">> := #{<<"y">> := false}, + <<"c">> := [true] + }, Res00). + +%%------------------------------------------------------------------------------ +%% Test cases for compare +%%------------------------------------------------------------------------------ + +-define(TEST_SQL(SQL), + emqx_rule_sqltester:test( + #{<<"rawsql">> => SQL, + <<"ctx">> => #{<<"payload">> => <<"{}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_compare_undefined(_Config) -> + Sql00 = "select " + " * " + "from \"t/#\" " + "where dev != undefined ", + %% no match + ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00)), + + Sql01 = "select " + " 'd' as dev " + "from \"t/#\" " + "where dev != undefined ", + {ok, Res01} = ?TEST_SQL(Sql01), + %% pass + ?assertMatch(#{}, Res01), + + Sql02 = "select " + " * " + "from \"t/#\" " + "where dev != 'undefined' ", + {ok, Res02} = ?TEST_SQL(Sql02), + %% pass + ?assertMatch(#{}, Res02). + +t_sqlparse_compare_null_null(_Config) -> + %% test undefined == undefined + Sql00 = "select " + " a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := true + }, Res00), + + %% test undefined != undefined + Sql01 = "select " + " a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := false + }, Res01), + + %% test undefined > undefined + Sql02 = "select " + " a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test undefined < undefined + Sql03 = "select " + " a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test undefined <= undefined + Sql04 = "select " + " a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := true + }, Res04), + + %% test undefined >= undefined + Sql05 = "select " + " a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := true + }, Res05). + +t_sqlparse_compare_null_notnull(_Config) -> + %% test undefined == b + Sql00 = "select " + " 'b' as b, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := false + }, Res00), + + %% test undefined != b + Sql01 = "select " + " 'b' as b, a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + %% test undefined > b + Sql02 = "select " + " 'b' as b, a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test undefined < b + Sql03 = "select " + " 'b' as b, a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test undefined <= b + Sql04 = "select " + " 'b' as b, a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test undefined >= b + Sql05 = "select " + " 'b' as b, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05). + +t_sqlparse_compare_notnull_null(_Config) -> + %% test 'a' == undefined + Sql00 = "select " + " 'a' as a, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := false + }, Res00), + + %% test 'a' != undefined + Sql01 = "select " + " 'a' as a, a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + %% test 'a' > undefined + Sql02 = "select " + " 'a' as a, a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test 'a' < undefined + Sql03 = "select " + " 'a' as a, a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test 'a' <= undefined + Sql04 = "select " + " 'a' as a, a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test 'a' >= undefined + Sql05 = "select " + " 'a' as a, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05). + +t_sqlparse_compare(_Config) -> + Sql00 = "select " + " 'a' as a, 'a' as b, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := true + }, Res00), + + Sql01 = "select " + " is_null(a) as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + Sql02 = "select " + " 1 as a, 2 as b, a < b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := true + }, Res02), + + Sql03 = "select " + " 1 as a, 2 as b, a > b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + Sql04 = "select " + " 1 as a, 2 as b, a = b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test 'a' >= undefined + Sql05 = "select " + " 1 as a, 2 as b, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05), + + %% test 'a' >= undefined + Sql06 = "select " + " 1 as a, 2 as b, a <= b as c " + "from \"t/#\" ", + {ok, Res06} = ?TEST_SQL(Sql06), + ?assertMatch(#{<<"c">> := true + }, Res06). + + + +t_sqlparse_new_map(_Config) -> + %% construct a range without 'as' + Sql00 = "select " + " map_new() as a, map_new() as b, " + " map_new() as x.y, map_new() as c[-0] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assertMatch(#{<<"a">> := #{}, <<"b">> := #{}, + <<"x">> := #{<<"y">> := #{}}, + <<"c">> := [#{}] + }, Res00). + + +t_sqlparse_invalid_json(_Config) -> + Sql02 = "select " + " payload.a[1..4] as c " + "from \"t/#\" ", + ?assertMatch({error, {select_and_transform_error, {error,{decode_json_failed,_},_}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql02, + <<"ctx">> => + #{<<"payload">> => <<"{\"x\":[0,1,2,3,}">>, + <<"topic">> => <<"t/a">>}})), + + + Sql2 = "foreach payload.sensors " + "do item.cmd as msg_type " + "from \"t/#\" ", + ?assertMatch({error, {select_and_collect_error, {error,{decode_json_failed,_},_}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"} {\"cmd\":}]}">>, + <<"topic">> => <<"t/a">>}})). From 267946c3794b8c251a056e9f4cb975dd04b1b14f Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Sep 2022 09:44:03 +0800 Subject: [PATCH 04/12] fix: rm unused module `emqx_rule_date` was added before [v4.3.15-rc1] PR: emqx#7894 commit: 8558a62ee291021e521dc2c1f6efa418fa76469d The change was released in [v4.3.15-rc1] [v4.3.15-rc2] And unused at [v4.3.15-rc3] PR: emqx#8044 commit: 4fc5cb2817c4f1e8225c048bc04f4972148d0df4 We just keep the module from [v4.3.15-rc1] to [v4.3.20] and remove this module in newer version --- apps/emqx_rule_engine/src/emqx_rule_date.erl | 248 ------------------- 1 file changed, 248 deletions(-) delete mode 100644 apps/emqx_rule_engine/src/emqx_rule_date.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_date.erl b/apps/emqx_rule_engine/src/emqx_rule_date.erl deleted file mode 100644 index fb9cad4c3..000000000 --- a/apps/emqx_rule_engine/src/emqx_rule_date.erl +++ /dev/null @@ -1,248 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_date). - --export([date/3, date/4, parse_date/4]). - --export([ - is_int_char/1, - is_symbol_char/1, - is_m_char/1 -]). - --record(result, { - %%year() - year = "1970" :: string(), - %%month() - month = "1" :: string(), - %%day() - day = "1" :: string(), - %%hour() - hour = "0" :: string(), - %%minute() %% epoch in millisecond precision - minute = "0" :: string(), - %%second() %% epoch in millisecond precision - second = "0" :: string(), - %%integer() %% zone maybe some value - zone = "+00:00" :: string() -}). - -%% -type time_unit() :: 'microsecond' -%% | 'millisecond' -%% | 'nanosecond' -%% | 'second'. -%% -type offset() :: [byte()] | (Time :: integer()). -date(TimeUnit, Offset, FormatString) -> - date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)). - -date(TimeUnit, Offset, FormatString, TimeEpoch) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - Res = lists:map( - fun(Expr) -> - eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr) - end, - R - ), - lists:concat(Res). - -parse_date(TimeUnit, Offset, FormatString, InputString) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - IsZ = fun(V) -> - case V of - {tag, $Z} -> true; - _ -> false - end - end, - R1 = lists:filter(IsZ, R), - IfFun = fun(Con, A, B) -> - case Con of - [] -> A; - _ -> B - end - end, - Res = parse_input(FormatString, InputString), - Str = - Res#result.year ++ "-" ++ - Res#result.month ++ "-" ++ - Res#result.day ++ "T" ++ - Res#result.hour ++ ":" ++ - Res#result.minute ++ ":" ++ - Res#result.second ++ - IfFun(R1, Offset, Res#result.zone), - calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]). - -mlist(R) -> - %% %H Shows hour in 24-hour format [15] - [ - {$H, R#result.hour}, - %% %M Displays minutes [00-59] - {$M, R#result.minute}, - %% %S Displays seconds [00-59] - {$S, R#result.second}, - %% %y Displays year YYYY [2021] - {$y, R#result.year}, - %% %m Displays the number of the month [01-12] - {$m, R#result.month}, - %% %d Displays the number of the month [01-12] - {$d, R#result.day}, - %% %Z Displays Time zone - {$Z, R#result.zone} - ]. - -rmap(Result) -> - maps:from_list(mlist(Result)). - -support_char() -> "HMSymdZ". - -create_tag(Head, []) -> - Head; -create_tag(Head, [Val1 | RVal]) -> - case Val1 of - [] -> - create_tag(Head ++ [{st, [$%]}], RVal); - [H | Other] -> - case lists:member(H, support_char()) of - true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal); - false -> create_tag(Head ++ [{st, [$% | Val1]}], RVal) - end - end. - -eval_tag(_, {st, Str}) -> - Str; -eval_tag(Map, {tag, Char}) -> - maps:get(Char, Map, "undefined"). - -%% make_time(TimeUnit, Offset) -> -%% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)). -make_time(TimeUnit, Offset, TimeEpoch) -> - Res = calendar:system_time_to_rfc3339(TimeEpoch, - [{unit, TimeUnit}, {offset, Offset}]), - [Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T, - H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = Res, - IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, - {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), - #result{ - year = [Y1, Y2, Y3, Y4] - , month = [Mon1, Mon2] - , day = [D1, D2] - , hour = [H1, H2] - , minute = [Min1, Min2] - , second = [S1, S2] ++ FractionStr - , zone = UtcOffset - }. - -is_int_char(C) -> - C >= $0 andalso C =< $9. -is_symbol_char(C) -> - C =:= $- orelse C =:= $+. -is_m_char(C) -> - C =:= $:. - -parse_char_with_fun(_, []) -> - error(null_input); -parse_char_with_fun(ValidFun, [C | Other]) -> - Res = - case erlang:is_function(ValidFun) of - true -> ValidFun(C); - false -> erlang:apply(emqx_rule_date, ValidFun, [C]) - end, - case Res of - true -> {C, Other}; - false -> error({unexpected, [C | Other]}) - end. -parse_string([], Input) -> - {[], Input}; -parse_string([C | Other], Input) -> - {C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input), - {Res, Input2} = parse_string(Other, Input1), - {[C1 | Res], Input2}. - -parse_times(0, _, Input) -> - {[], Input}; -parse_times(Times, Fun, Input) -> - {C1, Input1} = parse_char_with_fun(Fun, Input), - {Res, Input2} = parse_times((Times - 1), Fun, Input1), - {[C1 | Res], Input2}. - -parse_int_times(Times, Input) -> - parse_times(Times, is_int_char, Input). - -parse_fraction(Input) -> - IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, - lists:splitwith(IsFractionChar, Input). - -parse_second(Input) -> - {M, Input1} = parse_int_times(2, Input), - {M1, Input2} = parse_fraction(Input1), - {M ++ M1, Input2}. - -parse_zone(Input) -> - {S, Input1} = parse_char_with_fun(is_symbol_char, Input), - {M, Input2} = parse_int_times(2, Input1), - {C, Input3} = parse_char_with_fun(is_m_char, Input2), - {V, Input4} = parse_int_times(2, Input3), - {[S | M ++ [C | V]], Input4}. - -mlist1() -> - maps:from_list( - %% %H Shows hour in 24-hour format [15] - [ - {$H, fun(Input) -> parse_int_times(2, Input) end}, - %% %M Displays minutes [00-59] - {$M, fun(Input) -> parse_int_times(2, Input) end}, - %% %S Displays seconds [00-59] - {$S, fun(Input) -> parse_second(Input) end}, - %% %y Displays year YYYY [2021] - {$y, fun(Input) -> parse_int_times(4, Input) end}, - %% %m Displays the number of the month [01-12] - {$m, fun(Input) -> parse_int_times(2, Input) end}, - %% %d Displays the number of the month [01-12] - {$d, fun(Input) -> parse_int_times(2, Input) end}, - %% %Z Displays Time zone - {$Z, fun(Input) -> parse_zone(Input) end} - ] - ). - -update_result($H, Res, Str) -> Res#result{hour = Str}; -update_result($M, Res, Str) -> Res#result{minute = Str}; -update_result($S, Res, Str) -> Res#result{second = Str}; -update_result($y, Res, Str) -> Res#result{year = Str}; -update_result($m, Res, Str) -> Res#result{month = Str}; -update_result($d, Res, Str) -> Res#result{day = Str}; -update_result($Z, Res, Str) -> Res#result{zone = Str}. - -parse_tag(Res, {st, St}, InputString) -> - {_A, B} = parse_string(St, InputString), - {Res, B}; -parse_tag(Res, {tag, St}, InputString) -> - Fun = maps:get(St, mlist1()), - {A, B} = Fun(InputString), - NRes = update_result(St, Res, A), - {NRes, B}. - -parse_tags(Res, [], _) -> - Res; -parse_tags(Res, [Tag | Others], InputString) -> - {NRes, B} = parse_tag(Res, Tag, InputString), - parse_tags(NRes, Others, B). - -parse_input(FormatString, InputString) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - parse_tags(#result{}, R, InputString). From d8a022fb4579f505569fc141f07b8f3f4ebdeeae Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Sep 2022 14:36:02 +0800 Subject: [PATCH 05/12] fix(test): unstopped test client --- .../test/emqx_rule_engine_SUITE.erl | 14 +++++++++----- apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 42816e869..c6f5057f1 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -238,8 +238,12 @@ end_per_testcase(t_events, Config) -> ok = emqx_rule_registry:remove_rule(?config(hook_points_rules, Config)), ok = emqx_rule_registry:remove_action('hook-metrics-action'); end_per_testcase(Test, Config) - when Test =:= t_sqlselect_multi_actoins_1, - Test =:= t_sqlselect_multi_actoins_2 + when Test =:= t_sqlselect_multi_actoins_1 + ;Test =:= t_sqlselect_multi_actoins_1_1 + ;Test =:= t_sqlselect_multi_actoins_2 + ;Test =:= t_sqlselect_multi_actoins_3 + ;Test =:= t_sqlselect_multi_actoins_3_1 + ;Test =:= t_sqlselect_multi_actoins_4 -> emqtt:stop(?config(subclient, Config)), emqtt:stop(?config(connclient, Config)), @@ -1218,8 +1222,8 @@ t_match_atom_and_binary(_Config) -> {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), ct:sleep(100), - {ok, Client2} = emqtt:start_link([{username, <<"emqx2">>}]), - {ok, _} = emqtt:connect(Client2), + {ok, Client1} = emqtt:start_link([{username, <<"emqx2">>}]), + {ok, _} = emqtt:connect(Client1), receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(<<"t2">>, T), <<"user:", ConnAt/binary>> = Payload, @@ -1228,7 +1232,7 @@ t_match_atom_and_binary(_Config) -> ct:fail(wait_for_t2) end, - emqtt:stop(Client), + emqtt:stop(Client), emqtt:stop(Client1), emqx_rule_registry:remove_rule(TopicRule). t_metrics(_Config) -> diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index d93b747c0..5fd966d6c 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -474,7 +474,7 @@ t_sqlselect_3(_Config) -> ok end, - emqtt:stop(Client), + emqtt:stop(Client), emqtt:stop(Client1), emqx_rule_registry:remove_rule(TopicRule). %%------------------------------------------------------------------------------ From 0a5a0867e43d1790ac9a7f1d8668f35e07f5e224 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 29 Sep 2022 14:05:40 +0800 Subject: [PATCH 06/12] fix(rule_func): refine `+` error info when type implicit conversion --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 18e61967e..a4791a94d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -338,7 +338,13 @@ null() -> %% concat 2 strings '+'(X, Y) when is_binary(X), is_binary(Y) -> - concat(X, Y). + concat(X, Y); + +%% unsupported type implicit conversion +'+'(X, Y) + when (is_number(X) andalso is_binary(Y)) orelse + (is_binary(X) andalso is_number(Y)) -> + error(unsupported_type_implicit_conversion). '-'(X, Y) when is_number(X), is_number(Y) -> X - Y. From 32376c7cf962da1b4f6fe3f06f67c87523d872db Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 29 Sep 2022 16:21:40 +0800 Subject: [PATCH 07/12] fix(rule_func): refine num funcs error info type unsupported --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index a4791a94d..da345665b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -332,34 +332,46 @@ null() -> %% Arithmetic Funcs %%------------------------------------------------------------------------------ +-define(OPERATOR_TYPE_ERROR, unsupported_function_operator_type). + %% plus 2 numbers '+'(X, Y) when is_number(X), is_number(Y) -> X + Y; - %% concat 2 strings '+'(X, Y) when is_binary(X), is_binary(Y) -> concat(X, Y); - %% unsupported type implicit conversion '+'(X, Y) when (is_number(X) andalso is_binary(Y)) orelse (is_binary(X) andalso is_number(Y)) -> - error(unsupported_type_implicit_conversion). + error(unsupported_type_implicit_conversion); +'+'(_, _) -> + error(?OPERATOR_TYPE_ERROR). '-'(X, Y) when is_number(X), is_number(Y) -> - X - Y. + X - Y; +'-'(_, _) -> + error(?OPERATOR_TYPE_ERROR). '*'(X, Y) when is_number(X), is_number(Y) -> - X * Y. + X * Y; +'*'(_, _) -> + error(?OPERATOR_TYPE_ERROR). '/'(X, Y) when is_number(X), is_number(Y) -> - X / Y. + X / Y; +'/'(_, _) -> + error(?OPERATOR_TYPE_ERROR). 'div'(X, Y) when is_integer(X), is_integer(Y) -> - X div Y. + X div Y; +'div'(_, _) -> + error(?OPERATOR_TYPE_ERROR). mod(X, Y) when is_integer(X), is_integer(Y) -> - X rem Y. + X rem Y; +mod(_, _) -> + error(?OPERATOR_TYPE_ERROR). eq(X, Y) -> X == Y. From 27e19da06682344da2126ae258dd2d40217e40c3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 29 Sep 2022 16:23:13 +0800 Subject: [PATCH 08/12] test(rulesql): num funcs type cases --- .../test/emqx_rulesql_SUITE.erl | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index 5fd966d6c..33de9b7dd 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -236,8 +236,27 @@ t_sqlselect_00(_Config) -> #{<<"rawsql">> => Sql3, <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). + <<"topic">> => <<"t/a">>}})), + Sql4 = "select payload.msg1 + payload.msg2 as msg " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"msg">> := <<"hello world">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => + #{<<"payload">> => <<"{\"msg1\": \"hello\", \"msg2\": \" world\"}">>, + <<"topic">> => <<"t/1">>}})), + + Sql5 = "select payload.msg1 + payload.msg2 as msg " + "from \"t/#\" ", + ?assertMatch({error, {select_and_transform_error, {error, unsupported_type_implicit_conversion, _ST}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql5, + <<"ctx">> => + #{<<"payload">> => <<"{\"msg1\": \"hello\", \"msg2\": 1}">>, + <<"topic">> => <<"t/1">>}})). + +%% Verify SELECT with and with 'WHERE' t_sqlselect_01(_Config) -> ok = emqx_rule_engine:load_providers(), TopicRule1 = create_simple_repub_rule( @@ -312,7 +331,50 @@ t_sqlselect_02(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule1). +t_sqlselect_03(_Config) -> + %% Verify SELECT with and with 'WHERE' and `+` `=` and `or` condition in 'WHERE' clause + ok = emqx_rule_engine:load_providers(), + TopicRule1 = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"t3/#\", \"t1\" " + "WHERE payload.x + payload.y = 2 or payload.x + payload.y = \"11\""), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1, \"y\":1}">>, 0), + ct:sleep(100), + receive {publish, #{topic := T1, payload := Payload0}} -> + ?assertEqual(<<"t2">>, T1), + ?assertEqual(<<"{\"x\":1, \"y\":1}">>, Payload0) + after 1000 -> + ct:fail(wait_for_t2) + end, + receive {publish, #{topic := T2, payload := Payload1}} -> + ?assertEqual(<<"t2">>, T2), + ?assertEqual(<<"{\"x\":\"1\", \"y\":\"1\"}">>, Payload1) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1, \"y\":2}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := Payload2}} -> + ct:fail({unexpected_t2, Payload2}) + after 1000 -> + ok + end, + + emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1, \"y\":1}">>, 0), + receive {publish, #{topic := T3, payload := Payload3}} -> + ?assertEqual(<<"t2">>, T3), + ?assertEqual(<<"{\"x\":1, \"y\":1}">>, Payload3) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule1). t_sqlselect_1(_Config) -> ok = emqx_rule_engine:load_providers(), From 5047211950f223ad9972f364ccc2b5467472eaf7 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Sun, 9 Oct 2022 14:20:48 +0800 Subject: [PATCH 09/12] test: rulesql select fields and select from event message --- .../test/emqx_rulesql_SUITE.erl | 297 +++++++++++------- 1 file changed, 188 insertions(+), 109 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index 33de9b7dd..f4f329e87 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("emqx_rule_engine/include/rule_engine.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -51,15 +52,16 @@ groups() -> , t_sqlselect_02 , t_sqlselect_1 , t_sqlselect_2 - , t_sqlselect_2_1 - , t_sqlselect_2_2 - , t_sqlselect_2_3 - , t_sqlselect_3 ]}, {rulesql_select_events, [], - [ t_sqlparse_event_1 - , t_sqlparse_event_2 - , t_sqlparse_event_3 + [ t_sqlparse_event_client_connected_01 + , t_sqlparse_event_client_connected_02 + , t_sqlparse_event_client_disconnected + , t_sqlparse_event_session_subscribed + , t_sqlparse_event_session_unsubscribed + , t_sqlparse_event_message_delivered + , t_sqlparse_event_message_acked + , t_sqlparse_event_message_dropped ]}, {rulesql_select_metadata, [], [ t_sqlparse_select_matadata_1 @@ -159,16 +161,35 @@ end_per_testcase(_TestCase, _Config) -> t_sqlselect_0(_Config) -> %% Verify SELECT with and without 'AS' - Sql = "select * " - "from \"t/#\" " - "where payload.cmd.info = 'tt'", - ?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), + + Sql1 = "SELECT * " + "FROM \"t/#\" " + "WHERE payload.cmd.info = 'tt'", + %% all available fields by `select` from message publish, selecte other key will be `undefined` + Topic = <<"t/a">>, + Payload = <<"{\"cmd\": {\"info\":\"tt\"}}">>, + {ok, #{username := <<"u_emqx">>, + topic := Topic, + timestamp := TimeStamp, + qos := 1, + publish_received_at := TimeStamp, + peerhost := <<"127.0.0.1">>, + payload := Payload, + node := 'test@127.0.0.1', + metadata := #{rule_id := TestRuleId}, + id := MsgId, + flags := #{sys := true, event := true}, + clientid := <<"c_emqx">> + } + } = emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => Topic}}), + ?assert(is_binary(TestRuleId)), + ?assert(is_binary(MsgId)), + ?assert(is_integer(TimeStamp)), + Sql2 = "select payload.cmd as cmd " "from \"t/#\" " "where cmd.info = 'tt'", @@ -179,6 +200,7 @@ t_sqlselect_0(_Config) -> #{<<"payload">> => <<"{\"cmd\": {\"info\":\"tt\"}}">>, <<"topic">> => <<"t/a">>}})), + Sql3 = "select payload.cmd as cmd, cmd.info as info " "from \"t/#\" " "where cmd.info = 'tt' and info = 'tt'", @@ -205,14 +227,15 @@ t_sqlselect_0(_Config) -> t_sqlselect_00(_Config) -> %% Verify plus/subtract and unary_add_or_subtract - Sql = "select 1-1 as a " + Sql0 = "select 1 - 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, + #{<<"rawsql">> => Sql0, <<"ctx">> => #{<<"payload">> => <<"">>, <<"topic">> => <<"t/a">>}})), + Sql1 = "select -1 + 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 0}}, @@ -221,6 +244,7 @@ t_sqlselect_00(_Config) -> <<"ctx">> => #{<<"payload">> => <<"">>, <<"topic">> => <<"t/a">>}})), + Sql2 = "select 1 + 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 2}}, @@ -229,6 +253,7 @@ t_sqlselect_00(_Config) -> <<"ctx">> => #{<<"payload">> => <<"">>, <<"topic">> => <<"t/a">>}})), + Sql3 = "select +1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 1}}, @@ -431,58 +456,93 @@ t_sqlselect_2(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_2_1(_Config) -> +%%------------------------------------------------------------------------------ +%% Test cases for events +%%------------------------------------------------------------------------------ + +%% FROM $events/client_connected +t_sqlparse_event_client_connected_01(_Config) -> + Sql = "select *" + "from \"$events/client_connected\" ", + + %% all available fields by `select` from message publish, selecte other key will be `undefined` + {ok, #{clientid := <<"c_emqx">>, + username := <<"u_emqx">>, + timestamp := TimeStamp, + connected_at := TimeStamp, + peername := <<"127.0.0.1:12345">>, + metadata := #{rule_id := RuleId}, + %% default value + node := 'test@127.0.0.1', + sockname := <<"0.0.0.0:1883">>, + proto_name := <<"MQTT">>, + proto_ver := 5, + mountpoint := undefined, + keepalive := 60, + is_bridge := false, + expiry_interval := 3600, + event := 'client.connected', + clean_start := true + } + } = emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"clientid">> => <<"c_emqx">>, <<"peername">> => <<"127.0.0.1:12345">>, <<"username">> => <<"u_emqx">>}}), + ?assert(is_binary(RuleId)), + ?assert(is_integer(TimeStamp)). + +t_sqlparse_event_client_connected_02(_Config) -> ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg dropped + %% republish the client.connected msg TopicRule = create_simple_repub_rule( - <<"t2">>, + <<"repub/to/connected">>, "SELECT * " - "FROM \"$events/message_dropped\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + "FROM \"$events/client_connected\" " + "WHERE username = 'emqx1'", + <<"{clientid: ${clientid}}">>), + {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), {ok, _} = emqtt:connect(Client), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_nothing = Fun(), + {ok, _, _} = emqtt:subscribe(Client, <<"repub/to/connected">>, 0), + ct:sleep(200), + {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), + {ok, _} = emqtt:connect(Client1), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"repub/to/connected">>, T), + ?assertEqual(<<"{clientid: c_emqx1}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, - %% it should not keep republishing "t2" - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - received_nothing = Fun(), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, - emqtt:stop(Client), + emqtt:stop(Client), emqtt:stop(Client1), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_2_2(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg acked - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_acked\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), +%% FROM $events/client_disconnected +t_sqlparse_event_client_disconnected(_Config) -> + %% TODO + ok. - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). +%% FROM $events/session_subscribed +t_sqlparse_event_session_subscribed(_Config) -> + Sql = "select topic as tp " + "from \"$events/session_subscribed\" ", + ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). -t_sqlselect_2_3(_Config) -> +%% FROM $events/session_unsubscribed +t_sqlparse_event_session_unsubscribed(_Config) -> + %% TODO + ok. + +%% FROM $events/message_delivered +t_sqlparse_event_message_delivered(_Config) -> ok = emqx_rule_engine:load_providers(), %% recursively republish to t2, if the msg delivered TopicRule = create_simple_repub_rule( @@ -507,65 +567,84 @@ t_sqlselect_2_3(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_3(_Config) -> +%% FROM $events/message_acked +t_sqlparse_event_message_acked(_Config) -> ok = emqx_rule_engine:load_providers(), - %% republish the client.connected msg + %% republish to `repub/if/acked`, if the msg acked TopicRule = create_simple_repub_rule( - <<"t2">>, + <<"repub/if/acked">>, "SELECT * " - "FROM \"$events/client_connected\" " - "WHERE username = 'emqx1'", - <<"clientid=${clientid}">>), - {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), + "FROM \"$events/message_acked\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ct:sleep(200), - {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), - {ok, _} = emqtt:connect(Client1), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, + {ok, _, _} = emqtt:subscribe(Client, <<"repub/if/acked">>, ?QOS_1), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, + Fun = fun() -> + receive {publish, #{topic := <<"repub/if/acked">>, payload := _}} -> + received_acked + after 500 -> + received_nothing + end + end, - emqtt:stop(Client), emqtt:stop(Client1), + %% sub with max qos1 to generate ack packet + {ok, _, _} = emqtt:subscribe(Client, <<"any/topic">>, ?QOS_1), + + Payload = <<"{\"x\":1,\"y\":144}">>, + + %% even sub with qos1, but publish with qos0, no ack + emqtt:publish(Client, <<"any/topic">>, Payload, ?QOS_0), + received_nothing = Fun(), + + %% sub and pub both are qos1, acked + emqtt:publish(Client, <<"any/topic">>, Payload, ?QOS_1), + received_acked = Fun(), + received_nothing = Fun(), + + %% pub with qos2 but subscribed with qos1, acked + emqtt:publish(Client, <<"any/topic">>, Payload, ?QOS_2), + received_acked = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -%%------------------------------------------------------------------------------ -%% Test cases for events -%%------------------------------------------------------------------------------ +%% FROM $events/message_dropped +t_sqlparse_event_message_dropped(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% republish to `repub/if/dropped`, if any msg dropped + TopicRule = create_simple_repub_rule( + <<"repub/if/dropped">>, + "SELECT * " + "FROM \"$events/message_dropped\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), -t_sqlparse_event_1(_Config) -> - Sql = "select topic as tp " - "from \"$events/session_subscribed\" ", - ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). + %% this message will be dropped and then repub to `repub/if/dropped` + emqtt:publish(Client, <<"any/topic/1">>, <<"{\"x\":1,\"y\":144}">>, 0), -t_sqlparse_event_2(_Config) -> - Sql = "select clientid " - "from \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>}})). + Fun_t2 = fun() -> + receive {publish, #{topic := <<"repub/if/dropped">>, payload := _}} -> + received_repub + after 500 -> + received_nothing + end + end, -t_sqlparse_event_3(_Config) -> - Sql = "select clientid, topic as tp " - "from \"t/tt\", \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})). + %% No client subscribed `any/topic`, triggered repub rule. + %% But havn't sub `repub/if/dropped`, so the repub message will also be dropped with recursively republish. + received_nothing = Fun_t2(), + + {ok, _, _} = emqtt:subscribe(Client, <<"repub/if/dropped">>, 0), + + %% this message will be dropped and then repub to `repub/to/t` + emqtt:publish(Client, <<"any/topic/2">>, <<"{\"x\":1,\"y\":144}">>, 0), + + %% received subscribed `repub/to/t` + received_repub = Fun_t2(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). %%------------------------------------------------------------------------------ %% Test cases for `foreach` From 3e8c070b59ecac07ecf5f0464bb73b73c1808979 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Oct 2022 12:35:11 +0800 Subject: [PATCH 10/12] Revert "fix: rm unused module" This reverts commit 267946c3794b8c251a056e9f4cb975dd04b1b14f. Revert it temporary, we need fix `scripts/update-appup.sh` later. It only compares the current release with its predecessor. --- apps/emqx_rule_engine/src/emqx_rule_date.erl | 248 +++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 apps/emqx_rule_engine/src/emqx_rule_date.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_date.erl b/apps/emqx_rule_engine/src/emqx_rule_date.erl new file mode 100644 index 000000000..fb9cad4c3 --- /dev/null +++ b/apps/emqx_rule_engine/src/emqx_rule_date.erl @@ -0,0 +1,248 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rule_date). + +-export([date/3, date/4, parse_date/4]). + +-export([ + is_int_char/1, + is_symbol_char/1, + is_m_char/1 +]). + +-record(result, { + %%year() + year = "1970" :: string(), + %%month() + month = "1" :: string(), + %%day() + day = "1" :: string(), + %%hour() + hour = "0" :: string(), + %%minute() %% epoch in millisecond precision + minute = "0" :: string(), + %%second() %% epoch in millisecond precision + second = "0" :: string(), + %%integer() %% zone maybe some value + zone = "+00:00" :: string() +}). + +%% -type time_unit() :: 'microsecond' +%% | 'millisecond' +%% | 'nanosecond' +%% | 'second'. +%% -type offset() :: [byte()] | (Time :: integer()). +date(TimeUnit, Offset, FormatString) -> + date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)). + +date(TimeUnit, Offset, FormatString, TimeEpoch) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + Res = lists:map( + fun(Expr) -> + eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr) + end, + R + ), + lists:concat(Res). + +parse_date(TimeUnit, Offset, FormatString, InputString) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + IsZ = fun(V) -> + case V of + {tag, $Z} -> true; + _ -> false + end + end, + R1 = lists:filter(IsZ, R), + IfFun = fun(Con, A, B) -> + case Con of + [] -> A; + _ -> B + end + end, + Res = parse_input(FormatString, InputString), + Str = + Res#result.year ++ "-" ++ + Res#result.month ++ "-" ++ + Res#result.day ++ "T" ++ + Res#result.hour ++ ":" ++ + Res#result.minute ++ ":" ++ + Res#result.second ++ + IfFun(R1, Offset, Res#result.zone), + calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]). + +mlist(R) -> + %% %H Shows hour in 24-hour format [15] + [ + {$H, R#result.hour}, + %% %M Displays minutes [00-59] + {$M, R#result.minute}, + %% %S Displays seconds [00-59] + {$S, R#result.second}, + %% %y Displays year YYYY [2021] + {$y, R#result.year}, + %% %m Displays the number of the month [01-12] + {$m, R#result.month}, + %% %d Displays the number of the month [01-12] + {$d, R#result.day}, + %% %Z Displays Time zone + {$Z, R#result.zone} + ]. + +rmap(Result) -> + maps:from_list(mlist(Result)). + +support_char() -> "HMSymdZ". + +create_tag(Head, []) -> + Head; +create_tag(Head, [Val1 | RVal]) -> + case Val1 of + [] -> + create_tag(Head ++ [{st, [$%]}], RVal); + [H | Other] -> + case lists:member(H, support_char()) of + true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal); + false -> create_tag(Head ++ [{st, [$% | Val1]}], RVal) + end + end. + +eval_tag(_, {st, Str}) -> + Str; +eval_tag(Map, {tag, Char}) -> + maps:get(Char, Map, "undefined"). + +%% make_time(TimeUnit, Offset) -> +%% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)). +make_time(TimeUnit, Offset, TimeEpoch) -> + Res = calendar:system_time_to_rfc3339(TimeEpoch, + [{unit, TimeUnit}, {offset, Offset}]), + [Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T, + H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = Res, + IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, + {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), + #result{ + year = [Y1, Y2, Y3, Y4] + , month = [Mon1, Mon2] + , day = [D1, D2] + , hour = [H1, H2] + , minute = [Min1, Min2] + , second = [S1, S2] ++ FractionStr + , zone = UtcOffset + }. + +is_int_char(C) -> + C >= $0 andalso C =< $9. +is_symbol_char(C) -> + C =:= $- orelse C =:= $+. +is_m_char(C) -> + C =:= $:. + +parse_char_with_fun(_, []) -> + error(null_input); +parse_char_with_fun(ValidFun, [C | Other]) -> + Res = + case erlang:is_function(ValidFun) of + true -> ValidFun(C); + false -> erlang:apply(emqx_rule_date, ValidFun, [C]) + end, + case Res of + true -> {C, Other}; + false -> error({unexpected, [C | Other]}) + end. +parse_string([], Input) -> + {[], Input}; +parse_string([C | Other], Input) -> + {C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input), + {Res, Input2} = parse_string(Other, Input1), + {[C1 | Res], Input2}. + +parse_times(0, _, Input) -> + {[], Input}; +parse_times(Times, Fun, Input) -> + {C1, Input1} = parse_char_with_fun(Fun, Input), + {Res, Input2} = parse_times((Times - 1), Fun, Input1), + {[C1 | Res], Input2}. + +parse_int_times(Times, Input) -> + parse_times(Times, is_int_char, Input). + +parse_fraction(Input) -> + IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, + lists:splitwith(IsFractionChar, Input). + +parse_second(Input) -> + {M, Input1} = parse_int_times(2, Input), + {M1, Input2} = parse_fraction(Input1), + {M ++ M1, Input2}. + +parse_zone(Input) -> + {S, Input1} = parse_char_with_fun(is_symbol_char, Input), + {M, Input2} = parse_int_times(2, Input1), + {C, Input3} = parse_char_with_fun(is_m_char, Input2), + {V, Input4} = parse_int_times(2, Input3), + {[S | M ++ [C | V]], Input4}. + +mlist1() -> + maps:from_list( + %% %H Shows hour in 24-hour format [15] + [ + {$H, fun(Input) -> parse_int_times(2, Input) end}, + %% %M Displays minutes [00-59] + {$M, fun(Input) -> parse_int_times(2, Input) end}, + %% %S Displays seconds [00-59] + {$S, fun(Input) -> parse_second(Input) end}, + %% %y Displays year YYYY [2021] + {$y, fun(Input) -> parse_int_times(4, Input) end}, + %% %m Displays the number of the month [01-12] + {$m, fun(Input) -> parse_int_times(2, Input) end}, + %% %d Displays the number of the month [01-12] + {$d, fun(Input) -> parse_int_times(2, Input) end}, + %% %Z Displays Time zone + {$Z, fun(Input) -> parse_zone(Input) end} + ] + ). + +update_result($H, Res, Str) -> Res#result{hour = Str}; +update_result($M, Res, Str) -> Res#result{minute = Str}; +update_result($S, Res, Str) -> Res#result{second = Str}; +update_result($y, Res, Str) -> Res#result{year = Str}; +update_result($m, Res, Str) -> Res#result{month = Str}; +update_result($d, Res, Str) -> Res#result{day = Str}; +update_result($Z, Res, Str) -> Res#result{zone = Str}. + +parse_tag(Res, {st, St}, InputString) -> + {_A, B} = parse_string(St, InputString), + {Res, B}; +parse_tag(Res, {tag, St}, InputString) -> + Fun = maps:get(St, mlist1()), + {A, B} = Fun(InputString), + NRes = update_result(St, Res, A), + {NRes, B}. + +parse_tags(Res, [], _) -> + Res; +parse_tags(Res, [Tag | Others], InputString) -> + {NRes, B} = parse_tag(Res, Tag, InputString), + parse_tags(NRes, Others, B). + +parse_input(FormatString, InputString) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + parse_tags(#result{}, R, InputString). From 7f92c29ada1e235b3b45a1af2e8022fbd0ae7130 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Sep 2022 10:03:20 +0800 Subject: [PATCH 11/12] chore: bump appup.src for `emqx_rule_funcs.erl` --- .../src/emqx_rule_engine.appup.src | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index b46be85c7..430d1f7e3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,20 +2,23 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.15", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -24,7 +27,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -33,7 +37,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -43,7 +48,8 @@ {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -206,20 +212,23 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.15", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -228,7 +237,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -237,7 +247,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -247,7 +258,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, From 73d72eaccea0500aabc6bdcb68e76137e9bb11ac Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 11 Oct 2022 12:48:35 +0200 Subject: [PATCH 12/12] fix: demote rate limit log level to notice --- CHANGES-4.3.md | 1 + apps/emqx_exproto/src/emqx_exproto_conn.erl | 2 +- apps/emqx_stomp/src/emqx_stomp_connection.erl | 2 +- src/emqx_connection.erl | 2 +- src/emqx_ws_connection.erl | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index f136b7835..23899cb25 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -15,6 +15,7 @@ File format: ### Bug fixes - Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic. [#9024](https://github.com/emqx/emqx/pull/9024) +- "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134) ## v4.3.21 diff --git a/apps/emqx_exproto/src/emqx_exproto_conn.erl b/apps/emqx_exproto/src/emqx_exproto_conn.erl index 7c6eb6cb3..51abe86e9 100644 --- a/apps/emqx_exproto/src/emqx_exproto_conn.erl +++ b/apps/emqx_exproto/src/emqx_exproto_conn.erl @@ -643,7 +643,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, diff --git a/apps/emqx_stomp/src/emqx_stomp_connection.erl b/apps/emqx_stomp/src/emqx_stomp_connection.erl index e58b242e5..7177155fd 100644 --- a/apps/emqx_stomp/src/emqx_stomp_connection.erl +++ b/apps/emqx_stomp/src/emqx_stomp_connection.erl @@ -469,7 +469,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 7c9cfef0a..3cfcd8cdc 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -756,7 +756,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index a0c999ea9..0a29c1ee2 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -499,7 +499,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), NState = State#state{sockstate = blocked, limiter = Limiter1,