From 502f3e8d5c4f378a4902e72e1dafba69d6fad2ea Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 22 Sep 2022 13:33:41 +0800 Subject: [PATCH 01/44] test(mqtt_sn): improve test coverage to 90% --- apps/emqx_sn/test/emqx_sn_frame_SUITE.erl | 76 +++++++++++++- apps/emqx_sn/test/emqx_sn_misc_SUITE.erl | 62 ++++++++++++ apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 101 +++++++++++++++++++ apps/emqx_sn/test/emqx_sn_registry_SUITE.erl | 5 + 4 files changed, 241 insertions(+), 3 deletions(-) create mode 100644 apps/emqx_sn/test/emqx_sn_misc_SUITE.erl diff --git a/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl b/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl index d4500315c..813a1a343 100644 --- a/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("emqx_sn/include/emqx_sn.hrl"). -include_lib("eunit/include/eunit.hrl"). +-define(SHOW(X), ??X). -import(emqx_sn_frame, [ parse/1 , serialize/1 @@ -67,6 +68,14 @@ t_willtopic(_) -> Wt = #mqtt_sn_message{type = ?SN_WILLTOPIC, variable = {Flags, <<"WillTopic">>}}, ?assertEqual({ok, Wt}, parse(serialize(Wt))). +t_undefined_willtopic(_) -> + Wt = #mqtt_sn_message{type = ?SN_WILLTOPIC}, + ?assertEqual({ok, Wt}, parse(serialize(Wt))). + +t_willtopic_resp(_) -> + Wt = #mqtt_sn_message{type = ?SN_WILLTOPICRESP, variable = 0}, + ?assertEqual({ok, Wt}, parse(serialize(Wt))). + t_willmsgreq(_) -> WmReq = #mqtt_sn_message{type = ?SN_WILLMSGREQ}, ?assertEqual({ok, WmReq}, parse(serialize(WmReq))). @@ -88,6 +97,12 @@ t_publish(_) -> PubMsg = #mqtt_sn_message{type = ?SN_PUBLISH, variable = {Flags, 1, 2, <<"Payload">>}}, ?assertEqual({ok, PubMsg}, parse(serialize(PubMsg))). +t_publish_long_msg(_) -> + Flags = #mqtt_sn_flags{dup = false, qos = 1, retain = false, topic_id_type = 2#01}, + Payload = generate_random_binary(256 + rand:uniform(256)), + PubMsg = #mqtt_sn_message{type = ?SN_PUBLISH, variable = {Flags, 1, 2, Payload}}, + ?assertEqual({ok, PubMsg}, parse(serialize(PubMsg))). + t_puback(_) -> PubAck = #mqtt_sn_message{type = ?SN_PUBACK, variable = {1, 2, 0}}, ?assertEqual({ok, PubAck}, parse(serialize(PubAck))). @@ -105,9 +120,21 @@ t_pubcomp(_) -> ?assertEqual({ok, PubComp}, parse(serialize(PubComp))). t_subscribe(_) -> - Flags = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = 16#01}, + Flags = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = ?SN_PREDEFINED_TOPIC}, SubMsg = #mqtt_sn_message{type = ?SN_SUBSCRIBE, variable = {Flags, 16#4321, 16}}, - ?assertEqual({ok, SubMsg}, parse(serialize(SubMsg))). + ?assertEqual({ok, SubMsg}, parse(serialize(SubMsg))), + + Flags1 = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = ?SN_NORMAL_TOPIC}, + SubMsg1 = #mqtt_sn_message{type = ?SN_SUBSCRIBE, variable = {Flags1, 16#4321, <<"t/+">>}}, + ?assertEqual({ok, SubMsg1}, parse(serialize(SubMsg1))), + + Flags2 = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = ?SN_SHORT_TOPIC}, + SubMsg2 = #mqtt_sn_message{type = ?SN_SUBSCRIBE, variable = {Flags2, 16#4321, <<"t/+">>}}, + ?assertEqual({ok, SubMsg2}, parse(serialize(SubMsg2))), + + Flags3 = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = ?SN_RESERVED_TOPIC}, + SubMsg3 = #mqtt_sn_message{type = ?SN_SUBSCRIBE, variable = {Flags3, 16#4321, <<"t/+">>}}, + ?assertEqual({ok, SubMsg3}, parse(serialize(SubMsg3))). t_suback(_) -> Flags = #mqtt_sn_flags{qos = 1}, @@ -137,6 +164,10 @@ t_disconnect(_) -> Disconn = #mqtt_sn_message{type = ?SN_DISCONNECT}, ?assertEqual({ok, Disconn}, parse(serialize(Disconn))). +t_disconnect_duration(_) -> + Disconn = #mqtt_sn_message{type = ?SN_DISCONNECT, variable = 120}, + ?assertEqual({ok, Disconn}, parse(serialize(Disconn))). + t_willtopicupd(_) -> Flags = #mqtt_sn_flags{qos = 1, retain = true}, WtUpd = #mqtt_sn_message{type = ?SN_WILLTOPICUPD, variable = {Flags, <<"Topic">>}}, @@ -150,6 +181,43 @@ t_willmsgresp(_) -> UpdResp = #mqtt_sn_message{type = ?SN_WILLMSGRESP, variable = 0}, ?assertEqual({ok, UpdResp}, parse(serialize(UpdResp))). +t_invalid_inpacket(_) -> + Bin = <<2:8/big-integer, 16#F0:8/big-integer>>, + ?assertMatch({'EXIT', {unkown_message_type, _Stack}}, catch parse(Bin)). + +t_message_type(_) -> + TypeNames = [ {?SN_ADVERTISE, ?SHOW(SN_ADVERTISE)} + , {?SN_SEARCHGW, ?SHOW(SN_SEARCHGW)} + , {?SN_GWINFO, ?SHOW(SN_GWINFO)} + , {?SN_CONNECT, ?SHOW(SN_CONNECT)} + , {?SN_CONNACK, ?SHOW(SN_CONNACK)} + , {?SN_WILLTOPICREQ, ?SHOW(SN_WILLTOPICREQ)} + , {?SN_WILLTOPIC, ?SHOW(SN_WILLTOPIC)} + , {?SN_WILLMSGREQ, ?SHOW(SN_WILLMSGREQ)} + , {?SN_WILLMSG, ?SHOW(SN_WILLMSG)} + , {?SN_REGISTER, ?SHOW(SN_REGISTER)} + , {?SN_REGACK, ?SHOW(SN_REGACK)} + , {?SN_PUBLISH, ?SHOW(SN_PUBLISH)} + , {?SN_PUBACK, ?SHOW(SN_PUBACK)} + , {?SN_PUBCOMP, ?SHOW(SN_PUBCOMP)} + , {?SN_PUBREC, ?SHOW(SN_PUBREC)} + , {?SN_PUBREL, ?SHOW(SN_PUBREL)} + , {?SN_SUBSCRIBE, ?SHOW(SN_SUBSCRIBE)} + , {?SN_SUBACK, ?SHOW(SN_SUBACK)} + , {?SN_UNSUBSCRIBE, ?SHOW(SN_UNSUBSCRIBE)} + , {?SN_UNSUBACK, ?SHOW(SN_UNSUBACK)} + , {?SN_PINGREQ, ?SHOW(SN_PINGREQ)} + , {?SN_PINGRESP, ?SHOW(SN_PINGRESP)} + , {?SN_DISCONNECT, ?SHOW(SN_DISCONNECT)} + , {?SN_WILLTOPICUPD, ?SHOW(SN_WILLTOPICUPD)} + , {?SN_WILLTOPICRESP, ?SHOW(SN_WILLTOPICRESP)} + , {?SN_WILLMSGUPD, ?SHOW(SN_WILLMSGUPD)} + , {?SN_WILLMSGRESP, ?SHOW(SN_WILLMSGRESP)} + ], + {Types, Names} = lists:unzip(TypeNames), + ?assertEqual(Names, [emqx_sn_frame:message_type(Type) || Type <- Types]), + ok. + t_random_test(_) -> random_test_body(), random_test_body(), @@ -171,6 +239,9 @@ random_test_body() -> generate_random_binary() -> % The min packet length is 2 Len = rand:uniform(299) + 1, + generate_random_binary(Len). + +generate_random_binary(Len) -> gen_next(Len, <<>>). gen_next(0, Acc) -> @@ -178,4 +249,3 @@ gen_next(0, Acc) -> gen_next(N, Acc) -> Byte = rand:uniform(256) - 1, gen_next(N-1, <>). - diff --git a/apps/emqx_sn/test/emqx_sn_misc_SUITE.erl b/apps/emqx_sn/test/emqx_sn_misc_SUITE.erl new file mode 100644 index 000000000..127cb22e3 --- /dev/null +++ b/apps/emqx_sn/test/emqx_sn_misc_SUITE.erl @@ -0,0 +1,62 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_sn_misc_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_sn]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_sn]). + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- +t_sn_app(_) -> + ?assertMatch({'EXIT', {_, _}}, catch emqx_sn_app:start_listeners()), + ?assertMatch({error, _}, emqx_sn_app:stop_listener({udp, 9999, []})), + ?assertMatch({error, _}, emqx_sn_app:stop_listener({udp, {{0,0,0,0}, 9999}, []})), + ok. + +t_sn_broadcast(_) -> + ?assertEqual(ignored, gen_server:call(emqx_sn_broadcast, ignored)), + ?assertEqual(ok, gen_server:cast(emqx_sn_broadcast, ignored)), + ?assertEqual(ignored, erlang:send(emqx_sn_broadcast, ignored)), + ?assertEqual(broadcast_advertise, erlang:send(emqx_sn_broadcast, broadcast_advertise)), + ?assertEqual(ok, emqx_sn_broadcast:stop()). + +%%-------------------------------------------------------------------- +%% Helper funcs +%%-------------------------------------------------------------------- diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index 4912b4fcf..b064b630e 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -100,6 +100,17 @@ t_connect(_) -> send_connect_msg(Socket, <<"client_id_test1">>), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + %% unexpected advertise + Adv = ?SN_ADVERTISE_MSG(1, 100), + AdvPacket = emqx_sn_frame:serialize(Adv), + send_packet(Socket, AdvPacket), + timer:sleep(200), + + %% unexpected connect + ClientId = ?CLIENTID, + send_connect_msg(Socket, ClientId), + timer:sleep(200), + send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). @@ -412,6 +423,16 @@ t_publish_negqos_case09(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). +t_publish_negqos_case10(_) -> + QoS = ?QOS_NEG1, + MsgId = 1, + TopicId1 = ?PREDEF_TOPIC_ID1, + {ok, Socket} = gen_udp:open(0, [binary]), + Payload1 = <<20, 21, 22, 23>>, + send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), + timer:sleep(100), + gen_udp:close(Socket). + t_publish_qos0_case01(_) -> Dup = 0, QoS = 0, @@ -1075,6 +1096,60 @@ t_will_case06(_) -> gen_udp:close(Socket). +t_will_case07(_) -> + QoS = 1, + Duration = 1, + WillMsg = <<10, 11, 12, 13, 14>>, + WillTopic = <<"abc">>, + {ok, Socket} = gen_udp:open(0, [binary]), + ClientId = ?CLIENTID, + + ok = emqx_broker:subscribe(WillTopic), + + send_connect_msg_with_will(Socket, Duration, ClientId), + ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), + + %% unexpected advertise + Adv = ?SN_ADVERTISE_MSG(1, 100), + AdvPacket = emqx_sn_frame:serialize(Adv), + send_packet(Socket, AdvPacket), + timer:sleep(200), + + %% unexpected connect + send_connect_msg(Socket, ClientId), + timer:sleep(200), + + send_willtopic_msg(Socket, WillTopic, QoS), + ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), + + %% unexpected advertise + send_packet(Socket, AdvPacket), + timer:sleep(200), + + %% unexpected connect + send_connect_msg(Socket, ClientId), + timer:sleep(200), + + send_willmsg_msg(Socket, WillMsg), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + send_pingreq_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), + + % wait udp client keepalive timeout + timer:sleep(2000), + + receive + {deliver, WillTopic, #message{payload = WillMsg}} -> ok; + Msg -> ct:print("recevived --- unex: ~p", [Msg]) + after + 1000 -> ct:fail(wait_willmsg_timeout) + end, + send_disconnect_msg(Socket, undefined), + ?assertEqual(udp_receive_timeout, receive_response(Socket)), + + gen_udp:close(Socket). + t_asleep_test01_timeout(_) -> QoS = 1, Duration = 1, @@ -1960,6 +2035,7 @@ t_register_enqueue_delivering_messages(_) -> _ = emqx:publish(emqx_message:make(test, ?QOS_1, <<"topic-a">>, <<"m2">>)), send_regack_msg(NSocket, TopicIdA, RegMsgIdA, ?SN_RC_ACCEPTED), + send_regack_msg(NSocket, TopicIdA, RegMsgIdA, ?SN_RC_INVALID_TOPIC_ID), %% receive the queued messages @@ -1983,6 +2059,28 @@ t_register_enqueue_delivering_messages(_) -> gen_udp:close(NSocket1), restart_emqx_sn(#{subs_resume => false}). +t_code_change(_) -> + Old = [state, gwid, socket, socketpid, socketstate, socketname, peername, + channel, clientid, username, password, will_msg, keepalive_interval, + connpkt, asleep_timer, enable_stats, stats_timer, enable_qos3, + has_pending_pingresp, pending_topic_ids], + New = Old ++ [false, [], undefined], + + OldTulpe = erlang:list_to_tuple(Old), + NewTulpe = erlang:list_to_tuple(New), + + ?assertEqual({ok, name, NewTulpe}, + emqx_sn_gateway:code_change(1, name, OldTulpe, ["4.3.2"])), + + ?assertEqual({ok, name, NewTulpe}, + emqx_sn_gateway:code_change(1, name, NewTulpe, ["4.3.6"])), + + ?assertEqual({ok, name, NewTulpe}, + emqx_sn_gateway:code_change({down, 1}, name, NewTulpe, ["4.3.6"])), + + ?assertEqual({ok, name, OldTulpe}, + emqx_sn_gateway:code_change({down, 1}, name, NewTulpe, ["4.3.2"])). + %%-------------------------------------------------------------------- %% Helper funcs %%-------------------------------------------------------------------- @@ -2267,6 +2365,9 @@ send_disconnect_msg(Socket, Duration) -> ?LOG("send_disconnect_msg Duration=~p", [Duration]), ok = gen_udp:send(Socket, ?HOST, ?PORT, DisConnectPacket). +send_packet(Socket, Packet) -> + ok = gen_udp:send(Socket, ?HOST, ?PORT, Packet). + mid(Id) -> Id. tid(Id) -> Id. diff --git a/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl b/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl index 0d726f190..5d6bb9572 100644 --- a/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl @@ -108,6 +108,11 @@ t_deny_wildcard_topic(_Config) -> ?assertEqual({error, wildcard_topic}, ?REGISTRY:register_topic(<<"ClientId">>, <<"/TopicA/#">>)), ?assertEqual({error, wildcard_topic}, ?REGISTRY:register_topic(<<"ClientId">>, <<"/+/TopicB">>)). +t_gen_server(_) -> + ?assertEqual(ignored, gen_server:call(emqx_sn_registry, ignored)), + ?assertEqual(ok, gen_server:cast(emqx_sn_registry, ignored)), + ?assertEqual(ignored, erlang:send(emqx_sn_registry, ignored)). + %%-------------------------------------------------------------------- %% Helper funcs %%-------------------------------------------------------------------- From 5b14d7f70908f48b65da303ef76d33c112ea3762 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 26 Sep 2022 16:37:33 +0800 Subject: [PATCH 02/44] test(mqtt_sn): improve test coverage of `emqx_sn_gateway` --- apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 132 ++++++++++++++++++- 1 file changed, 130 insertions(+), 2 deletions(-) diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index b064b630e..60b27af2c 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -85,6 +85,15 @@ restart_emqx_sn(#{subs_resume := Bool}) -> _ = application:ensure_all_started(emqx_sn), ok. +recoverable_restart_emqx_sn(Setup) -> + AppEnvs = application:get_all_env(emqx_sn), + Setup(), + _ = application:stop(emqx_sn), + _ = application:ensure_all_started(emqx_sn), + fun() -> + application:set_env([{emqx_sn, AppEnvs}]) + end. + %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- @@ -388,6 +397,38 @@ t_subscribe_case08(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). +t_subscribe_case09(_) -> + Dup = 0, + QoS = 0, + Retain = 0, + CleanSession = 0, + ReturnCode = 0, + {ok, Socket} = gen_udp:open(0, [binary]), + ClientId = ?CLIENTID, + send_connect_msg(Socket, ClientId), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + TopicName1 = <<"t/+">>, + MsgId1 = 25, + TopicId0 = 0, + WillBit = 0, + send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket)), + + {ok, C} = emqtt:start_link(), + {ok, _} = emqtt:connect(C), + ok = emqtt:publish(C, <<"t/1">>, <<"Hello">>, 0), + timer:sleep(100), + ok = emqtt:disconnect(C), + + timer:sleep(50), + ?assertError(_, receive_publish(Socket)), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + gen_udp:close(Socket). + t_publish_negqos_case09(_) -> Dup = 0, QoS = 0, @@ -404,7 +445,6 @@ t_publish_negqos_case09(_) -> Topic = <<"abc">>, - send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), @@ -429,7 +469,7 @@ t_publish_negqos_case10(_) -> TopicId1 = ?PREDEF_TOPIC_ID1, {ok, Socket} = gen_udp:open(0, [binary]), Payload1 = <<20, 21, 22, 23>>, - send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), + send_publish_msg_predefined_topic(Socket, QoS, MsgId, TopicId1, Payload1), timer:sleep(100), gen_udp:close(Socket). @@ -1333,6 +1373,7 @@ t_asleep_test04_to_awake_qos1_dl_msg(_) -> ?assertError(_, receive_publish(Socket)), send_regack_msg(Socket, TopicIdNew, MsgId3), + send_regack_msg(Socket, TopicIdNew, MsgId3, ?SN_RC_INVALID_TOPIC_ID), UdpData2 = receive_response(Socket), MsgId_udp2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2), @@ -1662,6 +1703,33 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> gen_udp:close(Socket). +t_asleep_unexpected(_) -> + SleepDuration = 3, + {ok, Socket} = gen_udp:open(0, [binary]), + ClientId = ?CLIENTID, + send_connect_msg(Socket, ClientId), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + timer:sleep(200), + + % goto asleep state + send_disconnect_msg(Socket, SleepDuration), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + timer:sleep(100), + + send_pingreq_msg(Socket, undefined), + ?assertEqual(udp_receive_timeout, receive_response(Socket)), + + send_puback_msg(Socket, 5, 5, ?SN_RC_INVALID_TOPIC_ID), + send_pubrec_msg(Socket, 5), + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + send_disconnect_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + timer:sleep(100), + + gen_udp:close(Socket). + t_awake_test01_to_connected(_) -> QoS = 1, Keepalive_Duration = 3, @@ -2081,6 +2149,66 @@ t_code_change(_) -> ?assertEqual({ok, name, OldTulpe}, emqx_sn_gateway:code_change({down, 1}, name, NewTulpe, ["4.3.2"])). +t_topic_id_to_large(_) -> + Dup = 0, + QoS = 0, + Retain = 0, + Will = 0, + CleanSession = 0, + MsgId = 1, + {ok, Socket} = gen_udp:open(0, [binary]), + ClientId = ?CLIENTID, + send_connect_msg(Socket, ClientId), + + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + mnesia:dirty_write(emqx_sn_registry, {emqx_sn_registry, {ClientId, next_topic_id}, 16#FFFF}), + + TopicName1 = <<"abcD">>, + send_register_msg(Socket, TopicName1, MsgId), + ?assertEqual(<<7, ?SN_REGACK, 0:16, MsgId:16, 3:8>>, receive_response(Socket)), + + send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, + CleanSession:1, ?SN_NORMAL_TOPIC:2, 0:16, + MsgId:16, 2>>, receive_response(Socket)), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + gen_udp:close(Socket). + +t_idle_timeout(_) -> + Backup = recoverable_restart_emqx_sn(fun() -> + application:set_env(emqx_sn, idle_timeout, 500), + application:set_env(emqx_sn, enable_qos3, false) + end), + timer:sleep(200), + QoS = ?QOS_NEG1, + MsgId = 1, + TopicId1 = ?PREDEF_TOPIC_ID1, + {ok, Socket} = gen_udp:open(0, [binary]), + Payload1 = <<20, 21, 22, 23>>, + send_publish_msg_predefined_topic(Socket, QoS, MsgId, TopicId1, Payload1), + timer:sleep(1500), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(udp_receive_timeout, receive_response(Socket)), + gen_udp:close(Socket), + _ = recoverable_restart_emqx_sn(Backup), + timer:sleep(200), + ok. + +t_invalid_packet(_) -> + {ok, Socket} = gen_udp:open(0, [binary]), + send_connect_msg(Socket, <<"client_id_test1">>), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + send_packet(Socket, emqx_sn_frame_SUITE:generate_random_binary()), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(udp_receive_timeout, receive_response(Socket)), + gen_udp:close(Socket). + %%-------------------------------------------------------------------- %% Helper funcs %%-------------------------------------------------------------------- From 0a3c8d035c37add2b1ac58fc1c793d02135dd258 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 26 Sep 2022 16:46:05 +0800 Subject: [PATCH 03/44] fix(mqtt_sn): after receiving publish in `idle mode` the gateway may panic the `emqx_sn_gateway` will return `{keep_state, Timeout}` when received publish in `idle mode`, this is an incorrect format of `gen_statem` state return result, this return will replace the `state data` by the `Timeout`, the fix is to change it to `{keep_state_and_data, Timeout}` --- apps/emqx_sn/src/emqx_sn_gateway.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index 22fe51b6e..3f1a4f2d9 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -208,7 +208,7 @@ idle(cast, {incoming, ?SN_DISCONNECT_MSG(_Duration)}, State) -> idle(cast, {incoming, ?SN_PUBLISH_MSG(_Flag, _TopicId, _MsgId, _Data)}, State = #state{enable_qos3 = false}) -> ?LOG(debug, "The enable_qos3 is false, ignore the received publish with QoS=-1 in idle mode!"), - {keep_state, State#state.idle_timeout}; + {keep_state_and_data, State#state.idle_timeout}; idle(cast, {incoming, ?SN_PUBLISH_MSG(#mqtt_sn_flags{qos = ?QOS_NEG1, topic_id_type = TopicIdType @@ -226,7 +226,7 @@ idle(cast, {incoming, ?SN_PUBLISH_MSG(#mqtt_sn_flags{qos = ?QOS_NEG1, ok end, ?LOG(debug, "Client id=~p receives a publish with QoS=-1 in idle mode!", [ClientId]), - {keep_state, State#state.idle_timeout}; + {keep_state_and_data, State#state.idle_timeout}; idle(cast, {incoming, PingReq = ?SN_PINGREQ_MSG(_ClientId)}, State) -> handle_ping(PingReq, State); From 1e003457774cdbc53f947afa6a61b740b5fa7391 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 26 Sep 2022 17:04:55 +0800 Subject: [PATCH 04/44] chore(emqx_sn): bump version && update appup --- apps/emqx_sn/src/emqx_sn.app.src | 2 +- apps/emqx_sn/src/emqx_sn.appup.src | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_sn/src/emqx_sn.app.src b/apps/emqx_sn/src/emqx_sn.app.src index 68e0340f8..46c2df084 100644 --- a/apps/emqx_sn/src/emqx_sn.app.src +++ b/apps/emqx_sn/src/emqx_sn.app.src @@ -1,6 +1,6 @@ {application, emqx_sn, [{description, "EMQ X MQTT-SN Plugin"}, - {vsn, "4.3.7"}, % strict semver, bump manually! + {vsn, "4.3.8"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,esockd]}, diff --git a/apps/emqx_sn/src/emqx_sn.appup.src b/apps/emqx_sn/src/emqx_sn.appup.src index bab8a2b7a..c87bbb0a1 100644 --- a/apps/emqx_sn/src/emqx_sn.appup.src +++ b/apps/emqx_sn/src/emqx_sn.appup.src @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- {VSN, [ + {"4.3.7",[ + {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} + ]}, {"4.3.6",[ {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} ]}, @@ -29,6 +32,9 @@ {<<"4\\.3\\.[0-1]">>, [{restart_application,emqx_sn}]} ], [ + {"4.3.7",[ + {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} + ]}, {"4.3.6",[ {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} ]}, From 04c52aa5f484a05a0b06f1300535053bc970f726 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 26 Sep 2022 17:15:27 +0800 Subject: [PATCH 05/44] chore: update CHANGES-4.3.md --- CHANGES-4.3.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 2e2f5157c..219da0ceb 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -12,6 +12,10 @@ File format: ## v4.3.22 +### Bug fixes + +- Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic. [#9024](https://github.com/emqx/emqx/pull/9024) + ## v4.3.21 ### Enhancements From 29c76d16d79eff27fc504e8dc8ca6189e2970fc3 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 29 Sep 2022 13:40:01 +0800 Subject: [PATCH 06/44] fix: reset rule metrics crash if it has not initialized --- apps/emqx_rule_engine/src/emqx_rule_metrics.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index 624032056..e8a77e0db 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -328,6 +328,8 @@ handle_call({create_rule_metrics, Id}, _From, _ -> RuleSpeeds#{Id => #rule_speed{}} end}}; +handle_call({reset_speeds, _Id}, _From, State = #state{rule_speeds = undefined}) -> + {reply, ok, State}; handle_call({reset_speeds, Id}, _From, State = #state{rule_speeds = RuleSpeedMap}) -> {reply, ok, State#state{rule_speeds = maps:put(Id, #rule_speed{}, RuleSpeedMap)}}; From 7d2dd3d37daddde0f7fb2a60aa4617ff07480bab Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 29 Sep 2022 13:41:02 +0800 Subject: [PATCH 07/44] fix: deny POST an existing resource --- .../emqx_rule_engine/src/emqx_rule_engine_api.erl | 10 ++++++++++ .../test/emqx_rule_engine_SUITE.erl | 15 ++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 50fdcfe22..eec67e914 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -315,6 +315,16 @@ create_resource(#{}, Params) -> end. do_create_resource(Create, ParsedParams) -> + case maps:find(id, ParsedParams) of + {ok, ResId} -> + case emqx_rule_registry:find_resource(ResId) of + {ok, _} -> return({error, 400, <<"Already Exists">>}); + not_found -> do_create_resource2(Create, ParsedParams) + end; + error -> do_create_resource2(Create, ParsedParams) + end. + +do_create_resource2(Create, ParsedParams) -> case emqx_rule_engine:Create(ParsedParams) of ok -> return(ok); diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index c8f66ea00..dc95c0368 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -600,13 +600,18 @@ t_show_action_api(_Config) -> ok. t_crud_resources_api(_Config) -> + ResParams = [ + {<<"name">>, <<"Simple Resource">>}, + {<<"type">>, <<"built_in">>}, + {<<"config">>, [{<<"a">>, 1}]}, + {<<"description">>, <<"Simple Resource">>} + ], {ok, #{code := 0, data := Resources1}} = - emqx_rule_engine_api:create_resource(#{}, - [{<<"name">>, <<"Simple Resource">>}, - {<<"type">>, <<"built_in">>}, - {<<"config">>, [{<<"a">>, 1}]}, - {<<"description">>, <<"Simple Resource">>}]), + emqx_rule_engine_api:create_resource(#{}, ResParams), ResId = maps:get(id, Resources1), + %% create again using given resource id returns error + {ok, #{code := 400, message := <<"Already Exists">>}} = + emqx_rule_engine_api:create_resource(#{}, [{<<"id">>, ResId} | ResParams]), {ok, #{code := 0, data := Resources}} = emqx_rule_engine_api:list_resources(#{}, []), ?assert(length(Resources) > 0), {ok, #{code := 0, data := Resources2}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []), From 33e68c9d165ef09224c3200df6bb3069841340fc Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 30 Sep 2022 15:55:17 +0800 Subject: [PATCH 08/44] chore: update the CHANGES-4.3.md --- CHANGES-4.3.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 2e2f5157c..b1505844d 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -14,6 +14,11 @@ File format: ## v4.3.21 +### Bug fixes + +- Deny POST an existing resource id using HTTP API with error 400 "Already Exists". [#9079](https://github.com/emqx/emqx/pull/9079) +- Fix the issue that reseting rule metrics crashed under certain conditions. [#9079](https://github.com/emqx/emqx/pull/9079) + ### Enhancements - TLS listener memory usage optimization [#9005](https://github.com/emqx/emqx/pull/9005). From ab3ec9c176a60414cba794a30d15ce929fc76eba Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 30 Sep 2022 16:32:52 +0800 Subject: [PATCH 09/44] chore: update the appup.src --- .../src/emqx_rule_engine.appup.src | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 6aa35d2d6..b46be85c7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -3,17 +3,20 @@ {VSN, [{"4.3.15", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -22,6 +25,7 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -30,6 +34,7 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -39,6 +44,7 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -48,6 +54,7 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -200,17 +207,20 @@ {<<".*">>,[]}], [{"4.3.15", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -219,6 +229,7 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -227,6 +238,7 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -236,6 +248,7 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -245,6 +258,7 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, From 09b08d092d28def9bb74d669b6f3bdb96954f051 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 5 Oct 2022 15:10:10 +0200 Subject: [PATCH 10/44] chore: revert double exported pmap after merge --- src/emqx_misc.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index 5e5f59b09..d256569fb 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -45,8 +45,6 @@ , index_of/2 , maybe_parse_ip/1 , ipv6_probe/1 - , pmap/2 - , pmap/3 , ipv6_probe/2 , pmap/2 , pmap/3 From c9ff67a6376b6f26d42f408026b1cd4b8e7b3e2d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 5 Oct 2022 14:25:25 +0200 Subject: [PATCH 11/44] chore: add timetrap of 2 mins for pgsql test cases --- apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl index 2b644c235..99d92202b 100644 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl @@ -70,6 +70,9 @@ all() -> emqx_ct:all(?MODULE). +suite() -> + [{timetrap, {seconds, 120}}]. + init_per_suite(Config) -> emqx_ct_helpers:start_apps([emqx_auth_pgsql]), drop_acl(), From f24728ee7bf66e327f77014e57296c6c194d6bd0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 5 Oct 2022 19:24:42 +0200 Subject: [PATCH 12/44] chore: sync diverged modules modules from ee back to ce --- .ci/docker-compose-file/toxiproxy.json | 2 +- apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl | 10 +++++----- apps/emqx_exhook/docs/design-cn.md | 2 +- apps/emqx_exproto/docs/design-cn.md | 2 +- apps/emqx_rule_engine/include/rule_engine.hrl | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 7079b0599..8bc77bdbf 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -1,6 +1,6 @@ [ { - "name": "mongo", + "name": "mongo_single", "listen": "0.0.0.0:27017", "upstream": "mongo:27017", "enabled": true diff --git a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl index 0031bc8c4..557d662d3 100644 --- a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl +++ b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl @@ -686,7 +686,7 @@ heal_failure(FailureType, ProxyHost, ProxyPort) -> end. switch_proxy(Switch, ProxyHost, ProxyPort) -> - Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo", + Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo_single", Body = case Switch of off -> <<"{\"enabled\":false}">>; on -> <<"{\"enabled\":true}">> @@ -695,27 +695,27 @@ switch_proxy(Switch, ProxyHost, ProxyPort) -> [{body_format, binary}]). timeout_proxy(on, ProxyHost, ProxyPort) -> - Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo/toxics", + Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo_single/toxics", Body = <<"{\"name\":\"timeout\",\"type\":\"timeout\"," "\"stream\":\"upstream\",\"toxicity\":1.0," "\"attributes\":{\"timeout\":0}}">>, {ok, {{_, 200, _}, _, _}} = httpc:request(post, {Url, [], "application/json", Body}, [], [{body_format, binary}]); timeout_proxy(off, ProxyHost, ProxyPort) -> - Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo/toxics/timeout", + Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo_single/toxics/timeout", Body = <<>>, {ok, {{_, 204, _}, _, _}} = httpc:request(delete, {Url, [], "application/json", Body}, [], [{body_format, binary}]). latency_up_proxy(on, ProxyHost, ProxyPort) -> - Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo/toxics", + Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo_single/toxics", Body = <<"{\"name\":\"latency_up\",\"type\":\"latency\"," "\"stream\":\"upstream\",\"toxicity\":1.0," "\"attributes\":{\"latency\":20000,\"jitter\":3000}}">>, {ok, {{_, 200, _}, _, _}} = httpc:request(post, {Url, [], "application/json", Body}, [], [{body_format, binary}]); latency_up_proxy(off, ProxyHost, ProxyPort) -> - Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo/toxics/latency_up", + Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo_single/toxics/latency_up", Body = <<>>, {ok, {{_, 204, _}, _, _}} = httpc:request(delete, {Url, [], "application/json", Body}, [], [{body_format, binary}]). diff --git a/apps/emqx_exhook/docs/design-cn.md b/apps/emqx_exhook/docs/design-cn.md index 21ee333eb..b86d9a512 100644 --- a/apps/emqx_exhook/docs/design-cn.md +++ b/apps/emqx_exhook/docs/design-cn.md @@ -47,7 +47,7 @@ 用户需要实现的方法，和数据类型的定义在 `priv/protos/exhook.proto` 文件中： -```protobuff +```protobuf syntax = "proto3"; package emqx.exhook.v1; diff --git a/apps/emqx_exproto/docs/design-cn.md b/apps/emqx_exproto/docs/design-cn.md index 1e1ba9e31..50246d216 100644 --- a/apps/emqx_exproto/docs/design-cn.md +++ b/apps/emqx_exproto/docs/design-cn.md @@ -44,7 +44,7 @@ 详情参见：`priv/protos/exproto.proto`，例如接口的定义有： -```protobuff +```protobuf syntax = "proto3"; package emqx.exproto.v1; diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index 04cfeb06f..c6d8fb2b4 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -131,7 +131,7 @@ -record(resource_params, { id :: resource_id() - , params :: #{} %% the params got after initializing the resource + , params :: map() %% the params got after initializing the resource , status = #{is_alive => false} :: #{is_alive := boolean(), atom() => term()} }). From 280668ce1349055813ff31d3e3bb5c0971832198 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 6 Oct 2022 11:26:56 +0200 Subject: [PATCH 13/44] chore: portable stdout coloring in scripts BSD's (macos) version of echo does not support -e flag --- scripts/rel/cut4x.sh | 4 ++-- scripts/rel/sync-remotes.sh | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/rel/cut4x.sh b/scripts/rel/cut4x.sh index 58131c757..1b1af4a69 100755 --- a/scripts/rel/cut4x.sh +++ b/scripts/rel/cut4x.sh @@ -40,10 +40,10 @@ EOF } logerr() { - echo -e "\e[31mERROR: $1\e[39m" + echo "$(tput setaf 1)ERROR: $1$(tput sgr0)" } logmsg() { - echo -e "\e[33mINFO: $1\e[39m" + echo "INFO: $1" } REL_BRANCH_CE="${REL_BRANCH_CE:-release-v43}" diff --git a/scripts/rel/sync-remotes.sh b/scripts/rel/sync-remotes.sh index c79e7b957..550c65cf6 100755 --- a/scripts/rel/sync-remotes.sh +++ b/scripts/rel/sync-remotes.sh @@ -45,11 +45,10 @@ EOF } logerr() { - echo -e "\e[31mERROR: $1\e[39m" + echo "$(tput setaf 1)ERROR: $1$(tput sgr0)" } - logwarn() { - echo -e "\e[33mINFO: $1\e[39m" + echo "$(tput setaf 3)WARNING: $1$(tput sgr0)" } logmsg() { From 1bb7c23db13a629d4c5bf37ef7d29eb8f3590a77 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 8 Oct 2022 10:50:42 +0800 Subject: [PATCH 14/44] fix: discard the 'id' field when testing a resource --- apps/emqx_rule_engine/src/emqx_rule_engine_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index eec67e914..ea0a13824 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -306,7 +306,7 @@ show_action(#{name := Name}, _Params) -> create_resource(#{}, Params) -> case parse_resource_params(Params) of {ok, ParsedParams} -> - if_test(fun() -> do_create_resource(test_resource, ParsedParams) end, + if_test(fun() -> do_create_resource(test_resource, maps:without([id], ParsedParams)) end, fun() -> do_create_resource(create_resource, ParsedParams) end, Params); {error, Reason} -> From 48d8f7e7a90bde1599565ef6a39d2529acdf2839 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 10 Oct 2022 16:03:31 +0800 Subject: [PATCH 15/44] fix(channel): add warning log if the acl check of a subscribed topic failed --- src/emqx_channel.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index b8f3c5b2c..3e20c7fef 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1501,12 +1501,15 @@ check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, check_sub_acls(TopicFilters, Channel) -> check_sub_acls(TopicFilters, Channel, []). -check_sub_acls([ TopicFilter = {Topic, _} | More] , Channel, Acc) -> +check_sub_acls([ TopicFilter = {Topic, SubOpts} | More] , Channel, Acc) -> case check_sub_acl(Topic, Channel) of allow -> check_sub_acls(More, Channel, [ {TopicFilter, 0} | Acc]); deny -> - check_sub_acls(More, Channel, [ {TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) + ReasonCode = ?RC_NOT_AUTHORIZED, + ?LOG(warning, "Cannot subscribe ~s with options ~p due to ~s.", + [Topic, SubOpts, emqx_reason_codes:text(ReasonCode)]), + check_sub_acls(More, Channel, [ {TopicFilter, ReasonCode} | Acc]) end; check_sub_acls([], _Channel, Acc) -> lists:reverse(Acc). From ef04310c2746a160e14a6273bd225a07c07da28d Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 10 Oct 2022 16:04:57 +0800 Subject: [PATCH 16/44] chore: update CHANGES.md --- CHANGES-4.3.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 673dbef8a..f136b7835 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -39,6 +39,8 @@ File format: - Added a test to prevent a last will testament message to be published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894) +- Add warning log if the acl check of a subscribed topic failed. [#9124](https://github.com/emqx/emqx/pull/9124) + ### Bug fixes - Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) From fd7230353c459f3d67c5232aa602defb36a7d393 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Sep 2022 14:29:59 +0800 Subject: [PATCH 17/44] refactor(test): move rule_engine sql test cases into a separate file --- apps/emqx_rule_engine/rebar.config | 10 + .../test/emqx_rule_engine_SUITE.erl | 1443 +---------------- apps/emqx_rule_engine/test/emqx_rule_test.hrl | 29 + .../test/emqx_rule_test_lib.erl | 141 ++ .../test/emqx_rulesql_SUITE.erl | 1378 ++++++++++++++++ 5 files changed, 1564 insertions(+), 1437 deletions(-) create mode 100644 apps/emqx_rule_engine/test/emqx_rule_test.hrl create mode 100644 apps/emqx_rule_engine/test/emqx_rule_test_lib.erl create mode 100644 apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 097c18a3d..f9ad1b283 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1,5 +1,7 @@ +%% -*- mode: erlang -*- {deps, []}. +%% Comple Opts {erl_opts, [warn_unused_vars, warn_shadow_vars, warn_unused_import, @@ -18,6 +20,14 @@ warnings_as_errors, deprecated_functions ]}. +%% {erl_opts, [...]}, but for CT runs +%% NOT WORKING!!! +%% %% == Common Test == +%% {ct_compile_opts, [ export_all +%% , nowarn_export_all +%% ]}. +%% {ct_opts, []}. + {cover_enabled, true}. {cover_opts, [verbose]}. {cover_export_enabled, true}. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index dc95c0368..42816e869 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -25,6 +25,9 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include("emqx_rule_test.hrl"). +-import(emqx_rule_test_lib, [make_simple_resource_type/1]). + %%-define(PROPTEST(M,F), true = proper:quickcheck(M:F())). all() -> @@ -92,47 +95,7 @@ groups() -> t_resource_types ]}, {runtime, [], - [t_match_atom_and_binary, - t_sqlselect_0, - t_sqlselect_00, - t_sqlselect_01, - t_sqlselect_02, - t_sqlselect_1, - t_sqlselect_2, - t_sqlselect_2_1, - t_sqlselect_2_2, - t_sqlselect_2_3, - t_sqlselect_3, - t_sqlparse_event_1, - t_sqlparse_event_2, - t_sqlparse_event_3, - t_sqlparse_foreach_1, - t_sqlparse_foreach_2, - t_sqlparse_foreach_3, - t_sqlparse_foreach_4, - t_sqlparse_foreach_5, - t_sqlparse_foreach_6, - t_sqlparse_foreach_7, - t_sqlparse_foreach_8, - t_sqlparse_case_when_1, - t_sqlparse_case_when_2, - t_sqlparse_case_when_3, - t_sqlparse_array_index_1, - t_sqlparse_array_index_2, - t_sqlparse_array_index_3, - t_sqlparse_array_index_4, - t_sqlparse_array_index_5, - t_sqlparse_select_matadata_1, - t_sqlparse_array_range_1, - t_sqlparse_array_range_2, - t_sqlparse_true_false, - t_sqlparse_compare_undefined, - t_sqlparse_compare_null_null, - t_sqlparse_compare_null_notnull, - t_sqlparse_compare_notnull_null, - t_sqlparse_compare, - t_sqlparse_new_map, - t_sqlparse_invalid_json + [t_match_atom_and_binary ]}, {rule_metrics, [], [t_metrics, @@ -169,10 +132,6 @@ end_per_suite(_Config) -> stop_apps(), ok. -on_resource_create(_id, _) -> #{}. -on_resource_destroy(_id, _) -> ok. -on_get_resource_status(_id, _) -> #{is_alive => true}. - %%------------------------------------------------------------------------------ %% Group specific setup/teardown %%------------------------------------------------------------------------------ @@ -269,15 +228,7 @@ init_per_testcase(Test, Config) | Config]; init_per_testcase(_TestCase, Config) -> ok = emqx_rule_registry:register_resource_types( - [#resource_type{ - name = built_in, - provider = ?APP, - params_spec = #{}, - on_create = {?MODULE, on_resource_create}, - on_destroy = {?MODULE, on_resource_destroy}, - on_status = {?MODULE, on_get_resource_status}, - title = #{en => <<"Built-In Resource Type (debug)">>}, - description = #{en => <<"The built in resource type for debug purpose">>}}]), + [make_simple_debug_resource_type()]), %ct:pal("============ ~p", [ets:tab2list(emqx_resource_type)]), Config. @@ -1280,324 +1231,6 @@ t_match_atom_and_binary(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_0(_Config) -> - %% Verify SELECT with and without 'AS' - Sql = "select * " - "from \"t/#\" " - "where payload.cmd.info = 'tt'", - ?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "select payload.cmd as cmd " - "from \"t/#\" " - "where cmd.info = 'tt'", - ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "select payload.cmd as cmd, cmd.info as info " - "from \"t/#\" " - "where cmd.info = 'tt' and info = 'tt'", - ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, - <<"info">> := <<"tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), - %% cascaded as - Sql4 = "select payload.cmd as cmd, cmd.info as meta.info " - "from \"t/#\" " - "where cmd.info = 'tt' and meta.info = 'tt'", - ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, - <<"meta">> := #{<<"info">> := <<"tt">>}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlselect_00(_Config) -> - %% Verify plus/subtract and unary_add_or_subtract - Sql = "select 1-1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 0}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - Sql1 = "select -1 + 1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 0}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "select 1 + 1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 2}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "select +1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 1}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlselect_01(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule1 = create_simple_repub_rule( - <<"t2">>, - "SELECT json_decode(payload) as p, payload " - "FROM \"t3/#\", \"t1\" " - "WHERE p.x = 1"), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), - ct:sleep(100), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"{\"x\":1}">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, - - emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), - receive {publish, #{topic := T3, payload := Payload3}} -> - ?assertEqual(<<"t2">>, T3), - ?assertEqual(<<"{\"x\":1}">>, Payload3) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule1). - -t_sqlselect_02(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule1 = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"t3/#\", \"t1\" " - "WHERE payload.x = 1"), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), - ct:sleep(100), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"{\"x\":1}">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := Payload0}} -> - ct:fail({unexpected_t2, Payload0}) - after 1000 -> - ok - end, - - emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), - receive {publish, #{topic := T3, payload := Payload3}} -> - ?assertEqual(<<"t2">>, T3), - ?assertEqual(<<"{\"x\":1}">>, Payload3) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule1). - -t_sqlselect_1(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT json_decode(payload) as p, payload " - "FROM \"t1\" " - "WHERE p.x = 1 and p.y = 2"), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ct:sleep(200), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":2}">>, 0), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"{\"x\":1,\"y\":2}">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2 - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"t2\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_1(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg dropped - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_dropped\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_nothing = Fun(), - - %% it should not keep republishing "t2" - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_2(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg acked - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_acked\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_3(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg delivered - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_delivered\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_3(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% republish the client.connected msg - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/client_connected\" " - "WHERE username = 'emqx1'", - <<"clientid=${clientid}">>), - {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ct:sleep(200), - {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), - {ok, _} = emqtt:connect(Client1), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - t_metrics(_Config) -> ok = emqx_rule_engine:load_providers(), TopicRule = create_simple_repub_rule( @@ -1899,964 +1532,6 @@ t_sqlselect_multi_actoins_4(Config) -> emqx_rule_registry:remove_rule(Rule). -t_sqlparse_event_1(_Config) -> - Sql = "select topic as tp " - "from \"$events/session_subscribed\" ", - ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). - -t_sqlparse_event_2(_Config) -> - Sql = "select clientid " - "from \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>}})). - -t_sqlparse_event_3(_Config) -> - Sql = "select clientid, topic as tp " - "from \"t/tt\", \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})). - -t_sqlparse_foreach_1(_Config) -> - %% Verify foreach with and without 'AS' - Sql = "foreach payload.sensors as s " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"s">> := 1}, #{<<"s">> := 2}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "from \"t/#\" ", - ?assertMatch({ok,[#{item := 1}, #{item := 2}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "foreach payload.sensors " - "from \"t/#\" ", - ?assertMatch({ok,[#{item := #{<<"cmd">> := <<"1">>}, clientid := <<"c_a">>}, - #{item := #{ <<"cmd">> := <<"2">> - , <<"name">> := <<"ct">> - } - , clientid := <<"c_a">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{ <<"payload">> => <<"{\"sensors\": [{\"cmd\":\"1\"}, " - "{\"cmd\":\"2\",\"name\":\"ct\"}]}">> - , <<"clientid">> => <<"c_a">> - , <<"topic">> => <<"t/a">> - }})), - Sql4 = "foreach payload.sensors " - "from \"t/#\" ", - {ok,[#{metadata := #{rule_id := TRuleId}}, - #{metadata := #{rule_id := TRuleId}}]} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{ - <<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}}), - ?assert(is_binary(TRuleId)). - -t_sqlparse_foreach_2(_Config) -> - %% Verify foreach-do with and without 'AS' - Sql = "foreach payload.sensors as s " - "do s.cmd as msg_type " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "do item.cmd as msg_type " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "foreach payload.sensors " - "do item as item " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"item">> := 1},#{<<"item">> := 2}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_3(_Config) -> - %% Verify foreach-incase with and without 'AS' - Sql = "foreach payload.sensors as s " - "incase s.cmd != 1 " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"s">> := #{<<"cmd">> := 2}}, - #{<<"s">> := #{<<"cmd">> := 3}} - ]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "incase item.cmd != 1 " - "from \"t/#\" ", - ?assertMatch({ok,[#{item := #{<<"cmd">> := 2}}, - #{item := #{<<"cmd">> := 3}} - ]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_4(_Config) -> - %% Verify foreach-do-incase - Sql = "foreach payload.sensors as s " - "do s.cmd as msg_type, s.name as name " - "incase is_not_null(s.cmd) " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ ok - , [ #{<<"msg_type">> := <<"1">>, <<"name">> := <<"n1">>} - , #{<<"msg_type">> := <<"2">>} - ] - }, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\", \"name\":\"n1\"}, " - "{\"cmd\":\"2\"}, {\"name\":\"n3\"}]}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok,[]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_5(_Config) -> - %% Verify foreach on a empty-list or non-list variable - Sql = "foreach payload.sensors as s " - "do s.cmd as msg_type, s.name as name " - "from \"t/#\" ", - ?assertMatch({ok,[]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok,[]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": []}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "from \"t/#\" ", - ?assertMatch({ok,[]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": 1}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_6(_Config) -> - %% Verify foreach on a empty-list or non-list variable - Sql = "foreach json_decode(payload) " - "do item.id as zid, timestamp as t " - "from \"t/#\" ", - {ok, Res} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"[{\"id\": 5},{\"id\": 15}]">>, - <<"topic">> => <<"t/a">>}}), - [#{<<"t">> := Ts1, <<"zid">> := Zid1}, - #{<<"t">> := Ts2, <<"zid">> := Zid2}] = Res, - ?assertEqual(true, is_integer(Ts1)), - ?assertEqual(true, is_integer(Ts2)), - ?assert(Zid1 == 5 orelse Zid1 == 15), - ?assert(Zid2 == 5 orelse Zid2 == 15). - -t_sqlparse_foreach_7(_Config) -> - %% Verify foreach-do-incase and cascaded AS - Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " - "incase is_not_null(info.cmd) " - "from \"t/#\" " - "where s.page = '2' ", - Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " - "{\"info\":[{\"name\":\"cmd1\", \"cmd\":\"1\"}, {\"cmd\":\"2\"}]} } }">>, - ?assertMatch({ ok - , [ #{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>} - , #{<<"msg_type">> := <<"2">>} - ] - }, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " - "incase is_not_null(info.cmd) " - "from \"t/#\" " - "where s.page = '3' ", - ?assertMatch({error, nomatch}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_8(_Config) -> - %% Verify foreach-do-incase and cascaded AS - Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " - "incase is_map(info) " - "from \"t/#\" " - "where s.page = '2' ", - Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " - "{\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }">>, - ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})), - - Sql3 = "foreach json_decode(payload) as p, p.sensors as s," - " s.collection as c, sublist(2,1,c.info) as info " - "do info.cmd as msg_type, info.name as name " - "from \"t/#\" " - "where s.page = '2' ", - [?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => SqlN, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})) - || SqlN <- [Sql3]]. - -t_sqlparse_case_when_1(_Config) -> - %% case-when-else clause - Sql = "select " - " case when payload.x < 0 then 0 " - " when payload.x > 7 then 7 " - " else payload.x " - " end as y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"y">> := 1}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})), - ok. - -t_sqlparse_case_when_2(_Config) -> - % switch clause - Sql = "select " - " case payload.x when 1 then 2 " - " when 2 then 3 " - " else 4 " - " end as y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"y">> := 2}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 2}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 4}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_case_when_3(_Config) -> - %% case-when clause - Sql = "select " - " case when payload.x < 0 then 0 " - " when payload.x > 7 then 7 " - " end as y " - "from \"t/#\" ", - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 5}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})), - ok. - -t_sqlparse_array_index_1(_Config) -> - %% index get - Sql = "select " - " json_decode(payload) as p, " - " p[1] as a " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"a">> := #{<<"x">> := 1}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"[{\"x\": 1}]">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - %% index get without 'as' - Sql2 = "select " - " payload.x[2] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [3]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,3,4]}, - <<"topic">> => <<"t/a">>}})), - %% index get without 'as' again - Sql3 = "select " - " payload.x[2].y " - "from \"t/#\" ", - ?assertMatch( {ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 3}]}}} - , emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}}) - ), - - %% index get with 'as' - Sql4 = "select " - " payload.x[2].y as b " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_index_2(_Config) -> - %% array get with negative index - Sql1 = "select " - " payload.x[-2].y as b " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}})), - %% array append to head or tail of a list: - Sql2 = "select " - " payload.x as b, " - " 1 as c[-0], " - " 2 as c[-0], " - " b as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := 0, <<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => 0}, - <<"topic">> => <<"t/a">>}})), - %% construct an empty list: - Sql3 = "select " - " [] as c, " - " 1 as c[-0], " - " 2 as c[-0], " - " 0 as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - %% construct a list: - Sql4 = "select " - " [payload.a, \"topic\", 'c'] as c, " - " 1 as c[-0], " - " 2 as c[-0], " - " 0 as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,11,<<"t/a">>,<<"c">>,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":11}">>, - <<"topic">> => <<"t/a">> - }})). - -t_sqlparse_array_index_3(_Config) -> - %% array with json string payload: - Sql0 = "select " - "payload," - "payload.x[2].y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := [1,2]}, 3]}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), - %% same as above but don't select payload: - Sql1 = "select " - "payload.x[2].y as b " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := [1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), - %% same as above but add 'as' clause: - Sql2 = "select " - "payload.x[2].y as b.c " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := #{<<"c">> := [1,2]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_index_4(_Config) -> - %% array with json string payload: - Sql0 = "select " - "0 as payload.x[2].y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 0}]}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), - %% array with json string payload, and also select payload.x: - Sql1 = "select " - "payload.x, " - "0 as payload.x[2].y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := 0}, 3]}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_index_5(_Config) -> - Sql00 = "select " - " [1,2,3,4] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assert(lists:any(fun({_K, V}) -> - V =:= [1,2,3,4] - end, maps:to_list(Res00))). - -t_sqlparse_select_matadata_1(_Config) -> - %% array with json string payload: - Sql0 = "select " - "payload " - "from \"t/#\" ", - ?assertNotMatch({ok, #{<<"payload">> := <<"abc">>, metadata := _}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"abc">>, - <<"topic">> => <<"t/a">>}})), - Sql1 = "select " - "payload, metadata " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := <<"abc">>, <<"metadata">> := _}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"abc">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_range_1(_Config) -> - %% get a range of list - Sql0 = "select " - " payload.a[1..4] as c " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,1,2,3]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})), - %% get a range from non-list data - Sql02 = "select " - " payload.a[1..4] as c " - "from \"t/#\" ", - ?assertMatch({error, {select_and_transform_error, {error,{range_get,non_list_data},_}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => - #{<<"payload">> => <<"{\"x\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})), - - %% construct a range: - Sql1 = "select " - " [1..4] as c, " - " 5 as c[-0], " - " 6 as c[-0], " - " 0 as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,1,2,3,4,5,6]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_range_2(_Config) -> - %% construct a range without 'as' - Sql00 = "select " - " [1..4] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assert(lists:any(fun({_K, V}) -> - V =:= [1,2,3,4] - end, maps:to_list(Res00))), - %% construct a range without 'as' - Sql01 = "select " - " a[2..4] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"a">> := [2,3,4]}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql01, - <<"ctx">> => #{<<"a">> => [1,2,3,4,5], - <<"topic">> => <<"t/a">>}})), - %% get a range of list without 'as' - Sql02 = "select " - " payload.a[1..4] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"a">> := [0,1,2,3]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_true_false(_Config) -> - %% construct a range without 'as' - Sql00 = "select " - " true as a, false as b, " - " false as x.y, true as c[-0] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assertMatch(#{<<"a">> := true, <<"b">> := false, - <<"x">> := #{<<"y">> := false}, - <<"c">> := [true] - }, Res00). - --define(TEST_SQL(SQL), - emqx_rule_sqltester:test( - #{<<"rawsql">> => SQL, - <<"ctx">> => #{<<"payload">> => <<"{}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_compare_undefined(_Config) -> - Sql00 = "select " - " * " - "from \"t/#\" " - "where dev != undefined ", - %% no match - ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00)), - - Sql00_1 = "select " - " * " - "from \"t/#\" " - "where dev <> undefined ", - %% no match - ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00_1)), - - Sql01 = "select " - " 'd' as dev " - "from \"t/#\" " - "where dev != undefined ", - {ok, Res01} = ?TEST_SQL(Sql01), - %% pass - ?assertMatch(#{}, Res01), - - Sql01_1 = "select " - " 'd' as dev " - "from \"t/#\" " - "where dev <> undefined ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - %% pass - ?assertMatch(#{}, Res01_1), - - Sql02 = "select " - " * " - "from \"t/#\" " - "where dev != 'undefined' ", - {ok, Res02} = ?TEST_SQL(Sql02), - %% pass - ?assertMatch(#{}, Res02), - - Sql03 = "select " - " * " - "from \"t/#\" " - "where dev =~ 'undefined' ", - Res03 = ?TEST_SQL(Sql03), - %% no match - ?assertMatch({error, nomatch}, Res03). - -t_sqlparse_compare_null_null(_Config) -> - %% test undefined == undefined - Sql00 = "select " - " a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := true - }, Res00), - - %% test undefined != undefined - Sql01 = "select " - " a != b as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := false - }, Res01), - - %% test undefined <> undefined - Sql01_1 = "select " - " a <> b as c " - "from \"t/#\" ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - ?assertMatch(#{<<"c">> := false - }, Res01_1), - - %% test undefined > undefined - Sql02 = "select " - " a > b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := false - }, Res02), - - %% test undefined < undefined - Sql03 = "select " - " a < b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - %% test undefined <= undefined - Sql04 = "select " - " a <= b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := true - }, Res04), - - %% test undefined >= undefined - Sql05 = "select " - " a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := true - }, Res05), - - %% test undefined =~ undefined - Sql06 = "select " - " a =~ b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := true - }, Res06). - -t_sqlparse_compare_null_notnull(_Config) -> - %% test undefined == 'b' - Sql00 = "select " - " 'b' as b, a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := false - }, Res00), - - %% test undefined != 'b' - Sql01 = "select " - " 'b' as b, a != b as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := true - }, Res01), - - %% test undefined <> 'b' - Sql01_1 = "select " - " 'b' as b, a <> b as c " - "from \"t/#\" ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - ?assertMatch(#{<<"c">> := true - }, Res01_1), - - %% test undefined > 'b' - Sql02 = "select " - " 'b' as b, a > b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := false - }, Res02), - - %% test undefined < 'b' - Sql03 = "select " - " 'b' as b, a < b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - %% test undefined <= 'b' - Sql04 = "select " - " 'b' as b, a <= b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := false - }, Res04), - - %% test undefined >= 'b' - Sql05 = "select " - " 'b' as b, a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := false - }, Res05), - - %% test undefined =~ 'b' - Sql06 = "select " - " 'b' as b, a =~ b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := false - }, Res06). - -t_sqlparse_compare_notnull_null(_Config) -> - %% test 'a' == undefined - Sql00 = "select " - " 'a' as a, a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := false - }, Res00), - - %% test 'a' != undefined - Sql01 = "select " - " 'a' as a, a != b as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := true - }, Res01), - - %% test 'a' <> undefined - Sql01_1 = "select " - " 'a' as a, a <> b as c " - "from \"t/#\" ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - ?assertMatch(#{<<"c">> := true - }, Res01_1), - - %% test 'a' > undefined - Sql02 = "select " - " 'a' as a, a > b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := false - }, Res02), - - %% test 'a' < undefined - Sql03 = "select " - " 'a' as a, a < b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - %% test 'a' <= undefined - Sql04 = "select " - " 'a' as a, a <= b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := false - }, Res04), - - %% test 'a' >= undefined - Sql05 = "select " - " 'a' as a, a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := false - }, Res05), - - %% test 'a' =~ undefined - Sql06 = "select " - " 'a' as a, a =~ b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := false - }, Res06). - -t_sqlparse_compare(_Config) -> - Sql00 = "select " - " 'a' as a, 'a' as b, a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := true - }, Res00), - - Sql00_1 = "select " - " 'true' as a, true as b, a = b as c " - "from \"t/#\" ", - {ok, Res00_1} = ?TEST_SQL(Sql00_1), - ?assertMatch(#{<<"c">> := true - }, Res00_1), - - Sql01 = "select " - " is_null(a) as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := true - }, Res01), - - Sql02 = "select " - " 1 as a, 2 as b, a < b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := true - }, Res02), - - Sql03 = "select " - " 1 as a, 2 as b, a > b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - Sql04 = "select " - " 1 as a, 2 as b, a = b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := false - }, Res04), - - Sql04_0 = "select " - " 1 as a, 1 as b, a = b as c " - "from \"t/#\" ", - {ok, Res04_0} = ?TEST_SQL(Sql04_0), - ?assertMatch(#{<<"c">> := true - }, Res04_0), - - Sql04_1 = "select " - " 1 as a, '1' as b, a = b as c " - "from \"t/#\" ", - {ok, Res04_1} = ?TEST_SQL(Sql04_1), - ?assertMatch(#{<<"c">> := true - }, Res04_1), - - %% test 1 >= 2 - Sql05 = "select " - " 1 as a, 2 as b, a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := false - }, Res05), - - %% test 1 <= 2 - Sql06 = "select " - " 1 as a, 2 as b, a <= b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := true - }, Res06), - - %% test 1 != 2 - Sql07 = "select " - " 1 as a, 2 as b, a != b as c " - "from \"t/#\" ", - {ok, Res07} = ?TEST_SQL(Sql07), - ?assertMatch(#{<<"c">> := true - }, Res07), - - %% test 1 <> 2 - Sql07_1 = "select " - " 1 as a, 2 as b, a <> b as c " - "from \"t/#\" ", - {ok, Res07_1} = ?TEST_SQL(Sql07_1), - ?assertMatch(#{<<"c">> := true - }, Res07_1), - - %% test 't' =~ 't' - Sql08 = "select " - " 't' as a, 't' as b, a =~ b as c " - "from \"t/#\" ", - {ok, Res08} = ?TEST_SQL(Sql08), - ?assertMatch(#{<<"c">> := true - }, Res08). - -t_sqlparse_new_map(_Config) -> - %% construct a range without 'as' - Sql00 = "select " - " map_new() as a, map_new() as b, " - " map_new() as x.y, map_new() as c[-0] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assertMatch(#{<<"a">> := #{}, <<"b">> := #{}, - <<"x">> := #{<<"y">> := #{}}, - <<"c">> := [#{}] - }, Res00). - t_sqlparse_payload_as(_Config) -> %% https://github.com/emqx/emqx/issues/3866 Sql00 = "SELECT " @@ -2908,29 +1583,6 @@ t_sqlparse_nested_get(_Config) -> <<"payload">> => <<"{\"a\": {\"b\": 0}}">> }})). -t_sqlparse_invalid_json(_Config) -> - Sql02 = "select " - " payload.a[1..4] as c " - "from \"t/#\" ", - ?assertMatch({error, {select_and_transform_error, {error,{decode_json_failed,_},_}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => - #{<<"payload">> => <<"{\"x\":[0,1,2,3,}">>, - <<"topic">> => <<"t/a">>}})), - - - Sql2 = "foreach payload.sensors " - "do item.cmd as msg_type " - "from \"t/#\" ", - ?assertMatch({error, {select_and_collect_error, {error,{decode_json_failed,_},_}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"} {\"cmd\":}]}">>, - <<"topic">> => <<"t/a">>}})). - %%------------------------------------------------------------------------------ %% Internal helpers %%------------------------------------------------------------------------------ @@ -2966,20 +1618,6 @@ make_simple_rule(RuleId, SQL, ForTopics) when is_binary(RuleId) -> actions = [{'inspect', #{}}], description = <<"simple rule">>}. -create_simple_repub_rule(TargetTopic, SQL) -> - create_simple_repub_rule(TargetTopic, SQL, <<"${payload}">>). - -create_simple_repub_rule(TargetTopic, SQL, Template) -> - {ok, Rule} = emqx_rule_engine:create_rule( - #{rawsql => SQL, - actions => [#{name => 'republish', - args => #{<<"target_topic">> => TargetTopic, - <<"target_qos">> => -1, - <<"payload_tmpl">> => Template} - }], - description => <<"simple repub rule">>}), - Rule. - make_simple_action(ActionName) when is_atom(ActionName) -> #action{name = ActionName, app = ?APP, module = ?MODULE, on_create = simple_action_inspect, params_spec = #{}, @@ -3002,19 +1640,6 @@ make_simple_resource(ResId) -> config = #{}, description = <<"Simple Resource">>}. -make_simple_resource_type(ResTypeName) -> - #resource_type{name = ResTypeName, provider = ?APP, - params_spec = #{}, - on_create = {?MODULE, on_simple_resource_type_create}, - on_destroy = {?MODULE, on_simple_resource_type_destroy}, - on_status = {?MODULE, on_simple_resource_type_status}, - title = #{en => <<"Simple Resource Type">>}, - description = #{en => <<"Simple Resource Type">>}}. - -on_simple_resource_type_create(_Id, #{}) -> #{}. -on_simple_resource_type_destroy(_Id, #{}) -> ok. -on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}. - hook_metrics_action(_Id, _Params) -> fun(Data = #{event := EventName}, _Envs) -> ct:pal("applying hook_metrics_action: ~p", [Data]), @@ -3305,66 +1930,10 @@ verify_peername(PeerName) -> verify_ipaddr(IPAddrS) -> ?assertMatch({ok, _}, inet:parse_address(binary_to_list(IPAddrS))). -init_events_counters() -> - ets:new(events_record_tab, [named_table, bag, public]). - %%------------------------------------------------------------------------------ -%% Start Apps +%% Mock funcs %%------------------------------------------------------------------------------ -stop_apps() -> - stopped = mnesia:stop(), - [application:stop(App) || App <- [emqx_rule_engine, emqx]]. - -start_apps() -> - [start_apps(App, SchemaFile, ConfigFile) || - {App, SchemaFile, ConfigFile} - <- [{emqx, deps_path(emqx, "priv/emqx.schema"), - deps_path(emqx, "etc/emqx.conf")}, - {emqx_rule_engine, local_path("priv/emqx_rule_engine.schema"), - local_path("etc/emqx_rule_engine.conf")}]]. - -start_apps(App, SchemaFile, ConfigFile) -> - read_schema_configs(App, SchemaFile, ConfigFile), - set_special_configs(App), - {ok, _} = application:ensure_all_started(App). - -read_schema_configs(App, SchemaFile, ConfigFile) -> - ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]), - Schema = cuttlefish_schema:files([SchemaFile]), - Conf = conf_parse:file(ConfigFile), - NewConfig = cuttlefish_generator:map(Schema, Conf), - Vals = proplists:get_value(App, NewConfig, []), - [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. - -deps_path(App, RelativePath) -> - %% Note: not lib_dir because etc dir is not sym-link-ed to _build dir - %% but priv dir is - Path0 = code:priv_dir(App), - Path = case file:read_link(Path0) of - {ok, Resolved} -> Resolved; - {error, _} -> Path0 - end, - filename:join([Path, "..", RelativePath]). - -local_path(RelativePath) -> - deps_path(emqx_rule_engine, RelativePath). - -set_special_configs(emqx_rule_engine) -> - application:set_env(emqx_rule_engine, ignore_sys_message, true), - application:set_env(emqx_rule_engine, events, - [{'client.connected',on,1}, - {'client.disconnected',on,1}, - {'session.subscribed',on,1}, - {'session.unsubscribed',on,1}, - {'message.acked',on,1}, - {'message.dropped',on,1}, - {'message.delivered',on,1} - ]), - ok; -set_special_configs(_App) -> - ok. - mock_print() -> catch meck:unload(emqx_ctl), meck:new(emqx_ctl, [non_strict, passthrough]), diff --git a/apps/emqx_rule_engine/test/emqx_rule_test.hrl b/apps/emqx_rule_engine/test/emqx_rule_test.hrl new file mode 100644 index 000000000..748cc2e9a --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rule_test.hrl @@ -0,0 +1,29 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Test Suite funcs +-import(emqx_rule_test_lib, + [ stop_apps/0 + , start_apps/0 + ]). + +%% RULE helper funcs +-import(emqx_rule_test_lib, + [ create_simple_repub_rule/2 + , create_simple_repub_rule/3 + , make_simple_debug_resource_type/0 + , init_events_counters/0 + ]). diff --git a/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl b/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl new file mode 100644 index 000000000..24550fbc7 --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl @@ -0,0 +1,141 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rule_test_lib). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx_rule_engine/include/rule_engine.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +%%------------------------------------------------------------------------------ +%% Start Apps +%%------------------------------------------------------------------------------ + +stop_apps() -> + stopped = mnesia:stop(), + [application:stop(App) || App <- [emqx_rule_engine, emqx]]. + +start_apps() -> + [start_apps(App, SchemaFile, ConfigFile) || + {App, SchemaFile, ConfigFile} + <- [{emqx, deps_path(emqx, "priv/emqx.schema"), + deps_path(emqx, "etc/emqx.conf")}, + {emqx_rule_engine, local_path("priv/emqx_rule_engine.schema"), + local_path("etc/emqx_rule_engine.conf")}]]. + +%%-------------------------------------- +%% start apps helper funcs + +start_apps(App, SchemaFile, ConfigFile) -> + read_schema_configs(App, SchemaFile, ConfigFile), + set_special_configs(App), + {ok, _} = application:ensure_all_started(App). + +read_schema_configs(App, SchemaFile, ConfigFile) -> + ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]), + Schema = cuttlefish_schema:files([SchemaFile]), + Conf = conf_parse:file(ConfigFile), + NewConfig = cuttlefish_generator:map(Schema, Conf), + Vals = proplists:get_value(App, NewConfig, []), + [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. + +deps_path(App, RelativePath) -> + %% Note: not lib_dir because etc dir is not sym-link-ed to _build dir + %% but priv dir is + Path0 = code:priv_dir(App), + Path = case file:read_link(Path0) of + {ok, Resolved} -> Resolved; + {error, _} -> Path0 + end, + filename:join([Path, "..", RelativePath]). + +local_path(RelativePath) -> + deps_path(emqx_rule_engine, RelativePath). + +set_special_configs(emqx_rule_engine) -> + application:set_env(emqx_rule_engine, ignore_sys_message, true), + application:set_env(emqx_rule_engine, events, + [{'client.connected',on,1}, + {'client.disconnected',on,1}, + {'session.subscribed',on,1}, + {'session.unsubscribed',on,1}, + {'message.acked',on,1}, + {'message.dropped',on,1}, + {'message.delivered',on,1} + ]), + ok; +set_special_configs(_App) -> + ok. + +%%------------------------------------------------------------------------------ +%% rule test helper funcs +%%------------------------------------------------------------------------------ + +create_simple_repub_rule(TargetTopic, SQL) -> + create_simple_repub_rule(TargetTopic, SQL, <<"${payload}">>). + +create_simple_repub_rule(TargetTopic, SQL, Template) -> + {ok, Rule} = emqx_rule_engine:create_rule( + #{rawsql => SQL, + actions => [#{name => 'republish', + args => #{<<"target_topic">> => TargetTopic, + <<"target_qos">> => -1, + <<"payload_tmpl">> => Template} + }], + description => <<"simple repub rule">>}), + Rule. + +make_simple_debug_resource_type() -> + #resource_type{ + name = built_in, + provider = ?APP, + params_spec = #{}, + on_create = {?MODULE, on_resource_create}, + on_destroy = {?MODULE, on_resource_destroy}, + on_status = {?MODULE, on_get_resource_status}, + title = #{en => <<"Built-In Resource Type (debug)">>}, + description = #{en => <<"The built in resource type for debug purpose">>}}. + +make_simple_resource_type(ResTypeName) -> + #resource_type{ + name = ResTypeName, + provider = ?APP, + params_spec = #{}, + on_create = {?MODULE, on_simple_resource_type_create}, + on_destroy = {?MODULE, on_simple_resource_type_destroy}, + on_status = {?MODULE, on_simple_resource_type_status}, + title = #{en => <<"Simple Resource Type">>}, + description = #{en => <<"Simple Resource Type">>}}. + +init_events_counters() -> + ets:new(events_record_tab, [named_table, bag, public]). + +%%------------------------------------------------------------------------------ +%% Internal helper funcs +%%------------------------------------------------------------------------------ + +on_resource_create(_id, _) -> #{}. +on_resource_destroy(_id, _) -> ok. +on_get_resource_status(_id, _) -> #{is_alive => true}. + +on_simple_resource_type_create(_Id, #{}) -> #{}. +on_simple_resource_type_destroy(_Id, #{}) -> ok. +on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}. diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl new file mode 100644 index 000000000..d93b747c0 --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -0,0 +1,1378 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rulesql_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx_rule_engine/include/rule_engine.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-include("emqx_rule_test.hrl"). + +all() -> + [ {group, rulesql_select} + , {group, rulesql_select_events} + , {group, rulesql_select_metadata} + , {group, rulesql_foreach} + , {group, rulesql_case_when} + , {group, rulesql_array_index} + , {group, rulesql_array_range} + , {group, rulesql_compare} + , {group, rulesql_boolean} + , {group, rulesql_others} + ]. + +suite() -> + [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}]. + +groups() -> + [{rulesql_select, [], + [ t_sqlselect_0 + , t_sqlselect_00 + , t_sqlselect_01 + , t_sqlselect_02 + , t_sqlselect_1 + , t_sqlselect_2 + , t_sqlselect_2_1 + , t_sqlselect_2_2 + , t_sqlselect_2_3 + , t_sqlselect_3 + ]}, + {rulesql_select_events, [], + [ t_sqlparse_event_1 + , t_sqlparse_event_2 + , t_sqlparse_event_3 + ]}, + {rulesql_select_metadata, [], + [ t_sqlparse_select_matadata_1 + ]}, + {rulesql_foreach, [], + [ t_sqlparse_foreach_1 + , t_sqlparse_foreach_2 + , t_sqlparse_foreach_3 + , t_sqlparse_foreach_4 + , t_sqlparse_foreach_5 + , t_sqlparse_foreach_6 + , t_sqlparse_foreach_7 + , t_sqlparse_foreach_8 + ]}, + {rulesql_case_when, [], + [ t_sqlparse_case_when_1 + , t_sqlparse_case_when_2 + , t_sqlparse_case_when_3 + ]}, + {rulesql_array_index, [], + [ t_sqlparse_array_index_1 + , t_sqlparse_array_index_2 + , t_sqlparse_array_index_3 + , t_sqlparse_array_index_4 + , t_sqlparse_array_index_5 + ]}, + {rulesql_array_range, [], + [ t_sqlparse_array_range_1 + , t_sqlparse_array_range_2 + ]}, + {rulesql_compare, [], + [ t_sqlparse_compare_undefined + , t_sqlparse_compare_null_null + , t_sqlparse_compare_null_notnull + , t_sqlparse_compare_notnull_null + , t_sqlparse_compare + ]}, + {rulesql_boolean, [], + [ t_sqlparse_true_false + ]}, + {rulesql_others, [], + [ t_sqlparse_new_map + , t_sqlparse_invalid_json + ]} + ]. + +%%------------------------------------------------------------------------------ +%% Overall setup/teardown +%%------------------------------------------------------------------------------ + +init_per_suite(Config) -> + ok = ekka_mnesia:start(), + ok = emqx_rule_registry:mnesia(boot), + start_apps(), + Config. + +end_per_suite(_Config) -> + stop_apps(), + ok. + +on_resource_create(_id, _) -> #{}. +on_resource_destroy(_id, _) -> ok. +on_get_resource_status(_id, _) -> #{is_alive => true}. + +%%------------------------------------------------------------------------------ +%% Group specific setup/teardown +%%------------------------------------------------------------------------------ + +group(_Groupname) -> + []. + +init_per_group(registry, Config) -> + Config; +init_per_group(_Groupname, Config) -> + Config. + +end_per_group(_Groupname, _Config) -> + ok. + +%%------------------------------------------------------------------------------ +%% Testcase specific setup/teardown +%%------------------------------------------------------------------------------ + +init_per_testcase(_TestCase, Config) -> + init_events_counters(), + ok = emqx_rule_registry:register_resource_types( + [make_simple_debug_resource_type()]), + %ct:pal("============ ~p", [ets:tab2list(emqx_resource_type)]), + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%%------------------------------------------------------------------------------ +%% Test cases for `select` +%%------------------------------------------------------------------------------ + +t_sqlselect_0(_Config) -> + %% Verify SELECT with and without 'AS' + Sql = "select * " + "from \"t/#\" " + "where payload.cmd.info = 'tt'", + ?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "select payload.cmd as cmd " + "from \"t/#\" " + "where cmd.info = 'tt'", + ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "select payload.cmd as cmd, cmd.info as info " + "from \"t/#\" " + "where cmd.info = 'tt' and info = 'tt'", + ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, + <<"info">> := <<"tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})), + %% cascaded as + Sql4 = "select payload.cmd as cmd, cmd.info as meta.info " + "from \"t/#\" " + "where cmd.info = 'tt' and meta.info = 'tt'", + ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, + <<"meta">> := #{<<"info">> := <<"tt">>}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlselect_00(_Config) -> + %% Verify plus/subtract and unary_add_or_subtract + Sql = "select 1-1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 0}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + Sql1 = "select -1 + 1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 0}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "select 1 + 1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 2}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "select +1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 1}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlselect_01(_Config) -> + ok = emqx_rule_engine:load_providers(), + TopicRule1 = create_simple_repub_rule( + <<"t2">>, + "SELECT json_decode(payload) as p, payload " + "FROM \"t3/#\", \"t1\" " + "WHERE p.x = 1"), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), + ct:sleep(100), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"{\"x\":1}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, + + emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), + receive {publish, #{topic := T3, payload := Payload3}} -> + ?assertEqual(<<"t2">>, T3), + ?assertEqual(<<"{\"x\":1}">>, Payload3) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule1). + +t_sqlselect_02(_Config) -> + ok = emqx_rule_engine:load_providers(), + TopicRule1 = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"t3/#\", \"t1\" " + "WHERE payload.x = 1"), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), + ct:sleep(100), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"{\"x\":1}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := Payload0}} -> + ct:fail({unexpected_t2, Payload0}) + after 1000 -> + ok + end, + + emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), + receive {publish, #{topic := T3, payload := Payload3}} -> + ?assertEqual(<<"t2">>, T3), + ?assertEqual(<<"{\"x\":1}">>, Payload3) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule1). + + + +t_sqlselect_1(_Config) -> + ok = emqx_rule_engine:load_providers(), + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT json_decode(payload) as p, payload " + "FROM \"t1\" " + "WHERE p.x = 1 and p.y = 2"), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + ct:sleep(200), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":2}">>, 0), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"{\"x\":1,\"y\":2}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2 + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"t2\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_t2 = Fun(), + received_t2 = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2_1(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2, if the msg dropped + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/message_dropped\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_nothing = Fun(), + + %% it should not keep republishing "t2" + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2_2(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2, if the msg acked + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/message_acked\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1), + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_t2 = Fun(), + received_t2 = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2_3(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2, if the msg delivered + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/message_delivered\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_t2 = Fun(), + received_t2 = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_3(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% republish the client.connected msg + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/client_connected\" " + "WHERE username = 'emqx1'", + <<"clientid=${clientid}">>), + {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + ct:sleep(200), + {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), + {ok, _} = emqtt:connect(Client1), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"clientid=c_emqx1">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +%%------------------------------------------------------------------------------ +%% Test cases for events +%%------------------------------------------------------------------------------ + +t_sqlparse_event_1(_Config) -> + Sql = "select topic as tp " + "from \"$events/session_subscribed\" ", + ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). + +t_sqlparse_event_2(_Config) -> + Sql = "select clientid " + "from \"$events/client_connected\" ", + ?assertMatch({ok,#{<<"clientid">> := <<"abc">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"clientid">> => <<"abc">>}})). + +t_sqlparse_event_3(_Config) -> + Sql = "select clientid, topic as tp " + "from \"t/tt\", \"$events/client_connected\" ", + ?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})). + +%%------------------------------------------------------------------------------ +%% Test cases for `foreach` +%%------------------------------------------------------------------------------ + +t_sqlparse_foreach_1(_Config) -> + %% Verify foreach with and without 'AS' + Sql = "foreach payload.sensors as s " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"s">> := 1}, #{<<"s">> := 2}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "from \"t/#\" ", + ?assertMatch({ok,[#{item := 1}, #{item := 2}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "foreach payload.sensors " + "from \"t/#\" ", + ?assertMatch({ok,[#{item := #{<<"cmd">> := <<"1">>}, clientid := <<"c_a">>}, + #{item := #{ <<"cmd">> := <<"2">> + , <<"name">> := <<"ct">> + } + , clientid := <<"c_a">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{ <<"payload">> => <<"{\"sensors\": [{\"cmd\":\"1\"}, " + "{\"cmd\":\"2\",\"name\":\"ct\"}]}">> + , <<"clientid">> => <<"c_a">> + , <<"topic">> => <<"t/a">> + }})), + Sql4 = "foreach payload.sensors " + "from \"t/#\" ", + {ok,[#{metadata := #{rule_id := TRuleId}}, + #{metadata := #{rule_id := TRuleId}}]} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => #{ + <<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}}), + ?assert(is_binary(TRuleId)). + +t_sqlparse_foreach_2(_Config) -> + %% Verify foreach-do with and without 'AS' + Sql = "foreach payload.sensors as s " + "do s.cmd as msg_type " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "do item.cmd as msg_type " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "foreach payload.sensors " + "do item as item " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"item">> := 1},#{<<"item">> := 2}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_3(_Config) -> + %% Verify foreach-incase with and without 'AS' + Sql = "foreach payload.sensors as s " + "incase s.cmd != 1 " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"s">> := #{<<"cmd">> := 2}}, + #{<<"s">> := #{<<"cmd">> := 3}} + ]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "incase item.cmd != 1 " + "from \"t/#\" ", + ?assertMatch({ok,[#{item := #{<<"cmd">> := 2}}, + #{item := #{<<"cmd">> := 3}} + ]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_4(_Config) -> + %% Verify foreach-do-incase + Sql = "foreach payload.sensors as s " + "do s.cmd as msg_type, s.name as name " + "incase is_not_null(s.cmd) " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ ok + , [ #{<<"msg_type">> := <<"1">>, <<"name">> := <<"n1">>} + , #{<<"msg_type">> := <<"2">>} + ] + }, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\", \"name\":\"n1\"}, " + "{\"cmd\":\"2\"}, {\"name\":\"n3\"}]}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok,[]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_5(_Config) -> + %% Verify foreach on a empty-list or non-list variable + Sql = "foreach payload.sensors as s " + "do s.cmd as msg_type, s.name as name " + "from \"t/#\" ", + ?assertMatch({ok,[]}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok,[]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": []}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "from \"t/#\" ", + ?assertMatch({ok,[]}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": 1}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_6(_Config) -> + %% Verify foreach on a empty-list or non-list variable + Sql = "foreach json_decode(payload) " + "do item.id as zid, timestamp as t " + "from \"t/#\" ", + {ok, Res} = emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"[{\"id\": 5},{\"id\": 15}]">>, + <<"topic">> => <<"t/a">>}}), + [#{<<"t">> := Ts1, <<"zid">> := Zid1}, + #{<<"t">> := Ts2, <<"zid">> := Zid2}] = Res, + ?assertEqual(true, is_integer(Ts1)), + ?assertEqual(true, is_integer(Ts2)), + ?assert(Zid1 == 5 orelse Zid1 == 15), + ?assert(Zid2 == 5 orelse Zid2 == 15). + +t_sqlparse_foreach_7(_Config) -> + %% Verify foreach-do-incase and cascaded AS + Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " + "do info.cmd as msg_type, info.name as name " + "incase is_not_null(info.cmd) " + "from \"t/#\" " + "where s.page = '2' ", + Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " + "{\"info\":[{\"name\":\"cmd1\", \"cmd\":\"1\"}, {\"cmd\":\"2\"}]} } }">>, + ?assertMatch({ ok + , [ #{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>} + , #{<<"msg_type">> := <<"2">>} + ] + }, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " + "do info.cmd as msg_type, info.name as name " + "incase is_not_null(info.cmd) " + "from \"t/#\" " + "where s.page = '3' ", + ?assertMatch({error, nomatch}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_8(_Config) -> + %% Verify foreach-do-incase and cascaded AS + Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " + "do info.cmd as msg_type, info.name as name " + "incase is_map(info) " + "from \"t/#\" " + "where s.page = '2' ", + Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " + "{\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }">>, + ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})), + + Sql3 = "foreach json_decode(payload) as p, p.sensors as s," + " s.collection as c, sublist(2,1,c.info) as info " + "do info.cmd as msg_type, info.name as name " + "from \"t/#\" " + "where s.page = '2' ", + [?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => SqlN, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})) + || SqlN <- [Sql3]]. + +%%------------------------------------------------------------------------------ +%% Test cases for `case..when..` +%%------------------------------------------------------------------------------ + +t_sqlparse_case_when_1(_Config) -> + %% case-when-else clause + Sql = "select " + " case when payload.x < 0 then 0 " + " when payload.x > 7 then 7 " + " else payload.x " + " end as y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"y">> := 1}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, + <<"topic">> => <<"t/a">>}})), + ok. + +t_sqlparse_case_when_2(_Config) -> + % switch clause + Sql = "select " + " case payload.x when 1 then 2 " + " when 2 then 3 " + " else 4 " + " end as y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"y">> := 2}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 3}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 2}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 4}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_case_when_3(_Config) -> + %% case-when clause + Sql = "select " + " case when payload.x < 0 then 0 " + " when payload.x > 7 then 7 " + " end as y " + "from \"t/#\" ", + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 5}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, + <<"topic">> => <<"t/a">>}})), + ok. + +%%------------------------------------------------------------------------------ +%% Test cases for array index +%%------------------------------------------------------------------------------ + +t_sqlparse_array_index_1(_Config) -> + %% index get + Sql = "select " + " json_decode(payload) as p, " + " p[1] as a " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"a">> := #{<<"x">> := 1}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"[{\"x\": 1}]">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + %% index get without 'as' + Sql2 = "select " + " payload.x[2] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [3]}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,3,4]}, + <<"topic">> => <<"t/a">>}})), + %% index get without 'as' again + Sql3 = "select " + " payload.x[2].y " + "from \"t/#\" ", + ?assertMatch( {ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 3}]}}} + , emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, + <<"topic">> => <<"t/a">>}}) + ), + + %% index get with 'as' + Sql4 = "select " + " payload.x[2].y as b " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_index_2(_Config) -> + %% array get with negative index + Sql1 = "select " + " payload.x[-2].y as b " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, + <<"topic">> => <<"t/a">>}})), + %% array append to head or tail of a list: + Sql2 = "select " + " payload.x as b, " + " 1 as c[-0], " + " 2 as c[-0], " + " b as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := 0, <<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => 0}, + <<"topic">> => <<"t/a">>}})), + %% construct an empty list: + Sql3 = "select " + " [] as c, " + " 1 as c[-0], " + " 2 as c[-0], " + " 0 as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + %% construct a list: + Sql4 = "select " + " [payload.a, \"topic\", 'c'] as c, " + " 1 as c[-0], " + " 2 as c[-0], " + " 0 as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,11,<<"t/a">>,<<"c">>,1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => #{<<"payload">> => <<"{\"a\":11}">>, + <<"topic">> => <<"t/a">> + }})). + +t_sqlparse_array_index_3(_Config) -> + %% array with json string payload: + Sql0 = "select " + "payload," + "payload.x[2].y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := [1,2]}, 3]}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})), + %% same as above but don't select payload: + Sql1 = "select " + "payload.x[2].y as b " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := [1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})), + %% same as above but add 'as' clause: + Sql2 = "select " + "payload.x[2].y as b.c " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := #{<<"c">> := [1,2]}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_index_4(_Config) -> + %% array with json string payload: + Sql0 = "select " + "0 as payload.x[2].y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 0}]}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})), + %% array with json string payload, and also select payload.x: + Sql1 = "select " + "payload.x, " + "0 as payload.x[2].y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := 0}, 3]}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_index_5(_Config) -> + Sql00 = "select " + " [1,2,3,4] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assert(lists:any(fun({_K, V}) -> + V =:= [1,2,3,4] + end, maps:to_list(Res00))). + +%%------------------------------------------------------------------------------ +%% Test cases for rule metadata +%%------------------------------------------------------------------------------ + +t_sqlparse_select_matadata_1(_Config) -> + %% array with json string payload: + Sql0 = "select " + "payload " + "from \"t/#\" ", + ?assertNotMatch({ok, #{<<"payload">> := <<"abc">>, metadata := _}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"abc">>, + <<"topic">> => <<"t/a">>}})), + Sql1 = "select " + "payload, metadata " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := <<"abc">>, <<"metadata">> := _}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"abc">>, + <<"topic">> => <<"t/a">>}})). + +%%------------------------------------------------------------------------------ +%% Test cases for array range +%%------------------------------------------------------------------------------ + +t_sqlparse_array_range_1(_Config) -> + %% get a range of list + Sql0 = "select " + " payload.a[1..4] as c " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,1,2,3]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, + <<"topic">> => <<"t/a">>}})), + %% get a range from non-list data + Sql02 = "select " + " payload.a[1..4] as c " + "from \"t/#\" ", + ?assertMatch({error, {select_and_transform_error, {error,{range_get,non_list_data},_}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql02, + <<"ctx">> => + #{<<"payload">> => <<"{\"x\":[0,1,2,3,4,5]}">>, + <<"topic">> => <<"t/a">>}})), + + %% construct a range: + Sql1 = "select " + " [1..4] as c, " + " 5 as c[-0], " + " 6 as c[-0], " + " 0 as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,1,2,3,4,5,6]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_range_2(_Config) -> + %% construct a range without 'as' + Sql00 = "select " + " [1..4] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assert(lists:any(fun({_K, V}) -> + V =:= [1,2,3,4] + end, maps:to_list(Res00))), + %% construct a range without 'as' + Sql01 = "select " + " a[2..4] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"a">> := [2,3,4]}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql01, + <<"ctx">> => #{<<"a">> => [1,2,3,4,5], + <<"topic">> => <<"t/a">>}})), + %% get a range of list without 'as' + Sql02 = "select " + " payload.a[1..4] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"a">> := [0,1,2,3]}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql02, + <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, + <<"topic">> => <<"t/a">>}})). + +%%------------------------------------------------------------------------------ +%% Test cases for boolean +%%------------------------------------------------------------------------------ + +t_sqlparse_true_false(_Config) -> + %% construct a range without 'as' + Sql00 = "select " + " true as a, false as b, " + " false as x.y, true as c[-0] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assertMatch(#{<<"a">> := true, <<"b">> := false, + <<"x">> := #{<<"y">> := false}, + <<"c">> := [true] + }, Res00). + +%%------------------------------------------------------------------------------ +%% Test cases for compare +%%------------------------------------------------------------------------------ + +-define(TEST_SQL(SQL), + emqx_rule_sqltester:test( + #{<<"rawsql">> => SQL, + <<"ctx">> => #{<<"payload">> => <<"{}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_compare_undefined(_Config) -> + Sql00 = "select " + " * " + "from \"t/#\" " + "where dev != undefined ", + %% no match + ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00)), + + Sql01 = "select " + " 'd' as dev " + "from \"t/#\" " + "where dev != undefined ", + {ok, Res01} = ?TEST_SQL(Sql01), + %% pass + ?assertMatch(#{}, Res01), + + Sql02 = "select " + " * " + "from \"t/#\" " + "where dev != 'undefined' ", + {ok, Res02} = ?TEST_SQL(Sql02), + %% pass + ?assertMatch(#{}, Res02). + +t_sqlparse_compare_null_null(_Config) -> + %% test undefined == undefined + Sql00 = "select " + " a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := true + }, Res00), + + %% test undefined != undefined + Sql01 = "select " + " a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := false + }, Res01), + + %% test undefined > undefined + Sql02 = "select " + " a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test undefined < undefined + Sql03 = "select " + " a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test undefined <= undefined + Sql04 = "select " + " a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := true + }, Res04), + + %% test undefined >= undefined + Sql05 = "select " + " a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := true + }, Res05). + +t_sqlparse_compare_null_notnull(_Config) -> + %% test undefined == b + Sql00 = "select " + " 'b' as b, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := false + }, Res00), + + %% test undefined != b + Sql01 = "select " + " 'b' as b, a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + %% test undefined > b + Sql02 = "select " + " 'b' as b, a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test undefined < b + Sql03 = "select " + " 'b' as b, a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test undefined <= b + Sql04 = "select " + " 'b' as b, a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test undefined >= b + Sql05 = "select " + " 'b' as b, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05). + +t_sqlparse_compare_notnull_null(_Config) -> + %% test 'a' == undefined + Sql00 = "select " + " 'a' as a, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := false + }, Res00), + + %% test 'a' != undefined + Sql01 = "select " + " 'a' as a, a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + %% test 'a' > undefined + Sql02 = "select " + " 'a' as a, a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test 'a' < undefined + Sql03 = "select " + " 'a' as a, a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test 'a' <= undefined + Sql04 = "select " + " 'a' as a, a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test 'a' >= undefined + Sql05 = "select " + " 'a' as a, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05). + +t_sqlparse_compare(_Config) -> + Sql00 = "select " + " 'a' as a, 'a' as b, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := true + }, Res00), + + Sql01 = "select " + " is_null(a) as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + Sql02 = "select " + " 1 as a, 2 as b, a < b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := true + }, Res02), + + Sql03 = "select " + " 1 as a, 2 as b, a > b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + Sql04 = "select " + " 1 as a, 2 as b, a = b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test 'a' >= undefined + Sql05 = "select " + " 1 as a, 2 as b, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05), + + %% test 'a' >= undefined + Sql06 = "select " + " 1 as a, 2 as b, a <= b as c " + "from \"t/#\" ", + {ok, Res06} = ?TEST_SQL(Sql06), + ?assertMatch(#{<<"c">> := true + }, Res06). + + + +t_sqlparse_new_map(_Config) -> + %% construct a range without 'as' + Sql00 = "select " + " map_new() as a, map_new() as b, " + " map_new() as x.y, map_new() as c[-0] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assertMatch(#{<<"a">> := #{}, <<"b">> := #{}, + <<"x">> := #{<<"y">> := #{}}, + <<"c">> := [#{}] + }, Res00). + + +t_sqlparse_invalid_json(_Config) -> + Sql02 = "select " + " payload.a[1..4] as c " + "from \"t/#\" ", + ?assertMatch({error, {select_and_transform_error, {error,{decode_json_failed,_},_}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql02, + <<"ctx">> => + #{<<"payload">> => <<"{\"x\":[0,1,2,3,}">>, + <<"topic">> => <<"t/a">>}})), + + + Sql2 = "foreach payload.sensors " + "do item.cmd as msg_type " + "from \"t/#\" ", + ?assertMatch({error, {select_and_collect_error, {error,{decode_json_failed,_},_}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"} {\"cmd\":}]}">>, + <<"topic">> => <<"t/a">>}})). From 267946c3794b8c251a056e9f4cb975dd04b1b14f Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Sep 2022 09:44:03 +0800 Subject: [PATCH 18/44] fix: rm unused module `emqx_rule_date` was added before [v4.3.15-rc1] PR: emqx#7894 commit: 8558a62ee291021e521dc2c1f6efa418fa76469d The change was released in [v4.3.15-rc1] [v4.3.15-rc2] And unused at [v4.3.15-rc3] PR: emqx#8044 commit: 4fc5cb2817c4f1e8225c048bc04f4972148d0df4 We just keep the module from [v4.3.15-rc1] to [v4.3.20] and remove this module in newer version --- apps/emqx_rule_engine/src/emqx_rule_date.erl | 248 ------------------- 1 file changed, 248 deletions(-) delete mode 100644 apps/emqx_rule_engine/src/emqx_rule_date.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_date.erl b/apps/emqx_rule_engine/src/emqx_rule_date.erl deleted file mode 100644 index fb9cad4c3..000000000 --- a/apps/emqx_rule_engine/src/emqx_rule_date.erl +++ /dev/null @@ -1,248 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_date). - --export([date/3, date/4, parse_date/4]). - --export([ - is_int_char/1, - is_symbol_char/1, - is_m_char/1 -]). - --record(result, { - %%year() - year = "1970" :: string(), - %%month() - month = "1" :: string(), - %%day() - day = "1" :: string(), - %%hour() - hour = "0" :: string(), - %%minute() %% epoch in millisecond precision - minute = "0" :: string(), - %%second() %% epoch in millisecond precision - second = "0" :: string(), - %%integer() %% zone maybe some value - zone = "+00:00" :: string() -}). - -%% -type time_unit() :: 'microsecond' -%% | 'millisecond' -%% | 'nanosecond' -%% | 'second'. -%% -type offset() :: [byte()] | (Time :: integer()). -date(TimeUnit, Offset, FormatString) -> - date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)). - -date(TimeUnit, Offset, FormatString, TimeEpoch) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - Res = lists:map( - fun(Expr) -> - eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr) - end, - R - ), - lists:concat(Res). - -parse_date(TimeUnit, Offset, FormatString, InputString) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - IsZ = fun(V) -> - case V of - {tag, $Z} -> true; - _ -> false - end - end, - R1 = lists:filter(IsZ, R), - IfFun = fun(Con, A, B) -> - case Con of - [] -> A; - _ -> B - end - end, - Res = parse_input(FormatString, InputString), - Str = - Res#result.year ++ "-" ++ - Res#result.month ++ "-" ++ - Res#result.day ++ "T" ++ - Res#result.hour ++ ":" ++ - Res#result.minute ++ ":" ++ - Res#result.second ++ - IfFun(R1, Offset, Res#result.zone), - calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]). - -mlist(R) -> - %% %H Shows hour in 24-hour format [15] - [ - {$H, R#result.hour}, - %% %M Displays minutes [00-59] - {$M, R#result.minute}, - %% %S Displays seconds [00-59] - {$S, R#result.second}, - %% %y Displays year YYYY [2021] - {$y, R#result.year}, - %% %m Displays the number of the month [01-12] - {$m, R#result.month}, - %% %d Displays the number of the month [01-12] - {$d, R#result.day}, - %% %Z Displays Time zone - {$Z, R#result.zone} - ]. - -rmap(Result) -> - maps:from_list(mlist(Result)). - -support_char() -> "HMSymdZ". - -create_tag(Head, []) -> - Head; -create_tag(Head, [Val1 | RVal]) -> - case Val1 of - [] -> - create_tag(Head ++ [{st, [$%]}], RVal); - [H | Other] -> - case lists:member(H, support_char()) of - true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal); - false -> create_tag(Head ++ [{st, [$% | Val1]}], RVal) - end - end. - -eval_tag(_, {st, Str}) -> - Str; -eval_tag(Map, {tag, Char}) -> - maps:get(Char, Map, "undefined"). - -%% make_time(TimeUnit, Offset) -> -%% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)). -make_time(TimeUnit, Offset, TimeEpoch) -> - Res = calendar:system_time_to_rfc3339(TimeEpoch, - [{unit, TimeUnit}, {offset, Offset}]), - [Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T, - H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = Res, - IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, - {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), - #result{ - year = [Y1, Y2, Y3, Y4] - , month = [Mon1, Mon2] - , day = [D1, D2] - , hour = [H1, H2] - , minute = [Min1, Min2] - , second = [S1, S2] ++ FractionStr - , zone = UtcOffset - }. - -is_int_char(C) -> - C >= $0 andalso C =< $9. -is_symbol_char(C) -> - C =:= $- orelse C =:= $+. -is_m_char(C) -> - C =:= $:. - -parse_char_with_fun(_, []) -> - error(null_input); -parse_char_with_fun(ValidFun, [C | Other]) -> - Res = - case erlang:is_function(ValidFun) of - true -> ValidFun(C); - false -> erlang:apply(emqx_rule_date, ValidFun, [C]) - end, - case Res of - true -> {C, Other}; - false -> error({unexpected, [C | Other]}) - end. -parse_string([], Input) -> - {[], Input}; -parse_string([C | Other], Input) -> - {C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input), - {Res, Input2} = parse_string(Other, Input1), - {[C1 | Res], Input2}. - -parse_times(0, _, Input) -> - {[], Input}; -parse_times(Times, Fun, Input) -> - {C1, Input1} = parse_char_with_fun(Fun, Input), - {Res, Input2} = parse_times((Times - 1), Fun, Input1), - {[C1 | Res], Input2}. - -parse_int_times(Times, Input) -> - parse_times(Times, is_int_char, Input). - -parse_fraction(Input) -> - IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, - lists:splitwith(IsFractionChar, Input). - -parse_second(Input) -> - {M, Input1} = parse_int_times(2, Input), - {M1, Input2} = parse_fraction(Input1), - {M ++ M1, Input2}. - -parse_zone(Input) -> - {S, Input1} = parse_char_with_fun(is_symbol_char, Input), - {M, Input2} = parse_int_times(2, Input1), - {C, Input3} = parse_char_with_fun(is_m_char, Input2), - {V, Input4} = parse_int_times(2, Input3), - {[S | M ++ [C | V]], Input4}. - -mlist1() -> - maps:from_list( - %% %H Shows hour in 24-hour format [15] - [ - {$H, fun(Input) -> parse_int_times(2, Input) end}, - %% %M Displays minutes [00-59] - {$M, fun(Input) -> parse_int_times(2, Input) end}, - %% %S Displays seconds [00-59] - {$S, fun(Input) -> parse_second(Input) end}, - %% %y Displays year YYYY [2021] - {$y, fun(Input) -> parse_int_times(4, Input) end}, - %% %m Displays the number of the month [01-12] - {$m, fun(Input) -> parse_int_times(2, Input) end}, - %% %d Displays the number of the month [01-12] - {$d, fun(Input) -> parse_int_times(2, Input) end}, - %% %Z Displays Time zone - {$Z, fun(Input) -> parse_zone(Input) end} - ] - ). - -update_result($H, Res, Str) -> Res#result{hour = Str}; -update_result($M, Res, Str) -> Res#result{minute = Str}; -update_result($S, Res, Str) -> Res#result{second = Str}; -update_result($y, Res, Str) -> Res#result{year = Str}; -update_result($m, Res, Str) -> Res#result{month = Str}; -update_result($d, Res, Str) -> Res#result{day = Str}; -update_result($Z, Res, Str) -> Res#result{zone = Str}. - -parse_tag(Res, {st, St}, InputString) -> - {_A, B} = parse_string(St, InputString), - {Res, B}; -parse_tag(Res, {tag, St}, InputString) -> - Fun = maps:get(St, mlist1()), - {A, B} = Fun(InputString), - NRes = update_result(St, Res, A), - {NRes, B}. - -parse_tags(Res, [], _) -> - Res; -parse_tags(Res, [Tag | Others], InputString) -> - {NRes, B} = parse_tag(Res, Tag, InputString), - parse_tags(NRes, Others, B). - -parse_input(FormatString, InputString) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - parse_tags(#result{}, R, InputString). From d8a022fb4579f505569fc141f07b8f3f4ebdeeae Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Sep 2022 14:36:02 +0800 Subject: [PATCH 19/44] fix(test): unstopped test client --- .../test/emqx_rule_engine_SUITE.erl | 14 +++++++++----- apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 42816e869..c6f5057f1 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -238,8 +238,12 @@ end_per_testcase(t_events, Config) -> ok = emqx_rule_registry:remove_rule(?config(hook_points_rules, Config)), ok = emqx_rule_registry:remove_action('hook-metrics-action'); end_per_testcase(Test, Config) - when Test =:= t_sqlselect_multi_actoins_1, - Test =:= t_sqlselect_multi_actoins_2 + when Test =:= t_sqlselect_multi_actoins_1 + ;Test =:= t_sqlselect_multi_actoins_1_1 + ;Test =:= t_sqlselect_multi_actoins_2 + ;Test =:= t_sqlselect_multi_actoins_3 + ;Test =:= t_sqlselect_multi_actoins_3_1 + ;Test =:= t_sqlselect_multi_actoins_4 -> emqtt:stop(?config(subclient, Config)), emqtt:stop(?config(connclient, Config)), @@ -1218,8 +1222,8 @@ t_match_atom_and_binary(_Config) -> {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), ct:sleep(100), - {ok, Client2} = emqtt:start_link([{username, <<"emqx2">>}]), - {ok, _} = emqtt:connect(Client2), + {ok, Client1} = emqtt:start_link([{username, <<"emqx2">>}]), + {ok, _} = emqtt:connect(Client1), receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(<<"t2">>, T), <<"user:", ConnAt/binary>> = Payload, @@ -1228,7 +1232,7 @@ t_match_atom_and_binary(_Config) -> ct:fail(wait_for_t2) end, - emqtt:stop(Client), + emqtt:stop(Client), emqtt:stop(Client1), emqx_rule_registry:remove_rule(TopicRule). t_metrics(_Config) -> diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index d93b747c0..5fd966d6c 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -474,7 +474,7 @@ t_sqlselect_3(_Config) -> ok end, - emqtt:stop(Client), + emqtt:stop(Client), emqtt:stop(Client1), emqx_rule_registry:remove_rule(TopicRule). %%------------------------------------------------------------------------------ From 0a5a0867e43d1790ac9a7f1d8668f35e07f5e224 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 29 Sep 2022 14:05:40 +0800 Subject: [PATCH 20/44] fix(rule_func): refine `+` error info when type implicit conversion --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 18e61967e..a4791a94d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -338,7 +338,13 @@ null() -> %% concat 2 strings '+'(X, Y) when is_binary(X), is_binary(Y) -> - concat(X, Y). + concat(X, Y); + +%% unsupported type implicit conversion +'+'(X, Y) + when (is_number(X) andalso is_binary(Y)) orelse + (is_binary(X) andalso is_number(Y)) -> + error(unsupported_type_implicit_conversion). '-'(X, Y) when is_number(X), is_number(Y) -> X - Y. From 32376c7cf962da1b4f6fe3f06f67c87523d872db Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 29 Sep 2022 16:21:40 +0800 Subject: [PATCH 21/44] fix(rule_func): refine num funcs error info type unsupported --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index a4791a94d..da345665b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -332,34 +332,46 @@ null() -> %% Arithmetic Funcs %%------------------------------------------------------------------------------ +-define(OPERATOR_TYPE_ERROR, unsupported_function_operator_type). + %% plus 2 numbers '+'(X, Y) when is_number(X), is_number(Y) -> X + Y; - %% concat 2 strings '+'(X, Y) when is_binary(X), is_binary(Y) -> concat(X, Y); - %% unsupported type implicit conversion '+'(X, Y) when (is_number(X) andalso is_binary(Y)) orelse (is_binary(X) andalso is_number(Y)) -> - error(unsupported_type_implicit_conversion). + error(unsupported_type_implicit_conversion); +'+'(_, _) -> + error(?OPERATOR_TYPE_ERROR). '-'(X, Y) when is_number(X), is_number(Y) -> - X - Y. + X - Y; +'-'(_, _) -> + error(?OPERATOR_TYPE_ERROR). '*'(X, Y) when is_number(X), is_number(Y) -> - X * Y. + X * Y; +'*'(_, _) -> + error(?OPERATOR_TYPE_ERROR). '/'(X, Y) when is_number(X), is_number(Y) -> - X / Y. + X / Y; +'/'(_, _) -> + error(?OPERATOR_TYPE_ERROR). 'div'(X, Y) when is_integer(X), is_integer(Y) -> - X div Y. + X div Y; +'div'(_, _) -> + error(?OPERATOR_TYPE_ERROR). mod(X, Y) when is_integer(X), is_integer(Y) -> - X rem Y. + X rem Y; +mod(_, _) -> + error(?OPERATOR_TYPE_ERROR). eq(X, Y) -> X == Y. From 27e19da06682344da2126ae258dd2d40217e40c3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 29 Sep 2022 16:23:13 +0800 Subject: [PATCH 22/44] test(rulesql): num funcs type cases --- .../test/emqx_rulesql_SUITE.erl | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index 5fd966d6c..33de9b7dd 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -236,8 +236,27 @@ t_sqlselect_00(_Config) -> #{<<"rawsql">> => Sql3, <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). + <<"topic">> => <<"t/a">>}})), + Sql4 = "select payload.msg1 + payload.msg2 as msg " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"msg">> := <<"hello world">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => + #{<<"payload">> => <<"{\"msg1\": \"hello\", \"msg2\": \" world\"}">>, + <<"topic">> => <<"t/1">>}})), + + Sql5 = "select payload.msg1 + payload.msg2 as msg " + "from \"t/#\" ", + ?assertMatch({error, {select_and_transform_error, {error, unsupported_type_implicit_conversion, _ST}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql5, + <<"ctx">> => + #{<<"payload">> => <<"{\"msg1\": \"hello\", \"msg2\": 1}">>, + <<"topic">> => <<"t/1">>}})). + +%% Verify SELECT with and with 'WHERE' t_sqlselect_01(_Config) -> ok = emqx_rule_engine:load_providers(), TopicRule1 = create_simple_repub_rule( @@ -312,7 +331,50 @@ t_sqlselect_02(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule1). +t_sqlselect_03(_Config) -> + %% Verify SELECT with and with 'WHERE' and `+` `=` and `or` condition in 'WHERE' clause + ok = emqx_rule_engine:load_providers(), + TopicRule1 = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"t3/#\", \"t1\" " + "WHERE payload.x + payload.y = 2 or payload.x + payload.y = \"11\""), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1, \"y\":1}">>, 0), + ct:sleep(100), + receive {publish, #{topic := T1, payload := Payload0}} -> + ?assertEqual(<<"t2">>, T1), + ?assertEqual(<<"{\"x\":1, \"y\":1}">>, Payload0) + after 1000 -> + ct:fail(wait_for_t2) + end, + receive {publish, #{topic := T2, payload := Payload1}} -> + ?assertEqual(<<"t2">>, T2), + ?assertEqual(<<"{\"x\":\"1\", \"y\":\"1\"}">>, Payload1) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1, \"y\":2}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := Payload2}} -> + ct:fail({unexpected_t2, Payload2}) + after 1000 -> + ok + end, + + emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1, \"y\":1}">>, 0), + receive {publish, #{topic := T3, payload := Payload3}} -> + ?assertEqual(<<"t2">>, T3), + ?assertEqual(<<"{\"x\":1, \"y\":1}">>, Payload3) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule1). t_sqlselect_1(_Config) -> ok = emqx_rule_engine:load_providers(), From 5047211950f223ad9972f364ccc2b5467472eaf7 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Sun, 9 Oct 2022 14:20:48 +0800 Subject: [PATCH 23/44] test: rulesql select fields and select from event message --- .../test/emqx_rulesql_SUITE.erl | 297 +++++++++++------- 1 file changed, 188 insertions(+), 109 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index 33de9b7dd..f4f329e87 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("emqx_rule_engine/include/rule_engine.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -51,15 +52,16 @@ groups() -> , t_sqlselect_02 , t_sqlselect_1 , t_sqlselect_2 - , t_sqlselect_2_1 - , t_sqlselect_2_2 - , t_sqlselect_2_3 - , t_sqlselect_3 ]}, {rulesql_select_events, [], - [ t_sqlparse_event_1 - , t_sqlparse_event_2 - , t_sqlparse_event_3 + [ t_sqlparse_event_client_connected_01 + , t_sqlparse_event_client_connected_02 + , t_sqlparse_event_client_disconnected + , t_sqlparse_event_session_subscribed + , t_sqlparse_event_session_unsubscribed + , t_sqlparse_event_message_delivered + , t_sqlparse_event_message_acked + , t_sqlparse_event_message_dropped ]}, {rulesql_select_metadata, [], [ t_sqlparse_select_matadata_1 @@ -159,16 +161,35 @@ end_per_testcase(_TestCase, _Config) -> t_sqlselect_0(_Config) -> %% Verify SELECT with and without 'AS' - Sql = "select * " - "from \"t/#\" " - "where payload.cmd.info = 'tt'", - ?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), + + Sql1 = "SELECT * " + "FROM \"t/#\" " + "WHERE payload.cmd.info = 'tt'", + %% all available fields by `select` from message publish, selecte other key will be `undefined` + Topic = <<"t/a">>, + Payload = <<"{\"cmd\": {\"info\":\"tt\"}}">>, + {ok, #{username := <<"u_emqx">>, + topic := Topic, + timestamp := TimeStamp, + qos := 1, + publish_received_at := TimeStamp, + peerhost := <<"127.0.0.1">>, + payload := Payload, + node := 'test@127.0.0.1', + metadata := #{rule_id := TestRuleId}, + id := MsgId, + flags := #{sys := true, event := true}, + clientid := <<"c_emqx">> + } + } = emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => Topic}}), + ?assert(is_binary(TestRuleId)), + ?assert(is_binary(MsgId)), + ?assert(is_integer(TimeStamp)), + Sql2 = "select payload.cmd as cmd " "from \"t/#\" " "where cmd.info = 'tt'", @@ -179,6 +200,7 @@ t_sqlselect_0(_Config) -> #{<<"payload">> => <<"{\"cmd\": {\"info\":\"tt\"}}">>, <<"topic">> => <<"t/a">>}})), + Sql3 = "select payload.cmd as cmd, cmd.info as info " "from \"t/#\" " "where cmd.info = 'tt' and info = 'tt'", @@ -205,14 +227,15 @@ t_sqlselect_0(_Config) -> t_sqlselect_00(_Config) -> %% Verify plus/subtract and unary_add_or_subtract - Sql = "select 1-1 as a " + Sql0 = "select 1 - 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, + #{<<"rawsql">> => Sql0, <<"ctx">> => #{<<"payload">> => <<"">>, <<"topic">> => <<"t/a">>}})), + Sql1 = "select -1 + 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 0}}, @@ -221,6 +244,7 @@ t_sqlselect_00(_Config) -> <<"ctx">> => #{<<"payload">> => <<"">>, <<"topic">> => <<"t/a">>}})), + Sql2 = "select 1 + 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 2}}, @@ -229,6 +253,7 @@ t_sqlselect_00(_Config) -> <<"ctx">> => #{<<"payload">> => <<"">>, <<"topic">> => <<"t/a">>}})), + Sql3 = "select +1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 1}}, @@ -431,58 +456,93 @@ t_sqlselect_2(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_2_1(_Config) -> +%%------------------------------------------------------------------------------ +%% Test cases for events +%%------------------------------------------------------------------------------ + +%% FROM $events/client_connected +t_sqlparse_event_client_connected_01(_Config) -> + Sql = "select *" + "from \"$events/client_connected\" ", + + %% all available fields by `select` from message publish, selecte other key will be `undefined` + {ok, #{clientid := <<"c_emqx">>, + username := <<"u_emqx">>, + timestamp := TimeStamp, + connected_at := TimeStamp, + peername := <<"127.0.0.1:12345">>, + metadata := #{rule_id := RuleId}, + %% default value + node := 'test@127.0.0.1', + sockname := <<"0.0.0.0:1883">>, + proto_name := <<"MQTT">>, + proto_ver := 5, + mountpoint := undefined, + keepalive := 60, + is_bridge := false, + expiry_interval := 3600, + event := 'client.connected', + clean_start := true + } + } = emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"clientid">> => <<"c_emqx">>, <<"peername">> => <<"127.0.0.1:12345">>, <<"username">> => <<"u_emqx">>}}), + ?assert(is_binary(RuleId)), + ?assert(is_integer(TimeStamp)). + +t_sqlparse_event_client_connected_02(_Config) -> ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg dropped + %% republish the client.connected msg TopicRule = create_simple_repub_rule( - <<"t2">>, + <<"repub/to/connected">>, "SELECT * " - "FROM \"$events/message_dropped\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + "FROM \"$events/client_connected\" " + "WHERE username = 'emqx1'", + <<"{clientid: ${clientid}}">>), + {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), {ok, _} = emqtt:connect(Client), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_nothing = Fun(), + {ok, _, _} = emqtt:subscribe(Client, <<"repub/to/connected">>, 0), + ct:sleep(200), + {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), + {ok, _} = emqtt:connect(Client1), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"repub/to/connected">>, T), + ?assertEqual(<<"{clientid: c_emqx1}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, - %% it should not keep republishing "t2" - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - received_nothing = Fun(), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, - emqtt:stop(Client), + emqtt:stop(Client), emqtt:stop(Client1), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_2_2(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg acked - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_acked\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), +%% FROM $events/client_disconnected +t_sqlparse_event_client_disconnected(_Config) -> + %% TODO + ok. - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). +%% FROM $events/session_subscribed +t_sqlparse_event_session_subscribed(_Config) -> + Sql = "select topic as tp " + "from \"$events/session_subscribed\" ", + ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). -t_sqlselect_2_3(_Config) -> +%% FROM $events/session_unsubscribed +t_sqlparse_event_session_unsubscribed(_Config) -> + %% TODO + ok. + +%% FROM $events/message_delivered +t_sqlparse_event_message_delivered(_Config) -> ok = emqx_rule_engine:load_providers(), %% recursively republish to t2, if the msg delivered TopicRule = create_simple_repub_rule( @@ -507,65 +567,84 @@ t_sqlselect_2_3(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_3(_Config) -> +%% FROM $events/message_acked +t_sqlparse_event_message_acked(_Config) -> ok = emqx_rule_engine:load_providers(), - %% republish the client.connected msg + %% republish to `repub/if/acked`, if the msg acked TopicRule = create_simple_repub_rule( - <<"t2">>, + <<"repub/if/acked">>, "SELECT * " - "FROM \"$events/client_connected\" " - "WHERE username = 'emqx1'", - <<"clientid=${clientid}">>), - {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), + "FROM \"$events/message_acked\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ct:sleep(200), - {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), - {ok, _} = emqtt:connect(Client1), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, + {ok, _, _} = emqtt:subscribe(Client, <<"repub/if/acked">>, ?QOS_1), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, + Fun = fun() -> + receive {publish, #{topic := <<"repub/if/acked">>, payload := _}} -> + received_acked + after 500 -> + received_nothing + end + end, - emqtt:stop(Client), emqtt:stop(Client1), + %% sub with max qos1 to generate ack packet + {ok, _, _} = emqtt:subscribe(Client, <<"any/topic">>, ?QOS_1), + + Payload = <<"{\"x\":1,\"y\":144}">>, + + %% even sub with qos1, but publish with qos0, no ack + emqtt:publish(Client, <<"any/topic">>, Payload, ?QOS_0), + received_nothing = Fun(), + + %% sub and pub both are qos1, acked + emqtt:publish(Client, <<"any/topic">>, Payload, ?QOS_1), + received_acked = Fun(), + received_nothing = Fun(), + + %% pub with qos2 but subscribed with qos1, acked + emqtt:publish(Client, <<"any/topic">>, Payload, ?QOS_2), + received_acked = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -%%------------------------------------------------------------------------------ -%% Test cases for events -%%------------------------------------------------------------------------------ +%% FROM $events/message_dropped +t_sqlparse_event_message_dropped(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% republish to `repub/if/dropped`, if any msg dropped + TopicRule = create_simple_repub_rule( + <<"repub/if/dropped">>, + "SELECT * " + "FROM \"$events/message_dropped\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), -t_sqlparse_event_1(_Config) -> - Sql = "select topic as tp " - "from \"$events/session_subscribed\" ", - ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). + %% this message will be dropped and then repub to `repub/if/dropped` + emqtt:publish(Client, <<"any/topic/1">>, <<"{\"x\":1,\"y\":144}">>, 0), -t_sqlparse_event_2(_Config) -> - Sql = "select clientid " - "from \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>}})). + Fun_t2 = fun() -> + receive {publish, #{topic := <<"repub/if/dropped">>, payload := _}} -> + received_repub + after 500 -> + received_nothing + end + end, -t_sqlparse_event_3(_Config) -> - Sql = "select clientid, topic as tp " - "from \"t/tt\", \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})). + %% No client subscribed `any/topic`, triggered repub rule. + %% But havn't sub `repub/if/dropped`, so the repub message will also be dropped with recursively republish. + received_nothing = Fun_t2(), + + {ok, _, _} = emqtt:subscribe(Client, <<"repub/if/dropped">>, 0), + + %% this message will be dropped and then repub to `repub/to/t` + emqtt:publish(Client, <<"any/topic/2">>, <<"{\"x\":1,\"y\":144}">>, 0), + + %% received subscribed `repub/to/t` + received_repub = Fun_t2(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). %%------------------------------------------------------------------------------ %% Test cases for `foreach` From 3e8c070b59ecac07ecf5f0464bb73b73c1808979 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Oct 2022 12:35:11 +0800 Subject: [PATCH 24/44] Revert "fix: rm unused module" This reverts commit 267946c3794b8c251a056e9f4cb975dd04b1b14f. Revert it temporary, we need fix `scripts/update-appup.sh` later. It only compares the current release with its predecessor. --- apps/emqx_rule_engine/src/emqx_rule_date.erl | 248 +++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 apps/emqx_rule_engine/src/emqx_rule_date.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_date.erl b/apps/emqx_rule_engine/src/emqx_rule_date.erl new file mode 100644 index 000000000..fb9cad4c3 --- /dev/null +++ b/apps/emqx_rule_engine/src/emqx_rule_date.erl @@ -0,0 +1,248 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rule_date). + +-export([date/3, date/4, parse_date/4]). + +-export([ + is_int_char/1, + is_symbol_char/1, + is_m_char/1 +]). + +-record(result, { + %%year() + year = "1970" :: string(), + %%month() + month = "1" :: string(), + %%day() + day = "1" :: string(), + %%hour() + hour = "0" :: string(), + %%minute() %% epoch in millisecond precision + minute = "0" :: string(), + %%second() %% epoch in millisecond precision + second = "0" :: string(), + %%integer() %% zone maybe some value + zone = "+00:00" :: string() +}). + +%% -type time_unit() :: 'microsecond' +%% | 'millisecond' +%% | 'nanosecond' +%% | 'second'. +%% -type offset() :: [byte()] | (Time :: integer()). +date(TimeUnit, Offset, FormatString) -> + date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)). + +date(TimeUnit, Offset, FormatString, TimeEpoch) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + Res = lists:map( + fun(Expr) -> + eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr) + end, + R + ), + lists:concat(Res). + +parse_date(TimeUnit, Offset, FormatString, InputString) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + IsZ = fun(V) -> + case V of + {tag, $Z} -> true; + _ -> false + end + end, + R1 = lists:filter(IsZ, R), + IfFun = fun(Con, A, B) -> + case Con of + [] -> A; + _ -> B + end + end, + Res = parse_input(FormatString, InputString), + Str = + Res#result.year ++ "-" ++ + Res#result.month ++ "-" ++ + Res#result.day ++ "T" ++ + Res#result.hour ++ ":" ++ + Res#result.minute ++ ":" ++ + Res#result.second ++ + IfFun(R1, Offset, Res#result.zone), + calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]). + +mlist(R) -> + %% %H Shows hour in 24-hour format [15] + [ + {$H, R#result.hour}, + %% %M Displays minutes [00-59] + {$M, R#result.minute}, + %% %S Displays seconds [00-59] + {$S, R#result.second}, + %% %y Displays year YYYY [2021] + {$y, R#result.year}, + %% %m Displays the number of the month [01-12] + {$m, R#result.month}, + %% %d Displays the number of the month [01-12] + {$d, R#result.day}, + %% %Z Displays Time zone + {$Z, R#result.zone} + ]. + +rmap(Result) -> + maps:from_list(mlist(Result)). + +support_char() -> "HMSymdZ". + +create_tag(Head, []) -> + Head; +create_tag(Head, [Val1 | RVal]) -> + case Val1 of + [] -> + create_tag(Head ++ [{st, [$%]}], RVal); + [H | Other] -> + case lists:member(H, support_char()) of + true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal); + false -> create_tag(Head ++ [{st, [$% | Val1]}], RVal) + end + end. + +eval_tag(_, {st, Str}) -> + Str; +eval_tag(Map, {tag, Char}) -> + maps:get(Char, Map, "undefined"). + +%% make_time(TimeUnit, Offset) -> +%% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)). +make_time(TimeUnit, Offset, TimeEpoch) -> + Res = calendar:system_time_to_rfc3339(TimeEpoch, + [{unit, TimeUnit}, {offset, Offset}]), + [Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T, + H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = Res, + IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, + {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), + #result{ + year = [Y1, Y2, Y3, Y4] + , month = [Mon1, Mon2] + , day = [D1, D2] + , hour = [H1, H2] + , minute = [Min1, Min2] + , second = [S1, S2] ++ FractionStr + , zone = UtcOffset + }. + +is_int_char(C) -> + C >= $0 andalso C =< $9. +is_symbol_char(C) -> + C =:= $- orelse C =:= $+. +is_m_char(C) -> + C =:= $:. + +parse_char_with_fun(_, []) -> + error(null_input); +parse_char_with_fun(ValidFun, [C | Other]) -> + Res = + case erlang:is_function(ValidFun) of + true -> ValidFun(C); + false -> erlang:apply(emqx_rule_date, ValidFun, [C]) + end, + case Res of + true -> {C, Other}; + false -> error({unexpected, [C | Other]}) + end. +parse_string([], Input) -> + {[], Input}; +parse_string([C | Other], Input) -> + {C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input), + {Res, Input2} = parse_string(Other, Input1), + {[C1 | Res], Input2}. + +parse_times(0, _, Input) -> + {[], Input}; +parse_times(Times, Fun, Input) -> + {C1, Input1} = parse_char_with_fun(Fun, Input), + {Res, Input2} = parse_times((Times - 1), Fun, Input1), + {[C1 | Res], Input2}. + +parse_int_times(Times, Input) -> + parse_times(Times, is_int_char, Input). + +parse_fraction(Input) -> + IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, + lists:splitwith(IsFractionChar, Input). + +parse_second(Input) -> + {M, Input1} = parse_int_times(2, Input), + {M1, Input2} = parse_fraction(Input1), + {M ++ M1, Input2}. + +parse_zone(Input) -> + {S, Input1} = parse_char_with_fun(is_symbol_char, Input), + {M, Input2} = parse_int_times(2, Input1), + {C, Input3} = parse_char_with_fun(is_m_char, Input2), + {V, Input4} = parse_int_times(2, Input3), + {[S | M ++ [C | V]], Input4}. + +mlist1() -> + maps:from_list( + %% %H Shows hour in 24-hour format [15] + [ + {$H, fun(Input) -> parse_int_times(2, Input) end}, + %% %M Displays minutes [00-59] + {$M, fun(Input) -> parse_int_times(2, Input) end}, + %% %S Displays seconds [00-59] + {$S, fun(Input) -> parse_second(Input) end}, + %% %y Displays year YYYY [2021] + {$y, fun(Input) -> parse_int_times(4, Input) end}, + %% %m Displays the number of the month [01-12] + {$m, fun(Input) -> parse_int_times(2, Input) end}, + %% %d Displays the number of the month [01-12] + {$d, fun(Input) -> parse_int_times(2, Input) end}, + %% %Z Displays Time zone + {$Z, fun(Input) -> parse_zone(Input) end} + ] + ). + +update_result($H, Res, Str) -> Res#result{hour = Str}; +update_result($M, Res, Str) -> Res#result{minute = Str}; +update_result($S, Res, Str) -> Res#result{second = Str}; +update_result($y, Res, Str) -> Res#result{year = Str}; +update_result($m, Res, Str) -> Res#result{month = Str}; +update_result($d, Res, Str) -> Res#result{day = Str}; +update_result($Z, Res, Str) -> Res#result{zone = Str}. + +parse_tag(Res, {st, St}, InputString) -> + {_A, B} = parse_string(St, InputString), + {Res, B}; +parse_tag(Res, {tag, St}, InputString) -> + Fun = maps:get(St, mlist1()), + {A, B} = Fun(InputString), + NRes = update_result(St, Res, A), + {NRes, B}. + +parse_tags(Res, [], _) -> + Res; +parse_tags(Res, [Tag | Others], InputString) -> + {NRes, B} = parse_tag(Res, Tag, InputString), + parse_tags(NRes, Others, B). + +parse_input(FormatString, InputString) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + parse_tags(#result{}, R, InputString). From 7f92c29ada1e235b3b45a1af2e8022fbd0ae7130 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Sep 2022 10:03:20 +0800 Subject: [PATCH 25/44] chore: bump appup.src for `emqx_rule_funcs.erl` --- .../src/emqx_rule_engine.appup.src | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index b46be85c7..430d1f7e3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,20 +2,23 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.15", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -24,7 +27,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -33,7 +37,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -43,7 +48,8 @@ {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -206,20 +212,23 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.15", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -228,7 +237,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -237,7 +247,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -247,7 +258,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, From 73d72eaccea0500aabc6bdcb68e76137e9bb11ac Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 11 Oct 2022 12:48:35 +0200 Subject: [PATCH 26/44] fix: demote rate limit log level to notice --- CHANGES-4.3.md | 1 + apps/emqx_exproto/src/emqx_exproto_conn.erl | 2 +- apps/emqx_stomp/src/emqx_stomp_connection.erl | 2 +- src/emqx_connection.erl | 2 +- src/emqx_ws_connection.erl | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index f136b7835..23899cb25 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -15,6 +15,7 @@ File format: ### Bug fixes - Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic. [#9024](https://github.com/emqx/emqx/pull/9024) +- "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134) ## v4.3.21 diff --git a/apps/emqx_exproto/src/emqx_exproto_conn.erl b/apps/emqx_exproto/src/emqx_exproto_conn.erl index 7c6eb6cb3..51abe86e9 100644 --- a/apps/emqx_exproto/src/emqx_exproto_conn.erl +++ b/apps/emqx_exproto/src/emqx_exproto_conn.erl @@ -643,7 +643,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, diff --git a/apps/emqx_stomp/src/emqx_stomp_connection.erl b/apps/emqx_stomp/src/emqx_stomp_connection.erl index e58b242e5..7177155fd 100644 --- a/apps/emqx_stomp/src/emqx_stomp_connection.erl +++ b/apps/emqx_stomp/src/emqx_stomp_connection.erl @@ -469,7 +469,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 7c9cfef0a..3cfcd8cdc 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -756,7 +756,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index a0c999ea9..0a29c1ee2 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -499,7 +499,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), NState = State#state{sockstate = blocked, limiter = Limiter1, From 53bc2d9d584001a1cdd27bb8f5c6f862c1538130 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 12 Oct 2022 14:53:45 +0300 Subject: [PATCH 27/44] fix(jwt): restore legacy emqx_auth_jwt hook interface --- CHANGES-4.3.md | 1 + apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 8 ++++++-- apps/emqx_auth_jwt/src/emqx_auth_jwt.erl | 5 +++++ apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl | 16 ++++++++++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 23899cb25..2290147da 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -16,6 +16,7 @@ File format: - Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic. [#9024](https://github.com/emqx/emqx/pull/9024) - "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134) +- Restore legacy `emqx_auth_jwt` interface to keep hooks working after relup. [##9144](https://github.com/emqx/emqx/pull/9144) ## v4.3.21 diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index 6f07c4449..bea59cbbb 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.6",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + [{"4.3.6", + [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, @@ -13,7 +15,9 @@ {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [{"4.3.6",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + [ {"4.3.6", + [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl index 26fd34365..0b3f17b04 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl @@ -23,6 +23,7 @@ -logger_header("[JWT]"). -export([ check_auth/3 + , check/3 , check_acl/5 , description/0 ]). @@ -33,6 +34,10 @@ %% Authentication callbacks %%-------------------------------------------------------------------- +%% for compatibility with old versions +check(ClientInfo, AuthResult, State) -> + ?MODULE:check_auth(ClientInfo, AuthResult, State). + check_auth(ClientInfo, AuthResult, #{from := From, checklists := Checklists}) -> case maps:find(From, ClientInfo) of error -> diff --git a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl index a0be3768f..62f753904 100644 --- a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl @@ -472,3 +472,19 @@ t_check_jwt_acl_no_exp(_Config) -> emqtt:subscribe(C, <<"a/b">>, 0)), ok = emqtt:disconnect(C). + +t_check_compatibility(init, _Config) -> ok. +t_check_compatibility(_Config) -> + + %% We literary want emqx_auth_jwt:check call emqx_auth_jwt:check_auth, so check with meck + + ok = meck:new(emqx_auth_jwt, [passthrough, no_history]), + ok = meck:expect(emqx_auth_jwt, check_auth, fun(a, b, c) -> ok end), + + ?assertEqual( + ok, + emqx_auth_jwt:check(a, b, c) + ), + + meck:validate(emqx_auth_jwt), + meck:unload(emqx_auth_jwt). From 458b9b2e4bf1458ebbacf7848ce3afd017938377 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 15 Oct 2022 14:59:48 +0200 Subject: [PATCH 28/44] chore: prepare new release version 4.3.22-alpha.1 --- deploy/charts/emqx/Chart.yaml | 4 ++-- include/emqx_release.hrl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 4569834df..28cafe9b3 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -13,8 +13,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 4.3.21 +version: 4.3.22 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 4.3.21 +appVersion: 4.3.22 diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 43c74b248..e7d50d7fc 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.21"}). +-define(EMQX_RELEASE, {opensource, "4.3.22-alpha.1"}). -else. From 20e0d140265af9b3b13b4daccceb9804ffb9e413 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 15 Oct 2022 15:01:57 +0200 Subject: [PATCH 29/44] chore: bump app versions --- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- src/emqx.app.src | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 05d4d3754..e0550fa78 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.3.7"}, % strict semver, bump manually! + {vsn, "4.3.8"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 40be93b33..93f2c6fb8 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.12"}, %% 4.3.3 is used by ee + {vsn, "4.3.13"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index dffcc7024..7989c9542 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.16"}, % strict semver, bump manually! + {vsn, "4.3.17"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index e35e570d3..13afb9e67 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_stomp, [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.3.6"}, % strict semver, bump manually! + {vsn, "4.3.7"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_stomp_sup]}, {applications, [kernel,stdlib]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 092e5d3b6..07c67545b 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.17"}, % strict semver, bump manually! + {vsn, "4.3.18"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/src/emqx.app.src b/src/emqx.app.src index 433c75326..99715d1a2 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.22"}, % strict semver, bump manually! + {vsn, "4.3.23"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel From 11026b7644aa98eb3a996696bb22d40df25b61ae Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 16 Oct 2022 08:43:45 +0200 Subject: [PATCH 30/44] chore: re-generate appup for v4.3.22 --- apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 6 ++++-- apps/emqx_exproto/src/emqx_exproto.appup.src | 6 ++++-- .../src/emqx_rule_engine.appup.src | 12 ++++++++++-- apps/emqx_stomp/src/emqx_stomp.appup.src | 6 ++++-- src/emqx.appup.src | 14 ++++++++++++-- 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index bea59cbbb..7ee2349c3 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.6", + [{"4.3.7",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {"4.3.6", [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.3.5", @@ -15,7 +16,8 @@ {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [ {"4.3.6", + [{"4.3.7",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {"4.3.6", [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.3.5", diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index f31e4a969..e97296a68 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + [{"4.3.12",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + {"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, @@ -19,7 +20,8 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + [{"4.3.12",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + {"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index ab85f908e..94a79e2db 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,11 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.15", + [{"4.3.16", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, @@ -218,7 +222,11 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.15", + [{"4.3.16", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.appup.src b/apps/emqx_stomp/src/emqx_stomp.appup.src index 69e571934..499b80f38 100644 --- a/apps/emqx_stomp/src/emqx_stomp.appup.src +++ b/apps/emqx_stomp/src/emqx_stomp.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.5", + [{"4.3.6",[{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + {"4.3.5", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.4", @@ -23,7 +24,8 @@ [{restart_application,emqx_stomp}, {apply,{emqx_stomp,force_clear_after_app_stoped,[]}}]}, {<<".*">>,[]}], - [{"4.3.5", + [{"4.3.6",[{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + {"4.3.5", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.4", diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 51576061c..e8ee98afb 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.21", + [{"4.3.22", + [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {"4.3.21", [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {add_module,emqx_secret}, @@ -862,7 +867,12 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.21", + [{"4.3.22", + [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {"4.3.21", [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, From 5bc822cc78b83ba35fd40715be32ec57d6f08d0b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 17 Oct 2022 12:55:22 +0200 Subject: [PATCH 31/44] chore: delete duplicated module from appup --- .../src/emqx_rule_engine.appup.src | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 94a79e2db..ed783047d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -8,7 +8,6 @@ {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, @@ -16,7 +15,6 @@ {"4.3.14", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -25,7 +23,6 @@ {"4.3.13", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -36,7 +33,6 @@ {"4.3.12", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -47,7 +43,6 @@ {"4.3.11", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -59,7 +54,6 @@ {"4.3.10", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -70,7 +64,6 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -229,7 +222,6 @@ {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, @@ -237,7 +229,6 @@ {"4.3.14", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -246,7 +237,6 @@ {"4.3.13", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -257,7 +247,6 @@ {"4.3.12", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -268,7 +257,6 @@ {"4.3.11", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -280,7 +268,6 @@ {"4.3.10", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -291,7 +278,6 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, From f15845fb002cc527a229daedcdd9d1d2733946d3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 17 Oct 2022 15:04:33 +0200 Subject: [PATCH 32/44] chore: add code-owners --- .github/CODEOWNERS | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..0bed63887 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,43 @@ +## MQTT & Core +src/ @qzhuyan +include/ @qzhuyan +etc/ @qzhuyan +test/ @qzhuyan + +## CI +.github/ @id +.ci/ @id +scripts/ @id +build @id +deploy/ @id + +## Authenticatio & ACL +apps/emqx_auth_*/ @savonarola +apps/emqx_psk_file/ @savonarola +apps/emqx_retainer/ @savonarola +apps/emqx_sasl/ @savonarola + +## Gateway +apps/emqx_coap/ @HJianBo +apps/emqx_exhook/ @HJianBo +apps/emqx_exproto/ @HJianBo +apps/emqx_lua_hook/ @HJianBo +apps/emqx_lwm2m/ @HJianBo + +## OPs +apps/emqx_management/ @zhongwencool +apps/emqx_recon/ @zhongwencool +apps/emqx_plugin_libs/ @zhongwencool +apps/emqx_prometheus/ @zhongwencool +apps/emqx_recon/ @zhongwencool + + +## Data integration +apps/emqx_rule_engine/ @thalesmg +apps/emqx_web_hook/ @thalesmg + +## External Plugins +lib-extra/ @zmstone + +## Default +* @zmstone From a2869327430cf53ecbc8e7c7bc6616b427f4b1c1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 17 Oct 2022 13:54:07 +0200 Subject: [PATCH 33/44] docs: split change logs We will start collecting change logs and release notes using scripts. --- CHANGES-4.3.md | 15 ++------------- changes/v4.3.22-en.md | 12 ++++++++++++ changes/v4.3.22-zh.md | 12 ++++++++++++ 3 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 changes/v4.3.22-en.md create mode 100644 changes/v4.3.22-zh.md diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 84be29a1f..c812713db 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,18 +10,7 @@ File format: - One list item per change topic Change log ends with a list of GitHub PRs -## v4.3.22 - -### Enhancements - -- Add a warning log if the ACL check failed for subscription. [#9124](https://github.com/emqx/emqx/pull/9124) - This is to make the ACL deny logging for subscription behave the same as for publish. - -### Bug fixes - -- Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic. [#9024](https://github.com/emqx/emqx/pull/9024) -- "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134) -- Restore legacy `emqx_auth_jwt` interface to keep hooks working after relup. [##9144](https://github.com/emqx/emqx/pull/9144) +## For 4.3.22 and later versions, please find details in `changes` dir ## v4.3.21 @@ -63,7 +52,7 @@ File format: - Fix HTTP client library to handle SSL socket passive signal. [#9145](https://github.com/emqx/emqx/pull/9145) - Hide redis password in error logs [#9071](https://github.com/emqx/emqx/pull/9071) - In this change, it also included more changes in redis client: + More changes in redis client included in this release: - Improve redis connection error logging [eredis #19](https://github.com/emqx/eredis/pull/19). Also added support for eredis to accept an anonymous function as password instead of passing around plaintext args which may get dumpped to crash logs (hard to predict where). diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md new file mode 100644 index 000000000..3cb84e5f4 --- /dev/null +++ b/changes/v4.3.22-en.md @@ -0,0 +1,12 @@ +### Enhancements + +- Add a warning log if the ACL check failed for subscription [#9124](https://github.com/emqx/emqx/pull/9124). + This is to make the ACL deny logging for subscription behave the same as for publish. + +### Bug fixes + +- Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic [#9024](https://github.com/emqx/emqx/pull/9024). + +- "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134). + +- Restore old `emqx_auth_jwt` module API, so the hook callback functions registered in older version will not be invalidated after hot-upgrade [#9144](https://github.com/emqx/emqx/pull/9144). diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md new file mode 100644 index 000000000..e8f534780 --- /dev/null +++ b/changes/v4.3.22-zh.md @@ -0,0 +1,12 @@ +### 增强 + +- 订阅时，如果 ACL 检查不通过，打印一个警告日志 [#9124](https://github.com/emqx/emqx/pull/9124)。 + 该行为的改变主要是为了跟发布失败时的行为保持一致。 + +### 修复 + +- 修复 emqx-sn 插件在“空闲”状态下收到消息发布请求时可能崩溃的情况 [#9024](https://github.com/emqx/emqx/pull/9024)。 + +- 限速 “Pause due to rate limit” 的日志级别从原先的 `warning` 降级到 `notice` [#9134](https://github.com/emqx/emqx/pull/9134)。 + +- 保留老的 `emqx_auth_jwt` 模块的接口函数，保障热升级之前添加的回调函数在热升级之后也不会失效 [#9144](https://github.com/emqx/emqx/pull/9144)。 From b0ee2fdd3cacd1ba1a838569373a9d6ff09c469b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 18 Oct 2022 17:58:55 +0800 Subject: [PATCH 34/44] chore: workflow steps with path --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index c64af4f93..322f1dbf4 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -31,9 +31,9 @@ jobs: path: source fetch-depth: 0 - id: detect-profiles - working-directory: source uses: ./.github/actions/detect-profiles with: + path: source ci_git_token: ${{ secrets.CI_GIT_TOKEN }} - name: get_all_deps if: endsWith(github.repository, 'emqx') From e0e8a5a7073360f9137739e9e9eba626fe5b3722 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 20 Oct 2022 10:37:32 +0800 Subject: [PATCH 35/44] chore: re-generate appup for v4.3.22 --- src/emqx.appup.src | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index e8ee98afb..2629f8771 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.22", - [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, @@ -868,7 +869,8 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.22", - [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, From 524ba512d8f359dcd3a0fc167d3f37220f92f5ca Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 20 Oct 2022 11:25:22 +0800 Subject: [PATCH 36/44] chore: update mongodb driver to v3.0.14 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 68a37ef10..2bdc3d952 100644 --- a/rebar.config +++ b/rebar.config @@ -61,7 +61,7 @@ , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.1"}}} , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.1"}}} - , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}} + , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.14"}}} , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} ]}. From bfe238fa1c9f78dff182dd522ac9a73e2b25db14 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 20 Oct 2022 14:20:30 +0800 Subject: [PATCH 37/44] fix: set precision of counter 'max_speed' to 2 digits --- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 2 ++ apps/emqx_rule_engine/src/emqx_rule_metrics.erl | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index ed783047d..cbd15a5eb 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -4,6 +4,7 @@ [{"4.3.16", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, @@ -218,6 +219,7 @@ [{"4.3.16", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index e4c539164..414fc15e0 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -500,11 +500,10 @@ calculate_speed(CurrVal, #rule_speed{max = MaxSpeed0, last_v = LastVal, last5m_smpl = Last5MinSamples, tick = Tick + 1}. format_rule_speed(#rule_speed{max = Max, current = Current, last5m = Last5Min}) -> - #{max => Max, current => precision(Current, 2), last5m => precision(Last5Min, 2)}. + #{max => round2(Max), current => round2(Current), last5m => round2(Last5Min)}. -precision(Float, N) -> - Base = math:pow(10, N), - round(Float * Base) / Base. +round2(Float) -> + round(Float * 100) / 100. %%------------------------------------------------------------------------------ %% Metrics Definitions From f8686ffc629a72ecba4c8bea0dd78d8b62df0a31 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 20 Oct 2022 16:48:48 +0800 Subject: [PATCH 38/44] chore: update the change logs --- changes/v4.3.22-en.md | 5 +++++ changes/v4.3.22-zh.md | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 3cb84e5f4..77a197624 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -5,6 +5,11 @@ ### Bug fixes +- Improve the display of rule's 'Maximum Speed' counter to only reserve 2 decimal places. [#9185](https://github.com/emqx/emqx/pull/9185) + This is to avoid displaying floats like `0.30000000000000004` on the dashboard. + +- Fix the issue that emqx prints too many error logs when connecting to mongodb but auth failed. [#9184](https://github.com/emqx/emqx/pull/9184) + - Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic [#9024](https://github.com/emqx/emqx/pull/9024). - "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134). diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index e8f534780..58c82588d 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -5,6 +5,11 @@ ### 修复 +- 改进规则的 "最大执行速度" 的计数，只保留小数点之后 2 位 [#9185](https://github.com/emqx/emqx/pull/9185) + 避免在 dashboard 上展示类似这样的浮点数：`0.30000000000000004`。 + +- 修复在尝试连接 MongoDB 数据库过程中，如果认证失败会不停打印错误日志的问题。[#9184](https://github.com/emqx/emqx/pull/9184) + - 修复 emqx-sn 插件在“空闲”状态下收到消息发布请求时可能崩溃的情况 [#9024](https://github.com/emqx/emqx/pull/9024)。 - 限速 “Pause due to rate limit” 的日志级别从原先的 `warning` 降级到 `notice` [#9134](https://github.com/emqx/emqx/pull/9134)。 From 3b59f1440b01605790183b0859c91f5304609fbc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 22 Oct 2022 15:14:58 +0200 Subject: [PATCH 39/44] chore: bump lc from 0.3.1 to 0.3.2 there is no beam change in 0.3.2 (comparing to 0.3.1) --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 2bdc3d952..86436101b 100644 --- a/rebar.config +++ b/rebar.config @@ -60,7 +60,7 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.1"}}} - , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.1"}}} + , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.14"}}} , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} From 075228aadc519ff710c91f96d7766fe6b0aefe78 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 19 Oct 2022 10:20:31 +0800 Subject: [PATCH 40/44] chore: fix local actions path --- .github/workflows/build_packages.yaml | 3 +-- .github/workflows/release.yaml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 322f1dbf4..b46eb9973 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -31,9 +31,8 @@ jobs: path: source fetch-depth: 0 - id: detect-profiles - uses: ./.github/actions/detect-profiles + uses: ./source/.github/actions/detect-profiles with: - path: source ci_git_token: ${{ secrets.CI_GIT_TOKEN }} - name: get_all_deps if: endsWith(github.repository, 'emqx') diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 71e5fe48a..5b438ffa0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,8 +18,7 @@ jobs: path: source fetch-depth: 0 - id: detect-profiles - working-directory: source - uses: ./.github/actions/detect-profiles + uses: ./source/.github/actions/detect-profiles with: ci_git_token: ${{ secrets.CI_GIT_TOKEN }} From 34d73960e729a3217c5ea60d73ee0857793b0954 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 19 Oct 2022 10:42:09 +0800 Subject: [PATCH 41/44] ci: fix macos erlang-otp cache hitting --- .github/actions/package-macos/action.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 3efe0ef81..1a7e4fb9c 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -35,6 +35,8 @@ runs: with: path: ~/.kerl/${{ inputs.otp }} key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit + restore-keys: | + otp-install-${{ inputs.otp }}-${{ inputs.os }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' shell: bash @@ -93,4 +95,5 @@ runs: exit 1 fi cd .. + # test with a spaces in path rm -rf "emqx home" From 78fc9ff64ed193575340d0c3064e834ec943412e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 24 Oct 2022 11:28:39 +0200 Subject: [PATCH 42/44] chore: add leading slash for paths in codeowners --- .github/CODEOWNERS | 52 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0bed63887..e2631f3ac 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,43 +1,43 @@ ## MQTT & Core -src/ @qzhuyan -include/ @qzhuyan -etc/ @qzhuyan -test/ @qzhuyan +/src/ @qzhuyan +/include/ @qzhuyan +/etc/ @qzhuyan +/test/ @qzhuyan ## CI -.github/ @id -.ci/ @id -scripts/ @id -build @id -deploy/ @id +/.github/ @id +/.ci/ @id +/scripts/ @id +/build @id +/deploy/ @id ## Authenticatio & ACL -apps/emqx_auth_*/ @savonarola -apps/emqx_psk_file/ @savonarola -apps/emqx_retainer/ @savonarola -apps/emqx_sasl/ @savonarola +/apps/emqx_auth_*/ @savonarola +/apps/emqx_psk_file/ @savonarola +/apps/emqx_retainer/ @savonarola +/apps/emqx_sasl/ @savonarola ## Gateway -apps/emqx_coap/ @HJianBo -apps/emqx_exhook/ @HJianBo -apps/emqx_exproto/ @HJianBo -apps/emqx_lua_hook/ @HJianBo -apps/emqx_lwm2m/ @HJianBo +/apps/emqx_coap/ @HJianBo +/apps/emqx_exhook/ @HJianBo +/apps/emqx_exproto/ @HJianBo +/apps/emqx_lua_hook/ @HJianBo +/apps/emqx_lwm2m/ @HJianBo ## OPs -apps/emqx_management/ @zhongwencool -apps/emqx_recon/ @zhongwencool -apps/emqx_plugin_libs/ @zhongwencool -apps/emqx_prometheus/ @zhongwencool -apps/emqx_recon/ @zhongwencool +/apps/emqx_management/ @zhongwencool +/apps/emqx_recon/ @zhongwencool +/apps/emqx_plugin_libs/ @zhongwencool +/apps/emqx_prometheus/ @zhongwencool +/apps/emqx_recon/ @zhongwencool ## Data integration -apps/emqx_rule_engine/ @thalesmg -apps/emqx_web_hook/ @thalesmg +/apps/emqx_rule_engine/ @thalesmg +/apps/emqx_web_hook/ @thalesmg ## External Plugins -lib-extra/ @zmstone +/lib-extra/ @zmstone ## Default * @zmstone From c0b5b887c35c1d324a98e877423245f21cdae956 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 21 Oct 2022 14:58:04 -0300 Subject: [PATCH 43/44] fix(mgmt_api): return 503 when emqx is not running in `/status` --- .../src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt_http.erl | 6 +- .../test/emqx_mgmt_api_SUITE.erl | 69 +++++++++++++++++++ .../test/emqx_mgmt_api_test_helpers.erl | 12 +++- changes/v4.3.22-en.md | 2 + changes/v4.3.22-zh.md | 2 + 6 files changed, 89 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 1003a1140..5ab803c51 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.18"}, % strict semver, bump manually! + {vsn, "4.3.19"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index fae1bd86a..8cddc11b2 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -114,7 +114,11 @@ handle_request(<<"GET">>, <<"/status">>, Req) -> end, Status = io_lib:format("Node ~s is ~s~nemqx is ~s", [node(), InternalStatus, AppStatus]), - cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, Status, Req); + StatusCode = case AppStatus of + running -> 200; + not_running -> 503 + end, + cowboy_req:reply(StatusCode, #{<<"content-type">> => <<"text/plain">>}, Status, Req); handle_request(_Method, _Path, Req) -> cowboy_req:reply(400, #{<<"content-type">> => <<"text/plain">>}, <<"Not found.">>, Req). diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index fda5c2adf..6b15bcfff 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -25,6 +25,8 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_management/include/emqx_mgmt.hrl"). +-define(HOST, "http://127.0.0.1:8081/"). + -import(emqx_mgmt_api_test_helpers, [request_api/3, request_api/4, @@ -44,9 +46,31 @@ end_per_suite(Config) -> emqx_ct_helpers:stop_apps([emqx_management]), Config. +init_per_testcase(t_status_ok, Config) -> + ok = emqx_rule_registry:mnesia(boot), + ok = emqx_dashboard_admin:mnesia(boot), + application:ensure_all_started(emqx_rule_engine), + application:ensure_all_started(emqx_dashboard), + Config; +init_per_testcase(t_status_not_ok, Config) -> + ok = emqx_rule_registry:mnesia(boot), + ok = emqx_dashboard_admin:mnesia(boot), + application:ensure_all_started(emqx_rule_engine), + application:ensure_all_started(emqx_dashboard), + application:stop(emqx), + Config; init_per_testcase(_, Config) -> Config. +end_per_testcase(t_status_ok, _Config) -> + application:stop(emqx_rule_engine), + application:stop(emqx_dashboard), + ok; +end_per_testcase(t_status_not_ok, _Config) -> + application:stop(emqx_rule_engine), + application:stop(emqx_dashboard), + application:ensure_all_started(emqx), + ok; end_per_testcase(_, Config) -> Config. @@ -656,6 +680,51 @@ t_data_import_content(_) -> application:stop(emqx_rule_engine), application:stop(emqx_dashboard). +t_status_ok(_Config) -> + {ok, #{ body := Resp + , status_code := StatusCode + }} = do_request(#{method => get, path => ["status"], headers => [], + body => no_body}), + ?assertMatch( + {match, _}, + re:run(Resp, <<"emqx is running$">>)), + ?assertEqual(200, StatusCode), + ok. + +t_status_not_ok(_Config) -> + {ok, #{ body := Resp + , status_code := StatusCode + }} = do_request(#{method => get, path => ["status"], headers => [], + body => no_body}), + ?assertMatch( + {match, _}, + re:run(Resp, <<"emqx is not_running$">>)), + ?assertEqual(503, StatusCode), + ok. + +do_request(Opts) -> + #{ path := Path + , method := Method + , headers := Headers + , body := Body0 + } = Opts, + URL = ?HOST ++ filename:join(Path), + Request = case Body0 of + no_body -> {URL, Headers}; + {Encoding, Body} -> {URL, Headers, Encoding, Body} + end, + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{_, StatusCode, _}, Headers1, Body1}} -> + Body2 = case emqx_json:safe_decode(Body1, [return_maps]) of + {ok, Json} -> Json; + {error, _} -> Body1 + end, + {ok, #{status_code => StatusCode, headers => Headers1, body => Body2}} + end. + filter(List, Key, Value) -> lists:filter(fun(Item) -> maps:get(Key, Item) == Value diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl b/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl index a943ca760..952d71b9b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl @@ -36,13 +36,21 @@ request_api(Method, Url, QueryParams, Auth, []) -> "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth]}); + Headers = case Auth of + no_auth -> []; + Header -> [Header] + end, + do_request_api(Method, {NewUrl, Headers}); request_api(Method, Url, QueryParams, Auth, Body) -> NewUrl = case QueryParams of "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + Headers = case Auth of + no_auth -> []; + Header -> [Header] + end, + do_request_api(Method, {NewUrl, Headers, "application/json", emqx_json:encode(Body)}). do_request_api(Method, Request)-> ct:pal("Method: ~p, Request: ~p", [Method, Request]), diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 77a197624..7851139e2 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -15,3 +15,5 @@ - "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134). - Restore old `emqx_auth_jwt` module API, so the hook callback functions registered in older version will not be invalidated after hot-upgrade [#9144](https://github.com/emqx/emqx/pull/9144). + +- Fixed the response status code for the `/status` endpoint. Before the fix, it always returned 200 even if the EMQX application was not running. Now it returns 503 in that case. diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 58c82588d..6865a8614 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -15,3 +15,5 @@ - 限速 “Pause due to rate limit” 的日志级别从原先的 `warning` 降级到 `notice` [#9134](https://github.com/emqx/emqx/pull/9134)。 - 保留老的 `emqx_auth_jwt` 模块的接口函数，保障热升级之前添加的回调函数在热升级之后也不会失效 [#9144](https://github.com/emqx/emqx/pull/9144)。 + +- 修正了`/status`端点的响应状态代码。 在修复之前，它总是返回200，即使EMQX应用程序没有运行。 现在它在这种情况下返回503。 From 51f2414eaa70e1f631d9a72e92620946899a9fd8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 24 Oct 2022 15:24:49 -0300 Subject: [PATCH 44/44] docs: improve changelog Co-authored-by: Zaiming (Stone) Shi --- changes/v4.3.22-en.md | 3 ++- changes/v4.3.22-zh.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 7851139e2..db76d3423 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -16,4 +16,5 @@ - Restore old `emqx_auth_jwt` module API, so the hook callback functions registered in older version will not be invalidated after hot-upgrade [#9144](https://github.com/emqx/emqx/pull/9144). -- Fixed the response status code for the `/status` endpoint. Before the fix, it always returned 200 even if the EMQX application was not running. Now it returns 503 in that case. +- Fixed the response status code for the `/status` endpoint [#9210](https://github.com/emqx/emqx/pull/9210). + Before the fix, it always returned `200` even if the EMQX application was not running. Now it returns `503` in that case. diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 6865a8614..e7d343db0 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -16,4 +16,5 @@ - 保留老的 `emqx_auth_jwt` 模块的接口函数，保障热升级之前添加的回调函数在热升级之后也不会失效 [#9144](https://github.com/emqx/emqx/pull/9144)。 -- 修正了`/status`端点的响应状态代码。 在修复之前，它总是返回200，即使EMQX应用程序没有运行。 现在它在这种情况下返回503。 +- 修正了 `/status` API 的响应状态代码 [#9210](https://github.com/emqx/emqx/pull/9210)。 + 在修复之前，它总是返回 `200`，即使 EMQX 应用程序没有运行。 现在它在这种情况下返回 `503`。