From 502f3e8d5c4f378a4902e72e1dafba69d6fad2ea Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 22 Sep 2022 13:33:41 +0800 Subject: [PATCH 01/50] test(mqtt_sn): improve test coverage to 90% --- apps/emqx_sn/test/emqx_sn_frame_SUITE.erl | 76 +++++++++++++- apps/emqx_sn/test/emqx_sn_misc_SUITE.erl | 62 ++++++++++++ apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 101 +++++++++++++++++++ apps/emqx_sn/test/emqx_sn_registry_SUITE.erl | 5 + 4 files changed, 241 insertions(+), 3 deletions(-) create mode 100644 apps/emqx_sn/test/emqx_sn_misc_SUITE.erl diff --git a/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl b/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl index d4500315c..813a1a343 100644 --- a/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("emqx_sn/include/emqx_sn.hrl"). -include_lib("eunit/include/eunit.hrl"). +-define(SHOW(X), ??X). -import(emqx_sn_frame, [ parse/1 , serialize/1 @@ -67,6 +68,14 @@ t_willtopic(_) -> Wt = #mqtt_sn_message{type = ?SN_WILLTOPIC, variable = {Flags, <<"WillTopic">>}}, ?assertEqual({ok, Wt}, parse(serialize(Wt))). +t_undefined_willtopic(_) -> + Wt = #mqtt_sn_message{type = ?SN_WILLTOPIC}, + ?assertEqual({ok, Wt}, parse(serialize(Wt))). + +t_willtopic_resp(_) -> + Wt = #mqtt_sn_message{type = ?SN_WILLTOPICRESP, variable = 0}, + ?assertEqual({ok, Wt}, parse(serialize(Wt))). + t_willmsgreq(_) -> WmReq = #mqtt_sn_message{type = ?SN_WILLMSGREQ}, ?assertEqual({ok, WmReq}, parse(serialize(WmReq))). @@ -88,6 +97,12 @@ t_publish(_) -> PubMsg = #mqtt_sn_message{type = ?SN_PUBLISH, variable = {Flags, 1, 2, <<"Payload">>}}, ?assertEqual({ok, PubMsg}, parse(serialize(PubMsg))). +t_publish_long_msg(_) -> + Flags = #mqtt_sn_flags{dup = false, qos = 1, retain = false, topic_id_type = 2#01}, + Payload = generate_random_binary(256 + rand:uniform(256)), + PubMsg = #mqtt_sn_message{type = ?SN_PUBLISH, variable = {Flags, 1, 2, Payload}}, + ?assertEqual({ok, PubMsg}, parse(serialize(PubMsg))). + t_puback(_) -> PubAck = #mqtt_sn_message{type = ?SN_PUBACK, variable = {1, 2, 0}}, ?assertEqual({ok, PubAck}, parse(serialize(PubAck))). @@ -105,9 +120,21 @@ t_pubcomp(_) -> ?assertEqual({ok, PubComp}, parse(serialize(PubComp))). t_subscribe(_) -> - Flags = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = 16#01}, + Flags = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = ?SN_PREDEFINED_TOPIC}, SubMsg = #mqtt_sn_message{type = ?SN_SUBSCRIBE, variable = {Flags, 16#4321, 16}}, - ?assertEqual({ok, SubMsg}, parse(serialize(SubMsg))). + ?assertEqual({ok, SubMsg}, parse(serialize(SubMsg))), + + Flags1 = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = ?SN_NORMAL_TOPIC}, + SubMsg1 = #mqtt_sn_message{type = ?SN_SUBSCRIBE, variable = {Flags1, 16#4321, <<"t/+">>}}, + ?assertEqual({ok, SubMsg1}, parse(serialize(SubMsg1))), + + Flags2 = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = ?SN_SHORT_TOPIC}, + SubMsg2 = #mqtt_sn_message{type = ?SN_SUBSCRIBE, variable = {Flags2, 16#4321, <<"t/+">>}}, + ?assertEqual({ok, SubMsg2}, parse(serialize(SubMsg2))), + + Flags3 = #mqtt_sn_flags{dup = false, qos = 1, topic_id_type = ?SN_RESERVED_TOPIC}, + SubMsg3 = #mqtt_sn_message{type = ?SN_SUBSCRIBE, variable = {Flags3, 16#4321, <<"t/+">>}}, + ?assertEqual({ok, SubMsg3}, parse(serialize(SubMsg3))). t_suback(_) -> Flags = #mqtt_sn_flags{qos = 1}, @@ -137,6 +164,10 @@ t_disconnect(_) -> Disconn = #mqtt_sn_message{type = ?SN_DISCONNECT}, ?assertEqual({ok, Disconn}, parse(serialize(Disconn))). +t_disconnect_duration(_) -> + Disconn = #mqtt_sn_message{type = ?SN_DISCONNECT, variable = 120}, + ?assertEqual({ok, Disconn}, parse(serialize(Disconn))). + t_willtopicupd(_) -> Flags = #mqtt_sn_flags{qos = 1, retain = true}, WtUpd = #mqtt_sn_message{type = ?SN_WILLTOPICUPD, variable = {Flags, <<"Topic">>}}, @@ -150,6 +181,43 @@ t_willmsgresp(_) -> UpdResp = #mqtt_sn_message{type = ?SN_WILLMSGRESP, variable = 0}, ?assertEqual({ok, UpdResp}, parse(serialize(UpdResp))). +t_invalid_inpacket(_) -> + Bin = <<2:8/big-integer, 16#F0:8/big-integer>>, + ?assertMatch({'EXIT', {unkown_message_type, _Stack}}, catch parse(Bin)). + +t_message_type(_) -> + TypeNames = [ {?SN_ADVERTISE, ?SHOW(SN_ADVERTISE)} + , {?SN_SEARCHGW, ?SHOW(SN_SEARCHGW)} + , {?SN_GWINFO, ?SHOW(SN_GWINFO)} + , {?SN_CONNECT, ?SHOW(SN_CONNECT)} + , {?SN_CONNACK, ?SHOW(SN_CONNACK)} + , {?SN_WILLTOPICREQ, ?SHOW(SN_WILLTOPICREQ)} + , {?SN_WILLTOPIC, ?SHOW(SN_WILLTOPIC)} + , {?SN_WILLMSGREQ, ?SHOW(SN_WILLMSGREQ)} + , {?SN_WILLMSG, ?SHOW(SN_WILLMSG)} + , {?SN_REGISTER, ?SHOW(SN_REGISTER)} + , {?SN_REGACK, ?SHOW(SN_REGACK)} + , {?SN_PUBLISH, ?SHOW(SN_PUBLISH)} + , {?SN_PUBACK, ?SHOW(SN_PUBACK)} + , {?SN_PUBCOMP, ?SHOW(SN_PUBCOMP)} + , {?SN_PUBREC, ?SHOW(SN_PUBREC)} + , {?SN_PUBREL, ?SHOW(SN_PUBREL)} + , {?SN_SUBSCRIBE, ?SHOW(SN_SUBSCRIBE)} + , {?SN_SUBACK, ?SHOW(SN_SUBACK)} + , {?SN_UNSUBSCRIBE, ?SHOW(SN_UNSUBSCRIBE)} + , {?SN_UNSUBACK, ?SHOW(SN_UNSUBACK)} + , {?SN_PINGREQ, ?SHOW(SN_PINGREQ)} + , {?SN_PINGRESP, ?SHOW(SN_PINGRESP)} + , {?SN_DISCONNECT, ?SHOW(SN_DISCONNECT)} + , {?SN_WILLTOPICUPD, ?SHOW(SN_WILLTOPICUPD)} + , {?SN_WILLTOPICRESP, ?SHOW(SN_WILLTOPICRESP)} + , {?SN_WILLMSGUPD, ?SHOW(SN_WILLMSGUPD)} + , {?SN_WILLMSGRESP, ?SHOW(SN_WILLMSGRESP)} + ], + {Types, Names} = lists:unzip(TypeNames), + ?assertEqual(Names, [emqx_sn_frame:message_type(Type) || Type <- Types]), + ok. + t_random_test(_) -> random_test_body(), random_test_body(), @@ -171,6 +239,9 @@ random_test_body() -> generate_random_binary() -> % The min packet length is 2 Len = rand:uniform(299) + 1, + generate_random_binary(Len). + +generate_random_binary(Len) -> gen_next(Len, <<>>). gen_next(0, Acc) -> @@ -178,4 +249,3 @@ gen_next(0, Acc) -> gen_next(N, Acc) -> Byte = rand:uniform(256) - 1, gen_next(N-1, <>). - diff --git a/apps/emqx_sn/test/emqx_sn_misc_SUITE.erl b/apps/emqx_sn/test/emqx_sn_misc_SUITE.erl new file mode 100644 index 000000000..127cb22e3 --- /dev/null +++ b/apps/emqx_sn/test/emqx_sn_misc_SUITE.erl @@ -0,0 +1,62 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_sn_misc_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_sn]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_sn]). + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- +t_sn_app(_) -> + ?assertMatch({'EXIT', {_, _}}, catch emqx_sn_app:start_listeners()), + ?assertMatch({error, _}, emqx_sn_app:stop_listener({udp, 9999, []})), + ?assertMatch({error, _}, emqx_sn_app:stop_listener({udp, {{0,0,0,0}, 9999}, []})), + ok. + +t_sn_broadcast(_) -> + ?assertEqual(ignored, gen_server:call(emqx_sn_broadcast, ignored)), + ?assertEqual(ok, gen_server:cast(emqx_sn_broadcast, ignored)), + ?assertEqual(ignored, erlang:send(emqx_sn_broadcast, ignored)), + ?assertEqual(broadcast_advertise, erlang:send(emqx_sn_broadcast, broadcast_advertise)), + ?assertEqual(ok, emqx_sn_broadcast:stop()). + +%%-------------------------------------------------------------------- +%% Helper funcs +%%-------------------------------------------------------------------- diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index 4912b4fcf..b064b630e 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -100,6 +100,17 @@ t_connect(_) -> send_connect_msg(Socket, <<"client_id_test1">>), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + %% unexpected advertise + Adv = ?SN_ADVERTISE_MSG(1, 100), + AdvPacket = emqx_sn_frame:serialize(Adv), + send_packet(Socket, AdvPacket), + timer:sleep(200), + + %% unexpected connect + ClientId = ?CLIENTID, + send_connect_msg(Socket, ClientId), + timer:sleep(200), + send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). @@ -412,6 +423,16 @@ t_publish_negqos_case09(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). +t_publish_negqos_case10(_) -> + QoS = ?QOS_NEG1, + MsgId = 1, + TopicId1 = ?PREDEF_TOPIC_ID1, + {ok, Socket} = gen_udp:open(0, [binary]), + Payload1 = <<20, 21, 22, 23>>, + send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), + timer:sleep(100), + gen_udp:close(Socket). + t_publish_qos0_case01(_) -> Dup = 0, QoS = 0, @@ -1075,6 +1096,60 @@ t_will_case06(_) -> gen_udp:close(Socket). +t_will_case07(_) -> + QoS = 1, + Duration = 1, + WillMsg = <<10, 11, 12, 13, 14>>, + WillTopic = <<"abc">>, + {ok, Socket} = gen_udp:open(0, [binary]), + ClientId = ?CLIENTID, + + ok = emqx_broker:subscribe(WillTopic), + + send_connect_msg_with_will(Socket, Duration, ClientId), + ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), + + %% unexpected advertise + Adv = ?SN_ADVERTISE_MSG(1, 100), + AdvPacket = emqx_sn_frame:serialize(Adv), + send_packet(Socket, AdvPacket), + timer:sleep(200), + + %% unexpected connect + send_connect_msg(Socket, ClientId), + timer:sleep(200), + + send_willtopic_msg(Socket, WillTopic, QoS), + ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), + + %% unexpected advertise + send_packet(Socket, AdvPacket), + timer:sleep(200), + + %% unexpected connect + send_connect_msg(Socket, ClientId), + timer:sleep(200), + + send_willmsg_msg(Socket, WillMsg), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + send_pingreq_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), + + % wait udp client keepalive timeout + timer:sleep(2000), + + receive + {deliver, WillTopic, #message{payload = WillMsg}} -> ok; + Msg -> ct:print("recevived --- unex: ~p", [Msg]) + after + 1000 -> ct:fail(wait_willmsg_timeout) + end, + send_disconnect_msg(Socket, undefined), + ?assertEqual(udp_receive_timeout, receive_response(Socket)), + + gen_udp:close(Socket). + t_asleep_test01_timeout(_) -> QoS = 1, Duration = 1, @@ -1960,6 +2035,7 @@ t_register_enqueue_delivering_messages(_) -> _ = emqx:publish(emqx_message:make(test, ?QOS_1, <<"topic-a">>, <<"m2">>)), send_regack_msg(NSocket, TopicIdA, RegMsgIdA, ?SN_RC_ACCEPTED), + send_regack_msg(NSocket, TopicIdA, RegMsgIdA, ?SN_RC_INVALID_TOPIC_ID), %% receive the queued messages @@ -1983,6 +2059,28 @@ t_register_enqueue_delivering_messages(_) -> gen_udp:close(NSocket1), restart_emqx_sn(#{subs_resume => false}). +t_code_change(_) -> + Old = [state, gwid, socket, socketpid, socketstate, socketname, peername, + channel, clientid, username, password, will_msg, keepalive_interval, + connpkt, asleep_timer, enable_stats, stats_timer, enable_qos3, + has_pending_pingresp, pending_topic_ids], + New = Old ++ [false, [], undefined], + + OldTulpe = erlang:list_to_tuple(Old), + NewTulpe = erlang:list_to_tuple(New), + + ?assertEqual({ok, name, NewTulpe}, + emqx_sn_gateway:code_change(1, name, OldTulpe, ["4.3.2"])), + + ?assertEqual({ok, name, NewTulpe}, + emqx_sn_gateway:code_change(1, name, NewTulpe, ["4.3.6"])), + + ?assertEqual({ok, name, NewTulpe}, + emqx_sn_gateway:code_change({down, 1}, name, NewTulpe, ["4.3.6"])), + + ?assertEqual({ok, name, OldTulpe}, + emqx_sn_gateway:code_change({down, 1}, name, NewTulpe, ["4.3.2"])). + %%-------------------------------------------------------------------- %% Helper funcs %%-------------------------------------------------------------------- @@ -2267,6 +2365,9 @@ send_disconnect_msg(Socket, Duration) -> ?LOG("send_disconnect_msg Duration=~p", [Duration]), ok = gen_udp:send(Socket, ?HOST, ?PORT, DisConnectPacket). +send_packet(Socket, Packet) -> + ok = gen_udp:send(Socket, ?HOST, ?PORT, Packet). + mid(Id) -> Id. tid(Id) -> Id. diff --git a/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl b/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl index 0d726f190..5d6bb9572 100644 --- a/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl @@ -108,6 +108,11 @@ t_deny_wildcard_topic(_Config) -> ?assertEqual({error, wildcard_topic}, ?REGISTRY:register_topic(<<"ClientId">>, <<"/TopicA/#">>)), ?assertEqual({error, wildcard_topic}, ?REGISTRY:register_topic(<<"ClientId">>, <<"/+/TopicB">>)). +t_gen_server(_) -> + ?assertEqual(ignored, gen_server:call(emqx_sn_registry, ignored)), + ?assertEqual(ok, gen_server:cast(emqx_sn_registry, ignored)), + ?assertEqual(ignored, erlang:send(emqx_sn_registry, ignored)). + %%-------------------------------------------------------------------- %% Helper funcs %%-------------------------------------------------------------------- From 5b14d7f70908f48b65da303ef76d33c112ea3762 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 26 Sep 2022 16:37:33 +0800 Subject: [PATCH 02/50] test(mqtt_sn): improve test coverage of `emqx_sn_gateway` --- apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 132 ++++++++++++++++++- 1 file changed, 130 insertions(+), 2 deletions(-) diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index b064b630e..60b27af2c 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -85,6 +85,15 @@ restart_emqx_sn(#{subs_resume := Bool}) -> _ = application:ensure_all_started(emqx_sn), ok. +recoverable_restart_emqx_sn(Setup) -> + AppEnvs = application:get_all_env(emqx_sn), + Setup(), + _ = application:stop(emqx_sn), + _ = application:ensure_all_started(emqx_sn), + fun() -> + application:set_env([{emqx_sn, AppEnvs}]) + end. + %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- @@ -388,6 +397,38 @@ t_subscribe_case08(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). +t_subscribe_case09(_) -> + Dup = 0, + QoS = 0, + Retain = 0, + CleanSession = 0, + ReturnCode = 0, + {ok, Socket} = gen_udp:open(0, [binary]), + ClientId = ?CLIENTID, + send_connect_msg(Socket, ClientId), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + TopicName1 = <<"t/+">>, + MsgId1 = 25, + TopicId0 = 0, + WillBit = 0, + send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, + receive_response(Socket)), + + {ok, C} = emqtt:start_link(), + {ok, _} = emqtt:connect(C), + ok = emqtt:publish(C, <<"t/1">>, <<"Hello">>, 0), + timer:sleep(100), + ok = emqtt:disconnect(C), + + timer:sleep(50), + ?assertError(_, receive_publish(Socket)), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + gen_udp:close(Socket). + t_publish_negqos_case09(_) -> Dup = 0, QoS = 0, @@ -404,7 +445,6 @@ t_publish_negqos_case09(_) -> Topic = <<"abc">>, - send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), @@ -429,7 +469,7 @@ t_publish_negqos_case10(_) -> TopicId1 = ?PREDEF_TOPIC_ID1, {ok, Socket} = gen_udp:open(0, [binary]), Payload1 = <<20, 21, 22, 23>>, - send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), + send_publish_msg_predefined_topic(Socket, QoS, MsgId, TopicId1, Payload1), timer:sleep(100), gen_udp:close(Socket). @@ -1333,6 +1373,7 @@ t_asleep_test04_to_awake_qos1_dl_msg(_) -> ?assertError(_, receive_publish(Socket)), send_regack_msg(Socket, TopicIdNew, MsgId3), + send_regack_msg(Socket, TopicIdNew, MsgId3, ?SN_RC_INVALID_TOPIC_ID), UdpData2 = receive_response(Socket), MsgId_udp2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2), @@ -1662,6 +1703,33 @@ t_asleep_test09_to_awake_again_qos1_dl_msg(_) -> gen_udp:close(Socket). +t_asleep_unexpected(_) -> + SleepDuration = 3, + {ok, Socket} = gen_udp:open(0, [binary]), + ClientId = ?CLIENTID, + send_connect_msg(Socket, ClientId), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + timer:sleep(200), + + % goto asleep state + send_disconnect_msg(Socket, SleepDuration), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + timer:sleep(100), + + send_pingreq_msg(Socket, undefined), + ?assertEqual(udp_receive_timeout, receive_response(Socket)), + + send_puback_msg(Socket, 5, 5, ?SN_RC_INVALID_TOPIC_ID), + send_pubrec_msg(Socket, 5), + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + send_disconnect_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + timer:sleep(100), + + gen_udp:close(Socket). + t_awake_test01_to_connected(_) -> QoS = 1, Keepalive_Duration = 3, @@ -2081,6 +2149,66 @@ t_code_change(_) -> ?assertEqual({ok, name, OldTulpe}, emqx_sn_gateway:code_change({down, 1}, name, NewTulpe, ["4.3.2"])). +t_topic_id_to_large(_) -> + Dup = 0, + QoS = 0, + Retain = 0, + Will = 0, + CleanSession = 0, + MsgId = 1, + {ok, Socket} = gen_udp:open(0, [binary]), + ClientId = ?CLIENTID, + send_connect_msg(Socket, ClientId), + + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + mnesia:dirty_write(emqx_sn_registry, {emqx_sn_registry, {ClientId, next_topic_id}, 16#FFFF}), + + TopicName1 = <<"abcD">>, + send_register_msg(Socket, TopicName1, MsgId), + ?assertEqual(<<7, ?SN_REGACK, 0:16, MsgId:16, 3:8>>, receive_response(Socket)), + + send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, + CleanSession:1, ?SN_NORMAL_TOPIC:2, 0:16, + MsgId:16, 2>>, receive_response(Socket)), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + gen_udp:close(Socket). + +t_idle_timeout(_) -> + Backup = recoverable_restart_emqx_sn(fun() -> + application:set_env(emqx_sn, idle_timeout, 500), + application:set_env(emqx_sn, enable_qos3, false) + end), + timer:sleep(200), + QoS = ?QOS_NEG1, + MsgId = 1, + TopicId1 = ?PREDEF_TOPIC_ID1, + {ok, Socket} = gen_udp:open(0, [binary]), + Payload1 = <<20, 21, 22, 23>>, + send_publish_msg_predefined_topic(Socket, QoS, MsgId, TopicId1, Payload1), + timer:sleep(1500), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(udp_receive_timeout, receive_response(Socket)), + gen_udp:close(Socket), + _ = recoverable_restart_emqx_sn(Backup), + timer:sleep(200), + ok. + +t_invalid_packet(_) -> + {ok, Socket} = gen_udp:open(0, [binary]), + send_connect_msg(Socket, <<"client_id_test1">>), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + send_packet(Socket, emqx_sn_frame_SUITE:generate_random_binary()), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(udp_receive_timeout, receive_response(Socket)), + gen_udp:close(Socket). + %%-------------------------------------------------------------------- %% Helper funcs %%-------------------------------------------------------------------- From 0a3c8d035c37add2b1ac58fc1c793d02135dd258 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 26 Sep 2022 16:46:05 +0800 Subject: [PATCH 03/50] fix(mqtt_sn): after receiving publish in `idle mode` the gateway may panic the `emqx_sn_gateway` will return `{keep_state, Timeout}` when received publish in `idle mode`, this is an incorrect format of `gen_statem` state return result, this return will replace the `state data` by the `Timeout`, the fix is to change it to `{keep_state_and_data, Timeout}` --- apps/emqx_sn/src/emqx_sn_gateway.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index 22fe51b6e..3f1a4f2d9 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -208,7 +208,7 @@ idle(cast, {incoming, ?SN_DISCONNECT_MSG(_Duration)}, State) -> idle(cast, {incoming, ?SN_PUBLISH_MSG(_Flag, _TopicId, _MsgId, _Data)}, State = #state{enable_qos3 = false}) -> ?LOG(debug, "The enable_qos3 is false, ignore the received publish with QoS=-1 in idle mode!"), - {keep_state, State#state.idle_timeout}; + {keep_state_and_data, State#state.idle_timeout}; idle(cast, {incoming, ?SN_PUBLISH_MSG(#mqtt_sn_flags{qos = ?QOS_NEG1, topic_id_type = TopicIdType @@ -226,7 +226,7 @@ idle(cast, {incoming, ?SN_PUBLISH_MSG(#mqtt_sn_flags{qos = ?QOS_NEG1, ok end, ?LOG(debug, "Client id=~p receives a publish with QoS=-1 in idle mode!", [ClientId]), - {keep_state, State#state.idle_timeout}; + {keep_state_and_data, State#state.idle_timeout}; idle(cast, {incoming, PingReq = ?SN_PINGREQ_MSG(_ClientId)}, State) -> handle_ping(PingReq, State); From 1e003457774cdbc53f947afa6a61b740b5fa7391 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 26 Sep 2022 17:04:55 +0800 Subject: [PATCH 04/50] chore(emqx_sn): bump version && update appup --- apps/emqx_sn/src/emqx_sn.app.src | 2 +- apps/emqx_sn/src/emqx_sn.appup.src | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_sn/src/emqx_sn.app.src b/apps/emqx_sn/src/emqx_sn.app.src index 68e0340f8..46c2df084 100644 --- a/apps/emqx_sn/src/emqx_sn.app.src +++ b/apps/emqx_sn/src/emqx_sn.app.src @@ -1,6 +1,6 @@ {application, emqx_sn, [{description, "EMQ X MQTT-SN Plugin"}, - {vsn, "4.3.7"}, % strict semver, bump manually! + {vsn, "4.3.8"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,esockd]}, diff --git a/apps/emqx_sn/src/emqx_sn.appup.src b/apps/emqx_sn/src/emqx_sn.appup.src index bab8a2b7a..c87bbb0a1 100644 --- a/apps/emqx_sn/src/emqx_sn.appup.src +++ b/apps/emqx_sn/src/emqx_sn.appup.src @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- {VSN, [ + {"4.3.7",[ + {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} + ]}, {"4.3.6",[ {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} ]}, @@ -29,6 +32,9 @@ {<<"4\\.3\\.[0-1]">>, [{restart_application,emqx_sn}]} ], [ + {"4.3.7",[ + {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} + ]}, {"4.3.6",[ {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} ]}, From 04c52aa5f484a05a0b06f1300535053bc970f726 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 26 Sep 2022 17:15:27 +0800 Subject: [PATCH 05/50] chore: update CHANGES-4.3.md --- CHANGES-4.3.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 2e2f5157c..219da0ceb 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -12,6 +12,10 @@ File format: ## v4.3.22 +### Bug fixes + +- Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic. [#9024](https://github.com/emqx/emqx/pull/9024) + ## v4.3.21 ### Enhancements From 29c76d16d79eff27fc504e8dc8ca6189e2970fc3 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 29 Sep 2022 13:40:01 +0800 Subject: [PATCH 06/50] fix: reset rule metrics crash if it has not initialized --- apps/emqx_rule_engine/src/emqx_rule_metrics.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index 624032056..e8a77e0db 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -328,6 +328,8 @@ handle_call({create_rule_metrics, Id}, _From, _ -> RuleSpeeds#{Id => #rule_speed{}} end}}; +handle_call({reset_speeds, _Id}, _From, State = #state{rule_speeds = undefined}) -> + {reply, ok, State}; handle_call({reset_speeds, Id}, _From, State = #state{rule_speeds = RuleSpeedMap}) -> {reply, ok, State#state{rule_speeds = maps:put(Id, #rule_speed{}, RuleSpeedMap)}}; From 7d2dd3d37daddde0f7fb2a60aa4617ff07480bab Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 29 Sep 2022 13:41:02 +0800 Subject: [PATCH 07/50] fix: deny POST an existing resource --- .../emqx_rule_engine/src/emqx_rule_engine_api.erl | 10 ++++++++++ .../test/emqx_rule_engine_SUITE.erl | 15 ++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 50fdcfe22..eec67e914 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -315,6 +315,16 @@ create_resource(#{}, Params) -> end. do_create_resource(Create, ParsedParams) -> + case maps:find(id, ParsedParams) of + {ok, ResId} -> + case emqx_rule_registry:find_resource(ResId) of + {ok, _} -> return({error, 400, <<"Already Exists">>}); + not_found -> do_create_resource2(Create, ParsedParams) + end; + error -> do_create_resource2(Create, ParsedParams) + end. + +do_create_resource2(Create, ParsedParams) -> case emqx_rule_engine:Create(ParsedParams) of ok -> return(ok); diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index c8f66ea00..dc95c0368 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -600,13 +600,18 @@ t_show_action_api(_Config) -> ok. t_crud_resources_api(_Config) -> + ResParams = [ + {<<"name">>, <<"Simple Resource">>}, + {<<"type">>, <<"built_in">>}, + {<<"config">>, [{<<"a">>, 1}]}, + {<<"description">>, <<"Simple Resource">>} + ], {ok, #{code := 0, data := Resources1}} = - emqx_rule_engine_api:create_resource(#{}, - [{<<"name">>, <<"Simple Resource">>}, - {<<"type">>, <<"built_in">>}, - {<<"config">>, [{<<"a">>, 1}]}, - {<<"description">>, <<"Simple Resource">>}]), + emqx_rule_engine_api:create_resource(#{}, ResParams), ResId = maps:get(id, Resources1), + %% create again using given resource id returns error + {ok, #{code := 400, message := <<"Already Exists">>}} = + emqx_rule_engine_api:create_resource(#{}, [{<<"id">>, ResId} | ResParams]), {ok, #{code := 0, data := Resources}} = emqx_rule_engine_api:list_resources(#{}, []), ?assert(length(Resources) > 0), {ok, #{code := 0, data := Resources2}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []), From 33e68c9d165ef09224c3200df6bb3069841340fc Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 30 Sep 2022 15:55:17 +0800 Subject: [PATCH 08/50] chore: update the CHANGES-4.3.md --- CHANGES-4.3.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 2e2f5157c..b1505844d 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -14,6 +14,11 @@ File format: ## v4.3.21 +### Bug fixes + +- Deny POST an existing resource id using HTTP API with error 400 "Already Exists". [#9079](https://github.com/emqx/emqx/pull/9079) +- Fix the issue that reseting rule metrics crashed under certain conditions. [#9079](https://github.com/emqx/emqx/pull/9079) + ### Enhancements - TLS listener memory usage optimization [#9005](https://github.com/emqx/emqx/pull/9005). From ab3ec9c176a60414cba794a30d15ce929fc76eba Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 30 Sep 2022 16:32:52 +0800 Subject: [PATCH 09/50] chore: update the appup.src --- .../src/emqx_rule_engine.appup.src | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 6aa35d2d6..b46be85c7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -3,17 +3,20 @@ {VSN, [{"4.3.15", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -22,6 +25,7 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -30,6 +34,7 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -39,6 +44,7 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -48,6 +54,7 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -200,17 +207,20 @@ {<<".*">>,[]}], [{"4.3.15", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -219,6 +229,7 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -227,6 +238,7 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -236,6 +248,7 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -245,6 +258,7 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, From 09b08d092d28def9bb74d669b6f3bdb96954f051 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 5 Oct 2022 15:10:10 +0200 Subject: [PATCH 10/50] chore: revert double exported pmap after merge --- src/emqx_misc.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index 5e5f59b09..d256569fb 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -45,8 +45,6 @@ , index_of/2 , maybe_parse_ip/1 , ipv6_probe/1 - , pmap/2 - , pmap/3 , ipv6_probe/2 , pmap/2 , pmap/3 From c9ff67a6376b6f26d42f408026b1cd4b8e7b3e2d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 5 Oct 2022 14:25:25 +0200 Subject: [PATCH 11/50] chore: add timetrap of 2 mins for pgsql test cases --- apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl index 2b644c235..99d92202b 100644 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl @@ -70,6 +70,9 @@ all() -> emqx_ct:all(?MODULE). +suite() -> + [{timetrap, {seconds, 120}}]. + init_per_suite(Config) -> emqx_ct_helpers:start_apps([emqx_auth_pgsql]), drop_acl(), From f24728ee7bf66e327f77014e57296c6c194d6bd0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 5 Oct 2022 19:24:42 +0200 Subject: [PATCH 12/50] chore: sync diverged modules modules from ee back to ce --- .ci/docker-compose-file/toxiproxy.json | 2 +- apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl | 10 +++++----- apps/emqx_exhook/docs/design-cn.md | 2 +- apps/emqx_exproto/docs/design-cn.md | 2 +- apps/emqx_rule_engine/include/rule_engine.hrl | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 7079b0599..8bc77bdbf 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -1,6 +1,6 @@ [ { - "name": "mongo", + "name": "mongo_single", "listen": "0.0.0.0:27017", "upstream": "mongo:27017", "enabled": true diff --git a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl index 0031bc8c4..557d662d3 100644 --- a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl +++ b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl @@ -686,7 +686,7 @@ heal_failure(FailureType, ProxyHost, ProxyPort) -> end. switch_proxy(Switch, ProxyHost, ProxyPort) -> - Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo", + Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo_single", Body = case Switch of off -> <<"{\"enabled\":false}">>; on -> <<"{\"enabled\":true}">> @@ -695,27 +695,27 @@ switch_proxy(Switch, ProxyHost, ProxyPort) -> [{body_format, binary}]). timeout_proxy(on, ProxyHost, ProxyPort) -> - Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo/toxics", + Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo_single/toxics", Body = <<"{\"name\":\"timeout\",\"type\":\"timeout\"," "\"stream\":\"upstream\",\"toxicity\":1.0," "\"attributes\":{\"timeout\":0}}">>, {ok, {{_, 200, _}, _, _}} = httpc:request(post, {Url, [], "application/json", Body}, [], [{body_format, binary}]); timeout_proxy(off, ProxyHost, ProxyPort) -> - Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo/toxics/timeout", + Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo_single/toxics/timeout", Body = <<>>, {ok, {{_, 204, _}, _, _}} = httpc:request(delete, {Url, [], "application/json", Body}, [], [{body_format, binary}]). latency_up_proxy(on, ProxyHost, ProxyPort) -> - Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo/toxics", + Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo_single/toxics", Body = <<"{\"name\":\"latency_up\",\"type\":\"latency\"," "\"stream\":\"upstream\",\"toxicity\":1.0," "\"attributes\":{\"latency\":20000,\"jitter\":3000}}">>, {ok, {{_, 200, _}, _, _}} = httpc:request(post, {Url, [], "application/json", Body}, [], [{body_format, binary}]); latency_up_proxy(off, ProxyHost, ProxyPort) -> - Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo/toxics/latency_up", + Url = "http://" ++ ProxyHost ++ ":" ++ integer_to_list(ProxyPort) ++ "/proxies/mongo_single/toxics/latency_up", Body = <<>>, {ok, {{_, 204, _}, _, _}} = httpc:request(delete, {Url, [], "application/json", Body}, [], [{body_format, binary}]). diff --git a/apps/emqx_exhook/docs/design-cn.md b/apps/emqx_exhook/docs/design-cn.md index 21ee333eb..b86d9a512 100644 --- a/apps/emqx_exhook/docs/design-cn.md +++ b/apps/emqx_exhook/docs/design-cn.md @@ -47,7 +47,7 @@ 用户需要实现的方法,和数据类型的定义在 `priv/protos/exhook.proto` 文件中: -```protobuff +```protobuf syntax = "proto3"; package emqx.exhook.v1; diff --git a/apps/emqx_exproto/docs/design-cn.md b/apps/emqx_exproto/docs/design-cn.md index 1e1ba9e31..50246d216 100644 --- a/apps/emqx_exproto/docs/design-cn.md +++ b/apps/emqx_exproto/docs/design-cn.md @@ -44,7 +44,7 @@ 详情参见:`priv/protos/exproto.proto`,例如接口的定义有: -```protobuff +```protobuf syntax = "proto3"; package emqx.exproto.v1; diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index 04cfeb06f..c6d8fb2b4 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -131,7 +131,7 @@ -record(resource_params, { id :: resource_id() - , params :: #{} %% the params got after initializing the resource + , params :: map() %% the params got after initializing the resource , status = #{is_alive => false} :: #{is_alive := boolean(), atom() => term()} }). From 280668ce1349055813ff31d3e3bb5c0971832198 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 6 Oct 2022 11:26:56 +0200 Subject: [PATCH 13/50] chore: portable stdout coloring in scripts BSD's (macos) version of echo does not support -e flag --- scripts/rel/cut4x.sh | 4 ++-- scripts/rel/sync-remotes.sh | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/rel/cut4x.sh b/scripts/rel/cut4x.sh index 58131c757..1b1af4a69 100755 --- a/scripts/rel/cut4x.sh +++ b/scripts/rel/cut4x.sh @@ -40,10 +40,10 @@ EOF } logerr() { - echo -e "\e[31mERROR: $1\e[39m" + echo "$(tput setaf 1)ERROR: $1$(tput sgr0)" } logmsg() { - echo -e "\e[33mINFO: $1\e[39m" + echo "INFO: $1" } REL_BRANCH_CE="${REL_BRANCH_CE:-release-v43}" diff --git a/scripts/rel/sync-remotes.sh b/scripts/rel/sync-remotes.sh index c79e7b957..550c65cf6 100755 --- a/scripts/rel/sync-remotes.sh +++ b/scripts/rel/sync-remotes.sh @@ -45,11 +45,10 @@ EOF } logerr() { - echo -e "\e[31mERROR: $1\e[39m" + echo "$(tput setaf 1)ERROR: $1$(tput sgr0)" } - logwarn() { - echo -e "\e[33mINFO: $1\e[39m" + echo "$(tput setaf 3)WARNING: $1$(tput sgr0)" } logmsg() { From 1bb7c23db13a629d4c5bf37ef7d29eb8f3590a77 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 8 Oct 2022 10:50:42 +0800 Subject: [PATCH 14/50] fix: discard the 'id' field when testing a resource --- apps/emqx_rule_engine/src/emqx_rule_engine_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index eec67e914..ea0a13824 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -306,7 +306,7 @@ show_action(#{name := Name}, _Params) -> create_resource(#{}, Params) -> case parse_resource_params(Params) of {ok, ParsedParams} -> - if_test(fun() -> do_create_resource(test_resource, ParsedParams) end, + if_test(fun() -> do_create_resource(test_resource, maps:without([id], ParsedParams)) end, fun() -> do_create_resource(create_resource, ParsedParams) end, Params); {error, Reason} -> From 48d8f7e7a90bde1599565ef6a39d2529acdf2839 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 10 Oct 2022 16:03:31 +0800 Subject: [PATCH 15/50] fix(channel): add warning log if the acl check of a subscribed topic failed --- src/emqx_channel.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index b8f3c5b2c..3e20c7fef 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1501,12 +1501,15 @@ check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, check_sub_acls(TopicFilters, Channel) -> check_sub_acls(TopicFilters, Channel, []). -check_sub_acls([ TopicFilter = {Topic, _} | More] , Channel, Acc) -> +check_sub_acls([ TopicFilter = {Topic, SubOpts} | More] , Channel, Acc) -> case check_sub_acl(Topic, Channel) of allow -> check_sub_acls(More, Channel, [ {TopicFilter, 0} | Acc]); deny -> - check_sub_acls(More, Channel, [ {TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) + ReasonCode = ?RC_NOT_AUTHORIZED, + ?LOG(warning, "Cannot subscribe ~s with options ~p due to ~s.", + [Topic, SubOpts, emqx_reason_codes:text(ReasonCode)]), + check_sub_acls(More, Channel, [ {TopicFilter, ReasonCode} | Acc]) end; check_sub_acls([], _Channel, Acc) -> lists:reverse(Acc). From ef04310c2746a160e14a6273bd225a07c07da28d Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 10 Oct 2022 16:04:57 +0800 Subject: [PATCH 16/50] chore: update CHANGES.md --- CHANGES-4.3.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 673dbef8a..f136b7835 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -39,6 +39,8 @@ File format: - Added a test to prevent a last will testament message to be published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894) +- Add warning log if the acl check of a subscribed topic failed. [#9124](https://github.com/emqx/emqx/pull/9124) + ### Bug fixes - Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) From fd7230353c459f3d67c5232aa602defb36a7d393 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Sep 2022 14:29:59 +0800 Subject: [PATCH 17/50] refactor(test): move rule_engine sql test cases into a separate file --- apps/emqx_rule_engine/rebar.config | 10 + .../test/emqx_rule_engine_SUITE.erl | 1443 +---------------- apps/emqx_rule_engine/test/emqx_rule_test.hrl | 29 + .../test/emqx_rule_test_lib.erl | 141 ++ .../test/emqx_rulesql_SUITE.erl | 1378 ++++++++++++++++ 5 files changed, 1564 insertions(+), 1437 deletions(-) create mode 100644 apps/emqx_rule_engine/test/emqx_rule_test.hrl create mode 100644 apps/emqx_rule_engine/test/emqx_rule_test_lib.erl create mode 100644 apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 097c18a3d..f9ad1b283 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1,5 +1,7 @@ +%% -*- mode: erlang -*- {deps, []}. +%% Comple Opts {erl_opts, [warn_unused_vars, warn_shadow_vars, warn_unused_import, @@ -18,6 +20,14 @@ warnings_as_errors, deprecated_functions ]}. +%% {erl_opts, [...]}, but for CT runs +%% NOT WORKING!!! +%% %% == Common Test == +%% {ct_compile_opts, [ export_all +%% , nowarn_export_all +%% ]}. +%% {ct_opts, []}. + {cover_enabled, true}. {cover_opts, [verbose]}. {cover_export_enabled, true}. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index dc95c0368..42816e869 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -25,6 +25,9 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include("emqx_rule_test.hrl"). +-import(emqx_rule_test_lib, [make_simple_resource_type/1]). + %%-define(PROPTEST(M,F), true = proper:quickcheck(M:F())). all() -> @@ -92,47 +95,7 @@ groups() -> t_resource_types ]}, {runtime, [], - [t_match_atom_and_binary, - t_sqlselect_0, - t_sqlselect_00, - t_sqlselect_01, - t_sqlselect_02, - t_sqlselect_1, - t_sqlselect_2, - t_sqlselect_2_1, - t_sqlselect_2_2, - t_sqlselect_2_3, - t_sqlselect_3, - t_sqlparse_event_1, - t_sqlparse_event_2, - t_sqlparse_event_3, - t_sqlparse_foreach_1, - t_sqlparse_foreach_2, - t_sqlparse_foreach_3, - t_sqlparse_foreach_4, - t_sqlparse_foreach_5, - t_sqlparse_foreach_6, - t_sqlparse_foreach_7, - t_sqlparse_foreach_8, - t_sqlparse_case_when_1, - t_sqlparse_case_when_2, - t_sqlparse_case_when_3, - t_sqlparse_array_index_1, - t_sqlparse_array_index_2, - t_sqlparse_array_index_3, - t_sqlparse_array_index_4, - t_sqlparse_array_index_5, - t_sqlparse_select_matadata_1, - t_sqlparse_array_range_1, - t_sqlparse_array_range_2, - t_sqlparse_true_false, - t_sqlparse_compare_undefined, - t_sqlparse_compare_null_null, - t_sqlparse_compare_null_notnull, - t_sqlparse_compare_notnull_null, - t_sqlparse_compare, - t_sqlparse_new_map, - t_sqlparse_invalid_json + [t_match_atom_and_binary ]}, {rule_metrics, [], [t_metrics, @@ -169,10 +132,6 @@ end_per_suite(_Config) -> stop_apps(), ok. -on_resource_create(_id, _) -> #{}. -on_resource_destroy(_id, _) -> ok. -on_get_resource_status(_id, _) -> #{is_alive => true}. - %%------------------------------------------------------------------------------ %% Group specific setup/teardown %%------------------------------------------------------------------------------ @@ -269,15 +228,7 @@ init_per_testcase(Test, Config) | Config]; init_per_testcase(_TestCase, Config) -> ok = emqx_rule_registry:register_resource_types( - [#resource_type{ - name = built_in, - provider = ?APP, - params_spec = #{}, - on_create = {?MODULE, on_resource_create}, - on_destroy = {?MODULE, on_resource_destroy}, - on_status = {?MODULE, on_get_resource_status}, - title = #{en => <<"Built-In Resource Type (debug)">>}, - description = #{en => <<"The built in resource type for debug purpose">>}}]), + [make_simple_debug_resource_type()]), %ct:pal("============ ~p", [ets:tab2list(emqx_resource_type)]), Config. @@ -1280,324 +1231,6 @@ t_match_atom_and_binary(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_0(_Config) -> - %% Verify SELECT with and without 'AS' - Sql = "select * " - "from \"t/#\" " - "where payload.cmd.info = 'tt'", - ?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "select payload.cmd as cmd " - "from \"t/#\" " - "where cmd.info = 'tt'", - ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "select payload.cmd as cmd, cmd.info as info " - "from \"t/#\" " - "where cmd.info = 'tt' and info = 'tt'", - ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, - <<"info">> := <<"tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), - %% cascaded as - Sql4 = "select payload.cmd as cmd, cmd.info as meta.info " - "from \"t/#\" " - "where cmd.info = 'tt' and meta.info = 'tt'", - ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, - <<"meta">> := #{<<"info">> := <<"tt">>}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlselect_00(_Config) -> - %% Verify plus/subtract and unary_add_or_subtract - Sql = "select 1-1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 0}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - Sql1 = "select -1 + 1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 0}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "select 1 + 1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 2}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "select +1 as a " - "from \"t/#\" ", - ?assertMatch({ok,#{<<"a">> := 1}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlselect_01(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule1 = create_simple_repub_rule( - <<"t2">>, - "SELECT json_decode(payload) as p, payload " - "FROM \"t3/#\", \"t1\" " - "WHERE p.x = 1"), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), - ct:sleep(100), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"{\"x\":1}">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, - - emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), - receive {publish, #{topic := T3, payload := Payload3}} -> - ?assertEqual(<<"t2">>, T3), - ?assertEqual(<<"{\"x\":1}">>, Payload3) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule1). - -t_sqlselect_02(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule1 = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"t3/#\", \"t1\" " - "WHERE payload.x = 1"), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), - ct:sleep(100), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"{\"x\":1}">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := Payload0}} -> - ct:fail({unexpected_t2, Payload0}) - after 1000 -> - ok - end, - - emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), - receive {publish, #{topic := T3, payload := Payload3}} -> - ?assertEqual(<<"t2">>, T3), - ?assertEqual(<<"{\"x\":1}">>, Payload3) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule1). - -t_sqlselect_1(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT json_decode(payload) as p, payload " - "FROM \"t1\" " - "WHERE p.x = 1 and p.y = 2"), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ct:sleep(200), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":2}">>, 0), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"{\"x\":1,\"y\":2}">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2 - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"t2\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_1(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg dropped - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_dropped\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_nothing = Fun(), - - %% it should not keep republishing "t2" - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_2(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg acked - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_acked\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_3(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg delivered - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_delivered\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_3(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% republish the client.connected msg - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/client_connected\" " - "WHERE username = 'emqx1'", - <<"clientid=${clientid}">>), - {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ct:sleep(200), - {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), - {ok, _} = emqtt:connect(Client1), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - t_metrics(_Config) -> ok = emqx_rule_engine:load_providers(), TopicRule = create_simple_repub_rule( @@ -1899,964 +1532,6 @@ t_sqlselect_multi_actoins_4(Config) -> emqx_rule_registry:remove_rule(Rule). -t_sqlparse_event_1(_Config) -> - Sql = "select topic as tp " - "from \"$events/session_subscribed\" ", - ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). - -t_sqlparse_event_2(_Config) -> - Sql = "select clientid " - "from \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>}})). - -t_sqlparse_event_3(_Config) -> - Sql = "select clientid, topic as tp " - "from \"t/tt\", \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})). - -t_sqlparse_foreach_1(_Config) -> - %% Verify foreach with and without 'AS' - Sql = "foreach payload.sensors as s " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"s">> := 1}, #{<<"s">> := 2}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "from \"t/#\" ", - ?assertMatch({ok,[#{item := 1}, #{item := 2}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "foreach payload.sensors " - "from \"t/#\" ", - ?assertMatch({ok,[#{item := #{<<"cmd">> := <<"1">>}, clientid := <<"c_a">>}, - #{item := #{ <<"cmd">> := <<"2">> - , <<"name">> := <<"ct">> - } - , clientid := <<"c_a">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{ <<"payload">> => <<"{\"sensors\": [{\"cmd\":\"1\"}, " - "{\"cmd\":\"2\",\"name\":\"ct\"}]}">> - , <<"clientid">> => <<"c_a">> - , <<"topic">> => <<"t/a">> - }})), - Sql4 = "foreach payload.sensors " - "from \"t/#\" ", - {ok,[#{metadata := #{rule_id := TRuleId}}, - #{metadata := #{rule_id := TRuleId}}]} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{ - <<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}}), - ?assert(is_binary(TRuleId)). - -t_sqlparse_foreach_2(_Config) -> - %% Verify foreach-do with and without 'AS' - Sql = "foreach payload.sensors as s " - "do s.cmd as msg_type " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "do item.cmd as msg_type " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), - Sql3 = "foreach payload.sensors " - "do item as item " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"item">> := 1},#{<<"item">> := 2}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_3(_Config) -> - %% Verify foreach-incase with and without 'AS' - Sql = "foreach payload.sensors as s " - "incase s.cmd != 1 " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"s">> := #{<<"cmd">> := 2}}, - #{<<"s">> := #{<<"cmd">> := 3}} - ]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "incase item.cmd != 1 " - "from \"t/#\" ", - ?assertMatch({ok,[#{item := #{<<"cmd">> := 2}}, - #{item := #{<<"cmd">> := 3}} - ]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_4(_Config) -> - %% Verify foreach-do-incase - Sql = "foreach payload.sensors as s " - "do s.cmd as msg_type, s.name as name " - "incase is_not_null(s.cmd) " - "from \"t/#\" ", - ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ ok - , [ #{<<"msg_type">> := <<"1">>, <<"name">> := <<"n1">>} - , #{<<"msg_type">> := <<"2">>} - ] - }, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\", \"name\":\"n1\"}, " - "{\"cmd\":\"2\"}, {\"name\":\"n3\"}]}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok,[]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_5(_Config) -> - %% Verify foreach on a empty-list or non-list variable - Sql = "foreach payload.sensors as s " - "do s.cmd as msg_type, s.name as name " - "from \"t/#\" ", - ?assertMatch({ok,[]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok,[]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": []}">>, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach payload.sensors " - "from \"t/#\" ", - ?assertMatch({ok,[]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": 1}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_6(_Config) -> - %% Verify foreach on a empty-list or non-list variable - Sql = "foreach json_decode(payload) " - "do item.id as zid, timestamp as t " - "from \"t/#\" ", - {ok, Res} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"[{\"id\": 5},{\"id\": 15}]">>, - <<"topic">> => <<"t/a">>}}), - [#{<<"t">> := Ts1, <<"zid">> := Zid1}, - #{<<"t">> := Ts2, <<"zid">> := Zid2}] = Res, - ?assertEqual(true, is_integer(Ts1)), - ?assertEqual(true, is_integer(Ts2)), - ?assert(Zid1 == 5 orelse Zid1 == 15), - ?assert(Zid2 == 5 orelse Zid2 == 15). - -t_sqlparse_foreach_7(_Config) -> - %% Verify foreach-do-incase and cascaded AS - Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " - "incase is_not_null(info.cmd) " - "from \"t/#\" " - "where s.page = '2' ", - Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " - "{\"info\":[{\"name\":\"cmd1\", \"cmd\":\"1\"}, {\"cmd\":\"2\"}]} } }">>, - ?assertMatch({ ok - , [ #{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>} - , #{<<"msg_type">> := <<"2">>} - ] - }, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})), - Sql2 = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " - "incase is_not_null(info.cmd) " - "from \"t/#\" " - "where s.page = '3' ", - ?assertMatch({error, nomatch}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_foreach_8(_Config) -> - %% Verify foreach-do-incase and cascaded AS - Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " - "incase is_map(info) " - "from \"t/#\" " - "where s.page = '2' ", - Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " - "{\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }">>, - ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})), - - Sql3 = "foreach json_decode(payload) as p, p.sensors as s," - " s.collection as c, sublist(2,1,c.info) as info " - "do info.cmd as msg_type, info.name as name " - "from \"t/#\" " - "where s.page = '2' ", - [?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => SqlN, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})) - || SqlN <- [Sql3]]. - -t_sqlparse_case_when_1(_Config) -> - %% case-when-else clause - Sql = "select " - " case when payload.x < 0 then 0 " - " when payload.x > 7 then 7 " - " else payload.x " - " end as y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"y">> := 1}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})), - ok. - -t_sqlparse_case_when_2(_Config) -> - % switch clause - Sql = "select " - " case payload.x when 1 then 2 " - " when 2 then 3 " - " else 4 " - " end as y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"y">> := 2}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 2}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 4}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_case_when_3(_Config) -> - %% case-when clause - Sql = "select " - " case when payload.x < 0 then 0 " - " when payload.x > 7 then 7 " - " end as y " - "from \"t/#\" ", - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 5}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})), - ok. - -t_sqlparse_array_index_1(_Config) -> - %% index get - Sql = "select " - " json_decode(payload) as p, " - " p[1] as a " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"a">> := #{<<"x">> := 1}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"[{\"x\": 1}]">>, - <<"topic">> => <<"t/a">>}})), - ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), - %% index get without 'as' - Sql2 = "select " - " payload.x[2] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [3]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,3,4]}, - <<"topic">> => <<"t/a">>}})), - %% index get without 'as' again - Sql3 = "select " - " payload.x[2].y " - "from \"t/#\" ", - ?assertMatch( {ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 3}]}}} - , emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}}) - ), - - %% index get with 'as' - Sql4 = "select " - " payload.x[2].y as b " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_index_2(_Config) -> - %% array get with negative index - Sql1 = "select " - " payload.x[-2].y as b " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}})), - %% array append to head or tail of a list: - Sql2 = "select " - " payload.x as b, " - " 1 as c[-0], " - " 2 as c[-0], " - " b as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := 0, <<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => 0}, - <<"topic">> => <<"t/a">>}})), - %% construct an empty list: - Sql3 = "select " - " [] as c, " - " 1 as c[-0], " - " 2 as c[-0], " - " 0 as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), - %% construct a list: - Sql4 = "select " - " [payload.a, \"topic\", 'c'] as c, " - " 1 as c[-0], " - " 2 as c[-0], " - " 0 as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,11,<<"t/a">>,<<"c">>,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":11}">>, - <<"topic">> => <<"t/a">> - }})). - -t_sqlparse_array_index_3(_Config) -> - %% array with json string payload: - Sql0 = "select " - "payload," - "payload.x[2].y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := [1,2]}, 3]}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), - %% same as above but don't select payload: - Sql1 = "select " - "payload.x[2].y as b " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := [1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), - %% same as above but add 'as' clause: - Sql2 = "select " - "payload.x[2].y as b.c " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"b">> := #{<<"c">> := [1,2]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_index_4(_Config) -> - %% array with json string payload: - Sql0 = "select " - "0 as payload.x[2].y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 0}]}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), - %% array with json string payload, and also select payload.x: - Sql1 = "select " - "payload.x, " - "0 as payload.x[2].y " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := 0}, 3]}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_index_5(_Config) -> - Sql00 = "select " - " [1,2,3,4] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assert(lists:any(fun({_K, V}) -> - V =:= [1,2,3,4] - end, maps:to_list(Res00))). - -t_sqlparse_select_matadata_1(_Config) -> - %% array with json string payload: - Sql0 = "select " - "payload " - "from \"t/#\" ", - ?assertNotMatch({ok, #{<<"payload">> := <<"abc">>, metadata := _}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"abc">>, - <<"topic">> => <<"t/a">>}})), - Sql1 = "select " - "payload, metadata " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := <<"abc">>, <<"metadata">> := _}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"abc">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_range_1(_Config) -> - %% get a range of list - Sql0 = "select " - " payload.a[1..4] as c " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,1,2,3]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})), - %% get a range from non-list data - Sql02 = "select " - " payload.a[1..4] as c " - "from \"t/#\" ", - ?assertMatch({error, {select_and_transform_error, {error,{range_get,non_list_data},_}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => - #{<<"payload">> => <<"{\"x\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})), - - %% construct a range: - Sql1 = "select " - " [1..4] as c, " - " 5 as c[-0], " - " 6 as c[-0], " - " 0 as c[0] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"c">> := [0,1,2,3,4,5,6]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_array_range_2(_Config) -> - %% construct a range without 'as' - Sql00 = "select " - " [1..4] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assert(lists:any(fun({_K, V}) -> - V =:= [1,2,3,4] - end, maps:to_list(Res00))), - %% construct a range without 'as' - Sql01 = "select " - " a[2..4] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"a">> := [2,3,4]}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql01, - <<"ctx">> => #{<<"a">> => [1,2,3,4,5], - <<"topic">> => <<"t/a">>}})), - %% get a range of list without 'as' - Sql02 = "select " - " payload.a[1..4] " - "from \"t/#\" ", - ?assertMatch({ok, #{<<"payload">> := #{<<"a">> := [0,1,2,3]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_true_false(_Config) -> - %% construct a range without 'as' - Sql00 = "select " - " true as a, false as b, " - " false as x.y, true as c[-0] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assertMatch(#{<<"a">> := true, <<"b">> := false, - <<"x">> := #{<<"y">> := false}, - <<"c">> := [true] - }, Res00). - --define(TEST_SQL(SQL), - emqx_rule_sqltester:test( - #{<<"rawsql">> => SQL, - <<"ctx">> => #{<<"payload">> => <<"{}">>, - <<"topic">> => <<"t/a">>}})). - -t_sqlparse_compare_undefined(_Config) -> - Sql00 = "select " - " * " - "from \"t/#\" " - "where dev != undefined ", - %% no match - ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00)), - - Sql00_1 = "select " - " * " - "from \"t/#\" " - "where dev <> undefined ", - %% no match - ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00_1)), - - Sql01 = "select " - " 'd' as dev " - "from \"t/#\" " - "where dev != undefined ", - {ok, Res01} = ?TEST_SQL(Sql01), - %% pass - ?assertMatch(#{}, Res01), - - Sql01_1 = "select " - " 'd' as dev " - "from \"t/#\" " - "where dev <> undefined ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - %% pass - ?assertMatch(#{}, Res01_1), - - Sql02 = "select " - " * " - "from \"t/#\" " - "where dev != 'undefined' ", - {ok, Res02} = ?TEST_SQL(Sql02), - %% pass - ?assertMatch(#{}, Res02), - - Sql03 = "select " - " * " - "from \"t/#\" " - "where dev =~ 'undefined' ", - Res03 = ?TEST_SQL(Sql03), - %% no match - ?assertMatch({error, nomatch}, Res03). - -t_sqlparse_compare_null_null(_Config) -> - %% test undefined == undefined - Sql00 = "select " - " a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := true - }, Res00), - - %% test undefined != undefined - Sql01 = "select " - " a != b as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := false - }, Res01), - - %% test undefined <> undefined - Sql01_1 = "select " - " a <> b as c " - "from \"t/#\" ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - ?assertMatch(#{<<"c">> := false - }, Res01_1), - - %% test undefined > undefined - Sql02 = "select " - " a > b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := false - }, Res02), - - %% test undefined < undefined - Sql03 = "select " - " a < b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - %% test undefined <= undefined - Sql04 = "select " - " a <= b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := true - }, Res04), - - %% test undefined >= undefined - Sql05 = "select " - " a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := true - }, Res05), - - %% test undefined =~ undefined - Sql06 = "select " - " a =~ b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := true - }, Res06). - -t_sqlparse_compare_null_notnull(_Config) -> - %% test undefined == 'b' - Sql00 = "select " - " 'b' as b, a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := false - }, Res00), - - %% test undefined != 'b' - Sql01 = "select " - " 'b' as b, a != b as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := true - }, Res01), - - %% test undefined <> 'b' - Sql01_1 = "select " - " 'b' as b, a <> b as c " - "from \"t/#\" ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - ?assertMatch(#{<<"c">> := true - }, Res01_1), - - %% test undefined > 'b' - Sql02 = "select " - " 'b' as b, a > b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := false - }, Res02), - - %% test undefined < 'b' - Sql03 = "select " - " 'b' as b, a < b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - %% test undefined <= 'b' - Sql04 = "select " - " 'b' as b, a <= b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := false - }, Res04), - - %% test undefined >= 'b' - Sql05 = "select " - " 'b' as b, a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := false - }, Res05), - - %% test undefined =~ 'b' - Sql06 = "select " - " 'b' as b, a =~ b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := false - }, Res06). - -t_sqlparse_compare_notnull_null(_Config) -> - %% test 'a' == undefined - Sql00 = "select " - " 'a' as a, a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := false - }, Res00), - - %% test 'a' != undefined - Sql01 = "select " - " 'a' as a, a != b as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := true - }, Res01), - - %% test 'a' <> undefined - Sql01_1 = "select " - " 'a' as a, a <> b as c " - "from \"t/#\" ", - {ok, Res01_1} = ?TEST_SQL(Sql01_1), - ?assertMatch(#{<<"c">> := true - }, Res01_1), - - %% test 'a' > undefined - Sql02 = "select " - " 'a' as a, a > b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := false - }, Res02), - - %% test 'a' < undefined - Sql03 = "select " - " 'a' as a, a < b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - %% test 'a' <= undefined - Sql04 = "select " - " 'a' as a, a <= b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := false - }, Res04), - - %% test 'a' >= undefined - Sql05 = "select " - " 'a' as a, a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := false - }, Res05), - - %% test 'a' =~ undefined - Sql06 = "select " - " 'a' as a, a =~ b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := false - }, Res06). - -t_sqlparse_compare(_Config) -> - Sql00 = "select " - " 'a' as a, 'a' as b, a = b as c " - "from \"t/#\" ", - {ok, Res00} = ?TEST_SQL(Sql00), - ?assertMatch(#{<<"c">> := true - }, Res00), - - Sql00_1 = "select " - " 'true' as a, true as b, a = b as c " - "from \"t/#\" ", - {ok, Res00_1} = ?TEST_SQL(Sql00_1), - ?assertMatch(#{<<"c">> := true - }, Res00_1), - - Sql01 = "select " - " is_null(a) as c " - "from \"t/#\" ", - {ok, Res01} = ?TEST_SQL(Sql01), - ?assertMatch(#{<<"c">> := true - }, Res01), - - Sql02 = "select " - " 1 as a, 2 as b, a < b as c " - "from \"t/#\" ", - {ok, Res02} = ?TEST_SQL(Sql02), - ?assertMatch(#{<<"c">> := true - }, Res02), - - Sql03 = "select " - " 1 as a, 2 as b, a > b as c " - "from \"t/#\" ", - {ok, Res03} = ?TEST_SQL(Sql03), - ?assertMatch(#{<<"c">> := false - }, Res03), - - Sql04 = "select " - " 1 as a, 2 as b, a = b as c " - "from \"t/#\" ", - {ok, Res04} = ?TEST_SQL(Sql04), - ?assertMatch(#{<<"c">> := false - }, Res04), - - Sql04_0 = "select " - " 1 as a, 1 as b, a = b as c " - "from \"t/#\" ", - {ok, Res04_0} = ?TEST_SQL(Sql04_0), - ?assertMatch(#{<<"c">> := true - }, Res04_0), - - Sql04_1 = "select " - " 1 as a, '1' as b, a = b as c " - "from \"t/#\" ", - {ok, Res04_1} = ?TEST_SQL(Sql04_1), - ?assertMatch(#{<<"c">> := true - }, Res04_1), - - %% test 1 >= 2 - Sql05 = "select " - " 1 as a, 2 as b, a >= b as c " - "from \"t/#\" ", - {ok, Res05} = ?TEST_SQL(Sql05), - ?assertMatch(#{<<"c">> := false - }, Res05), - - %% test 1 <= 2 - Sql06 = "select " - " 1 as a, 2 as b, a <= b as c " - "from \"t/#\" ", - {ok, Res06} = ?TEST_SQL(Sql06), - ?assertMatch(#{<<"c">> := true - }, Res06), - - %% test 1 != 2 - Sql07 = "select " - " 1 as a, 2 as b, a != b as c " - "from \"t/#\" ", - {ok, Res07} = ?TEST_SQL(Sql07), - ?assertMatch(#{<<"c">> := true - }, Res07), - - %% test 1 <> 2 - Sql07_1 = "select " - " 1 as a, 2 as b, a <> b as c " - "from \"t/#\" ", - {ok, Res07_1} = ?TEST_SQL(Sql07_1), - ?assertMatch(#{<<"c">> := true - }, Res07_1), - - %% test 't' =~ 't' - Sql08 = "select " - " 't' as a, 't' as b, a =~ b as c " - "from \"t/#\" ", - {ok, Res08} = ?TEST_SQL(Sql08), - ?assertMatch(#{<<"c">> := true - }, Res08). - -t_sqlparse_new_map(_Config) -> - %% construct a range without 'as' - Sql00 = "select " - " map_new() as a, map_new() as b, " - " map_new() as x.y, map_new() as c[-0] " - "from \"t/#\" ", - {ok, Res00} = - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), - ?assertMatch(#{<<"a">> := #{}, <<"b">> := #{}, - <<"x">> := #{<<"y">> := #{}}, - <<"c">> := [#{}] - }, Res00). - t_sqlparse_payload_as(_Config) -> %% https://github.com/emqx/emqx/issues/3866 Sql00 = "SELECT " @@ -2908,29 +1583,6 @@ t_sqlparse_nested_get(_Config) -> <<"payload">> => <<"{\"a\": {\"b\": 0}}">> }})). -t_sqlparse_invalid_json(_Config) -> - Sql02 = "select " - " payload.a[1..4] as c " - "from \"t/#\" ", - ?assertMatch({error, {select_and_transform_error, {error,{decode_json_failed,_},_}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => - #{<<"payload">> => <<"{\"x\":[0,1,2,3,}">>, - <<"topic">> => <<"t/a">>}})), - - - Sql2 = "foreach payload.sensors " - "do item.cmd as msg_type " - "from \"t/#\" ", - ?assertMatch({error, {select_and_collect_error, {error,{decode_json_failed,_},_}}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => - <<"{\"sensors\": [{\"cmd\":\"1\"} {\"cmd\":}]}">>, - <<"topic">> => <<"t/a">>}})). - %%------------------------------------------------------------------------------ %% Internal helpers %%------------------------------------------------------------------------------ @@ -2966,20 +1618,6 @@ make_simple_rule(RuleId, SQL, ForTopics) when is_binary(RuleId) -> actions = [{'inspect', #{}}], description = <<"simple rule">>}. -create_simple_repub_rule(TargetTopic, SQL) -> - create_simple_repub_rule(TargetTopic, SQL, <<"${payload}">>). - -create_simple_repub_rule(TargetTopic, SQL, Template) -> - {ok, Rule} = emqx_rule_engine:create_rule( - #{rawsql => SQL, - actions => [#{name => 'republish', - args => #{<<"target_topic">> => TargetTopic, - <<"target_qos">> => -1, - <<"payload_tmpl">> => Template} - }], - description => <<"simple repub rule">>}), - Rule. - make_simple_action(ActionName) when is_atom(ActionName) -> #action{name = ActionName, app = ?APP, module = ?MODULE, on_create = simple_action_inspect, params_spec = #{}, @@ -3002,19 +1640,6 @@ make_simple_resource(ResId) -> config = #{}, description = <<"Simple Resource">>}. -make_simple_resource_type(ResTypeName) -> - #resource_type{name = ResTypeName, provider = ?APP, - params_spec = #{}, - on_create = {?MODULE, on_simple_resource_type_create}, - on_destroy = {?MODULE, on_simple_resource_type_destroy}, - on_status = {?MODULE, on_simple_resource_type_status}, - title = #{en => <<"Simple Resource Type">>}, - description = #{en => <<"Simple Resource Type">>}}. - -on_simple_resource_type_create(_Id, #{}) -> #{}. -on_simple_resource_type_destroy(_Id, #{}) -> ok. -on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}. - hook_metrics_action(_Id, _Params) -> fun(Data = #{event := EventName}, _Envs) -> ct:pal("applying hook_metrics_action: ~p", [Data]), @@ -3305,66 +1930,10 @@ verify_peername(PeerName) -> verify_ipaddr(IPAddrS) -> ?assertMatch({ok, _}, inet:parse_address(binary_to_list(IPAddrS))). -init_events_counters() -> - ets:new(events_record_tab, [named_table, bag, public]). - %%------------------------------------------------------------------------------ -%% Start Apps +%% Mock funcs %%------------------------------------------------------------------------------ -stop_apps() -> - stopped = mnesia:stop(), - [application:stop(App) || App <- [emqx_rule_engine, emqx]]. - -start_apps() -> - [start_apps(App, SchemaFile, ConfigFile) || - {App, SchemaFile, ConfigFile} - <- [{emqx, deps_path(emqx, "priv/emqx.schema"), - deps_path(emqx, "etc/emqx.conf")}, - {emqx_rule_engine, local_path("priv/emqx_rule_engine.schema"), - local_path("etc/emqx_rule_engine.conf")}]]. - -start_apps(App, SchemaFile, ConfigFile) -> - read_schema_configs(App, SchemaFile, ConfigFile), - set_special_configs(App), - {ok, _} = application:ensure_all_started(App). - -read_schema_configs(App, SchemaFile, ConfigFile) -> - ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]), - Schema = cuttlefish_schema:files([SchemaFile]), - Conf = conf_parse:file(ConfigFile), - NewConfig = cuttlefish_generator:map(Schema, Conf), - Vals = proplists:get_value(App, NewConfig, []), - [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. - -deps_path(App, RelativePath) -> - %% Note: not lib_dir because etc dir is not sym-link-ed to _build dir - %% but priv dir is - Path0 = code:priv_dir(App), - Path = case file:read_link(Path0) of - {ok, Resolved} -> Resolved; - {error, _} -> Path0 - end, - filename:join([Path, "..", RelativePath]). - -local_path(RelativePath) -> - deps_path(emqx_rule_engine, RelativePath). - -set_special_configs(emqx_rule_engine) -> - application:set_env(emqx_rule_engine, ignore_sys_message, true), - application:set_env(emqx_rule_engine, events, - [{'client.connected',on,1}, - {'client.disconnected',on,1}, - {'session.subscribed',on,1}, - {'session.unsubscribed',on,1}, - {'message.acked',on,1}, - {'message.dropped',on,1}, - {'message.delivered',on,1} - ]), - ok; -set_special_configs(_App) -> - ok. - mock_print() -> catch meck:unload(emqx_ctl), meck:new(emqx_ctl, [non_strict, passthrough]), diff --git a/apps/emqx_rule_engine/test/emqx_rule_test.hrl b/apps/emqx_rule_engine/test/emqx_rule_test.hrl new file mode 100644 index 000000000..748cc2e9a --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rule_test.hrl @@ -0,0 +1,29 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Test Suite funcs +-import(emqx_rule_test_lib, + [ stop_apps/0 + , start_apps/0 + ]). + +%% RULE helper funcs +-import(emqx_rule_test_lib, + [ create_simple_repub_rule/2 + , create_simple_repub_rule/3 + , make_simple_debug_resource_type/0 + , init_events_counters/0 + ]). diff --git a/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl b/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl new file mode 100644 index 000000000..24550fbc7 --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl @@ -0,0 +1,141 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rule_test_lib). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx_rule_engine/include/rule_engine.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +%%------------------------------------------------------------------------------ +%% Start Apps +%%------------------------------------------------------------------------------ + +stop_apps() -> + stopped = mnesia:stop(), + [application:stop(App) || App <- [emqx_rule_engine, emqx]]. + +start_apps() -> + [start_apps(App, SchemaFile, ConfigFile) || + {App, SchemaFile, ConfigFile} + <- [{emqx, deps_path(emqx, "priv/emqx.schema"), + deps_path(emqx, "etc/emqx.conf")}, + {emqx_rule_engine, local_path("priv/emqx_rule_engine.schema"), + local_path("etc/emqx_rule_engine.conf")}]]. + +%%-------------------------------------- +%% start apps helper funcs + +start_apps(App, SchemaFile, ConfigFile) -> + read_schema_configs(App, SchemaFile, ConfigFile), + set_special_configs(App), + {ok, _} = application:ensure_all_started(App). + +read_schema_configs(App, SchemaFile, ConfigFile) -> + ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]), + Schema = cuttlefish_schema:files([SchemaFile]), + Conf = conf_parse:file(ConfigFile), + NewConfig = cuttlefish_generator:map(Schema, Conf), + Vals = proplists:get_value(App, NewConfig, []), + [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. + +deps_path(App, RelativePath) -> + %% Note: not lib_dir because etc dir is not sym-link-ed to _build dir + %% but priv dir is + Path0 = code:priv_dir(App), + Path = case file:read_link(Path0) of + {ok, Resolved} -> Resolved; + {error, _} -> Path0 + end, + filename:join([Path, "..", RelativePath]). + +local_path(RelativePath) -> + deps_path(emqx_rule_engine, RelativePath). + +set_special_configs(emqx_rule_engine) -> + application:set_env(emqx_rule_engine, ignore_sys_message, true), + application:set_env(emqx_rule_engine, events, + [{'client.connected',on,1}, + {'client.disconnected',on,1}, + {'session.subscribed',on,1}, + {'session.unsubscribed',on,1}, + {'message.acked',on,1}, + {'message.dropped',on,1}, + {'message.delivered',on,1} + ]), + ok; +set_special_configs(_App) -> + ok. + +%%------------------------------------------------------------------------------ +%% rule test helper funcs +%%------------------------------------------------------------------------------ + +create_simple_repub_rule(TargetTopic, SQL) -> + create_simple_repub_rule(TargetTopic, SQL, <<"${payload}">>). + +create_simple_repub_rule(TargetTopic, SQL, Template) -> + {ok, Rule} = emqx_rule_engine:create_rule( + #{rawsql => SQL, + actions => [#{name => 'republish', + args => #{<<"target_topic">> => TargetTopic, + <<"target_qos">> => -1, + <<"payload_tmpl">> => Template} + }], + description => <<"simple repub rule">>}), + Rule. + +make_simple_debug_resource_type() -> + #resource_type{ + name = built_in, + provider = ?APP, + params_spec = #{}, + on_create = {?MODULE, on_resource_create}, + on_destroy = {?MODULE, on_resource_destroy}, + on_status = {?MODULE, on_get_resource_status}, + title = #{en => <<"Built-In Resource Type (debug)">>}, + description = #{en => <<"The built in resource type for debug purpose">>}}. + +make_simple_resource_type(ResTypeName) -> + #resource_type{ + name = ResTypeName, + provider = ?APP, + params_spec = #{}, + on_create = {?MODULE, on_simple_resource_type_create}, + on_destroy = {?MODULE, on_simple_resource_type_destroy}, + on_status = {?MODULE, on_simple_resource_type_status}, + title = #{en => <<"Simple Resource Type">>}, + description = #{en => <<"Simple Resource Type">>}}. + +init_events_counters() -> + ets:new(events_record_tab, [named_table, bag, public]). + +%%------------------------------------------------------------------------------ +%% Internal helper funcs +%%------------------------------------------------------------------------------ + +on_resource_create(_id, _) -> #{}. +on_resource_destroy(_id, _) -> ok. +on_get_resource_status(_id, _) -> #{is_alive => true}. + +on_simple_resource_type_create(_Id, #{}) -> #{}. +on_simple_resource_type_destroy(_Id, #{}) -> ok. +on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}. diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl new file mode 100644 index 000000000..d93b747c0 --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -0,0 +1,1378 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rulesql_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx_rule_engine/include/rule_engine.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-include("emqx_rule_test.hrl"). + +all() -> + [ {group, rulesql_select} + , {group, rulesql_select_events} + , {group, rulesql_select_metadata} + , {group, rulesql_foreach} + , {group, rulesql_case_when} + , {group, rulesql_array_index} + , {group, rulesql_array_range} + , {group, rulesql_compare} + , {group, rulesql_boolean} + , {group, rulesql_others} + ]. + +suite() -> + [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}]. + +groups() -> + [{rulesql_select, [], + [ t_sqlselect_0 + , t_sqlselect_00 + , t_sqlselect_01 + , t_sqlselect_02 + , t_sqlselect_1 + , t_sqlselect_2 + , t_sqlselect_2_1 + , t_sqlselect_2_2 + , t_sqlselect_2_3 + , t_sqlselect_3 + ]}, + {rulesql_select_events, [], + [ t_sqlparse_event_1 + , t_sqlparse_event_2 + , t_sqlparse_event_3 + ]}, + {rulesql_select_metadata, [], + [ t_sqlparse_select_matadata_1 + ]}, + {rulesql_foreach, [], + [ t_sqlparse_foreach_1 + , t_sqlparse_foreach_2 + , t_sqlparse_foreach_3 + , t_sqlparse_foreach_4 + , t_sqlparse_foreach_5 + , t_sqlparse_foreach_6 + , t_sqlparse_foreach_7 + , t_sqlparse_foreach_8 + ]}, + {rulesql_case_when, [], + [ t_sqlparse_case_when_1 + , t_sqlparse_case_when_2 + , t_sqlparse_case_when_3 + ]}, + {rulesql_array_index, [], + [ t_sqlparse_array_index_1 + , t_sqlparse_array_index_2 + , t_sqlparse_array_index_3 + , t_sqlparse_array_index_4 + , t_sqlparse_array_index_5 + ]}, + {rulesql_array_range, [], + [ t_sqlparse_array_range_1 + , t_sqlparse_array_range_2 + ]}, + {rulesql_compare, [], + [ t_sqlparse_compare_undefined + , t_sqlparse_compare_null_null + , t_sqlparse_compare_null_notnull + , t_sqlparse_compare_notnull_null + , t_sqlparse_compare + ]}, + {rulesql_boolean, [], + [ t_sqlparse_true_false + ]}, + {rulesql_others, [], + [ t_sqlparse_new_map + , t_sqlparse_invalid_json + ]} + ]. + +%%------------------------------------------------------------------------------ +%% Overall setup/teardown +%%------------------------------------------------------------------------------ + +init_per_suite(Config) -> + ok = ekka_mnesia:start(), + ok = emqx_rule_registry:mnesia(boot), + start_apps(), + Config. + +end_per_suite(_Config) -> + stop_apps(), + ok. + +on_resource_create(_id, _) -> #{}. +on_resource_destroy(_id, _) -> ok. +on_get_resource_status(_id, _) -> #{is_alive => true}. + +%%------------------------------------------------------------------------------ +%% Group specific setup/teardown +%%------------------------------------------------------------------------------ + +group(_Groupname) -> + []. + +init_per_group(registry, Config) -> + Config; +init_per_group(_Groupname, Config) -> + Config. + +end_per_group(_Groupname, _Config) -> + ok. + +%%------------------------------------------------------------------------------ +%% Testcase specific setup/teardown +%%------------------------------------------------------------------------------ + +init_per_testcase(_TestCase, Config) -> + init_events_counters(), + ok = emqx_rule_registry:register_resource_types( + [make_simple_debug_resource_type()]), + %ct:pal("============ ~p", [ets:tab2list(emqx_resource_type)]), + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%%------------------------------------------------------------------------------ +%% Test cases for `select` +%%------------------------------------------------------------------------------ + +t_sqlselect_0(_Config) -> + %% Verify SELECT with and without 'AS' + Sql = "select * " + "from \"t/#\" " + "where payload.cmd.info = 'tt'", + ?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "select payload.cmd as cmd " + "from \"t/#\" " + "where cmd.info = 'tt'", + ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "select payload.cmd as cmd, cmd.info as info " + "from \"t/#\" " + "where cmd.info = 'tt' and info = 'tt'", + ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, + <<"info">> := <<"tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})), + %% cascaded as + Sql4 = "select payload.cmd as cmd, cmd.info as meta.info " + "from \"t/#\" " + "where cmd.info = 'tt' and meta.info = 'tt'", + ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, + <<"meta">> := #{<<"info">> := <<"tt">>}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => + #{<<"payload">> => + <<"{\"cmd\": {\"info\":\"tt\"}}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlselect_00(_Config) -> + %% Verify plus/subtract and unary_add_or_subtract + Sql = "select 1-1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 0}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + Sql1 = "select -1 + 1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 0}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "select 1 + 1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 2}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "select +1 as a " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"a">> := 1}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlselect_01(_Config) -> + ok = emqx_rule_engine:load_providers(), + TopicRule1 = create_simple_repub_rule( + <<"t2">>, + "SELECT json_decode(payload) as p, payload " + "FROM \"t3/#\", \"t1\" " + "WHERE p.x = 1"), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), + ct:sleep(100), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"{\"x\":1}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, + + emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), + receive {publish, #{topic := T3, payload := Payload3}} -> + ?assertEqual(<<"t2">>, T3), + ?assertEqual(<<"{\"x\":1}">>, Payload3) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule1). + +t_sqlselect_02(_Config) -> + ok = emqx_rule_engine:load_providers(), + TopicRule1 = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"t3/#\", \"t1\" " + "WHERE payload.x = 1"), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0), + ct:sleep(100), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"{\"x\":1}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := Payload0}} -> + ct:fail({unexpected_t2, Payload0}) + after 1000 -> + ok + end, + + emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0), + receive {publish, #{topic := T3, payload := Payload3}} -> + ?assertEqual(<<"t2">>, T3), + ?assertEqual(<<"{\"x\":1}">>, Payload3) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule1). + + + +t_sqlselect_1(_Config) -> + ok = emqx_rule_engine:load_providers(), + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT json_decode(payload) as p, payload " + "FROM \"t1\" " + "WHERE p.x = 1 and p.y = 2"), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + ct:sleep(200), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":2}">>, 0), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"{\"x\":1,\"y\":2}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2 + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"t2\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_t2 = Fun(), + received_t2 = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2_1(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2, if the msg dropped + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/message_dropped\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_nothing = Fun(), + + %% it should not keep republishing "t2" + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2_2(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2, if the msg acked + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/message_acked\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1), + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_t2 = Fun(), + received_t2 = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_2_3(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% recursively republish to t2, if the msg delivered + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/message_delivered\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), + Fun = fun() -> + receive {publish, #{topic := <<"t2">>, payload := _}} -> + received_t2 + after 500 -> + received_nothing + end + end, + received_t2 = Fun(), + received_t2 = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +t_sqlselect_3(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% republish the client.connected msg + TopicRule = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"$events/client_connected\" " + "WHERE username = 'emqx1'", + <<"clientid=${clientid}">>), + {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + ct:sleep(200), + {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), + {ok, _} = emqtt:connect(Client1), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"t2">>, T), + ?assertEqual(<<"clientid=c_emqx1">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). + +%%------------------------------------------------------------------------------ +%% Test cases for events +%%------------------------------------------------------------------------------ + +t_sqlparse_event_1(_Config) -> + Sql = "select topic as tp " + "from \"$events/session_subscribed\" ", + ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). + +t_sqlparse_event_2(_Config) -> + Sql = "select clientid " + "from \"$events/client_connected\" ", + ?assertMatch({ok,#{<<"clientid">> := <<"abc">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"clientid">> => <<"abc">>}})). + +t_sqlparse_event_3(_Config) -> + Sql = "select clientid, topic as tp " + "from \"t/tt\", \"$events/client_connected\" ", + ?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})). + +%%------------------------------------------------------------------------------ +%% Test cases for `foreach` +%%------------------------------------------------------------------------------ + +t_sqlparse_foreach_1(_Config) -> + %% Verify foreach with and without 'AS' + Sql = "foreach payload.sensors as s " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"s">> := 1}, #{<<"s">> := 2}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "from \"t/#\" ", + ?assertMatch({ok,[#{item := 1}, #{item := 2}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "foreach payload.sensors " + "from \"t/#\" ", + ?assertMatch({ok,[#{item := #{<<"cmd">> := <<"1">>}, clientid := <<"c_a">>}, + #{item := #{ <<"cmd">> := <<"2">> + , <<"name">> := <<"ct">> + } + , clientid := <<"c_a">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{ <<"payload">> => <<"{\"sensors\": [{\"cmd\":\"1\"}, " + "{\"cmd\":\"2\",\"name\":\"ct\"}]}">> + , <<"clientid">> => <<"c_a">> + , <<"topic">> => <<"t/a">> + }})), + Sql4 = "foreach payload.sensors " + "from \"t/#\" ", + {ok,[#{metadata := #{rule_id := TRuleId}}, + #{metadata := #{rule_id := TRuleId}}]} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => #{ + <<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}}), + ?assert(is_binary(TRuleId)). + +t_sqlparse_foreach_2(_Config) -> + %% Verify foreach-do with and without 'AS' + Sql = "foreach payload.sensors as s " + "do s.cmd as msg_type " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "do item.cmd as msg_type " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, + <<"topic">> => <<"t/a">>}})), + Sql3 = "foreach payload.sensors " + "do item as item " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"item">> := 1},#{<<"item">> := 2}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_3(_Config) -> + %% Verify foreach-incase with and without 'AS' + Sql = "foreach payload.sensors as s " + "incase s.cmd != 1 " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"s">> := #{<<"cmd">> := 2}}, + #{<<"s">> := #{<<"cmd">> := 3}} + ]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "incase item.cmd != 1 " + "from \"t/#\" ", + ?assertMatch({ok,[#{item := #{<<"cmd">> := 2}}, + #{item := #{<<"cmd">> := 3}} + ]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_4(_Config) -> + %% Verify foreach-do-incase + Sql = "foreach payload.sensors as s " + "do s.cmd as msg_type, s.name as name " + "incase is_not_null(s.cmd) " + "from \"t/#\" ", + ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ ok + , [ #{<<"msg_type">> := <<"1">>, <<"name">> := <<"n1">>} + , #{<<"msg_type">> := <<"2">>} + ] + }, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\", \"name\":\"n1\"}, " + "{\"cmd\":\"2\"}, {\"name\":\"n3\"}]}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok,[]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_5(_Config) -> + %% Verify foreach on a empty-list or non-list variable + Sql = "foreach payload.sensors as s " + "do s.cmd as msg_type, s.name as name " + "from \"t/#\" ", + ?assertMatch({ok,[]}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok,[]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": []}">>, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach payload.sensors " + "from \"t/#\" ", + ?assertMatch({ok,[]}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => <<"{\"sensors\": 1}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_6(_Config) -> + %% Verify foreach on a empty-list or non-list variable + Sql = "foreach json_decode(payload) " + "do item.id as zid, timestamp as t " + "from \"t/#\" ", + {ok, Res} = emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => <<"[{\"id\": 5},{\"id\": 15}]">>, + <<"topic">> => <<"t/a">>}}), + [#{<<"t">> := Ts1, <<"zid">> := Zid1}, + #{<<"t">> := Ts2, <<"zid">> := Zid2}] = Res, + ?assertEqual(true, is_integer(Ts1)), + ?assertEqual(true, is_integer(Ts2)), + ?assert(Zid1 == 5 orelse Zid1 == 15), + ?assert(Zid2 == 5 orelse Zid2 == 15). + +t_sqlparse_foreach_7(_Config) -> + %% Verify foreach-do-incase and cascaded AS + Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " + "do info.cmd as msg_type, info.name as name " + "incase is_not_null(info.cmd) " + "from \"t/#\" " + "where s.page = '2' ", + Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " + "{\"info\":[{\"name\":\"cmd1\", \"cmd\":\"1\"}, {\"cmd\":\"2\"}]} } }">>, + ?assertMatch({ ok + , [ #{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>} + , #{<<"msg_type">> := <<"2">>} + ] + }, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})), + Sql2 = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " + "do info.cmd as msg_type, info.name as name " + "incase is_not_null(info.cmd) " + "from \"t/#\" " + "where s.page = '3' ", + ?assertMatch({error, nomatch}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_foreach_8(_Config) -> + %% Verify foreach-do-incase and cascaded AS + Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " + "do info.cmd as msg_type, info.name as name " + "incase is_map(info) " + "from \"t/#\" " + "where s.page = '2' ", + Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " + "{\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }">>, + ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})), + + Sql3 = "foreach json_decode(payload) as p, p.sensors as s," + " s.collection as c, sublist(2,1,c.info) as info " + "do info.cmd as msg_type, info.name as name " + "from \"t/#\" " + "where s.page = '2' ", + [?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => SqlN, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => <<"t/a">>}})) + || SqlN <- [Sql3]]. + +%%------------------------------------------------------------------------------ +%% Test cases for `case..when..` +%%------------------------------------------------------------------------------ + +t_sqlparse_case_when_1(_Config) -> + %% case-when-else clause + Sql = "select " + " case when payload.x < 0 then 0 " + " when payload.x > 7 then 7 " + " else payload.x " + " end as y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"y">> := 1}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, + <<"topic">> => <<"t/a">>}})), + ok. + +t_sqlparse_case_when_2(_Config) -> + % switch clause + Sql = "select " + " case payload.x when 1 then 2 " + " when 2 then 3 " + " else 4 " + " end as y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"y">> := 2}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 3}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 2}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 4}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_case_when_3(_Config) -> + %% case-when clause + Sql = "select " + " case when payload.x < 0 then 0 " + " when payload.x > 7 then 7 " + " end as y " + "from \"t/#\" ", + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 5}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, + <<"topic">> => <<"t/a">>}})), + ok. + +%%------------------------------------------------------------------------------ +%% Test cases for array index +%%------------------------------------------------------------------------------ + +t_sqlparse_array_index_1(_Config) -> + %% index get + Sql = "select " + " json_decode(payload) as p, " + " p[1] as a " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"a">> := #{<<"x">> := 1}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"[{\"x\": 1}]">>, + <<"topic">> => <<"t/a">>}})), + ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, + <<"topic">> => <<"t/a">>}})), + %% index get without 'as' + Sql2 = "select " + " payload.x[2] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [3]}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,3,4]}, + <<"topic">> => <<"t/a">>}})), + %% index get without 'as' again + Sql3 = "select " + " payload.x[2].y " + "from \"t/#\" ", + ?assertMatch( {ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 3}]}}} + , emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, + <<"topic">> => <<"t/a">>}}) + ), + + %% index get with 'as' + Sql4 = "select " + " payload.x[2].y as b " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_index_2(_Config) -> + %% array get with negative index + Sql1 = "select " + " payload.x[-2].y as b " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, + <<"topic">> => <<"t/a">>}})), + %% array append to head or tail of a list: + Sql2 = "select " + " payload.x as b, " + " 1 as c[-0], " + " 2 as c[-0], " + " b as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := 0, <<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => #{<<"x">> => 0}, + <<"topic">> => <<"t/a">>}})), + %% construct an empty list: + Sql3 = "select " + " [] as c, " + " 1 as c[-0], " + " 2 as c[-0], " + " 0 as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql3, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})), + %% construct a list: + Sql4 = "select " + " [payload.a, \"topic\", 'c'] as c, " + " 1 as c[-0], " + " 2 as c[-0], " + " 0 as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,11,<<"t/a">>,<<"c">>,1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => #{<<"payload">> => <<"{\"a\":11}">>, + <<"topic">> => <<"t/a">> + }})). + +t_sqlparse_array_index_3(_Config) -> + %% array with json string payload: + Sql0 = "select " + "payload," + "payload.x[2].y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := [1,2]}, 3]}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})), + %% same as above but don't select payload: + Sql1 = "select " + "payload.x[2].y as b " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := [1,2]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})), + %% same as above but add 'as' clause: + Sql2 = "select " + "payload.x[2].y as b.c " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"b">> := #{<<"c">> := [1,2]}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_index_4(_Config) -> + %% array with json string payload: + Sql0 = "select " + "0 as payload.x[2].y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 0}]}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})), + %% array with json string payload, and also select payload.x: + Sql1 = "select " + "payload.x, " + "0 as payload.x[2].y " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := 0}, 3]}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_index_5(_Config) -> + Sql00 = "select " + " [1,2,3,4] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assert(lists:any(fun({_K, V}) -> + V =:= [1,2,3,4] + end, maps:to_list(Res00))). + +%%------------------------------------------------------------------------------ +%% Test cases for rule metadata +%%------------------------------------------------------------------------------ + +t_sqlparse_select_matadata_1(_Config) -> + %% array with json string payload: + Sql0 = "select " + "payload " + "from \"t/#\" ", + ?assertNotMatch({ok, #{<<"payload">> := <<"abc">>, metadata := _}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"abc">>, + <<"topic">> => <<"t/a">>}})), + Sql1 = "select " + "payload, metadata " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := <<"abc">>, <<"metadata">> := _}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"abc">>, + <<"topic">> => <<"t/a">>}})). + +%%------------------------------------------------------------------------------ +%% Test cases for array range +%%------------------------------------------------------------------------------ + +t_sqlparse_array_range_1(_Config) -> + %% get a range of list + Sql0 = "select " + " payload.a[1..4] as c " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,1,2,3]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql0, + <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, + <<"topic">> => <<"t/a">>}})), + %% get a range from non-list data + Sql02 = "select " + " payload.a[1..4] as c " + "from \"t/#\" ", + ?assertMatch({error, {select_and_transform_error, {error,{range_get,non_list_data},_}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql02, + <<"ctx">> => + #{<<"payload">> => <<"{\"x\":[0,1,2,3,4,5]}">>, + <<"topic">> => <<"t/a">>}})), + + %% construct a range: + Sql1 = "select " + " [1..4] as c, " + " 5 as c[-0], " + " 6 as c[-0], " + " 0 as c[0] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"c">> := [0,1,2,3,4,5,6]}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_array_range_2(_Config) -> + %% construct a range without 'as' + Sql00 = "select " + " [1..4] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assert(lists:any(fun({_K, V}) -> + V =:= [1,2,3,4] + end, maps:to_list(Res00))), + %% construct a range without 'as' + Sql01 = "select " + " a[2..4] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"a">> := [2,3,4]}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql01, + <<"ctx">> => #{<<"a">> => [1,2,3,4,5], + <<"topic">> => <<"t/a">>}})), + %% get a range of list without 'as' + Sql02 = "select " + " payload.a[1..4] " + "from \"t/#\" ", + ?assertMatch({ok, #{<<"payload">> := #{<<"a">> := [0,1,2,3]}}}, emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql02, + <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, + <<"topic">> => <<"t/a">>}})). + +%%------------------------------------------------------------------------------ +%% Test cases for boolean +%%------------------------------------------------------------------------------ + +t_sqlparse_true_false(_Config) -> + %% construct a range without 'as' + Sql00 = "select " + " true as a, false as b, " + " false as x.y, true as c[-0] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assertMatch(#{<<"a">> := true, <<"b">> := false, + <<"x">> := #{<<"y">> := false}, + <<"c">> := [true] + }, Res00). + +%%------------------------------------------------------------------------------ +%% Test cases for compare +%%------------------------------------------------------------------------------ + +-define(TEST_SQL(SQL), + emqx_rule_sqltester:test( + #{<<"rawsql">> => SQL, + <<"ctx">> => #{<<"payload">> => <<"{}">>, + <<"topic">> => <<"t/a">>}})). + +t_sqlparse_compare_undefined(_Config) -> + Sql00 = "select " + " * " + "from \"t/#\" " + "where dev != undefined ", + %% no match + ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00)), + + Sql01 = "select " + " 'd' as dev " + "from \"t/#\" " + "where dev != undefined ", + {ok, Res01} = ?TEST_SQL(Sql01), + %% pass + ?assertMatch(#{}, Res01), + + Sql02 = "select " + " * " + "from \"t/#\" " + "where dev != 'undefined' ", + {ok, Res02} = ?TEST_SQL(Sql02), + %% pass + ?assertMatch(#{}, Res02). + +t_sqlparse_compare_null_null(_Config) -> + %% test undefined == undefined + Sql00 = "select " + " a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := true + }, Res00), + + %% test undefined != undefined + Sql01 = "select " + " a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := false + }, Res01), + + %% test undefined > undefined + Sql02 = "select " + " a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test undefined < undefined + Sql03 = "select " + " a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test undefined <= undefined + Sql04 = "select " + " a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := true + }, Res04), + + %% test undefined >= undefined + Sql05 = "select " + " a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := true + }, Res05). + +t_sqlparse_compare_null_notnull(_Config) -> + %% test undefined == b + Sql00 = "select " + " 'b' as b, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := false + }, Res00), + + %% test undefined != b + Sql01 = "select " + " 'b' as b, a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + %% test undefined > b + Sql02 = "select " + " 'b' as b, a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test undefined < b + Sql03 = "select " + " 'b' as b, a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test undefined <= b + Sql04 = "select " + " 'b' as b, a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test undefined >= b + Sql05 = "select " + " 'b' as b, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05). + +t_sqlparse_compare_notnull_null(_Config) -> + %% test 'a' == undefined + Sql00 = "select " + " 'a' as a, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := false + }, Res00), + + %% test 'a' != undefined + Sql01 = "select " + " 'a' as a, a != b as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + %% test 'a' > undefined + Sql02 = "select " + " 'a' as a, a > b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := false + }, Res02), + + %% test 'a' < undefined + Sql03 = "select " + " 'a' as a, a < b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + %% test 'a' <= undefined + Sql04 = "select " + " 'a' as a, a <= b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test 'a' >= undefined + Sql05 = "select " + " 'a' as a, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05). + +t_sqlparse_compare(_Config) -> + Sql00 = "select " + " 'a' as a, 'a' as b, a = b as c " + "from \"t/#\" ", + {ok, Res00} = ?TEST_SQL(Sql00), + ?assertMatch(#{<<"c">> := true + }, Res00), + + Sql01 = "select " + " is_null(a) as c " + "from \"t/#\" ", + {ok, Res01} = ?TEST_SQL(Sql01), + ?assertMatch(#{<<"c">> := true + }, Res01), + + Sql02 = "select " + " 1 as a, 2 as b, a < b as c " + "from \"t/#\" ", + {ok, Res02} = ?TEST_SQL(Sql02), + ?assertMatch(#{<<"c">> := true + }, Res02), + + Sql03 = "select " + " 1 as a, 2 as b, a > b as c " + "from \"t/#\" ", + {ok, Res03} = ?TEST_SQL(Sql03), + ?assertMatch(#{<<"c">> := false + }, Res03), + + Sql04 = "select " + " 1 as a, 2 as b, a = b as c " + "from \"t/#\" ", + {ok, Res04} = ?TEST_SQL(Sql04), + ?assertMatch(#{<<"c">> := false + }, Res04), + + %% test 'a' >= undefined + Sql05 = "select " + " 1 as a, 2 as b, a >= b as c " + "from \"t/#\" ", + {ok, Res05} = ?TEST_SQL(Sql05), + ?assertMatch(#{<<"c">> := false + }, Res05), + + %% test 'a' >= undefined + Sql06 = "select " + " 1 as a, 2 as b, a <= b as c " + "from \"t/#\" ", + {ok, Res06} = ?TEST_SQL(Sql06), + ?assertMatch(#{<<"c">> := true + }, Res06). + + + +t_sqlparse_new_map(_Config) -> + %% construct a range without 'as' + Sql00 = "select " + " map_new() as a, map_new() as b, " + " map_new() as x.y, map_new() as c[-0] " + "from \"t/#\" ", + {ok, Res00} = + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql00, + <<"ctx">> => #{<<"payload">> => <<"">>, + <<"topic">> => <<"t/a">>}}), + ?assertMatch(#{<<"a">> := #{}, <<"b">> := #{}, + <<"x">> := #{<<"y">> := #{}}, + <<"c">> := [#{}] + }, Res00). + + +t_sqlparse_invalid_json(_Config) -> + Sql02 = "select " + " payload.a[1..4] as c " + "from \"t/#\" ", + ?assertMatch({error, {select_and_transform_error, {error,{decode_json_failed,_},_}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql02, + <<"ctx">> => + #{<<"payload">> => <<"{\"x\":[0,1,2,3,}">>, + <<"topic">> => <<"t/a">>}})), + + + Sql2 = "foreach payload.sensors " + "do item.cmd as msg_type " + "from \"t/#\" ", + ?assertMatch({error, {select_and_collect_error, {error,{decode_json_failed,_},_}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql2, + <<"ctx">> => + #{<<"payload">> => + <<"{\"sensors\": [{\"cmd\":\"1\"} {\"cmd\":}]}">>, + <<"topic">> => <<"t/a">>}})). From 267946c3794b8c251a056e9f4cb975dd04b1b14f Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Sep 2022 09:44:03 +0800 Subject: [PATCH 18/50] fix: rm unused module `emqx_rule_date` was added before [v4.3.15-rc1] PR: emqx#7894 commit: 8558a62ee291021e521dc2c1f6efa418fa76469d The change was released in [v4.3.15-rc1] [v4.3.15-rc2] And unused at [v4.3.15-rc3] PR: emqx#8044 commit: 4fc5cb2817c4f1e8225c048bc04f4972148d0df4 We just keep the module from [v4.3.15-rc1] to [v4.3.20] and remove this module in newer version --- apps/emqx_rule_engine/src/emqx_rule_date.erl | 248 ------------------- 1 file changed, 248 deletions(-) delete mode 100644 apps/emqx_rule_engine/src/emqx_rule_date.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_date.erl b/apps/emqx_rule_engine/src/emqx_rule_date.erl deleted file mode 100644 index fb9cad4c3..000000000 --- a/apps/emqx_rule_engine/src/emqx_rule_date.erl +++ /dev/null @@ -1,248 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_date). - --export([date/3, date/4, parse_date/4]). - --export([ - is_int_char/1, - is_symbol_char/1, - is_m_char/1 -]). - --record(result, { - %%year() - year = "1970" :: string(), - %%month() - month = "1" :: string(), - %%day() - day = "1" :: string(), - %%hour() - hour = "0" :: string(), - %%minute() %% epoch in millisecond precision - minute = "0" :: string(), - %%second() %% epoch in millisecond precision - second = "0" :: string(), - %%integer() %% zone maybe some value - zone = "+00:00" :: string() -}). - -%% -type time_unit() :: 'microsecond' -%% | 'millisecond' -%% | 'nanosecond' -%% | 'second'. -%% -type offset() :: [byte()] | (Time :: integer()). -date(TimeUnit, Offset, FormatString) -> - date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)). - -date(TimeUnit, Offset, FormatString, TimeEpoch) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - Res = lists:map( - fun(Expr) -> - eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr) - end, - R - ), - lists:concat(Res). - -parse_date(TimeUnit, Offset, FormatString, InputString) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - IsZ = fun(V) -> - case V of - {tag, $Z} -> true; - _ -> false - end - end, - R1 = lists:filter(IsZ, R), - IfFun = fun(Con, A, B) -> - case Con of - [] -> A; - _ -> B - end - end, - Res = parse_input(FormatString, InputString), - Str = - Res#result.year ++ "-" ++ - Res#result.month ++ "-" ++ - Res#result.day ++ "T" ++ - Res#result.hour ++ ":" ++ - Res#result.minute ++ ":" ++ - Res#result.second ++ - IfFun(R1, Offset, Res#result.zone), - calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]). - -mlist(R) -> - %% %H Shows hour in 24-hour format [15] - [ - {$H, R#result.hour}, - %% %M Displays minutes [00-59] - {$M, R#result.minute}, - %% %S Displays seconds [00-59] - {$S, R#result.second}, - %% %y Displays year YYYY [2021] - {$y, R#result.year}, - %% %m Displays the number of the month [01-12] - {$m, R#result.month}, - %% %d Displays the number of the month [01-12] - {$d, R#result.day}, - %% %Z Displays Time zone - {$Z, R#result.zone} - ]. - -rmap(Result) -> - maps:from_list(mlist(Result)). - -support_char() -> "HMSymdZ". - -create_tag(Head, []) -> - Head; -create_tag(Head, [Val1 | RVal]) -> - case Val1 of - [] -> - create_tag(Head ++ [{st, [$%]}], RVal); - [H | Other] -> - case lists:member(H, support_char()) of - true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal); - false -> create_tag(Head ++ [{st, [$% | Val1]}], RVal) - end - end. - -eval_tag(_, {st, Str}) -> - Str; -eval_tag(Map, {tag, Char}) -> - maps:get(Char, Map, "undefined"). - -%% make_time(TimeUnit, Offset) -> -%% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)). -make_time(TimeUnit, Offset, TimeEpoch) -> - Res = calendar:system_time_to_rfc3339(TimeEpoch, - [{unit, TimeUnit}, {offset, Offset}]), - [Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T, - H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = Res, - IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, - {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), - #result{ - year = [Y1, Y2, Y3, Y4] - , month = [Mon1, Mon2] - , day = [D1, D2] - , hour = [H1, H2] - , minute = [Min1, Min2] - , second = [S1, S2] ++ FractionStr - , zone = UtcOffset - }. - -is_int_char(C) -> - C >= $0 andalso C =< $9. -is_symbol_char(C) -> - C =:= $- orelse C =:= $+. -is_m_char(C) -> - C =:= $:. - -parse_char_with_fun(_, []) -> - error(null_input); -parse_char_with_fun(ValidFun, [C | Other]) -> - Res = - case erlang:is_function(ValidFun) of - true -> ValidFun(C); - false -> erlang:apply(emqx_rule_date, ValidFun, [C]) - end, - case Res of - true -> {C, Other}; - false -> error({unexpected, [C | Other]}) - end. -parse_string([], Input) -> - {[], Input}; -parse_string([C | Other], Input) -> - {C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input), - {Res, Input2} = parse_string(Other, Input1), - {[C1 | Res], Input2}. - -parse_times(0, _, Input) -> - {[], Input}; -parse_times(Times, Fun, Input) -> - {C1, Input1} = parse_char_with_fun(Fun, Input), - {Res, Input2} = parse_times((Times - 1), Fun, Input1), - {[C1 | Res], Input2}. - -parse_int_times(Times, Input) -> - parse_times(Times, is_int_char, Input). - -parse_fraction(Input) -> - IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, - lists:splitwith(IsFractionChar, Input). - -parse_second(Input) -> - {M, Input1} = parse_int_times(2, Input), - {M1, Input2} = parse_fraction(Input1), - {M ++ M1, Input2}. - -parse_zone(Input) -> - {S, Input1} = parse_char_with_fun(is_symbol_char, Input), - {M, Input2} = parse_int_times(2, Input1), - {C, Input3} = parse_char_with_fun(is_m_char, Input2), - {V, Input4} = parse_int_times(2, Input3), - {[S | M ++ [C | V]], Input4}. - -mlist1() -> - maps:from_list( - %% %H Shows hour in 24-hour format [15] - [ - {$H, fun(Input) -> parse_int_times(2, Input) end}, - %% %M Displays minutes [00-59] - {$M, fun(Input) -> parse_int_times(2, Input) end}, - %% %S Displays seconds [00-59] - {$S, fun(Input) -> parse_second(Input) end}, - %% %y Displays year YYYY [2021] - {$y, fun(Input) -> parse_int_times(4, Input) end}, - %% %m Displays the number of the month [01-12] - {$m, fun(Input) -> parse_int_times(2, Input) end}, - %% %d Displays the number of the month [01-12] - {$d, fun(Input) -> parse_int_times(2, Input) end}, - %% %Z Displays Time zone - {$Z, fun(Input) -> parse_zone(Input) end} - ] - ). - -update_result($H, Res, Str) -> Res#result{hour = Str}; -update_result($M, Res, Str) -> Res#result{minute = Str}; -update_result($S, Res, Str) -> Res#result{second = Str}; -update_result($y, Res, Str) -> Res#result{year = Str}; -update_result($m, Res, Str) -> Res#result{month = Str}; -update_result($d, Res, Str) -> Res#result{day = Str}; -update_result($Z, Res, Str) -> Res#result{zone = Str}. - -parse_tag(Res, {st, St}, InputString) -> - {_A, B} = parse_string(St, InputString), - {Res, B}; -parse_tag(Res, {tag, St}, InputString) -> - Fun = maps:get(St, mlist1()), - {A, B} = Fun(InputString), - NRes = update_result(St, Res, A), - {NRes, B}. - -parse_tags(Res, [], _) -> - Res; -parse_tags(Res, [Tag | Others], InputString) -> - {NRes, B} = parse_tag(Res, Tag, InputString), - parse_tags(NRes, Others, B). - -parse_input(FormatString, InputString) -> - [Head | Other] = string:split(FormatString, "%", all), - R = create_tag([{st, Head}], Other), - parse_tags(#result{}, R, InputString). From d8a022fb4579f505569fc141f07b8f3f4ebdeeae Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 28 Sep 2022 14:36:02 +0800 Subject: [PATCH 19/50] fix(test): unstopped test client --- .../test/emqx_rule_engine_SUITE.erl | 14 +++++++++----- apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 42816e869..c6f5057f1 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -238,8 +238,12 @@ end_per_testcase(t_events, Config) -> ok = emqx_rule_registry:remove_rule(?config(hook_points_rules, Config)), ok = emqx_rule_registry:remove_action('hook-metrics-action'); end_per_testcase(Test, Config) - when Test =:= t_sqlselect_multi_actoins_1, - Test =:= t_sqlselect_multi_actoins_2 + when Test =:= t_sqlselect_multi_actoins_1 + ;Test =:= t_sqlselect_multi_actoins_1_1 + ;Test =:= t_sqlselect_multi_actoins_2 + ;Test =:= t_sqlselect_multi_actoins_3 + ;Test =:= t_sqlselect_multi_actoins_3_1 + ;Test =:= t_sqlselect_multi_actoins_4 -> emqtt:stop(?config(subclient, Config)), emqtt:stop(?config(connclient, Config)), @@ -1218,8 +1222,8 @@ t_match_atom_and_binary(_Config) -> {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), ct:sleep(100), - {ok, Client2} = emqtt:start_link([{username, <<"emqx2">>}]), - {ok, _} = emqtt:connect(Client2), + {ok, Client1} = emqtt:start_link([{username, <<"emqx2">>}]), + {ok, _} = emqtt:connect(Client1), receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(<<"t2">>, T), <<"user:", ConnAt/binary>> = Payload, @@ -1228,7 +1232,7 @@ t_match_atom_and_binary(_Config) -> ct:fail(wait_for_t2) end, - emqtt:stop(Client), + emqtt:stop(Client), emqtt:stop(Client1), emqx_rule_registry:remove_rule(TopicRule). t_metrics(_Config) -> diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index d93b747c0..5fd966d6c 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -474,7 +474,7 @@ t_sqlselect_3(_Config) -> ok end, - emqtt:stop(Client), + emqtt:stop(Client), emqtt:stop(Client1), emqx_rule_registry:remove_rule(TopicRule). %%------------------------------------------------------------------------------ From 0a5a0867e43d1790ac9a7f1d8668f35e07f5e224 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 29 Sep 2022 14:05:40 +0800 Subject: [PATCH 20/50] fix(rule_func): refine `+` error info when type implicit conversion --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 18e61967e..a4791a94d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -338,7 +338,13 @@ null() -> %% concat 2 strings '+'(X, Y) when is_binary(X), is_binary(Y) -> - concat(X, Y). + concat(X, Y); + +%% unsupported type implicit conversion +'+'(X, Y) + when (is_number(X) andalso is_binary(Y)) orelse + (is_binary(X) andalso is_number(Y)) -> + error(unsupported_type_implicit_conversion). '-'(X, Y) when is_number(X), is_number(Y) -> X - Y. From 32376c7cf962da1b4f6fe3f06f67c87523d872db Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 29 Sep 2022 16:21:40 +0800 Subject: [PATCH 21/50] fix(rule_func): refine num funcs error info type unsupported --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index a4791a94d..da345665b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -332,34 +332,46 @@ null() -> %% Arithmetic Funcs %%------------------------------------------------------------------------------ +-define(OPERATOR_TYPE_ERROR, unsupported_function_operator_type). + %% plus 2 numbers '+'(X, Y) when is_number(X), is_number(Y) -> X + Y; - %% concat 2 strings '+'(X, Y) when is_binary(X), is_binary(Y) -> concat(X, Y); - %% unsupported type implicit conversion '+'(X, Y) when (is_number(X) andalso is_binary(Y)) orelse (is_binary(X) andalso is_number(Y)) -> - error(unsupported_type_implicit_conversion). + error(unsupported_type_implicit_conversion); +'+'(_, _) -> + error(?OPERATOR_TYPE_ERROR). '-'(X, Y) when is_number(X), is_number(Y) -> - X - Y. + X - Y; +'-'(_, _) -> + error(?OPERATOR_TYPE_ERROR). '*'(X, Y) when is_number(X), is_number(Y) -> - X * Y. + X * Y; +'*'(_, _) -> + error(?OPERATOR_TYPE_ERROR). '/'(X, Y) when is_number(X), is_number(Y) -> - X / Y. + X / Y; +'/'(_, _) -> + error(?OPERATOR_TYPE_ERROR). 'div'(X, Y) when is_integer(X), is_integer(Y) -> - X div Y. + X div Y; +'div'(_, _) -> + error(?OPERATOR_TYPE_ERROR). mod(X, Y) when is_integer(X), is_integer(Y) -> - X rem Y. + X rem Y; +mod(_, _) -> + error(?OPERATOR_TYPE_ERROR). eq(X, Y) -> X == Y. From 27e19da06682344da2126ae258dd2d40217e40c3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 29 Sep 2022 16:23:13 +0800 Subject: [PATCH 22/50] test(rulesql): num funcs type cases --- .../test/emqx_rulesql_SUITE.erl | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index 5fd966d6c..33de9b7dd 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -236,8 +236,27 @@ t_sqlselect_00(_Config) -> #{<<"rawsql">> => Sql3, <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). + <<"topic">> => <<"t/a">>}})), + Sql4 = "select payload.msg1 + payload.msg2 as msg " + "from \"t/#\" ", + ?assertMatch({ok,#{<<"msg">> := <<"hello world">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql4, + <<"ctx">> => + #{<<"payload">> => <<"{\"msg1\": \"hello\", \"msg2\": \" world\"}">>, + <<"topic">> => <<"t/1">>}})), + + Sql5 = "select payload.msg1 + payload.msg2 as msg " + "from \"t/#\" ", + ?assertMatch({error, {select_and_transform_error, {error, unsupported_type_implicit_conversion, _ST}}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql5, + <<"ctx">> => + #{<<"payload">> => <<"{\"msg1\": \"hello\", \"msg2\": 1}">>, + <<"topic">> => <<"t/1">>}})). + +%% Verify SELECT with and with 'WHERE' t_sqlselect_01(_Config) -> ok = emqx_rule_engine:load_providers(), TopicRule1 = create_simple_repub_rule( @@ -312,7 +331,50 @@ t_sqlselect_02(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule1). +t_sqlselect_03(_Config) -> + %% Verify SELECT with and with 'WHERE' and `+` `=` and `or` condition in 'WHERE' clause + ok = emqx_rule_engine:load_providers(), + TopicRule1 = create_simple_repub_rule( + <<"t2">>, + "SELECT * " + "FROM \"t3/#\", \"t1\" " + "WHERE payload.x + payload.y = 2 or payload.x + payload.y = \"11\""), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1, \"y\":1}">>, 0), + ct:sleep(100), + receive {publish, #{topic := T1, payload := Payload0}} -> + ?assertEqual(<<"t2">>, T1), + ?assertEqual(<<"{\"x\":1, \"y\":1}">>, Payload0) + after 1000 -> + ct:fail(wait_for_t2) + end, + receive {publish, #{topic := T2, payload := Payload1}} -> + ?assertEqual(<<"t2">>, T2), + ?assertEqual(<<"{\"x\":\"1\", \"y\":\"1\"}">>, Payload1) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1, \"y\":2}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := Payload2}} -> + ct:fail({unexpected_t2, Payload2}) + after 1000 -> + ok + end, + + emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1, \"y\":1}">>, 0), + receive {publish, #{topic := T3, payload := Payload3}} -> + ?assertEqual(<<"t2">>, T3), + ?assertEqual(<<"{\"x\":1, \"y\":1}">>, Payload3) + after 1000 -> + ct:fail(wait_for_t2) + end, + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule1). t_sqlselect_1(_Config) -> ok = emqx_rule_engine:load_providers(), From 5047211950f223ad9972f364ccc2b5467472eaf7 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Sun, 9 Oct 2022 14:20:48 +0800 Subject: [PATCH 23/50] test: rulesql select fields and select from event message --- .../test/emqx_rulesql_SUITE.erl | 297 +++++++++++------- 1 file changed, 188 insertions(+), 109 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index 33de9b7dd..f4f329e87 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("emqx_rule_engine/include/rule_engine.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -51,15 +52,16 @@ groups() -> , t_sqlselect_02 , t_sqlselect_1 , t_sqlselect_2 - , t_sqlselect_2_1 - , t_sqlselect_2_2 - , t_sqlselect_2_3 - , t_sqlselect_3 ]}, {rulesql_select_events, [], - [ t_sqlparse_event_1 - , t_sqlparse_event_2 - , t_sqlparse_event_3 + [ t_sqlparse_event_client_connected_01 + , t_sqlparse_event_client_connected_02 + , t_sqlparse_event_client_disconnected + , t_sqlparse_event_session_subscribed + , t_sqlparse_event_session_unsubscribed + , t_sqlparse_event_message_delivered + , t_sqlparse_event_message_acked + , t_sqlparse_event_message_dropped ]}, {rulesql_select_metadata, [], [ t_sqlparse_select_matadata_1 @@ -159,16 +161,35 @@ end_per_testcase(_TestCase, _Config) -> t_sqlselect_0(_Config) -> %% Verify SELECT with and without 'AS' - Sql = "select * " - "from \"t/#\" " - "where payload.cmd.info = 'tt'", - ?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => - <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), + + Sql1 = "SELECT * " + "FROM \"t/#\" " + "WHERE payload.cmd.info = 'tt'", + %% all available fields by `select` from message publish, selecte other key will be `undefined` + Topic = <<"t/a">>, + Payload = <<"{\"cmd\": {\"info\":\"tt\"}}">>, + {ok, #{username := <<"u_emqx">>, + topic := Topic, + timestamp := TimeStamp, + qos := 1, + publish_received_at := TimeStamp, + peerhost := <<"127.0.0.1">>, + payload := Payload, + node := 'test@127.0.0.1', + metadata := #{rule_id := TestRuleId}, + id := MsgId, + flags := #{sys := true, event := true}, + clientid := <<"c_emqx">> + } + } = emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql1, + <<"ctx">> => + #{<<"payload">> => Payload, + <<"topic">> => Topic}}), + ?assert(is_binary(TestRuleId)), + ?assert(is_binary(MsgId)), + ?assert(is_integer(TimeStamp)), + Sql2 = "select payload.cmd as cmd " "from \"t/#\" " "where cmd.info = 'tt'", @@ -179,6 +200,7 @@ t_sqlselect_0(_Config) -> #{<<"payload">> => <<"{\"cmd\": {\"info\":\"tt\"}}">>, <<"topic">> => <<"t/a">>}})), + Sql3 = "select payload.cmd as cmd, cmd.info as info " "from \"t/#\" " "where cmd.info = 'tt' and info = 'tt'", @@ -205,14 +227,15 @@ t_sqlselect_0(_Config) -> t_sqlselect_00(_Config) -> %% Verify plus/subtract and unary_add_or_subtract - Sql = "select 1-1 as a " + Sql0 = "select 1 - 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, + #{<<"rawsql">> => Sql0, <<"ctx">> => #{<<"payload">> => <<"">>, <<"topic">> => <<"t/a">>}})), + Sql1 = "select -1 + 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 0}}, @@ -221,6 +244,7 @@ t_sqlselect_00(_Config) -> <<"ctx">> => #{<<"payload">> => <<"">>, <<"topic">> => <<"t/a">>}})), + Sql2 = "select 1 + 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 2}}, @@ -229,6 +253,7 @@ t_sqlselect_00(_Config) -> <<"ctx">> => #{<<"payload">> => <<"">>, <<"topic">> => <<"t/a">>}})), + Sql3 = "select +1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 1}}, @@ -431,58 +456,93 @@ t_sqlselect_2(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_2_1(_Config) -> +%%------------------------------------------------------------------------------ +%% Test cases for events +%%------------------------------------------------------------------------------ + +%% FROM $events/client_connected +t_sqlparse_event_client_connected_01(_Config) -> + Sql = "select *" + "from \"$events/client_connected\" ", + + %% all available fields by `select` from message publish, selecte other key will be `undefined` + {ok, #{clientid := <<"c_emqx">>, + username := <<"u_emqx">>, + timestamp := TimeStamp, + connected_at := TimeStamp, + peername := <<"127.0.0.1:12345">>, + metadata := #{rule_id := RuleId}, + %% default value + node := 'test@127.0.0.1', + sockname := <<"0.0.0.0:1883">>, + proto_name := <<"MQTT">>, + proto_ver := 5, + mountpoint := undefined, + keepalive := 60, + is_bridge := false, + expiry_interval := 3600, + event := 'client.connected', + clean_start := true + } + } = emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"clientid">> => <<"c_emqx">>, <<"peername">> => <<"127.0.0.1:12345">>, <<"username">> => <<"u_emqx">>}}), + ?assert(is_binary(RuleId)), + ?assert(is_integer(TimeStamp)). + +t_sqlparse_event_client_connected_02(_Config) -> ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg dropped + %% republish the client.connected msg TopicRule = create_simple_repub_rule( - <<"t2">>, + <<"repub/to/connected">>, "SELECT * " - "FROM \"$events/message_dropped\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + "FROM \"$events/client_connected\" " + "WHERE username = 'emqx1'", + <<"{clientid: ${clientid}}">>), + {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), {ok, _} = emqtt:connect(Client), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_nothing = Fun(), + {ok, _, _} = emqtt:subscribe(Client, <<"repub/to/connected">>, 0), + ct:sleep(200), + {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), + {ok, _} = emqtt:connect(Client1), + receive {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(<<"repub/to/connected">>, T), + ?assertEqual(<<"{clientid: c_emqx1}">>, Payload) + after 1000 -> + ct:fail(wait_for_t2) + end, - %% it should not keep republishing "t2" - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - received_nothing = Fun(), + emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), + receive {publish, #{topic := <<"t2">>, payload := _}} -> + ct:fail(unexpected_t2) + after 1000 -> + ok + end, - emqtt:stop(Client), + emqtt:stop(Client), emqtt:stop(Client1), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_2_2(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg acked - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_acked\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), +%% FROM $events/client_disconnected +t_sqlparse_event_client_disconnected(_Config) -> + %% TODO + ok. - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). +%% FROM $events/session_subscribed +t_sqlparse_event_session_subscribed(_Config) -> + Sql = "select topic as tp " + "from \"$events/session_subscribed\" ", + ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, + emqx_rule_sqltester:test( + #{<<"rawsql">> => Sql, + <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). -t_sqlselect_2_3(_Config) -> +%% FROM $events/session_unsubscribed +t_sqlparse_event_session_unsubscribed(_Config) -> + %% TODO + ok. + +%% FROM $events/message_delivered +t_sqlparse_event_message_delivered(_Config) -> ok = emqx_rule_engine:load_providers(), %% recursively republish to t2, if the msg delivered TopicRule = create_simple_repub_rule( @@ -507,65 +567,84 @@ t_sqlselect_2_3(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_3(_Config) -> +%% FROM $events/message_acked +t_sqlparse_event_message_acked(_Config) -> ok = emqx_rule_engine:load_providers(), - %% republish the client.connected msg + %% republish to `repub/if/acked`, if the msg acked TopicRule = create_simple_repub_rule( - <<"t2">>, + <<"repub/if/acked">>, "SELECT * " - "FROM \"$events/client_connected\" " - "WHERE username = 'emqx1'", - <<"clientid=${clientid}">>), - {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), + "FROM \"$events/message_acked\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - ct:sleep(200), - {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), - {ok, _} = emqtt:connect(Client1), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, + {ok, _, _} = emqtt:subscribe(Client, <<"repub/if/acked">>, ?QOS_1), - emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":1}">>, 0), - receive {publish, #{topic := <<"t2">>, payload := _}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, + Fun = fun() -> + receive {publish, #{topic := <<"repub/if/acked">>, payload := _}} -> + received_acked + after 500 -> + received_nothing + end + end, - emqtt:stop(Client), emqtt:stop(Client1), + %% sub with max qos1 to generate ack packet + {ok, _, _} = emqtt:subscribe(Client, <<"any/topic">>, ?QOS_1), + + Payload = <<"{\"x\":1,\"y\":144}">>, + + %% even sub with qos1, but publish with qos0, no ack + emqtt:publish(Client, <<"any/topic">>, Payload, ?QOS_0), + received_nothing = Fun(), + + %% sub and pub both are qos1, acked + emqtt:publish(Client, <<"any/topic">>, Payload, ?QOS_1), + received_acked = Fun(), + received_nothing = Fun(), + + %% pub with qos2 but subscribed with qos1, acked + emqtt:publish(Client, <<"any/topic">>, Payload, ?QOS_2), + received_acked = Fun(), + received_nothing = Fun(), + + emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -%%------------------------------------------------------------------------------ -%% Test cases for events -%%------------------------------------------------------------------------------ +%% FROM $events/message_dropped +t_sqlparse_event_message_dropped(_Config) -> + ok = emqx_rule_engine:load_providers(), + %% republish to `repub/if/dropped`, if any msg dropped + TopicRule = create_simple_repub_rule( + <<"repub/if/dropped">>, + "SELECT * " + "FROM \"$events/message_dropped\" "), + {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), + {ok, _} = emqtt:connect(Client), -t_sqlparse_event_1(_Config) -> - Sql = "select topic as tp " - "from \"$events/session_subscribed\" ", - ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). + %% this message will be dropped and then repub to `repub/if/dropped` + emqtt:publish(Client, <<"any/topic/1">>, <<"{\"x\":1,\"y\":144}">>, 0), -t_sqlparse_event_2(_Config) -> - Sql = "select clientid " - "from \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>}})). + Fun_t2 = fun() -> + receive {publish, #{topic := <<"repub/if/dropped">>, payload := _}} -> + received_repub + after 500 -> + received_nothing + end + end, -t_sqlparse_event_3(_Config) -> - Sql = "select clientid, topic as tp " - "from \"t/tt\", \"$events/client_connected\" ", - ?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, - emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})). + %% No client subscribed `any/topic`, triggered repub rule. + %% But havn't sub `repub/if/dropped`, so the repub message will also be dropped with recursively republish. + received_nothing = Fun_t2(), + + {ok, _, _} = emqtt:subscribe(Client, <<"repub/if/dropped">>, 0), + + %% this message will be dropped and then repub to `repub/to/t` + emqtt:publish(Client, <<"any/topic/2">>, <<"{\"x\":1,\"y\":144}">>, 0), + + %% received subscribed `repub/to/t` + received_repub = Fun_t2(), + + emqtt:stop(Client), + emqx_rule_registry:remove_rule(TopicRule). %%------------------------------------------------------------------------------ %% Test cases for `foreach` From 3e8c070b59ecac07ecf5f0464bb73b73c1808979 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Oct 2022 12:35:11 +0800 Subject: [PATCH 24/50] Revert "fix: rm unused module" This reverts commit 267946c3794b8c251a056e9f4cb975dd04b1b14f. Revert it temporary, we need fix `scripts/update-appup.sh` later. It only compares the current release with its predecessor. --- apps/emqx_rule_engine/src/emqx_rule_date.erl | 248 +++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 apps/emqx_rule_engine/src/emqx_rule_date.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_date.erl b/apps/emqx_rule_engine/src/emqx_rule_date.erl new file mode 100644 index 000000000..fb9cad4c3 --- /dev/null +++ b/apps/emqx_rule_engine/src/emqx_rule_date.erl @@ -0,0 +1,248 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rule_date). + +-export([date/3, date/4, parse_date/4]). + +-export([ + is_int_char/1, + is_symbol_char/1, + is_m_char/1 +]). + +-record(result, { + %%year() + year = "1970" :: string(), + %%month() + month = "1" :: string(), + %%day() + day = "1" :: string(), + %%hour() + hour = "0" :: string(), + %%minute() %% epoch in millisecond precision + minute = "0" :: string(), + %%second() %% epoch in millisecond precision + second = "0" :: string(), + %%integer() %% zone maybe some value + zone = "+00:00" :: string() +}). + +%% -type time_unit() :: 'microsecond' +%% | 'millisecond' +%% | 'nanosecond' +%% | 'second'. +%% -type offset() :: [byte()] | (Time :: integer()). +date(TimeUnit, Offset, FormatString) -> + date(TimeUnit, Offset, FormatString, erlang:system_time(TimeUnit)). + +date(TimeUnit, Offset, FormatString, TimeEpoch) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + Res = lists:map( + fun(Expr) -> + eval_tag(rmap(make_time(TimeUnit, Offset, TimeEpoch)), Expr) + end, + R + ), + lists:concat(Res). + +parse_date(TimeUnit, Offset, FormatString, InputString) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + IsZ = fun(V) -> + case V of + {tag, $Z} -> true; + _ -> false + end + end, + R1 = lists:filter(IsZ, R), + IfFun = fun(Con, A, B) -> + case Con of + [] -> A; + _ -> B + end + end, + Res = parse_input(FormatString, InputString), + Str = + Res#result.year ++ "-" ++ + Res#result.month ++ "-" ++ + Res#result.day ++ "T" ++ + Res#result.hour ++ ":" ++ + Res#result.minute ++ ":" ++ + Res#result.second ++ + IfFun(R1, Offset, Res#result.zone), + calendar:rfc3339_to_system_time(Str, [{unit, TimeUnit}]). + +mlist(R) -> + %% %H Shows hour in 24-hour format [15] + [ + {$H, R#result.hour}, + %% %M Displays minutes [00-59] + {$M, R#result.minute}, + %% %S Displays seconds [00-59] + {$S, R#result.second}, + %% %y Displays year YYYY [2021] + {$y, R#result.year}, + %% %m Displays the number of the month [01-12] + {$m, R#result.month}, + %% %d Displays the number of the month [01-12] + {$d, R#result.day}, + %% %Z Displays Time zone + {$Z, R#result.zone} + ]. + +rmap(Result) -> + maps:from_list(mlist(Result)). + +support_char() -> "HMSymdZ". + +create_tag(Head, []) -> + Head; +create_tag(Head, [Val1 | RVal]) -> + case Val1 of + [] -> + create_tag(Head ++ [{st, [$%]}], RVal); + [H | Other] -> + case lists:member(H, support_char()) of + true -> create_tag(Head ++ [{tag, H}, {st, Other}], RVal); + false -> create_tag(Head ++ [{st, [$% | Val1]}], RVal) + end + end. + +eval_tag(_, {st, Str}) -> + Str; +eval_tag(Map, {tag, Char}) -> + maps:get(Char, Map, "undefined"). + +%% make_time(TimeUnit, Offset) -> +%% make_time(TimeUnit, Offset, erlang:system_time(TimeUnit)). +make_time(TimeUnit, Offset, TimeEpoch) -> + Res = calendar:system_time_to_rfc3339(TimeEpoch, + [{unit, TimeUnit}, {offset, Offset}]), + [Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T, + H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = Res, + IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, + {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), + #result{ + year = [Y1, Y2, Y3, Y4] + , month = [Mon1, Mon2] + , day = [D1, D2] + , hour = [H1, H2] + , minute = [Min1, Min2] + , second = [S1, S2] ++ FractionStr + , zone = UtcOffset + }. + +is_int_char(C) -> + C >= $0 andalso C =< $9. +is_symbol_char(C) -> + C =:= $- orelse C =:= $+. +is_m_char(C) -> + C =:= $:. + +parse_char_with_fun(_, []) -> + error(null_input); +parse_char_with_fun(ValidFun, [C | Other]) -> + Res = + case erlang:is_function(ValidFun) of + true -> ValidFun(C); + false -> erlang:apply(emqx_rule_date, ValidFun, [C]) + end, + case Res of + true -> {C, Other}; + false -> error({unexpected, [C | Other]}) + end. +parse_string([], Input) -> + {[], Input}; +parse_string([C | Other], Input) -> + {C1, Input1} = parse_char_with_fun(fun(V) -> V =:= C end, Input), + {Res, Input2} = parse_string(Other, Input1), + {[C1 | Res], Input2}. + +parse_times(0, _, Input) -> + {[], Input}; +parse_times(Times, Fun, Input) -> + {C1, Input1} = parse_char_with_fun(Fun, Input), + {Res, Input2} = parse_times((Times - 1), Fun, Input1), + {[C1 | Res], Input2}. + +parse_int_times(Times, Input) -> + parse_times(Times, is_int_char, Input). + +parse_fraction(Input) -> + IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, + lists:splitwith(IsFractionChar, Input). + +parse_second(Input) -> + {M, Input1} = parse_int_times(2, Input), + {M1, Input2} = parse_fraction(Input1), + {M ++ M1, Input2}. + +parse_zone(Input) -> + {S, Input1} = parse_char_with_fun(is_symbol_char, Input), + {M, Input2} = parse_int_times(2, Input1), + {C, Input3} = parse_char_with_fun(is_m_char, Input2), + {V, Input4} = parse_int_times(2, Input3), + {[S | M ++ [C | V]], Input4}. + +mlist1() -> + maps:from_list( + %% %H Shows hour in 24-hour format [15] + [ + {$H, fun(Input) -> parse_int_times(2, Input) end}, + %% %M Displays minutes [00-59] + {$M, fun(Input) -> parse_int_times(2, Input) end}, + %% %S Displays seconds [00-59] + {$S, fun(Input) -> parse_second(Input) end}, + %% %y Displays year YYYY [2021] + {$y, fun(Input) -> parse_int_times(4, Input) end}, + %% %m Displays the number of the month [01-12] + {$m, fun(Input) -> parse_int_times(2, Input) end}, + %% %d Displays the number of the month [01-12] + {$d, fun(Input) -> parse_int_times(2, Input) end}, + %% %Z Displays Time zone + {$Z, fun(Input) -> parse_zone(Input) end} + ] + ). + +update_result($H, Res, Str) -> Res#result{hour = Str}; +update_result($M, Res, Str) -> Res#result{minute = Str}; +update_result($S, Res, Str) -> Res#result{second = Str}; +update_result($y, Res, Str) -> Res#result{year = Str}; +update_result($m, Res, Str) -> Res#result{month = Str}; +update_result($d, Res, Str) -> Res#result{day = Str}; +update_result($Z, Res, Str) -> Res#result{zone = Str}. + +parse_tag(Res, {st, St}, InputString) -> + {_A, B} = parse_string(St, InputString), + {Res, B}; +parse_tag(Res, {tag, St}, InputString) -> + Fun = maps:get(St, mlist1()), + {A, B} = Fun(InputString), + NRes = update_result(St, Res, A), + {NRes, B}. + +parse_tags(Res, [], _) -> + Res; +parse_tags(Res, [Tag | Others], InputString) -> + {NRes, B} = parse_tag(Res, Tag, InputString), + parse_tags(NRes, Others, B). + +parse_input(FormatString, InputString) -> + [Head | Other] = string:split(FormatString, "%", all), + R = create_tag([{st, Head}], Other), + parse_tags(#result{}, R, InputString). From 7f92c29ada1e235b3b45a1af2e8022fbd0ae7130 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 27 Sep 2022 10:03:20 +0800 Subject: [PATCH 25/50] chore: bump appup.src for `emqx_rule_funcs.erl` --- .../src/emqx_rule_engine.appup.src | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index b46be85c7..430d1f7e3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,20 +2,23 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.15", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -24,7 +27,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -33,7 +37,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -43,7 +48,8 @@ {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -206,20 +212,23 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.15", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -228,7 +237,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -237,7 +247,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -247,7 +258,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, From 73d72eaccea0500aabc6bdcb68e76137e9bb11ac Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 11 Oct 2022 12:48:35 +0200 Subject: [PATCH 26/50] fix: demote rate limit log level to notice --- CHANGES-4.3.md | 1 + apps/emqx_exproto/src/emqx_exproto_conn.erl | 2 +- apps/emqx_stomp/src/emqx_stomp_connection.erl | 2 +- src/emqx_connection.erl | 2 +- src/emqx_ws_connection.erl | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index f136b7835..23899cb25 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -15,6 +15,7 @@ File format: ### Bug fixes - Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic. [#9024](https://github.com/emqx/emqx/pull/9024) +- "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134) ## v4.3.21 diff --git a/apps/emqx_exproto/src/emqx_exproto_conn.erl b/apps/emqx_exproto/src/emqx_exproto_conn.erl index 7c6eb6cb3..51abe86e9 100644 --- a/apps/emqx_exproto/src/emqx_exproto_conn.erl +++ b/apps/emqx_exproto/src/emqx_exproto_conn.erl @@ -643,7 +643,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, diff --git a/apps/emqx_stomp/src/emqx_stomp_connection.erl b/apps/emqx_stomp/src/emqx_stomp_connection.erl index e58b242e5..7177155fd 100644 --- a/apps/emqx_stomp/src/emqx_stomp_connection.erl +++ b/apps/emqx_stomp/src/emqx_stomp_connection.erl @@ -469,7 +469,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 7c9cfef0a..3cfcd8cdc 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -756,7 +756,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index a0c999ea9..0a29c1ee2 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -499,7 +499,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?LOG(notice, "Pause ~pms due to rate limit", [Time]), TRef = start_timer(Time, limit_timeout), NState = State#state{sockstate = blocked, limiter = Limiter1, From 53bc2d9d584001a1cdd27bb8f5c6f862c1538130 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 12 Oct 2022 14:53:45 +0300 Subject: [PATCH 27/50] fix(jwt): restore legacy emqx_auth_jwt hook interface --- CHANGES-4.3.md | 1 + apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 8 ++++++-- apps/emqx_auth_jwt/src/emqx_auth_jwt.erl | 5 +++++ apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl | 16 ++++++++++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 23899cb25..2290147da 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -16,6 +16,7 @@ File format: - Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic. [#9024](https://github.com/emqx/emqx/pull/9024) - "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134) +- Restore legacy `emqx_auth_jwt` interface to keep hooks working after relup. [##9144](https://github.com/emqx/emqx/pull/9144) ## v4.3.21 diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index 6f07c4449..bea59cbbb 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.6",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + [{"4.3.6", + [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, @@ -13,7 +15,9 @@ {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [{"4.3.6",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + [ {"4.3.6", + [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl index 26fd34365..0b3f17b04 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl @@ -23,6 +23,7 @@ -logger_header("[JWT]"). -export([ check_auth/3 + , check/3 , check_acl/5 , description/0 ]). @@ -33,6 +34,10 @@ %% Authentication callbacks %%-------------------------------------------------------------------- +%% for compatibility with old versions +check(ClientInfo, AuthResult, State) -> + ?MODULE:check_auth(ClientInfo, AuthResult, State). + check_auth(ClientInfo, AuthResult, #{from := From, checklists := Checklists}) -> case maps:find(From, ClientInfo) of error -> diff --git a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl index a0be3768f..62f753904 100644 --- a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl @@ -472,3 +472,19 @@ t_check_jwt_acl_no_exp(_Config) -> emqtt:subscribe(C, <<"a/b">>, 0)), ok = emqtt:disconnect(C). + +t_check_compatibility(init, _Config) -> ok. +t_check_compatibility(_Config) -> + + %% We literary want emqx_auth_jwt:check call emqx_auth_jwt:check_auth, so check with meck + + ok = meck:new(emqx_auth_jwt, [passthrough, no_history]), + ok = meck:expect(emqx_auth_jwt, check_auth, fun(a, b, c) -> ok end), + + ?assertEqual( + ok, + emqx_auth_jwt:check(a, b, c) + ), + + meck:validate(emqx_auth_jwt), + meck:unload(emqx_auth_jwt). From 81b1ea8c559bb9adc6c85126bad05d7bd0895368 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 14 Oct 2022 21:23:14 +0200 Subject: [PATCH 28/50] docs: refine change log text --- CHANGES-4.3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index f01b6078b..10f5078c9 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -76,7 +76,7 @@ File format: subscriber from another node in the cluster. Fixed in [#9122](https://github.com/emqx/emqx/pull/9122) -- Fix cannot reset metrics for fallback actions. [#9125](https://github.com/emqx/emqx/pull/9125) +- Fix rule engine fallback actions metrics reset. [#9125](https://github.com/emqx/emqx/pull/9125) ## v4.3.20 From 86da0f548e12a1486de9d34d783192414348fc7d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 15 Oct 2022 14:57:31 +0200 Subject: [PATCH 29/50] docs: refine v43 changelogs --- CHANGES-4.3.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 10f5078c9..a12c279f8 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -39,22 +39,24 @@ File format: Prior to this enhancement, one would have to set `broker.shared_dispatch_ack_enabled` to true to prevent sessions from buffering messages, however this acknowledgement comes with a cost. +- Prior to this fix, some of the time stamps were taken from the `os` module (system call), + while majority of other places are using `erlang` module (from Erlang virtual machine). + This inconsistent behaviour has caused some trouble for the Delayed Publish feature when OS time changes. + Now all time stamps are from `erlang` module. [#8908](https://github.com/emqx/emqx/pull/8908) ### Bug fixes - Fix HTTP client library to handle SSL socket passive signal. [#9145](https://github.com/emqx/emqx/pull/9145) -- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) - - Hide redis password in error logs [#9071](https://github.com/emqx/emqx/pull/9071) In this change, it also included more changes in redis client: - - Improve redis connection error logging [eredis:19](https://github.com/emqx/eredis/pull/19). + - Improve redis connection error logging [eredis #19](https://github.com/emqx/eredis/pull/19). Also added support for eredis to accept an anonymous function as password instead of passing around plaintext args which may get dumpped to crash logs (hard to predict where). This change also added `format_status` callback for `gen_server` states which hold plaintext password so the process termination log and `sys:get_status` will print '******' instead of the password to console. - - Avoid pool name clashing [eredis_cluster#22](https://github.com/emqx/eredis_cluster/pull/22) + - Avoid pool name clashing [eredis_cluster #22](https://github.com/emqx/eredis_cluster/pull/22) Same `format_status` callback is added here too for `gen_server`s which hold password in their state. From 458b9b2e4bf1458ebbacf7848ce3afd017938377 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 15 Oct 2022 14:59:48 +0200 Subject: [PATCH 30/50] chore: prepare new release version 4.3.22-alpha.1 --- deploy/charts/emqx/Chart.yaml | 4 ++-- include/emqx_release.hrl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 4569834df..28cafe9b3 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -13,8 +13,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 4.3.21 +version: 4.3.22 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 4.3.21 +appVersion: 4.3.22 diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 43c74b248..e7d50d7fc 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.21"}). +-define(EMQX_RELEASE, {opensource, "4.3.22-alpha.1"}). -else. From 20e0d140265af9b3b13b4daccceb9804ffb9e413 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 15 Oct 2022 15:01:57 +0200 Subject: [PATCH 31/50] chore: bump app versions --- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- src/emqx.app.src | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 05d4d3754..e0550fa78 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.3.7"}, % strict semver, bump manually! + {vsn, "4.3.8"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 40be93b33..93f2c6fb8 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.12"}, %% 4.3.3 is used by ee + {vsn, "4.3.13"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index dffcc7024..7989c9542 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.16"}, % strict semver, bump manually! + {vsn, "4.3.17"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index e35e570d3..13afb9e67 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_stomp, [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.3.6"}, % strict semver, bump manually! + {vsn, "4.3.7"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_stomp_sup]}, {applications, [kernel,stdlib]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 092e5d3b6..07c67545b 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.17"}, % strict semver, bump manually! + {vsn, "4.3.18"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/src/emqx.app.src b/src/emqx.app.src index 433c75326..99715d1a2 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.22"}, % strict semver, bump manually! + {vsn, "4.3.23"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel From 11026b7644aa98eb3a996696bb22d40df25b61ae Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 16 Oct 2022 08:43:45 +0200 Subject: [PATCH 32/50] chore: re-generate appup for v4.3.22 --- apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 6 ++++-- apps/emqx_exproto/src/emqx_exproto.appup.src | 6 ++++-- .../src/emqx_rule_engine.appup.src | 12 ++++++++++-- apps/emqx_stomp/src/emqx_stomp.appup.src | 6 ++++-- src/emqx.appup.src | 14 ++++++++++++-- 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index bea59cbbb..7ee2349c3 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.6", + [{"4.3.7",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {"4.3.6", [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.3.5", @@ -15,7 +16,8 @@ {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [ {"4.3.6", + [{"4.3.7",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {"4.3.6", [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.3.5", diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index f31e4a969..e97296a68 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + [{"4.3.12",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + {"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, @@ -19,7 +20,8 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + [{"4.3.12",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + {"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index ab85f908e..94a79e2db 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,11 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.15", + [{"4.3.16", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, @@ -218,7 +222,11 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.15", + [{"4.3.16", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.appup.src b/apps/emqx_stomp/src/emqx_stomp.appup.src index 69e571934..499b80f38 100644 --- a/apps/emqx_stomp/src/emqx_stomp.appup.src +++ b/apps/emqx_stomp/src/emqx_stomp.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.5", + [{"4.3.6",[{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + {"4.3.5", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.4", @@ -23,7 +24,8 @@ [{restart_application,emqx_stomp}, {apply,{emqx_stomp,force_clear_after_app_stoped,[]}}]}, {<<".*">>,[]}], - [{"4.3.5", + [{"4.3.6",[{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + {"4.3.5", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.4", diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 51576061c..e8ee98afb 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.21", + [{"4.3.22", + [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {"4.3.21", [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {add_module,emqx_secret}, @@ -862,7 +867,12 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.21", + [{"4.3.22", + [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {"4.3.21", [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, From 5bc822cc78b83ba35fd40715be32ec57d6f08d0b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 17 Oct 2022 12:55:22 +0200 Subject: [PATCH 33/50] chore: delete duplicated module from appup --- .../src/emqx_rule_engine.appup.src | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 94a79e2db..ed783047d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -8,7 +8,6 @@ {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, @@ -16,7 +15,6 @@ {"4.3.14", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -25,7 +23,6 @@ {"4.3.13", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -36,7 +33,6 @@ {"4.3.12", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -47,7 +43,6 @@ {"4.3.11", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -59,7 +54,6 @@ {"4.3.10", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -70,7 +64,6 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -229,7 +222,6 @@ {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, @@ -237,7 +229,6 @@ {"4.3.14", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -246,7 +237,6 @@ {"4.3.13", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -257,7 +247,6 @@ {"4.3.12", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -268,7 +257,6 @@ {"4.3.11", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, @@ -280,7 +268,6 @@ {"4.3.10", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -291,7 +278,6 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, From f15845fb002cc527a229daedcdd9d1d2733946d3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 17 Oct 2022 15:04:33 +0200 Subject: [PATCH 34/50] chore: add code-owners --- .github/CODEOWNERS | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..0bed63887 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,43 @@ +## MQTT & Core +src/ @qzhuyan +include/ @qzhuyan +etc/ @qzhuyan +test/ @qzhuyan + +## CI +.github/ @id +.ci/ @id +scripts/ @id +build @id +deploy/ @id + +## Authenticatio & ACL +apps/emqx_auth_*/ @savonarola +apps/emqx_psk_file/ @savonarola +apps/emqx_retainer/ @savonarola +apps/emqx_sasl/ @savonarola + +## Gateway +apps/emqx_coap/ @HJianBo +apps/emqx_exhook/ @HJianBo +apps/emqx_exproto/ @HJianBo +apps/emqx_lua_hook/ @HJianBo +apps/emqx_lwm2m/ @HJianBo + +## OPs +apps/emqx_management/ @zhongwencool +apps/emqx_recon/ @zhongwencool +apps/emqx_plugin_libs/ @zhongwencool +apps/emqx_prometheus/ @zhongwencool +apps/emqx_recon/ @zhongwencool + + +## Data integration +apps/emqx_rule_engine/ @thalesmg +apps/emqx_web_hook/ @thalesmg + +## External Plugins +lib-extra/ @zmstone + +## Default +* @zmstone From a2869327430cf53ecbc8e7c7bc6616b427f4b1c1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 17 Oct 2022 13:54:07 +0200 Subject: [PATCH 35/50] docs: split change logs We will start collecting change logs and release notes using scripts. --- CHANGES-4.3.md | 15 ++------------- changes/v4.3.22-en.md | 12 ++++++++++++ changes/v4.3.22-zh.md | 12 ++++++++++++ 3 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 changes/v4.3.22-en.md create mode 100644 changes/v4.3.22-zh.md diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 84be29a1f..c812713db 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,18 +10,7 @@ File format: - One list item per change topic Change log ends with a list of GitHub PRs -## v4.3.22 - -### Enhancements - -- Add a warning log if the ACL check failed for subscription. [#9124](https://github.com/emqx/emqx/pull/9124) - This is to make the ACL deny logging for subscription behave the same as for publish. - -### Bug fixes - -- Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic. [#9024](https://github.com/emqx/emqx/pull/9024) -- "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134) -- Restore legacy `emqx_auth_jwt` interface to keep hooks working after relup. [##9144](https://github.com/emqx/emqx/pull/9144) +## For 4.3.22 and later versions, please find details in `changes` dir ## v4.3.21 @@ -63,7 +52,7 @@ File format: - Fix HTTP client library to handle SSL socket passive signal. [#9145](https://github.com/emqx/emqx/pull/9145) - Hide redis password in error logs [#9071](https://github.com/emqx/emqx/pull/9071) - In this change, it also included more changes in redis client: + More changes in redis client included in this release: - Improve redis connection error logging [eredis #19](https://github.com/emqx/eredis/pull/19). Also added support for eredis to accept an anonymous function as password instead of passing around plaintext args which may get dumpped to crash logs (hard to predict where). diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md new file mode 100644 index 000000000..3cb84e5f4 --- /dev/null +++ b/changes/v4.3.22-en.md @@ -0,0 +1,12 @@ +### Enhancements + +- Add a warning log if the ACL check failed for subscription [#9124](https://github.com/emqx/emqx/pull/9124). + This is to make the ACL deny logging for subscription behave the same as for publish. + +### Bug fixes + +- Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic [#9024](https://github.com/emqx/emqx/pull/9024). + +- "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134). + +- Restore old `emqx_auth_jwt` module API, so the hook callback functions registered in older version will not be invalidated after hot-upgrade [#9144](https://github.com/emqx/emqx/pull/9144). diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md new file mode 100644 index 000000000..e8f534780 --- /dev/null +++ b/changes/v4.3.22-zh.md @@ -0,0 +1,12 @@ +### 增强 + +- 订阅时,如果 ACL 检查不通过,打印一个警告日志 [#9124](https://github.com/emqx/emqx/pull/9124)。 + 该行为的改变主要是为了跟发布失败时的行为保持一致。 + +### 修复 + +- 修复 emqx-sn 插件在“空闲”状态下收到消息发布请求时可能崩溃的情况 [#9024](https://github.com/emqx/emqx/pull/9024)。 + +- 限速 “Pause due to rate limit” 的日志级别从原先的 `warning` 降级到 `notice` [#9134](https://github.com/emqx/emqx/pull/9134)。 + +- 保留老的 `emqx_auth_jwt` 模块的接口函数,保障热升级之前添加的回调函数在热升级之后也不会失效 [#9144](https://github.com/emqx/emqx/pull/9144)。 From b0ee2fdd3cacd1ba1a838569373a9d6ff09c469b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 18 Oct 2022 17:58:55 +0800 Subject: [PATCH 36/50] chore: workflow steps with path --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index c64af4f93..322f1dbf4 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -31,9 +31,9 @@ jobs: path: source fetch-depth: 0 - id: detect-profiles - working-directory: source uses: ./.github/actions/detect-profiles with: + path: source ci_git_token: ${{ secrets.CI_GIT_TOKEN }} - name: get_all_deps if: endsWith(github.repository, 'emqx') From 73a5462cba3bec9435933573feafaf4341488c48 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 19 Oct 2022 17:09:37 +0200 Subject: [PATCH 37/50] fix(shared): do not redispatch shared messages for certain shutdown For takeover, there should be no message re-dispatch because the messages will be retried by the new session. For kick, messages should not be re-dispatched for security reason. i.e. if admin has identified that there are malicious messages stored in persisted sessions, killing the session should not cause messages to be re-dispatched --- src/emqx_session.erl | 12 ++- test/emqx_shared_sub_SUITE.erl | 134 +++++++++++++++++++++++++++++++-- 2 files changed, 138 insertions(+), 8 deletions(-) diff --git a/src/emqx_session.erl b/src/emqx_session.erl index d2379a3fd..41240aeaa 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -625,10 +625,11 @@ replay(ClientInfo, Session = #session{inflight = Inflight}) -> end. -spec(terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok). +terminate(ClientInfo, {shutdown, Reason}, Session) -> + terminate(ClientInfo, Reason, Session); terminate(ClientInfo, Reason, Session) -> run_terminate_hooks(ClientInfo, Reason, Session), - Reason =/= takeovered andalso - redispatch_shared_messages(Session), + maybe_redispatch_shared_messages(Reason, Session), ok. run_terminate_hooks(ClientInfo, discarded, Session) -> @@ -638,6 +639,13 @@ run_terminate_hooks(ClientInfo, takeovered, Session) -> run_terminate_hooks(ClientInfo, Reason, Session) -> run_hook('session.terminated', [ClientInfo, Reason, info(Session)]). +maybe_redispatch_shared_messages(takeovered, _Session) -> + ok; +maybe_redispatch_shared_messages(kicked, _Session) -> + ok; +maybe_redispatch_shared_messages(_Reason, Session) -> + redispatch_shared_messages(Session). + redispatch_shared_messages(#session{inflight = Inflight, mqueue = Q}) -> AllInflights = emqx_inflight:to_list(sort_fun(), Inflight), F = fun({_, {Msg, _Ts}}) -> diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index 2c4ecf265..0613ba37f 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -587,11 +587,16 @@ t_dispatch_qos2(Config) when is_list(Config) -> ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message3)), ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message4)), + %% assert client 2 receives two messages, they are eiter 1,3 or 2,4 depending + %% on if it's picked as the first one for round_robin MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1), MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2), - %% assert hello2 > hello1 or hello4 > hello3 - ?assert(MsgRec2 > MsgRec1), - + case MsgRec2 of + <<"hello3">> -> + ?assertEqual(<<"hello1">>, MsgRec1); + <<"hello4">> -> + ?assertEqual(<<"hello2">>, MsgRec1) + end, sys:resume(ConnPid1), %% emqtt subscriber automatically sends PUBREC, but since auto_ack is set to false %% so it will never send PUBCOMP, hence EMQX should not attempt to send @@ -604,8 +609,14 @@ t_dispatch_qos2(Config) when is_list(Config) -> kill_process(ConnPid1), %% client 2 should receive the message MsgRec4 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P4}}, P4), - %% assert hello2 > hello1 or hello4 > hello3 - ?assert(MsgRec4 > MsgRec3), + case MsgRec2 of + <<"hello3">> -> + ?assertEqual(<<"hello2">>, MsgRec3), + ?assertEqual(<<"hello4">>, MsgRec4); + <<"hello4">> -> + ?assertEqual(<<"hello1">>, MsgRec3), + ?assertEqual(<<"hello3">>, MsgRec4) + end, emqtt:stop(ConnPid2), ok. @@ -654,17 +665,128 @@ t_dispatch_qos0(Config) when is_list(Config) -> emqtt:stop(ConnPid2), ok. +t_session_takeover({init, Config}) when is_list(Config) -> + Config; +t_session_takeover({'end', Config}) when is_list(Config) -> + ok; +t_session_takeover(Config) when is_list(Config) -> + Topic = <<"t1/a">>, + ClientId = iolist_to_binary("c" ++ integer_to_list(erlang:system_time())), + Opts = [{clientid, ClientId}, + {auto_ack, true}, + {proto_ver, v5}, + {clean_start, false}, + {properties, #{'Session-Expiry-Interval' => 60}} + ], + {ok, ConnPid1} = emqtt:start_link(Opts), + %% with the same client ID, start another client + {ok, ConnPid2} = emqtt:start_link(Opts), + {ok, _} = emqtt:connect(ConnPid1), + emqtt:subscribe(ConnPid1, {<<"$share/t1/", Topic/binary>>, _QoS = 1}), + Message1 = emqx_message:make(<<"dummypub">>, 2, Topic, <<"hello1">>), + Message2 = emqx_message:make(<<"dummypub">>, 2, Topic, <<"hello2">>), + Message3 = emqx_message:make(<<"dummypub">>, 2, Topic, <<"hello3">>), + Message4 = emqx_message:make(<<"dummypub">>, 2, Topic, <<"hello4">>), + %% Make sure client1 is functioning + ?assertMatch([_], emqx:publish(Message1)), + {true, _} = last_message(<<"hello1">>, [ConnPid1]), + %% Kill client1 + emqtt:stop(ConnPid1), + %% publish another message (should end up in client1's session) + ?assertMatch([_], emqx:publish(Message2)), + %% connect client2 (with the same clientid) + {ok, _} = emqtt:connect(ConnPid2), %% should trigger session take over + ?assertMatch([_], emqx:publish(Message3)), + ?assertMatch([_], emqx:publish(Message4)), + {true, _} = last_message(<<"hello2">>, [ConnPid1]), + {true, _} = last_message(<<"hello3">>, [ConnPid1]), + {true, _} = last_message(<<"hello4">>, [ConnPid1]), + ?assertEqual([], collect_msgs(timer:seconds(2))), + emqtt:stop(ConnPid2), + ok. + + +t_session_kicked({init, Config}) when is_list(Config) -> + meck:new(emqx_zone, [passthrough, no_history]), + meck:expect(emqx_zone, max_inflight, fun(_Zone) -> 1 end), + Config; +t_session_kicked({'end', Config}) when is_list(Config) -> + meck:unload(emqx_zone); +t_session_kicked(Config) when is_list(Config) -> + ok = ensure_config(round_robin, _AckEnabled = false), + Topic = <<"foo/bar/1">>, + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + + {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]), + {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]), + {ok, _} = emqtt:connect(ConnPid1), + {ok, _} = emqtt:connect(ConnPid2), + + emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar/#">>, 2}), + emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar/#">>, 2}), + + Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>), + Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>), + Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>), + Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>), + ct:sleep(100), + + ok = sys:suspend(ConnPid1), + + %% One message is inflight + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message1)), + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message2)), + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message3)), + ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message4)), + + %% assert client 2 receives two messages, they are eiter 1,3 or 2,4 depending + %% on if it's picked as the first one for round_robin + MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1), + MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2), + case MsgRec2 of + <<"hello3">> -> + ?assertEqual(<<"hello1">>, MsgRec1); + <<"hello4">> -> + ?assertEqual(<<"hello2">>, MsgRec1) + end, + sys:resume(ConnPid1), + %% emqtt subscriber automatically sends PUBREC, but since auto_ack is set to false + %% so it will never send PUBCOMP, hence EMQX should not attempt to send + %% the 4th message yet since max_inflight is 1. + MsgRec3 = ?WAIT(2000, {publish, #{client_pid := ConnPid1, payload := P3}}, P3), + case MsgRec2 of + <<"hello3">> -> + ?assertEqual(<<"hello2">>, MsgRec3); + <<"hello4">> -> + ?assertEqual(<<"hello1">>, MsgRec3) + end, + %% no message expected + ?assertEqual([], collect_msgs(0)), + %% now kick client 1 + kill_process(ConnPid1, fun(_Pid) -> emqx_cm:kick_session(ClientId1) end), + %% client 2 should NOT receive the message + ?assertEqual([], collect_msgs(1000)), + emqtt:stop(ConnPid2), + ?assertEqual([], collect_msgs(0)), + ok. + %%-------------------------------------------------------------------- %% help functions %%-------------------------------------------------------------------- kill_process(Pid) -> + kill_process(Pid, fun(_) -> erlang:exit(Pid, kill) end). + +kill_process(Pid, WithFun) -> _ = unlink(Pid), _ = monitor(process, Pid), - erlang:exit(Pid, kill), + _ = WithFun(Pid), receive {'DOWN', _, process, Pid, _} -> ok + after 10_000 -> + error(timeout) end. collect_msgs(Timeout) -> From a163ffce7ce3b899a4a931009ba480ccce666adb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 19 Oct 2022 18:16:38 +0200 Subject: [PATCH 38/50] test: assert message receive pid is in the expected pids list --- test/emqx_shared_sub_SUITE.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index 0613ba37f..2112f0b8c 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -290,7 +290,7 @@ last_message(ExpectedPayload, Pids) -> last_message(ExpectedPayload, Pids, Timeout) -> receive {publish, #{client_pid := Pid, payload := ExpectedPayload}} -> - ct:pal("last_message: ~p ====== ~p, payload=~p", [Pids, Pid, ExpectedPayload]), + ?assert(lists:member(Pid, Pids)), {true, Pid} after Timeout -> ct:pal("not yet"), @@ -698,9 +698,9 @@ t_session_takeover(Config) when is_list(Config) -> {ok, _} = emqtt:connect(ConnPid2), %% should trigger session take over ?assertMatch([_], emqx:publish(Message3)), ?assertMatch([_], emqx:publish(Message4)), - {true, _} = last_message(<<"hello2">>, [ConnPid1]), - {true, _} = last_message(<<"hello3">>, [ConnPid1]), - {true, _} = last_message(<<"hello4">>, [ConnPid1]), + {true, _} = last_message(<<"hello2">>, [ConnPid2]), + {true, _} = last_message(<<"hello3">>, [ConnPid2]), + {true, _} = last_message(<<"hello4">>, [ConnPid2]), ?assertEqual([], collect_msgs(timer:seconds(2))), emqtt:stop(ConnPid2), ok. From e0e8a5a7073360f9137739e9e9eba626fe5b3722 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 20 Oct 2022 10:37:32 +0800 Subject: [PATCH 39/50] chore: re-generate appup for v4.3.22 --- src/emqx.appup.src | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index e8ee98afb..2629f8771 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.22", - [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, @@ -868,7 +869,8 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.22", - [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, From 524ba512d8f359dcd3a0fc167d3f37220f92f5ca Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 20 Oct 2022 11:25:22 +0800 Subject: [PATCH 40/50] chore: update mongodb driver to v3.0.14 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 68a37ef10..2bdc3d952 100644 --- a/rebar.config +++ b/rebar.config @@ -61,7 +61,7 @@ , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.1"}}} , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.1"}}} - , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}} + , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.14"}}} , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} ]}. From bfe238fa1c9f78dff182dd522ac9a73e2b25db14 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 20 Oct 2022 14:20:30 +0800 Subject: [PATCH 41/50] fix: set precision of counter 'max_speed' to 2 digits --- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 2 ++ apps/emqx_rule_engine/src/emqx_rule_metrics.erl | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index ed783047d..cbd15a5eb 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -4,6 +4,7 @@ [{"4.3.16", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, @@ -218,6 +219,7 @@ [{"4.3.16", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.15", [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index e4c539164..414fc15e0 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -500,11 +500,10 @@ calculate_speed(CurrVal, #rule_speed{max = MaxSpeed0, last_v = LastVal, last5m_smpl = Last5MinSamples, tick = Tick + 1}. format_rule_speed(#rule_speed{max = Max, current = Current, last5m = Last5Min}) -> - #{max => Max, current => precision(Current, 2), last5m => precision(Last5Min, 2)}. + #{max => round2(Max), current => round2(Current), last5m => round2(Last5Min)}. -precision(Float, N) -> - Base = math:pow(10, N), - round(Float * Base) / Base. +round2(Float) -> + round(Float * 100) / 100. %%------------------------------------------------------------------------------ %% Metrics Definitions From f8686ffc629a72ecba4c8bea0dd78d8b62df0a31 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 20 Oct 2022 16:48:48 +0800 Subject: [PATCH 42/50] chore: update the change logs --- changes/v4.3.22-en.md | 5 +++++ changes/v4.3.22-zh.md | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 3cb84e5f4..77a197624 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -5,6 +5,11 @@ ### Bug fixes +- Improve the display of rule's 'Maximum Speed' counter to only reserve 2 decimal places. [#9185](https://github.com/emqx/emqx/pull/9185) + This is to avoid displaying floats like `0.30000000000000004` on the dashboard. + +- Fix the issue that emqx prints too many error logs when connecting to mongodb but auth failed. [#9184](https://github.com/emqx/emqx/pull/9184) + - Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic [#9024](https://github.com/emqx/emqx/pull/9024). - "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134). diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index e8f534780..58c82588d 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -5,6 +5,11 @@ ### 修复 +- 改进规则的 "最大执行速度" 的计数,只保留小数点之后 2 位 [#9185](https://github.com/emqx/emqx/pull/9185) + 避免在 dashboard 上展示类似这样的浮点数:`0.30000000000000004`。 + +- 修复在尝试连接 MongoDB 数据库过程中,如果认证失败会不停打印错误日志的问题。[#9184](https://github.com/emqx/emqx/pull/9184) + - 修复 emqx-sn 插件在“空闲”状态下收到消息发布请求时可能崩溃的情况 [#9024](https://github.com/emqx/emqx/pull/9024)。 - 限速 “Pause due to rate limit” 的日志级别从原先的 `warning` 降级到 `notice` [#9134](https://github.com/emqx/emqx/pull/9134)。 From 3b59f1440b01605790183b0859c91f5304609fbc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 22 Oct 2022 15:14:58 +0200 Subject: [PATCH 43/50] chore: bump lc from 0.3.1 to 0.3.2 there is no beam change in 0.3.2 (comparing to 0.3.1) --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 2bdc3d952..86436101b 100644 --- a/rebar.config +++ b/rebar.config @@ -60,7 +60,7 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.1"}}} - , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.1"}}} + , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.14"}}} , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} From 075228aadc519ff710c91f96d7766fe6b0aefe78 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 19 Oct 2022 10:20:31 +0800 Subject: [PATCH 44/50] chore: fix local actions path --- .github/workflows/build_packages.yaml | 3 +-- .github/workflows/release.yaml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 322f1dbf4..b46eb9973 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -31,9 +31,8 @@ jobs: path: source fetch-depth: 0 - id: detect-profiles - uses: ./.github/actions/detect-profiles + uses: ./source/.github/actions/detect-profiles with: - path: source ci_git_token: ${{ secrets.CI_GIT_TOKEN }} - name: get_all_deps if: endsWith(github.repository, 'emqx') diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 71e5fe48a..5b438ffa0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,8 +18,7 @@ jobs: path: source fetch-depth: 0 - id: detect-profiles - working-directory: source - uses: ./.github/actions/detect-profiles + uses: ./source/.github/actions/detect-profiles with: ci_git_token: ${{ secrets.CI_GIT_TOKEN }} From 34d73960e729a3217c5ea60d73ee0857793b0954 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 19 Oct 2022 10:42:09 +0800 Subject: [PATCH 45/50] ci: fix macos erlang-otp cache hitting --- .github/actions/package-macos/action.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 3efe0ef81..1a7e4fb9c 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -35,6 +35,8 @@ runs: with: path: ~/.kerl/${{ inputs.otp }} key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit + restore-keys: | + otp-install-${{ inputs.otp }}-${{ inputs.os }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' shell: bash @@ -93,4 +95,5 @@ runs: exit 1 fi cd .. + # test with a spaces in path rm -rf "emqx home" From 78fc9ff64ed193575340d0c3064e834ec943412e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 24 Oct 2022 11:28:39 +0200 Subject: [PATCH 46/50] chore: add leading slash for paths in codeowners --- .github/CODEOWNERS | 52 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0bed63887..e2631f3ac 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,43 +1,43 @@ ## MQTT & Core -src/ @qzhuyan -include/ @qzhuyan -etc/ @qzhuyan -test/ @qzhuyan +/src/ @qzhuyan +/include/ @qzhuyan +/etc/ @qzhuyan +/test/ @qzhuyan ## CI -.github/ @id -.ci/ @id -scripts/ @id -build @id -deploy/ @id +/.github/ @id +/.ci/ @id +/scripts/ @id +/build @id +/deploy/ @id ## Authenticatio & ACL -apps/emqx_auth_*/ @savonarola -apps/emqx_psk_file/ @savonarola -apps/emqx_retainer/ @savonarola -apps/emqx_sasl/ @savonarola +/apps/emqx_auth_*/ @savonarola +/apps/emqx_psk_file/ @savonarola +/apps/emqx_retainer/ @savonarola +/apps/emqx_sasl/ @savonarola ## Gateway -apps/emqx_coap/ @HJianBo -apps/emqx_exhook/ @HJianBo -apps/emqx_exproto/ @HJianBo -apps/emqx_lua_hook/ @HJianBo -apps/emqx_lwm2m/ @HJianBo +/apps/emqx_coap/ @HJianBo +/apps/emqx_exhook/ @HJianBo +/apps/emqx_exproto/ @HJianBo +/apps/emqx_lua_hook/ @HJianBo +/apps/emqx_lwm2m/ @HJianBo ## OPs -apps/emqx_management/ @zhongwencool -apps/emqx_recon/ @zhongwencool -apps/emqx_plugin_libs/ @zhongwencool -apps/emqx_prometheus/ @zhongwencool -apps/emqx_recon/ @zhongwencool +/apps/emqx_management/ @zhongwencool +/apps/emqx_recon/ @zhongwencool +/apps/emqx_plugin_libs/ @zhongwencool +/apps/emqx_prometheus/ @zhongwencool +/apps/emqx_recon/ @zhongwencool ## Data integration -apps/emqx_rule_engine/ @thalesmg -apps/emqx_web_hook/ @thalesmg +/apps/emqx_rule_engine/ @thalesmg +/apps/emqx_web_hook/ @thalesmg ## External Plugins -lib-extra/ @zmstone +/lib-extra/ @zmstone ## Default * @zmstone From c0b5b887c35c1d324a98e877423245f21cdae956 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 21 Oct 2022 14:58:04 -0300 Subject: [PATCH 47/50] fix(mgmt_api): return 503 when emqx is not running in `/status` --- .../src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt_http.erl | 6 +- .../test/emqx_mgmt_api_SUITE.erl | 69 +++++++++++++++++++ .../test/emqx_mgmt_api_test_helpers.erl | 12 +++- changes/v4.3.22-en.md | 2 + changes/v4.3.22-zh.md | 2 + 6 files changed, 89 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 1003a1140..5ab803c51 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.18"}, % strict semver, bump manually! + {vsn, "4.3.19"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index fae1bd86a..8cddc11b2 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -114,7 +114,11 @@ handle_request(<<"GET">>, <<"/status">>, Req) -> end, Status = io_lib:format("Node ~s is ~s~nemqx is ~s", [node(), InternalStatus, AppStatus]), - cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, Status, Req); + StatusCode = case AppStatus of + running -> 200; + not_running -> 503 + end, + cowboy_req:reply(StatusCode, #{<<"content-type">> => <<"text/plain">>}, Status, Req); handle_request(_Method, _Path, Req) -> cowboy_req:reply(400, #{<<"content-type">> => <<"text/plain">>}, <<"Not found.">>, Req). diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index fda5c2adf..6b15bcfff 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -25,6 +25,8 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_management/include/emqx_mgmt.hrl"). +-define(HOST, "http://127.0.0.1:8081/"). + -import(emqx_mgmt_api_test_helpers, [request_api/3, request_api/4, @@ -44,9 +46,31 @@ end_per_suite(Config) -> emqx_ct_helpers:stop_apps([emqx_management]), Config. +init_per_testcase(t_status_ok, Config) -> + ok = emqx_rule_registry:mnesia(boot), + ok = emqx_dashboard_admin:mnesia(boot), + application:ensure_all_started(emqx_rule_engine), + application:ensure_all_started(emqx_dashboard), + Config; +init_per_testcase(t_status_not_ok, Config) -> + ok = emqx_rule_registry:mnesia(boot), + ok = emqx_dashboard_admin:mnesia(boot), + application:ensure_all_started(emqx_rule_engine), + application:ensure_all_started(emqx_dashboard), + application:stop(emqx), + Config; init_per_testcase(_, Config) -> Config. +end_per_testcase(t_status_ok, _Config) -> + application:stop(emqx_rule_engine), + application:stop(emqx_dashboard), + ok; +end_per_testcase(t_status_not_ok, _Config) -> + application:stop(emqx_rule_engine), + application:stop(emqx_dashboard), + application:ensure_all_started(emqx), + ok; end_per_testcase(_, Config) -> Config. @@ -656,6 +680,51 @@ t_data_import_content(_) -> application:stop(emqx_rule_engine), application:stop(emqx_dashboard). +t_status_ok(_Config) -> + {ok, #{ body := Resp + , status_code := StatusCode + }} = do_request(#{method => get, path => ["status"], headers => [], + body => no_body}), + ?assertMatch( + {match, _}, + re:run(Resp, <<"emqx is running$">>)), + ?assertEqual(200, StatusCode), + ok. + +t_status_not_ok(_Config) -> + {ok, #{ body := Resp + , status_code := StatusCode + }} = do_request(#{method => get, path => ["status"], headers => [], + body => no_body}), + ?assertMatch( + {match, _}, + re:run(Resp, <<"emqx is not_running$">>)), + ?assertEqual(503, StatusCode), + ok. + +do_request(Opts) -> + #{ path := Path + , method := Method + , headers := Headers + , body := Body0 + } = Opts, + URL = ?HOST ++ filename:join(Path), + Request = case Body0 of + no_body -> {URL, Headers}; + {Encoding, Body} -> {URL, Headers, Encoding, Body} + end, + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{_, StatusCode, _}, Headers1, Body1}} -> + Body2 = case emqx_json:safe_decode(Body1, [return_maps]) of + {ok, Json} -> Json; + {error, _} -> Body1 + end, + {ok, #{status_code => StatusCode, headers => Headers1, body => Body2}} + end. + filter(List, Key, Value) -> lists:filter(fun(Item) -> maps:get(Key, Item) == Value diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl b/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl index a943ca760..952d71b9b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl @@ -36,13 +36,21 @@ request_api(Method, Url, QueryParams, Auth, []) -> "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth]}); + Headers = case Auth of + no_auth -> []; + Header -> [Header] + end, + do_request_api(Method, {NewUrl, Headers}); request_api(Method, Url, QueryParams, Auth, Body) -> NewUrl = case QueryParams of "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + Headers = case Auth of + no_auth -> []; + Header -> [Header] + end, + do_request_api(Method, {NewUrl, Headers, "application/json", emqx_json:encode(Body)}). do_request_api(Method, Request)-> ct:pal("Method: ~p, Request: ~p", [Method, Request]), diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 77a197624..7851139e2 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -15,3 +15,5 @@ - "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134). - Restore old `emqx_auth_jwt` module API, so the hook callback functions registered in older version will not be invalidated after hot-upgrade [#9144](https://github.com/emqx/emqx/pull/9144). + +- Fixed the response status code for the `/status` endpoint. Before the fix, it always returned 200 even if the EMQX application was not running. Now it returns 503 in that case. diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 58c82588d..6865a8614 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -15,3 +15,5 @@ - 限速 “Pause due to rate limit” 的日志级别从原先的 `warning` 降级到 `notice` [#9134](https://github.com/emqx/emqx/pull/9134)。 - 保留老的 `emqx_auth_jwt` 模块的接口函数,保障热升级之前添加的回调函数在热升级之后也不会失效 [#9144](https://github.com/emqx/emqx/pull/9144)。 + +- 修正了`/status`端点的响应状态代码。 在修复之前,它总是返回200,即使EMQX应用程序没有运行。 现在它在这种情况下返回503。 From 51f2414eaa70e1f631d9a72e92620946899a9fd8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 24 Oct 2022 15:24:49 -0300 Subject: [PATCH 48/50] docs: improve changelog Co-authored-by: Zaiming (Stone) Shi --- changes/v4.3.22-en.md | 3 ++- changes/v4.3.22-zh.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 7851139e2..db76d3423 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -16,4 +16,5 @@ - Restore old `emqx_auth_jwt` module API, so the hook callback functions registered in older version will not be invalidated after hot-upgrade [#9144](https://github.com/emqx/emqx/pull/9144). -- Fixed the response status code for the `/status` endpoint. Before the fix, it always returned 200 even if the EMQX application was not running. Now it returns 503 in that case. +- Fixed the response status code for the `/status` endpoint [#9210](https://github.com/emqx/emqx/pull/9210). + Before the fix, it always returned `200` even if the EMQX application was not running. Now it returns `503` in that case. diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 6865a8614..e7d343db0 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -16,4 +16,5 @@ - 保留老的 `emqx_auth_jwt` 模块的接口函数,保障热升级之前添加的回调函数在热升级之后也不会失效 [#9144](https://github.com/emqx/emqx/pull/9144)。 -- 修正了`/status`端点的响应状态代码。 在修复之前,它总是返回200,即使EMQX应用程序没有运行。 现在它在这种情况下返回503。 +- 修正了 `/status` API 的响应状态代码 [#9210](https://github.com/emqx/emqx/pull/9210)。 + 在修复之前,它总是返回 `200`,即使 EMQX 应用程序没有运行。 现在它在这种情况下返回 `503`。 From a987d6fc9410d85bb1e8a235cbdbc08b62dc4f76 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 24 Oct 2022 17:10:31 -0300 Subject: [PATCH 49/50] test(rulesql): fix broken test --- apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index f4f329e87..7e49f0cb9 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -173,12 +173,12 @@ t_sqlselect_0(_Config) -> timestamp := TimeStamp, qos := 1, publish_received_at := TimeStamp, - peerhost := <<"127.0.0.1">>, + peerhost := <<"192.168.0.10">>, payload := Payload, node := 'test@127.0.0.1', metadata := #{rule_id := TestRuleId}, id := MsgId, - flags := #{sys := true, event := true}, + flags := #{}, clientid := <<"c_emqx">> } } = emqx_rule_sqltester:test( From 8206f39b7cbf01d633169e28e240c6903505ed65 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 24 Oct 2022 16:36:36 -0300 Subject: [PATCH 50/50] chore: update appups --- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- .../emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 10 ++++- .../src/emqx_management.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 40 +++++++++++++------ src/emqx.appup.src | 10 ++++- 5 files changed, 46 insertions(+), 18 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 1bbd434ac..2a50ac509 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.4.6"}, % strict semver, bump manually! + {vsn, "4.4.7"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index 2d15dc9e9..8e4ca7d29 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.5",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + [{"4.4.6",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {"4.4.5", + [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.4.4", [{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, @@ -13,7 +16,10 @@ {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, {<<"4\\.4\\.[0-1]">>,[{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [{"4.4.5",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + [{"4.4.6",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {"4.4.5", + [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.4.4", [{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 86e249abf..07abc4f5f 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.4.9"}, % strict semver, bump manually! + {vsn, "4.4.10"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,emqx_plugin_libs,minirest]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 6b880df7a..41a184e94 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,22 +2,28 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.10", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.9", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.8", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {<<"4\\.4\\.[6-7]">>, - [{load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -26,7 +32,8 @@ {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.5", - [{load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -36,7 +43,8 @@ {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.4.4", - [{load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, @@ -106,22 +114,28 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.10", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.9", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.4.8", - [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {<<"4\\.4\\.[6-7]">>, - [{load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -130,7 +144,8 @@ {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.4.5", - [{load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -140,7 +155,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.4.4", - [{load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index cca029104..8db8b0eaf 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,10 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.10", - [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.9", [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -275,7 +278,10 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.10", - [{load_module,emqx_cm,brutal_purge,soft_purge,[]}, + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.4.9", [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},