Merge pull request #9521 from zmstone/1209-sync-changes-from-ee-back-to-c

chore: sync changes from ee to ce

apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl

@@ -153,8 +153,11 @@ init_per_group(_Group, Config) ->
 
 end_per_group(resilience, Config) ->
     OriginalServer = ?config(original_server, Config),
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyPort = ?config(proxy_port, Config),
     application:set_env(emqx_auth_mongo, server, OriginalServer),
     emqx_ct_helpers:stop_apps([emqx_auth_mongo]),
+    reset_proxy(ProxyHost, ProxyPort),
     emqx_ct_helpers:start_apps([emqx_auth_mongo], fun set_special_confs/1),
     ok;
 end_per_group(_Group, _Config) ->

apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src

@@ -17,3 +17,4 @@
      {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]}.
+

apps/emqx_exhook/src/emqx_exhook.appup.src

@@ -1,9 +1,12 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.4",[{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}]},
+  [{"4.4.4",
+    [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}]},
    {"4.4.3",
-    [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]},
    {<<"4\\.4\\.[1-2]">>,
     [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]},
@@ -19,9 +22,12 @@
      {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
      {update,emqx_exhook_mngr,{advanced,["4.4.0"]}}]},
    {<<".*">>,[]}],
-  [{"4.4.4",[{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}]},
+  [{"4.4.4",
+    [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}]},
    {"4.4.3",
-    [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]},
    {<<"4\\.4\\.[1-2]">>,
     [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]},

apps/emqx_exhook/src/emqx_exhook_handler.erl

@@ -63,6 +63,7 @@
         , call_fold/3
         ]).
 
+
 -elvis([{elvis_style, god_modules, disable}]).
 
 -define(STOP_OR_OK(Res),

apps/emqx_retainer/src/emqx_retainer.appup.src

@@ -1,12 +1,15 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.1",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]},
-   {"4.4.0",[{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]},
-             {load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]},
+  [{"4.4.2",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]},
+   {"4.4.1",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]},
+   {"4.4.0",
+    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.1",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]},
-   {"4.4.0",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]},
-            {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]},
-   {<<".*">>,[]}]
-}.
+  [{"4.4.2",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]},
+   {"4.4.1",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]},
+   {"4.4.0",
+    [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_rule_engine/src/emqx_rule_engine.appup.src

@@ -1,7 +1,10 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.11",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.4.11",[
+    {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+   ]},
    {"4.4.10",
     [{add_module,emqx_rule_engine_jwt},
      {add_module,emqx_rule_engine_jwt_worker},
@@ -197,7 +200,10 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.11",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.4.11",[
+    {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
+   ]},
    {"4.4.10",
     [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},

apps/emqx_rule_engine/src/emqx_rule_engine.erl

@@ -195,36 +195,43 @@ create_rule(Params = #{rawsql := Sql, actions := ActArgs}) ->
         {ok, Select} ->
             RuleId = maps:get(id, Params, rule_id()),
             Enabled = maps:get(enabled, Params, true),
-            try prepare_actions(ActArgs, Enabled) of
-                Actions ->
-                    Rule = #rule{
-                        id = RuleId,
-                        rawsql = Sql,
-                        for = emqx_rule_sqlparser:select_from(Select),
-                        is_foreach = emqx_rule_sqlparser:select_is_foreach(Select),
-                        fields = emqx_rule_sqlparser:select_fields(Select),
-                        doeach = emqx_rule_sqlparser:select_doeach(Select),
-                        incase = emqx_rule_sqlparser:select_incase(Select),
-                        conditions = emqx_rule_sqlparser:select_where(Select),
-                        on_action_failed = maps:get(on_action_failed, Params, continue),
-                        actions = Actions,
-                        enabled = Enabled,
-                        created_at = erlang:system_time(millisecond),
-                        description = maps:get(description, Params, ""),
-                        state = normal
-                    },
-                    ok = emqx_rule_registry:add_rule(Rule),
-                    ok = emqx_rule_metrics:create_rule_metrics(RuleId),
-                    {ok, Rule}
-            catch
-                throw:{action_not_found, ActionName} ->
-                    {error, {action_not_found, ActionName}};
-                throw:Reason ->
-                    {error, Reason}
-            end;
+            Rule = #rule{
+                id = RuleId,
+                rawsql = Sql,
+                for = emqx_rule_sqlparser:select_from(Select),
+                is_foreach = emqx_rule_sqlparser:select_is_foreach(Select),
+                fields = emqx_rule_sqlparser:select_fields(Select),
+                doeach = emqx_rule_sqlparser:select_doeach(Select),
+                incase = emqx_rule_sqlparser:select_incase(Select),
+                conditions = emqx_rule_sqlparser:select_where(Select),
+                on_action_failed = maps:get(on_action_failed, Params, continue),
+                actions = [],
+                enabled = Enabled,
+                created_at = erlang:system_time(millisecond),
+                description = maps:get(description, Params, ""),
+                state = normal
+            },
+            do_create_rule(Rule, ActArgs);
         Reason -> {error, Reason}
     end.
 
+do_create_rule(Rule0 = #rule{id = RuleId, enabled = Enabled}, ActArgs) ->
+    try prepare_actions(ActArgs, Enabled) of
+        Actions ->
+            Rule = Rule0#rule{actions = Actions},
+            ok = emqx_rule_registry:add_rule(Rule),
+            ok = emqx_rule_metrics:create_rule_metrics(RuleId),
+            {ok, Rule}
+    catch
+        throw:{resource_not_initialized, _} when Enabled =:= true ->
+            %% try again with rule disabled
+            do_create_rule(Rule0#rule{enabled = false}, ActArgs);
+        throw:{action_not_found, ActionName} ->
+            {error, {action_not_found, ActionName}};
+        throw:Reason ->
+            {error, Reason}
+    end.
+
 -spec(update_rule(#{id := binary(), _=>_}) -> {ok, rule()} | {error, {not_found, rule_id()} | term()}).
 update_rule(Params = #{id := RuleId}) ->
     case emqx_rule_registry:get_rule(RuleId) of

apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl

@@ -25,15 +25,19 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
--include("emqx_rule_test.hrl").
--import(emqx_rule_test_lib, [make_simple_resource_type/1]).
-
-%% API request funcs
 -import(emqx_rule_test_lib,
         [ request_api/4
         , request_api/5
         , auth_header_/0
         , api_path/1
+        , stop_apps/0
+        , start_apps/0
+        , create_simple_repub_rule/2
+        , create_simple_repub_rule/3
+        , make_simple_debug_resource_type/0
+        , make_simple_resource_type/1
+        , make_bad_resource_type/0
+        , init_events_counters/0
         ]).
 
 %%-define(PROPTEST(M,F), true = proper:quickcheck(M:F())).
@@ -63,6 +67,7 @@ groups() ->
        t_reset_metrics,
        t_reset_metrics_fallbacks,
        t_create_resource,
+       t_create_rule_with_resource,
        t_clean_resource_alarms
       ]},
      {actions, [],
@@ -310,6 +315,41 @@ t_create_resource(_Config) ->
     emqx_rule_registry:remove_resource(ResId),
     ok.
 
+t_create_rule_with_resource(_) ->
+    emqx_rule_registry:register_resource_types([make_bad_resource_type()]),
+    ok = emqx_rule_engine:load_providers(),
+    {ok, #resource{id = BadResId}} = emqx_rule_engine:create_resource(
+            #{type => bad_resource,
+              config => #{},
+              description => <<"bad resource">>}),
+    {ok, #resource{id = GoodResId}} = emqx_rule_engine:create_resource(
+            #{type => built_in,
+              config => #{},
+              description => <<"bad resource">>}),
+
+    {ok, #rule{id = BadRuleId, enabled = Enabled}} = emqx_rule_engine:create_rule(
+                #{rawsql => "select clientid as c, username as u "
+                            "from \"t1\" ",
+                  actions => [#{name => 'inspect',
+                                args => #{<<"$resource">> => BadResId, a=>1, b=>2}}],
+                  description => <<"Inspect rule">>
+                }),
+    ?assertEqual(false, Enabled),
+
+    {ok, #rule{id = GoodRuleId, enabled = Enabled1}} = emqx_rule_engine:create_rule(
+                #{rawsql => "select clientid as c, username as u "
+                            "from \"t1\" ",
+                  actions => [#{name => 'inspect',
+                                args => #{<<"$resource">> => GoodResId, a=>1, b=>2}}],
+                  description => <<"Inspect rule">>
+                }),
+    ?assertEqual(true, Enabled1),
+    emqx_rule_registry:remove_rule(BadRuleId),
+    emqx_rule_registry:remove_rule(GoodRuleId),
+    emqx_rule_registry:remove_resource(BadResId),
+    emqx_rule_registry:remove_resource(GoodResId),
+    ok.
+
 t_clean_resource_alarms(_Config) ->
     lists:foreach(fun(ResId) ->
             clean_resource_alarms(ResId)
@@ -350,8 +390,7 @@ t_inspect_action(_Config) ->
                 #{rawsql => "select clientid as c, username as u "
                             "from \"t1\" ",
                   actions => [#{name => 'inspect',
-                                args => #{'$resource' => ResId, a=>1, b=>2}}],
-                  type => built_in,
+                                args => #{<<"$resource">> => ResId, a=>1, b=>2}}],
                   description => <<"Inspect rule">>
                   }),
     {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
@@ -369,7 +408,6 @@ t_reset_metrics(_Config) ->
             #{rawsql => "select clientid as c, username as u "
             "from \"t1\" ",
             actions => [#{name => 'inspect', args => #{a=>1, b=>2}}],
-            type => built_in,
             description => <<"Inspect rule">>
             }),
     {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
@@ -414,7 +452,6 @@ t_reset_metrics_fallbacks(_Config) ->
                               #{name => 'inspect', args => #{}, fallbacks => []},
                               #{name => 'inspect', args => #{}, fallbacks => []}
                           ]}],
-            type => built_in,
             description => <<"Inspect rule">>
             }),
     {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),

apps/emqx_rule_engine/test/emqx_rule_test.hrl

@@ -1,29 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
-%% Test Suite funcs
--import(emqx_rule_test_lib,
-        [ stop_apps/0
-        , start_apps/0
-        ]).
-
-%% RULE helper funcs
--import(emqx_rule_test_lib,
-        [ create_simple_repub_rule/2
-        , create_simple_repub_rule/3
-        , make_simple_debug_resource_type/0
-        , init_events_counters/0
-        ]).

apps/emqx_rule_engine/test/emqx_rule_test_lib.erl

@@ -125,6 +125,17 @@ make_simple_resource_type(ResTypeName) ->
        title = #{en => <<"Simple Resource Type">>},
        description = #{en => <<"Simple Resource Type">>}}.
 
+make_bad_resource_type() ->
+    #resource_type{
+       name = bad_resource,
+       provider = ?APP,
+       params_spec = #{},
+       on_create = {?MODULE, on_bad_resource_type_create},
+       on_destroy = {?MODULE, on_bad_resource_type_destroy},
+       on_status = {?MODULE, on_bad_resource_type_status},
+       title = #{en => <<"Bad Resource Type">>},
+       description = #{en => <<"Bad Resource Type">>}}.
+
 init_events_counters() ->
     ets:new(events_record_tab, [named_table, bag, public]).
 
@@ -200,3 +211,7 @@ on_get_resource_status(_id, _) -> #{is_alive => true}.
 on_simple_resource_type_create(_Id, #{}) -> #{}.
 on_simple_resource_type_destroy(_Id, #{}) -> ok.
 on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}.
+
+on_bad_resource_type_create(_Id, #{}) -> error(never_get_started).
+on_bad_resource_type_destroy(_Id, #{}) -> ok.
+on_bad_resource_type_status(_Id, #{}, #{}) -> #{is_alive => false}.

apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl

@@ -26,7 +26,14 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
--include("emqx_rule_test.hrl").
+-import(emqx_rule_test_lib,
+        [ stop_apps/0
+        , start_apps/0
+        , create_simple_repub_rule/2
+        , create_simple_repub_rule/3
+        , make_simple_debug_resource_type/0
+        , init_events_counters/0
+        ]).
 
 all() ->
     [ {group, rulesql_select}

apps/emqx_stomp/src/emqx_stomp.appup.src

@@ -8,18 +8,10 @@
    {"4.3.4",
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
-   {"4.3.3",
+   {<<"4\\.3\\.[1-3]">>,
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
-   {"4.3.2",
-    [{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
-     {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
-   {"4.3.1",
-    [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
-     {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]},
-     {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
     [{restart_application,emqx_stomp},
      {apply,{emqx_stomp,force_clear_after_app_stoped,[]}}]},
@@ -31,17 +23,9 @@
    {"4.3.4",
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
-   {"4.3.3",
+   {<<"4\\.3\\.[1-3]">>,
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
-   {"4.3.2",
-    [{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
-     {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
-   {"4.3.1",
-    [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
-     {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]},
-     {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
    {"4.3.0",[{restart_application,emqx_stomp}]},
    {<<".*">>,[]}]}.