From 0c3de088cb2472f1f90c7fbb4fdfe53f9ee134ff Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 9 Dec 2022 13:58:07 +0100 Subject: [PATCH] chore: sync changes from ee to ce --- .../test/emqx_auth_mongo_SUITE.erl | 3 + .../src/emqx_auth_pgsql.appup.src | 1 + apps/emqx_exhook/src/emqx_exhook_handler.erl | 1 + apps/emqx_retainer/src/emqx_retainer_sup.erl | 13 +--- .../src/emqx_rule_engine.appup.src | 10 ++- .../emqx_rule_engine/src/emqx_rule_engine.erl | 61 +++++++++++-------- .../test/emqx_rule_engine_SUITE.erl | 53 +++++++++++++--- apps/emqx_rule_engine/test/emqx_rule_test.hrl | 29 --------- .../test/emqx_rule_test_lib.erl | 15 +++++ .../test/emqx_rulesql_SUITE.erl | 9 ++- apps/emqx_stomp/src/emqx_stomp.appup.src | 20 +----- 11 files changed, 120 insertions(+), 95 deletions(-) delete mode 100644 apps/emqx_rule_engine/test/emqx_rule_test.hrl diff --git a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl index 557d662d3..9c4c2dbec 100644 --- a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl +++ b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl @@ -153,8 +153,11 @@ init_per_group(_Group, Config) -> end_per_group(resilience, Config) -> OriginalServer = ?config(original_server, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), application:set_env(emqx_auth_mongo, server, OriginalServer), emqx_ct_helpers:stop_apps([emqx_auth_mongo]), + reset_proxy(ProxyHost, ProxyPort), emqx_ct_helpers:start_apps([emqx_auth_mongo], fun set_special_confs/1), ok; end_per_group(_Group, _Config) -> diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src index b55528d38..d5a641a66 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src @@ -17,3 +17,4 @@ {load_module,emqx_auth_pgsql,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. + diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index c724b4de1..63ec0d840 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -63,6 +63,7 @@ , call_fold/3 ]). + -elvis([{elvis_style, god_modules, disable}]). -define(STOP_OR_OK(Res), diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl index ca16a98bb..4d1824588 100644 --- a/apps/emqx_retainer/src/emqx_retainer_sup.erl +++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl @@ -34,8 +34,9 @@ init([Env]) -> type => worker, modules => [emqx_retainer]} || not is_managed_by_modules()]}}. --ifdef(EMQX_ENTERPRISE). - +%% This function is added to enterprise branch only. +%% In enterprise edition, the retainer worker may start under modules supervisor +%% so we should avoid starting it under the application root supervisor. is_managed_by_modules() -> try case supervisor:get_childspec(emqx_modules_sup, emqx_retainer) of @@ -46,11 +47,3 @@ is_managed_by_modules() -> exit : {noproc, _} -> false end. - --else. - -is_managed_by_modules() -> - %% always false for opensource edition - false. - --endif. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 6e62e5df8..b74843a2f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.11",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{"4.4.11",[ + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + ]}, {"4.4.10", [{add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, @@ -197,7 +200,10 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.11",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{"4.4.11",[ + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]} + ]}, {"4.4.10", [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index c9f32b962..f3e1651d4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -195,36 +195,43 @@ create_rule(Params = #{rawsql := Sql, actions := ActArgs}) -> {ok, Select} -> RuleId = maps:get(id, Params, rule_id()), Enabled = maps:get(enabled, Params, true), - try prepare_actions(ActArgs, Enabled) of - Actions -> - Rule = #rule{ - id = RuleId, - rawsql = Sql, - for = emqx_rule_sqlparser:select_from(Select), - is_foreach = emqx_rule_sqlparser:select_is_foreach(Select), - fields = emqx_rule_sqlparser:select_fields(Select), - doeach = emqx_rule_sqlparser:select_doeach(Select), - incase = emqx_rule_sqlparser:select_incase(Select), - conditions = emqx_rule_sqlparser:select_where(Select), - on_action_failed = maps:get(on_action_failed, Params, continue), - actions = Actions, - enabled = Enabled, - created_at = erlang:system_time(millisecond), - description = maps:get(description, Params, ""), - state = normal - }, - ok = emqx_rule_registry:add_rule(Rule), - ok = emqx_rule_metrics:create_rule_metrics(RuleId), - {ok, Rule} - catch - throw:{action_not_found, ActionName} -> - {error, {action_not_found, ActionName}}; - throw:Reason -> - {error, Reason} - end; + Rule = #rule{ + id = RuleId, + rawsql = Sql, + for = emqx_rule_sqlparser:select_from(Select), + is_foreach = emqx_rule_sqlparser:select_is_foreach(Select), + fields = emqx_rule_sqlparser:select_fields(Select), + doeach = emqx_rule_sqlparser:select_doeach(Select), + incase = emqx_rule_sqlparser:select_incase(Select), + conditions = emqx_rule_sqlparser:select_where(Select), + on_action_failed = maps:get(on_action_failed, Params, continue), + actions = [], + enabled = Enabled, + created_at = erlang:system_time(millisecond), + description = maps:get(description, Params, ""), + state = normal + }, + do_create_rule(Rule, ActArgs); Reason -> {error, Reason} end. +do_create_rule(Rule0 = #rule{id = RuleId, enabled = Enabled}, ActArgs) -> + try prepare_actions(ActArgs, Enabled) of + Actions -> + Rule = Rule0#rule{actions = Actions}, + ok = emqx_rule_registry:add_rule(Rule), + ok = emqx_rule_metrics:create_rule_metrics(RuleId), + {ok, Rule} + catch + throw:{resource_not_initialized, _} when Enabled =:= true -> + %% try again with rule disabled + do_create_rule(Rule0#rule{enabled = false}, ActArgs); + throw:{action_not_found, ActionName} -> + {error, {action_not_found, ActionName}}; + throw:Reason -> + {error, Reason} + end. + -spec(update_rule(#{id := binary(), _=>_}) -> {ok, rule()} | {error, {not_found, rule_id()} | term()}). update_rule(Params = #{id := RuleId}) -> case emqx_rule_registry:get_rule(RuleId) of diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 855a86bc0..be689d5a7 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -25,15 +25,19 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --include("emqx_rule_test.hrl"). --import(emqx_rule_test_lib, [make_simple_resource_type/1]). - -%% API request funcs -import(emqx_rule_test_lib, [ request_api/4 , request_api/5 , auth_header_/0 , api_path/1 + , stop_apps/0 + , start_apps/0 + , create_simple_repub_rule/2 + , create_simple_repub_rule/3 + , make_simple_debug_resource_type/0 + , make_simple_resource_type/1 + , make_bad_resource_type/0 + , init_events_counters/0 ]). %%-define(PROPTEST(M,F), true = proper:quickcheck(M:F())). @@ -63,6 +67,7 @@ groups() -> t_reset_metrics, t_reset_metrics_fallbacks, t_create_resource, + t_create_rule_with_resource, t_clean_resource_alarms ]}, {actions, [], @@ -310,6 +315,41 @@ t_create_resource(_Config) -> emqx_rule_registry:remove_resource(ResId), ok. +t_create_rule_with_resource(_) -> + emqx_rule_registry:register_resource_types([make_bad_resource_type()]), + ok = emqx_rule_engine:load_providers(), + {ok, #resource{id = BadResId}} = emqx_rule_engine:create_resource( + #{type => bad_resource, + config => #{}, + description => <<"bad resource">>}), + {ok, #resource{id = GoodResId}} = emqx_rule_engine:create_resource( + #{type => built_in, + config => #{}, + description => <<"bad resource">>}), + + {ok, #rule{id = BadRuleId, enabled = Enabled}} = emqx_rule_engine:create_rule( + #{rawsql => "select clientid as c, username as u " + "from \"t1\" ", + actions => [#{name => 'inspect', + args => #{<<"$resource">> => BadResId, a=>1, b=>2}}], + description => <<"Inspect rule">> + }), + ?assertEqual(false, Enabled), + + {ok, #rule{id = GoodRuleId, enabled = Enabled1}} = emqx_rule_engine:create_rule( + #{rawsql => "select clientid as c, username as u " + "from \"t1\" ", + actions => [#{name => 'inspect', + args => #{<<"$resource">> => GoodResId, a=>1, b=>2}}], + description => <<"Inspect rule">> + }), + ?assertEqual(true, Enabled1), + emqx_rule_registry:remove_rule(BadRuleId), + emqx_rule_registry:remove_rule(GoodRuleId), + emqx_rule_registry:remove_resource(BadResId), + emqx_rule_registry:remove_resource(GoodResId), + ok. + t_clean_resource_alarms(_Config) -> lists:foreach(fun(ResId) -> clean_resource_alarms(ResId) @@ -350,8 +390,7 @@ t_inspect_action(_Config) -> #{rawsql => "select clientid as c, username as u " "from \"t1\" ", actions => [#{name => 'inspect', - args => #{'$resource' => ResId, a=>1, b=>2}}], - type => built_in, + args => #{<<"$resource">> => ResId, a=>1, b=>2}}], description => <<"Inspect rule">> }), {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), @@ -369,7 +408,6 @@ t_reset_metrics(_Config) -> #{rawsql => "select clientid as c, username as u " "from \"t1\" ", actions => [#{name => 'inspect', args => #{a=>1, b=>2}}], - type => built_in, description => <<"Inspect rule">> }), {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), @@ -414,7 +452,6 @@ t_reset_metrics_fallbacks(_Config) -> #{name => 'inspect', args => #{}, fallbacks => []}, #{name => 'inspect', args => #{}, fallbacks => []} ]}], - type => built_in, description => <<"Inspect rule">> }), {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), diff --git a/apps/emqx_rule_engine/test/emqx_rule_test.hrl b/apps/emqx_rule_engine/test/emqx_rule_test.hrl deleted file mode 100644 index 748cc2e9a..000000000 --- a/apps/emqx_rule_engine/test/emqx_rule_test.hrl +++ /dev/null @@ -1,29 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% Test Suite funcs --import(emqx_rule_test_lib, - [ stop_apps/0 - , start_apps/0 - ]). - -%% RULE helper funcs --import(emqx_rule_test_lib, - [ create_simple_repub_rule/2 - , create_simple_repub_rule/3 - , make_simple_debug_resource_type/0 - , init_events_counters/0 - ]). diff --git a/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl b/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl index e4ef2f5f5..26846417b 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl @@ -125,6 +125,17 @@ make_simple_resource_type(ResTypeName) -> title = #{en => <<"Simple Resource Type">>}, description = #{en => <<"Simple Resource Type">>}}. +make_bad_resource_type() -> + #resource_type{ + name = bad_resource, + provider = ?APP, + params_spec = #{}, + on_create = {?MODULE, on_bad_resource_type_create}, + on_destroy = {?MODULE, on_bad_resource_type_destroy}, + on_status = {?MODULE, on_bad_resource_type_status}, + title = #{en => <<"Bad Resource Type">>}, + description = #{en => <<"Bad Resource Type">>}}. + init_events_counters() -> ets:new(events_record_tab, [named_table, bag, public]). @@ -200,3 +211,7 @@ on_get_resource_status(_id, _) -> #{is_alive => true}. on_simple_resource_type_create(_Id, #{}) -> #{}. on_simple_resource_type_destroy(_Id, #{}) -> ok. on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}. + +on_bad_resource_type_create(_Id, #{}) -> error(never_get_started). +on_bad_resource_type_destroy(_Id, #{}) -> ok. +on_bad_resource_type_status(_Id, #{}, #{}) -> #{is_alive => false}. diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index 77554abf8..a0687f8be 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -26,7 +26,14 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --include("emqx_rule_test.hrl"). +-import(emqx_rule_test_lib, + [ stop_apps/0 + , start_apps/0 + , create_simple_repub_rule/2 + , create_simple_repub_rule/3 + , make_simple_debug_resource_type/0 + , init_events_counters/0 + ]). all() -> [ {group, rulesql_select} diff --git a/apps/emqx_stomp/src/emqx_stomp.appup.src b/apps/emqx_stomp/src/emqx_stomp.appup.src index 499b80f38..b6e491731 100644 --- a/apps/emqx_stomp/src/emqx_stomp.appup.src +++ b/apps/emqx_stomp/src/emqx_stomp.appup.src @@ -8,18 +8,10 @@ {"4.3.4", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, - {"4.3.3", + {<<"4\\.3\\.[1-3]">>, [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]}, - {"4.3.2", - [{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, - {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]}, - {"4.3.1", - [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, - {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{restart_application,emqx_stomp}, {apply,{emqx_stomp,force_clear_after_app_stoped,[]}}]}, @@ -31,17 +23,9 @@ {"4.3.4", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, - {"4.3.3", + {<<"4\\.3\\.[1-3]">>, [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]}, - {"4.3.2", - [{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, - {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]}, - {"4.3.1", - [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, - {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.0",[{restart_application,emqx_stomp}]}, {<<".*">>,[]}]}.