Merge remote-tracking branch 'origin/main-v4.3' into main-v4.3

2022-10-13 08:34:30 +02:00 · 2022-10-13 08:34:30 +02:00 · 198a1ed6cb
parent 7f9886e611 18db788662
commit 198a1ed6cb
13 changed files with 1778 additions and 1467 deletions
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@ -12,9 +12,15 @@ File format:

 ## v4.3.22

+### Enhancements
+
+- Add a warning log if the ACL check failed for subscription. [#9124](https://github.com/emqx/emqx/pull/9124)
+  This is to make the ACL deny logging for subscription behave the same as for publish.
+
 ### Bug fixes

 - Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic. [#9024](https://github.com/emqx/emqx/pull/9024)
+- "Pause due to rate limit" log level demoted from warning to notice [#9134](https://github.com/emqx/emqx/pull/9134)

 ## v4.3.21

--- a/apps/emqx_exproto/src/emqx_exproto_conn.erl
+++ b/apps/emqx_exproto/src/emqx_exproto_conn.erl
@ -643,7 +643,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) ->
        {ok, Limiter1} ->
            State#state{limiter = Limiter1};
        {pause, Time, Limiter1} ->
-            ?LOG(warning, "Pause ~pms due to rate limit", [Time]),
+            ?LOG(notice, "Pause ~pms due to rate limit", [Time]),
            TRef = start_timer(Time, limit_timeout),
            State#state{sockstate   = blocked,
                        limiter     = Limiter1,
--- a/apps/emqx_rule_engine/rebar.config
+++ b/apps/emqx_rule_engine/rebar.config
@ -1,5 +1,7 @@
+%% -*- mode: erlang -*-
 {deps, []}.

+%% Comple Opts
 {erl_opts, [warn_unused_vars,
            warn_shadow_vars,
            warn_unused_import,
@ -18,6 +20,14 @@
               warnings_as_errors, deprecated_functions
              ]}.

+%% {erl_opts, [...]}, but for CT runs
+%% NOT WORKING!!!
+%% %% == Common Test ==
+%% {ct_compile_opts, [ export_all
+%%                   , nowarn_export_all
+%%                   ]}.
+%% {ct_opts, []}.
+
 {cover_enabled, true}.
 {cover_opts, [verbose]}.
 {cover_export_enabled, true}.
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@ -2,20 +2,23 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
  [{"4.3.15",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
   {"4.3.14",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
   {"4.3.13",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
@ -24,7 +27,8 @@
     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
   {"4.3.12",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
@ -33,7 +37,8 @@
     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
   {"4.3.11",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
@ -43,7 +48,8 @@
     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
   {"4.3.10",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
@ -206,20 +212,23 @@
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
   {<<".*">>,[]}],
  [{"4.3.15",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
   {"4.3.14",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
   {"4.3.13",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
@ -228,7 +237,8 @@
     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
   {"4.3.12",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
@ -237,7 +247,8 @@
     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
   {"4.3.11",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
@ -247,7 +258,8 @@
     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
   {"4.3.10",
-    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
--- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
@ -332,28 +332,46 @@ null() ->
 %% Arithmetic Funcs
 %%------------------------------------------------------------------------------

+-define(OPERATOR_TYPE_ERROR, unsupported_function_operator_type).
+
 %% plus 2 numbers
 '+'(X, Y) when is_number(X), is_number(Y) ->
    X + Y;
-
 %% concat 2 strings
 '+'(X, Y) when is_binary(X), is_binary(Y) ->
-    concat(X, Y).
+    concat(X, Y);
+%% unsupported type implicit conversion
+'+'(X, Y)
+  when (is_number(X) andalso is_binary(Y)) orelse
+       (is_binary(X) andalso is_number(Y)) ->
+    error(unsupported_type_implicit_conversion);
+'+'(_, _) ->
+    error(?OPERATOR_TYPE_ERROR).

 '-'(X, Y) when is_number(X), is_number(Y) ->
-    X - Y.
+    X - Y;
+'-'(_, _) ->
+    error(?OPERATOR_TYPE_ERROR).

 '*'(X, Y) when is_number(X), is_number(Y) ->
-    X * Y.
+    X * Y;
+'*'(_, _) ->
+    error(?OPERATOR_TYPE_ERROR).

 '/'(X, Y) when is_number(X), is_number(Y) ->
-    X / Y.
+    X / Y;
+'/'(_, _) ->
+    error(?OPERATOR_TYPE_ERROR).

 'div'(X, Y) when is_integer(X), is_integer(Y) ->
-    X div Y.
+    X div Y;
+'div'(_, _) ->
+    error(?OPERATOR_TYPE_ERROR).

 mod(X, Y) when is_integer(X), is_integer(Y) ->
-    X rem Y.
+    X rem Y;
+mod(_, _) ->
+    error(?OPERATOR_TYPE_ERROR).

 eq(X, Y) ->
    X == Y.
--- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
--- a/apps/emqx_rule_engine/test/emqx_rule_test.hrl
+++ b/apps/emqx_rule_engine/test/emqx_rule_test.hrl
@ -0,0 +1,29 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% Test Suite funcs
+-import(emqx_rule_test_lib,
+        [ stop_apps/0
+        , start_apps/0
+        ]).
+
+%% RULE helper funcs
+-import(emqx_rule_test_lib,
+        [ create_simple_repub_rule/2
+        , create_simple_repub_rule/3
+        , make_simple_debug_resource_type/0
+        , init_events_counters/0
+        ]).
--- a/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_test_lib.erl
@ -0,0 +1,141 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_rule_test_lib).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("emqx_rule_engine/include/rule_engine.hrl").
+-include_lib("emqx/include/emqx.hrl").
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+%%------------------------------------------------------------------------------
+%% Start Apps
+%%------------------------------------------------------------------------------
+
+stop_apps() ->
+    stopped = mnesia:stop(),
+    [application:stop(App) || App <- [emqx_rule_engine, emqx]].
+
+start_apps() ->
+    [start_apps(App, SchemaFile, ConfigFile) ||
+        {App, SchemaFile, ConfigFile}
+            <- [{emqx, deps_path(emqx, "priv/emqx.schema"),
+                       deps_path(emqx, "etc/emqx.conf")},
+                {emqx_rule_engine, local_path("priv/emqx_rule_engine.schema"),
+                                   local_path("etc/emqx_rule_engine.conf")}]].
+
+%%--------------------------------------
+%% start apps helper funcs
+
+start_apps(App, SchemaFile, ConfigFile) ->
+    read_schema_configs(App, SchemaFile, ConfigFile),
+    set_special_configs(App),
+    {ok, _} = application:ensure_all_started(App).
+
+read_schema_configs(App, SchemaFile, ConfigFile) ->
+    ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]),
+    Schema = cuttlefish_schema:files([SchemaFile]),
+    Conf = conf_parse:file(ConfigFile),
+    NewConfig = cuttlefish_generator:map(Schema, Conf),
+    Vals = proplists:get_value(App, NewConfig, []),
+    [application:set_env(App, Par, Value) || {Par, Value} <- Vals].
+
+deps_path(App, RelativePath) ->
+    %% Note: not lib_dir because etc dir is not sym-link-ed to _build dir
+    %% but priv dir is
+    Path0 = code:priv_dir(App),
+    Path = case file:read_link(Path0) of
+               {ok, Resolved} -> Resolved;
+               {error, _} -> Path0
+           end,
+    filename:join([Path, "..", RelativePath]).
+
+local_path(RelativePath) ->
+    deps_path(emqx_rule_engine, RelativePath).
+
+set_special_configs(emqx_rule_engine) ->
+    application:set_env(emqx_rule_engine, ignore_sys_message, true),
+    application:set_env(emqx_rule_engine, events,
+                       [{'client.connected',on,1},
+                        {'client.disconnected',on,1},
+                        {'session.subscribed',on,1},
+                        {'session.unsubscribed',on,1},
+                        {'message.acked',on,1},
+                        {'message.dropped',on,1},
+                        {'message.delivered',on,1}
+                       ]),
+    ok;
+set_special_configs(_App) ->
+    ok.
+
+%%------------------------------------------------------------------------------
+%% rule test helper funcs
+%%------------------------------------------------------------------------------
+
+create_simple_repub_rule(TargetTopic, SQL) ->
+    create_simple_repub_rule(TargetTopic, SQL, <<"${payload}">>).
+
+create_simple_repub_rule(TargetTopic, SQL, Template) ->
+    {ok, Rule} = emqx_rule_engine:create_rule(
+                    #{rawsql => SQL,
+                      actions => [#{name => 'republish',
+                                    args => #{<<"target_topic">> => TargetTopic,
+                                              <<"target_qos">> => -1,
+                                              <<"payload_tmpl">> => Template}
+                                    }],
+                      description => <<"simple repub rule">>}),
+    Rule.
+
+make_simple_debug_resource_type() ->
+    #resource_type{
+       name = built_in,
+       provider = ?APP,
+       params_spec = #{},
+       on_create = {?MODULE, on_resource_create},
+       on_destroy = {?MODULE, on_resource_destroy},
+       on_status = {?MODULE, on_get_resource_status},
+       title = #{en => <<"Built-In Resource Type (debug)">>},
+       description = #{en => <<"The built in resource type for debug purpose">>}}.
+
+make_simple_resource_type(ResTypeName) ->
+    #resource_type{
+       name = ResTypeName,
+       provider = ?APP,
+       params_spec = #{},
+       on_create = {?MODULE, on_simple_resource_type_create},
+       on_destroy = {?MODULE, on_simple_resource_type_destroy},
+       on_status = {?MODULE, on_simple_resource_type_status},
+       title = #{en => <<"Simple Resource Type">>},
+       description = #{en => <<"Simple Resource Type">>}}.
+
+init_events_counters() ->
+    ets:new(events_record_tab, [named_table, bag, public]).
+
+%%------------------------------------------------------------------------------
+%% Internal helper funcs
+%%------------------------------------------------------------------------------
+
+on_resource_create(_id, _) -> #{}.
+on_resource_destroy(_id, _) -> ok.
+on_get_resource_status(_id, _) -> #{is_alive => true}.
+
+on_simple_resource_type_create(_Id, #{}) -> #{}.
+on_simple_resource_type_destroy(_Id, #{}) -> ok.
+on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}.
--- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl
--- a/apps/emqx_stomp/src/emqx_stomp_connection.erl
+++ b/apps/emqx_stomp/src/emqx_stomp_connection.erl
@ -469,7 +469,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) ->
        {ok, Limiter1} ->
            State#state{limiter = Limiter1};
        {pause, Time, Limiter1} ->
-            ?LOG(warning, "Pause ~pms due to rate limit", [Time]),
+            ?LOG(notice, "Pause ~pms due to rate limit", [Time]),
            TRef = start_timer(Time, limit_timeout),
            State#state{sockstate   = blocked,
                        limiter     = Limiter1,
--- a/src/emqx_channel.erl
+++ b/src/emqx_channel.erl
@ -1501,12 +1501,15 @@ check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos    = QoS,
 check_sub_acls(TopicFilters, Channel) ->
    check_sub_acls(TopicFilters, Channel, []).

-check_sub_acls([ TopicFilter = {Topic, _} | More] , Channel, Acc) ->
+check_sub_acls([ TopicFilter = {Topic, SubOpts} | More] , Channel, Acc) ->
    case check_sub_acl(Topic, Channel) of
        allow ->
            check_sub_acls(More, Channel, [ {TopicFilter, 0} | Acc]);
        deny ->
-            check_sub_acls(More, Channel, [ {TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
+            ReasonCode = ?RC_NOT_AUTHORIZED,
+            ?LOG(warning, "Cannot subscribe ~s with options ~p due to ~s.",
+                 [Topic, SubOpts, emqx_reason_codes:text(ReasonCode)]),
+            check_sub_acls(More, Channel, [ {TopicFilter, ReasonCode} | Acc])
    end;
 check_sub_acls([], _Channel, Acc) ->
    lists:reverse(Acc).
--- a/src/emqx_connection.erl
+++ b/src/emqx_connection.erl
@ -756,7 +756,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) ->
        {ok, Limiter1} ->
            State#state{limiter = Limiter1};
        {pause, Time, Limiter1} ->
-            ?LOG(warning, "Pause ~pms due to rate limit", [Time]),
+            ?LOG(notice, "Pause ~pms due to rate limit", [Time]),
            TRef = start_timer(Time, limit_timeout),
            State#state{sockstate   = blocked,
                        limiter     = Limiter1,
--- a/src/emqx_ws_connection.erl
+++ b/src/emqx_ws_connection.erl
@ -499,7 +499,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) ->
        {ok, Limiter1} ->
            State#state{limiter = Limiter1};
        {pause, Time, Limiter1} ->
-            ?LOG(warning, "Pause ~pms due to rate limit", [Time]),
+            ?LOG(notice, "Pause ~pms due to rate limit", [Time]),
            TRef = start_timer(Time, limit_timeout),
            NState = State#state{sockstate   = blocked,
                                 limiter     = Limiter1,