Merge pull request #10620 from terry-xiaoyu/rule-engine-export-columns-from-foreach

fix: cannot access columns exported by FOREACH in DO clause
2023-05-06 15:43:17 +08:00 · 2023-05-06 15:43:17 +08:00 · d9f8c096ed
parent 39b00ab602 6f959e5784
commit d9f8c096ed
11 changed files with 96 additions and 15 deletions
--- a/.ci/docker-compose-file/openldap/Dockerfile
+++ b/.ci/docker-compose-file/openldap/Dockerfile
@ -1,4 +1,4 @@
-FROM buildpack-deps:stretch
+FROM buildpack-deps:stable

 ARG LDAP_TAG=2.4.50

--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@ -1,6 +1,6 @@
 {application, emqx_rule_engine,
 [{description, "EMQX Rule Engine"},
-  {vsn, "4.4.18"}, % strict semver, bump manually!
+  {vsn, "4.4.19"}, % strict semver, bump manually!
  {modules, []},
  {registered, [emqx_rule_engine_sup, emqx_rule_registry, emqx_rule_engine_jwt_sup]},
  {applications, [kernel,stdlib,rulesql,getopt,jose]},
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@ -1,9 +1,13 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.17",[{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}]},
+  [{"4.4.18",[{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]},
+   {"4.4.17",
+    [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}]},
   {"4.4.16",
-    [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
@ -254,9 +258,13 @@
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]},
   {<<".*">>,[]}],
-  [{"4.4.17",[{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}]},
+  [{"4.4.18",[{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]},
+   {"4.4.17",
+    [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}]},
   {"4.4.16",
-    [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@ -103,12 +103,12 @@ do_apply_rule(#rule{id = RuleId,
    case ?RAISE(match_conditions(Conditions, ColumnsAndSelected),
                {match_conditions_error, {_EXCLASS_,_EXCPTION_,_ST_}}) of
        true ->
-            Collection2 = filter_collection(Input, InCase, DoEach, Collection),
+            Collection2 = filter_collection(ColumnsAndSelected, InCase, DoEach, Collection),
            case Collection2 of
                [] -> emqx_rule_metrics:inc_rules_no_result(RuleId);
                _ -> emqx_rule_metrics:inc_rules_passed(RuleId)
            end,
-            {ok, [take_actions(Actions, Coll, Input, OnFailed) || Coll <- Collection2]};
+            {ok, [take_actions(Actions, Coll, ColumnsAndSelected, OnFailed) || Coll <- Collection2]};
        false ->
            ok = emqx_rule_metrics:inc_rules_no_result(RuleId),
            {error, nomatch}
--- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl
@ -987,16 +987,19 @@ t_sqlparse_foreach_7(_Config) ->
                        #{<<"payload">> => Payload,
                          <<"topic">> => <<"t/a">>}})).

+-define(COLL, #{<<"info">> := [<<"haha">>, #{<<"name">> := <<"cmd1">>, <<"cmd">> := <<"1">>}]}).
 t_sqlparse_foreach_8(_Config) ->
    %% Verify foreach-do-incase and cascaded AS
    Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info "
-          "do info.cmd as msg_type, info.name as name "
+          "do info.cmd as msg_type, info.name as name, s, c "
          "incase is_map(info) "
          "from \"t/#\" "
          "where s.page = '2' ",
    Payload  = <<"{\"sensors\": {\"page\": 2, \"collection\": "
                 "{\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }">>,
-    ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]},
+    ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>,
+                        <<"s">> := #{<<"page">> := 2, <<"collection">> := ?COLL},
+                        <<"c">> := ?COLL}]},
                 emqx_rule_sqltester:test(
                    #{<<"rawsql">> => Sql,
                      <<"ctx">> =>
--- a/changes/v4.4.19-en.md
+++ b/changes/v4.4.19-en.md
@ -0,0 +1,32 @@
+# v4.4.19
+
+## Enhancements
+
+- Improving error logs related to Proxy Protocol [emqx/esockd#177](https://github.com/emqx/esockd/pull/177).
+
+  The sample logs before this improvement:
+  ```
+  2023-04-20T14:56:51.671735+08:00 [error] supervisor: 'esockd_connection_sup - <0.2537.0>', errorContext: connection_shutdown, reason: {invalid_proxy_info,<<"f\n">>}, offender: [{pid,<0.3192.0>},{name,connection},{mfargs,{...}}]
+
+  2023-04-20T14:57:01.348275+08:00 [error] supervisor: 'esockd_connection_sup - <0.2537.0>', errorContext: connection_shutdown, reason: {proxy_proto_timeout,5000}, offender: [{pid,<0.3194.0>},{name,connection},{mfargs,{...}}]
+  ```
+  After the improvement:
+  ```
+  2023-04-20T18:07:06.180134+08:00 [error] [esockd_proxy_protocol] The listener 127.0.0.1:8883 is working in proxy protocol mode, but received invalid proxy_protocol header, raw_bytes=<<"f\n">>
+
+  2023-04-20T18:10:17.205436+08:00 [error] [esockd_proxy_protocol] The listener 127.0.0.1:8883 is working in proxy protocol mode, but timed out while waiting for proxy_protocol header
+  ```
+
+## Bug fixes
+
+- Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` in the `DO` clause [#10620](https://github.com/emqx/emqx/pull/10620).
+
+  Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement:
+  ```
+  FOREACH payload.date as date, payload.array as elem
+  DO date, elem
+  FROM "t/#"
+  ```
+  Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement:
+  `[{"elem": "a","date": "undefined"}]`.
+  After the fix, the output of the SQL statement is: `[{"elem": "a","date": "2023-05-06"}]`
--- a/changes/v4.4.19-zh.md
+++ b/changes/v4.4.19-zh.md
@ -0,0 +1,32 @@
+# v4.4.19
+
+## 增强
+
+- 改进 Proxy Protocol 相关的错误日志 [emqx/esockd#177](https://github.com/emqx/esockd/pull/177)。
+
+  改进之前的日志样例:
+  ```
+  2023-04-20T14:56:51.671735+08:00 [error] supervisor: 'esockd_connection_sup - <0.2537.0>', errorContext: connection_shutdown, reason: {invalid_proxy_info,<<"f\n">>}, offender: [{pid,<0.3192.0>},{name,connection},{mfargs,{...}}]
+
+  2023-04-20T14:57:01.348275+08:00 [error] supervisor: 'esockd_connection_sup - <0.2537.0>', errorContext: connection_shutdown, reason: {proxy_proto_timeout,5000}, offender: [{pid,<0.3194.0>},{name,connection},{mfargs,{...}}]
+  ```
+  改进之后:
+  ```
+  2023-04-20T18:07:06.180134+08:00 [error] [esockd_proxy_protocol] The listener 127.0.0.1:8883 is working in proxy protocol mode, but received invalid proxy_protocol header, raw_bytes=<<"f\n">>
+
+  2023-04-20T18:10:17.205436+08:00 [error] [esockd_proxy_protocol] The listener 127.0.0.1:8883 is working in proxy protocol mode, but timed out while waiting for proxy_protocol header
+  ```
+
+## 修复
+
+- 修复规则引擎无法在 `DO` 子句中访问 `FOREACH` 导出的变量的问题 [#10620](https://github.com/emqx/emqx/pull/10620)。
+
+  给定消息：`{"date": "2023-05-06", "array": ["a"]}`，以及如下 SQL 语句：
+  ```
+  FOREACH payload.date as date, payload.array as elem
+  DO date, elem
+  FROM "t/#"
+  ```
+  修复前，以上 SQL 语句中 `FOREACH` 导出的 `date` 变量无法在 `DO` 子句中访问，导致以上 SQL 的输出为：
+  `[{"elem": "a","date": "undefined"}]`。
+  修复后，SQL 的输出为：`[{"elem": "a","date": "2023-05-06"}]`
--- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
@ -1,6 +1,6 @@
 {application, emqx_dashboard,
 [{description, "EMQX Web Dashboard"},
-  {vsn, "4.4.17"}, % strict semver, bump manually!
+  {vsn, "4.4.18"}, % strict semver, bump manually!
  {modules, []},
  {registered, [emqx_dashboard_sup]},
  {applications, [kernel,stdlib,mnesia,minirest]},
--- a/rebar.config
+++ b/rebar.config
@ -48,7 +48,7 @@
    , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.9.0"}}}
    , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
    , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.1"}}}
-    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.9"}}}
+    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.10"}}}
    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.11"}}}
    , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.0.1"}}}
    , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}}
--- a/src/emqx.app.src
+++ b/src/emqx.app.src
@ -6,7 +6,7 @@
  %% the emqx `release' version, which in turn is comprised of several
  %% apps, one of which is this.  See `emqx_release.hrl' for more
  %% info.
-  {vsn, "4.4.18"}, % strict semver, bump manually!
+  {vsn, "4.4.19"}, % strict semver, bump manually!
  {modules, []},
  {registered, []},
  {applications, [ kernel
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@ -1,7 +1,10 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.17",
+  [{"4.4.18",
+    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
+   {"4.4.17",
    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
     {load_module,emqx_app,brutal_purge,soft_purge,[]},
     {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
@ -537,7 +540,10 @@
     {apply,{application,set_env,
                         [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
   {<<".*">>,[]}],
-  [{"4.4.17",
+  [{"4.4.18",
+    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
+   {"4.4.17",
    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
     {load_module,emqx_app,brutal_purge,soft_purge,[]},
     {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},