From 154fe441de79e8a0b1be3a32ef24718a3d5c0ac7 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 6 May 2023 09:09:20 +0800 Subject: [PATCH 1/3] fix: cannot access columns exported by FOREACH in DO clause --- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 16 ++++++++++++---- apps/emqx_rule_engine/src/emqx_rule_runtime.erl | 4 ++-- .../emqx_rule_engine/test/emqx_rulesql_SUITE.erl | 7 +++++-- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- rebar.config | 2 +- src/emqx.app.src | 2 +- src/emqx.appup.src | 10 ++++++++-- 8 files changed, 31 insertions(+), 14 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 1eb151707..84b1de762 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQX Rule Engine"}, - {vsn, "4.4.18"}, % strict semver, bump manually! + {vsn, "4.4.19"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry, emqx_rule_engine_jwt_sup]}, {applications, [kernel,stdlib,rulesql,getopt,jose]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 2c065e7a9..c3b0154a1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,9 +1,13 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.17",[{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}]}, + [{"4.4.18",[{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]}, + {"4.4.17", + [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}]}, {"4.4.16", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, @@ -254,9 +258,13 @@ {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.17",[{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}]}, + [{"4.4.18",[{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]}, + {"4.4.17", + [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}]}, {"4.4.16", - [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index f8927dd05..56c64924f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -103,12 +103,12 @@ do_apply_rule(#rule{id = RuleId, case ?RAISE(match_conditions(Conditions, ColumnsAndSelected), {match_conditions_error, {_EXCLASS_,_EXCPTION_,_ST_}}) of true -> - Collection2 = filter_collection(Input, InCase, DoEach, Collection), + Collection2 = filter_collection(ColumnsAndSelected, InCase, DoEach, Collection), case Collection2 of [] -> emqx_rule_metrics:inc_rules_no_result(RuleId); _ -> emqx_rule_metrics:inc_rules_passed(RuleId) end, - {ok, [take_actions(Actions, Coll, Input, OnFailed) || Coll <- Collection2]}; + {ok, [take_actions(Actions, Coll, ColumnsAndSelected, OnFailed) || Coll <- Collection2]}; false -> ok = emqx_rule_metrics:inc_rules_no_result(RuleId), {error, nomatch} diff --git a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl index c57869ec5..51849a35f 100644 --- a/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rulesql_SUITE.erl @@ -987,16 +987,19 @@ t_sqlparse_foreach_7(_Config) -> #{<<"payload">> => Payload, <<"topic">> => <<"t/a">>}})). +-define(COLL, #{<<"info">> := [<<"haha">>, #{<<"name">> := <<"cmd1">>, <<"cmd">> := <<"1">>}]}). t_sqlparse_foreach_8(_Config) -> %% Verify foreach-do-incase and cascaded AS Sql = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " - "do info.cmd as msg_type, info.name as name " + "do info.cmd as msg_type, info.name as name, s, c " "incase is_map(info) " "from \"t/#\" " "where s.page = '2' ", Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": " "{\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }">>, - ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, + ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>, + <<"s">> := #{<<"page">> := 2, <<"collection">> := ?COLL}, + <<"c">> := ?COLL}]}, emqx_rule_sqltester:test( #{<<"rawsql">> => Sql, <<"ctx">> => diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index bc45aed88..a224d70ef 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQX Web Dashboard"}, - {vsn, "4.4.17"}, % strict semver, bump manually! + {vsn, "4.4.18"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/rebar.config b/rebar.config index 2b4e3d1b5..582307e61 100644 --- a/rebar.config +++ b/rebar.config @@ -48,7 +48,7 @@ , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.9.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.1"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.9"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.10"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.11"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.0.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} diff --git a/src/emqx.app.src b/src/emqx.app.src index 1ad866313..3ccbab2a7 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.4.18"}, % strict semver, bump manually! + {vsn, "4.4.19"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 9d85f019b..442bf0ef5 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.17", + [{"4.4.18", + [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {"4.4.17", [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, @@ -537,7 +540,10 @@ {apply,{application,set_env, [gen_rpc,insecure_auth_fallback_allowed,true]}}]}, {<<".*">>,[]}], - [{"4.4.17", + [{"4.4.18", + [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {"4.4.17", [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, From 58cbc7411b65ce5862abcdc5e98ecedc422d0dea Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 6 May 2023 14:12:12 +0800 Subject: [PATCH 2/3] fix: try to fix the ci for ldap --- .ci/docker-compose-file/openldap/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker-compose-file/openldap/Dockerfile b/.ci/docker-compose-file/openldap/Dockerfile index adbb80800..b42f12a9d 100644 --- a/.ci/docker-compose-file/openldap/Dockerfile +++ b/.ci/docker-compose-file/openldap/Dockerfile @@ -1,4 +1,4 @@ -FROM buildpack-deps:stretch +FROM buildpack-deps:stable ARG LDAP_TAG=2.4.50 From 6f959e57844db149a73f260f24e96a41c949f88e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 6 May 2023 15:20:28 +0800 Subject: [PATCH 3/3] chore: update the change logs for v4.4.19 --- changes/v4.4.19-en.md | 32 ++++++++++++++++++++++++++++++++ changes/v4.4.19-zh.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 changes/v4.4.19-en.md create mode 100644 changes/v4.4.19-zh.md diff --git a/changes/v4.4.19-en.md b/changes/v4.4.19-en.md new file mode 100644 index 000000000..9501c0f9f --- /dev/null +++ b/changes/v4.4.19-en.md @@ -0,0 +1,32 @@ +# v4.4.19 + +## Enhancements + +- Improving error logs related to Proxy Protocol [emqx/esockd#177](https://github.com/emqx/esockd/pull/177). + + The sample logs before this improvement: + ``` + 2023-04-20T14:56:51.671735+08:00 [error] supervisor: 'esockd_connection_sup - <0.2537.0>', errorContext: connection_shutdown, reason: {invalid_proxy_info,<<"f\n">>}, offender: [{pid,<0.3192.0>},{name,connection},{mfargs,{...}}] + + 2023-04-20T14:57:01.348275+08:00 [error] supervisor: 'esockd_connection_sup - <0.2537.0>', errorContext: connection_shutdown, reason: {proxy_proto_timeout,5000}, offender: [{pid,<0.3194.0>},{name,connection},{mfargs,{...}}] + ``` + After the improvement: + ``` + 2023-04-20T18:07:06.180134+08:00 [error] [esockd_proxy_protocol] The listener 127.0.0.1:8883 is working in proxy protocol mode, but received invalid proxy_protocol header, raw_bytes=<<"f\n">> + + 2023-04-20T18:10:17.205436+08:00 [error] [esockd_proxy_protocol] The listener 127.0.0.1:8883 is working in proxy protocol mode, but timed out while waiting for proxy_protocol header + ``` + +## Bug fixes + +- Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` in the `DO` clause [#10620](https://github.com/emqx/emqx/pull/10620). + + Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement: + ``` + FOREACH payload.date as date, payload.array as elem + DO date, elem + FROM "t/#" + ``` + Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement: + `[{"elem": "a","date": "undefined"}]`. + After the fix, the output of the SQL statement is: `[{"elem": "a","date": "2023-05-06"}]` diff --git a/changes/v4.4.19-zh.md b/changes/v4.4.19-zh.md new file mode 100644 index 000000000..37a54ecf2 --- /dev/null +++ b/changes/v4.4.19-zh.md @@ -0,0 +1,32 @@ +# v4.4.19 + +## 增强 + +- 改进 Proxy Protocol 相关的错误日志 [emqx/esockd#177](https://github.com/emqx/esockd/pull/177)。 + + 改进之前的日志样例: + ``` + 2023-04-20T14:56:51.671735+08:00 [error] supervisor: 'esockd_connection_sup - <0.2537.0>', errorContext: connection_shutdown, reason: {invalid_proxy_info,<<"f\n">>}, offender: [{pid,<0.3192.0>},{name,connection},{mfargs,{...}}] + + 2023-04-20T14:57:01.348275+08:00 [error] supervisor: 'esockd_connection_sup - <0.2537.0>', errorContext: connection_shutdown, reason: {proxy_proto_timeout,5000}, offender: [{pid,<0.3194.0>},{name,connection},{mfargs,{...}}] + ``` + 改进之后: + ``` + 2023-04-20T18:07:06.180134+08:00 [error] [esockd_proxy_protocol] The listener 127.0.0.1:8883 is working in proxy protocol mode, but received invalid proxy_protocol header, raw_bytes=<<"f\n">> + + 2023-04-20T18:10:17.205436+08:00 [error] [esockd_proxy_protocol] The listener 127.0.0.1:8883 is working in proxy protocol mode, but timed out while waiting for proxy_protocol header + ``` + +## 修复 + +- 修复规则引擎无法在 `DO` 子句中访问 `FOREACH` 导出的变量的问题 [#10620](https://github.com/emqx/emqx/pull/10620)。 + + 给定消息：`{"date": "2023-05-06", "array": ["a"]}`，以及如下 SQL 语句： + ``` + FOREACH payload.date as date, payload.array as elem + DO date, elem + FROM "t/#" + ``` + 修复前，以上 SQL 语句中 `FOREACH` 导出的 `date` 变量无法在 `DO` 子句中访问，导致以上 SQL 的输出为： + `[{"elem": "a","date": "undefined"}]`。 + 修复后，SQL 的输出为：`[{"elem": "a","date": "2023-05-06"}]`