From 8abf6474528351c0654bc72933534c3c5f631634 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 1 Jul 2022 18:23:29 +0800 Subject: [PATCH 01/34] fix(exclusive): add upgarde codes --- src/emqx_exclusive_subscription.erl | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/emqx_exclusive_subscription.erl b/src/emqx_exclusive_subscription.erl index ab26a6bd6..487256903 100644 --- a/src/emqx_exclusive_subscription.erl +++ b/src/emqx_exclusive_subscription.erl @@ -27,6 +27,9 @@ -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). +%% For upgrade +-export([on_add_module/0, on_delete_module/0]). + -export([ check_subscribe/2, unsubscribe/2 @@ -58,7 +61,22 @@ mnesia(boot) -> {storage_properties, StoreProps} ]); mnesia(copy) -> - ok = ekka_mnesia:copy_table(?TAB, ram_copies). + case ekka_mnesia:copy_table(?TAB, ram_copies) of + ok -> + ok; + {no_exists, _} -> + mnesia(boot) + end. + +%%-------------------------------------------------------------------- +%% Upgrade +%%-------------------------------------------------------------------- + +on_add_module() -> + mnesia(boot). + +on_delete_module() -> + mnesia:clear_table(?TAB). %%-------------------------------------------------------------------- %% APIs From 59ce864f453763cc51d518226175697d5bddc36c Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 1 Jul 2022 18:33:16 +0800 Subject: [PATCH 02/34] chore: add upgrade callback for exclusive_subscription --- src/emqx.appup.src | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 10c8b7ded..a64c3dbac 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -6,16 +6,18 @@ {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, - {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {add_module,emqx_exclusive_subscription}]}, + {load_module,emqx_topic,brutal_purge,soft_purge,[]}]}, {"4.3.15", [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, @@ -39,6 +41,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, @@ -65,6 +68,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -97,6 +101,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -133,6 +138,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -171,6 +177,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -209,6 +216,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -251,6 +259,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -293,6 +302,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -335,6 +345,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -377,6 +388,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -419,6 +431,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -461,6 +474,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -503,6 +517,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -545,6 +560,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -589,6 +605,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, + {apply, {emqx_exclusive_subscription, on_add_module, []}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -643,9 +660,11 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}]}, {"4.3.15", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -668,6 +687,7 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -693,6 +713,7 @@ {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -724,6 +745,7 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -758,6 +780,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -794,6 +817,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -830,6 +854,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -870,6 +895,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -910,6 +936,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -950,6 +977,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -990,6 +1018,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1030,6 +1059,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1070,6 +1100,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1110,6 +1141,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1150,6 +1182,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1192,6 +1225,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{delete_module,emqx_calendar}, + {apply, {emqx_exclusive_subscription, on_delete_module, []}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, From 03649cc3f323988826b418c0fdb60316045e5fcb Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 1 Jul 2022 13:05:29 -0300 Subject: [PATCH 03/34] chore: copy `.tool-versions` when calculating appups This can make it simpler to ensure the same OTP version that builds the current checkout will match the one that builds the appup base, since one may change the version locally. --- scripts/update-appup.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index 9a099b025..788c25ab0 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -92,6 +92,8 @@ else NEW_COPY='no' fi +TOOL_VERSIONS="$PWD/.tool-versions" + if [ "${SKIP_BUILD_BASE:-no}" = 'yes' ]; then echo "not building relup base ${PREV_DIR_BASE}/${PREV_TAG}" else @@ -103,6 +105,9 @@ else git reset --hard git clean -ffdx git checkout "${PREV_TAG}" + # copy current .tool-versions to ensure same OTP version, even if + # overridden. + cp "$TOOL_VERSIONS" ./ make "$PROFILE" popd fi From 84c6a27758b348f76a38ed98a790873c1080ef5f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 1 Jul 2022 13:19:41 -0300 Subject: [PATCH 04/34] ci: run xref checks in CI --- .github/workflows/build_slim_packages.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 529faa559..713ccc605 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -31,6 +31,8 @@ jobs: fi - name: fix-git-unsafe-repository run: git config --global --add safe.directory /__w/emqx/emqx + - name: make xref + run: make xref - name: build zip packages run: make ${EMQX_NAME}-zip - name: build deb/rpm packages From fcb2d3dc51124ad5abecdc22e51adcda5ee39932 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 1 Jul 2022 15:54:02 -0300 Subject: [PATCH 05/34] fix(mgmt): use correct arity fn --- apps/emqx_management/src/emqx_mgmt_cli.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 954a378b3..3e0e2dc00 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -580,7 +580,7 @@ acl(["cache-clean", "node", Node]) -> with_log(fun() -> for_node(fun emqx_mgmt:clean_acl_cache_all/1, Node) end, "ACL cache drain start"); acl(["cache-clean", "all"]) -> - with_log(fun emqx_mgmt:clean_acl_cache_all/1, + with_log(fun emqx_mgmt:clean_acl_cache_all/0, "ACL cache drain start"); acl(["cache-clean", ClientId]) -> emqx_mgmt:clean_acl_cache(ClientId); From 0e4626aab35385b3a861b330ceac26e97094bda2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 2 Jul 2022 11:35:47 +0200 Subject: [PATCH 06/34] ci: app version check script should also check include dir --- scripts/apps-version-check.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index a16d6a5ca..ffef46c87 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -24,6 +24,7 @@ while read -r app; do if [ "$old_app_version" = "$now_app_version" ]; then changed_lines="$(git diff "$latest_release"...HEAD --ignore-blank-lines -G "$no_comment_re" \ -- "$app_path/src" \ + -- "$app_path/include" \ -- ":(exclude)"$app_path/src/*.appup.src"" \ -- "$app_path/priv" \ -- "$app_path/c_src" | wc -l ) " From 6eadacd5a2a35f6681358175a188262bb6e88f03 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 4 Jul 2022 11:32:05 +0800 Subject: [PATCH 07/34] chore: bump rule-engine app vsn --- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 0d859a230..6c2fb96c8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.11"}, % strict semver, bump manually! + {vsn, "4.3.12"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index a43f0519f..be070305b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.10", + [{"4.3.11", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.10", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -159,7 +161,9 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.10", + [{"4.3.11", + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.10", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, From c1b3cc756ca333c827b2a221122e6e5b0690984a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 4 Jul 2022 13:57:00 +0800 Subject: [PATCH 08/34] chore: bump mgmt/dashboard vsn --- apps/emqx_management/src/emqx_management.app.src | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 3d9e6843b..937ec9db6 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.14"}, % strict semver, bump manually! + {vsn, "4.3.15"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index d0577fa74..7b7b76ede 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.12"}, % strict semver, bump manually! + {vsn, "4.3.13"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, From 1a6cf38c002e56a52e16844135eb9e25f8dac8fe Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 4 Jul 2022 14:25:32 -0300 Subject: [PATCH 09/34] fix: fix list difference precedence ```erlang 1> [a, b, c] ++ [d] -- [a]. [a,b,c,d] 2> ([a, b, c] ++ [d]) -- [a]. [b,c,d] ``` --- CHANGES-4.3.md | 7 +++++++ apps/emqx_management/src/emqx_mgmt_http.erl | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 059375efc..c32cd77b3 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,6 +10,13 @@ File format: - One list item per change topic Change log ends with a list of github PRs +## v4.3.17 + +### Bug fixes + +- Fixed issue where the dashboard APIs were being exposed under the + management listener. [#8411] + ## v4.3.16 ### Enhancements diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index 5ccc2cf5e..fae1bd86a 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -91,7 +91,7 @@ listener_name(Proto) -> http_handlers() -> Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), - [{"/api/v4", minirest:handler(#{apps => Plugins ++ [emqx_modules] -- ?EXCEPT_PLUGIN, + [{"/api/v4", minirest:handler(#{apps => (Plugins ++ [emqx_modules]) -- ?EXCEPT_PLUGIN, except => ?EXCEPT, filter => fun ?MODULE:filter/1}), [{authorization, fun ?MODULE:authorize_appid/1}]}]. From 5a9638b09b473045936dbe3c8a02e30fa85e02a6 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 6 Jul 2022 17:12:11 +0800 Subject: [PATCH 10/34] fix: get subscriptions crash when subid is undefined --- apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index aa3aeb3af..3f420780f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -122,7 +122,7 @@ format({_Subscriber, Topic, Options = #{share := Group}}) -> #{node => node(), topic => filename:join([<<"$share">>, Group, Topic]), clientid => maps:get(subid, Options), qos => QoS}; format({_Subscriber, Topic, Options}) -> QoS = maps:get(qos, Options), - #{node => node(), topic => Topic, clientid => maps:get(subid, Options), qos => QoS}. + #{node => node(), topic => Topic, clientid => maps:get(subid, Options, ""), qos => QoS}. %%-------------------------------------------------------------------- %% Query Function From 998f79070d5087e6b6ccfc1a5d656f0679fafdf8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Jul 2022 10:34:43 +0200 Subject: [PATCH 11/34] feat: add a quick deny option to allow_anonymous config prior to this change, allow_anonymous is only true | false when set to 'false', even if the client is connected without a username, the auth backends will be invoked. this commit introduced a new config value `false_quick_deny` to deny access immediately without involving auth backends --- priv/emqx.schema | 4 +- src/emqx.app.src | 2 +- src/emqx.appup.src | 80 ++++++++++++++++-------------- src/emqx_access_control.erl | 41 ++++++++++----- test/emqx_access_control_SUITE.erl | 6 +++ 5 files changed, 80 insertions(+), 53 deletions(-) diff --git a/priv/emqx.schema b/priv/emqx.schema index 475e0f6ad..a18df0a0a 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -807,7 +807,7 @@ end}. %% @doc Allow anonymous authentication. {mapping, "allow_anonymous", "emqx.allow_anonymous", [ {default, false}, - {datatype, {enum, [true, false]}} + {datatype, {enum, [true, false, false_quick_deny]}} ]}. %% @doc ACL nomatch. @@ -962,7 +962,7 @@ end}. ]}. {mapping, "zone.$name.allow_anonymous", "emqx.zones", [ - {datatype, {enum, [true, false]}} + {datatype, {enum, [true, false, false_quick_deny]}} ]}. {mapping, "zone.$name.acl_nomatch", "emqx.zones", [ diff --git a/src/emqx.app.src b/src/emqx.app.src index 5b5017b1a..5faeb3caf 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.17"}, % strict semver, bump manually! + {vsn, "4.3.18"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index a64c3dbac..2534585eb 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,13 +1,15 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.16", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{"4.3.17",[{load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, + {"4.3.16", + [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, @@ -17,7 +19,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, @@ -41,7 +43,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, @@ -68,7 +70,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -101,7 +103,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -138,7 +140,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -177,7 +179,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -216,7 +218,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -259,7 +261,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -302,7 +304,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -345,7 +347,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -388,7 +390,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -431,7 +433,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -474,7 +476,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -517,7 +519,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, @@ -560,7 +562,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -605,7 +607,7 @@ [{add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, - {apply, {emqx_exclusive_subscription, on_add_module, []}}, + {apply,{emqx_exclusive_subscription,on_add_module,[]}}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_broker,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -650,8 +652,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.16", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + [{"4.3.17",[{load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, + {"4.3.16", + [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, @@ -660,11 +664,11 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}]}, {"4.3.15", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -687,7 +691,7 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -713,7 +717,7 @@ {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -745,7 +749,7 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -780,7 +784,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -817,7 +821,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -854,7 +858,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -895,7 +899,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -936,7 +940,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -977,7 +981,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1018,7 +1022,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1059,7 +1063,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1100,7 +1104,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1141,7 +1145,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1182,7 +1186,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, @@ -1225,7 +1229,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{delete_module,emqx_calendar}, - {apply, {emqx_exclusive_subscription, on_delete_module, []}}, + {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index 93dde28c2..4675c7f81 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -33,15 +33,13 @@ -spec(authenticate(emqx_types:clientinfo()) -> {ok, result()} | {error, term()}). authenticate(ClientInfo = #{zone := Zone}) -> - AuthResult = default_auth_result(Zone), - case - begin ok = emqx_metrics:inc('client.authenticate'), - emqx_zone:get_env(Zone, bypass_auth_plugins, false) - end - of - true -> + ok = emqx_metrics:inc('client.authenticate'), + Username = maps:get(username, ClientInfo, undefined), + {MaybeStop, AuthResult} = default_auth_result(Username, Zone), + case MaybeStop of + stop -> return_auth_result(AuthResult); - false -> + continue -> return_auth_result(emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult)) end. @@ -91,10 +89,29 @@ inc_acl_metrics(cache_hit) -> emqx_metrics:inc('client.acl.cache_hit'). %% Auth -default_auth_result(Zone) -> - case emqx_zone:get_env(Zone, allow_anonymous, false) of - true -> #{auth_result => success, anonymous => true}; - false -> #{auth_result => not_authorized, anonymous => false} +default_auth_result(Username, Zone) -> + IsAnonymous = (Username =:= undefined orelse Username =:= <<>>), + AllowAnonymous = emqx_zone:get_env(Zone, allow_anonymous, false), + Bypass = emqx_zone:get_env(Zone, bypass_auth_plugins, false), + %% the `anonymous` filed in auth result does not mean the client is + %% connected without username, but if the auth result is based on + %% allowing anonymous access. + IsResultBasedOnAllowAnonymous = + case AllowAnonymous of + true -> true; + _ -> false + end, + Result = case AllowAnonymous of + true -> #{auth_result => success, anonymous => IsResultBasedOnAllowAnonymous}; + _ -> #{auth_result => not_authorized, anonymous => IsResultBasedOnAllowAnonymous} + end, + case {IsAnonymous, AllowAnonymous} of + {true, false_quick_deny} -> + {stop, Result}; + _ when Bypass -> + {stop, Result}; + _ -> + {continue, Result} end. -compile({inline, [return_auth_result/1]}). diff --git a/test/emqx_access_control_SUITE.erl b/test/emqx_access_control_SUITE.erl index 8b0d5778b..a03312853 100644 --- a/test/emqx_access_control_SUITE.erl +++ b/test/emqx_access_control_SUITE.erl @@ -38,6 +38,12 @@ t_authenticate(_) -> emqx_zone:set_env(zone, allow_anonymous, true), ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())). +t_authenticate_fast_fail(_) -> + emqx_zone:set_env(zone, allow_anonymous, false_quick_deny), + ?assertMatch({error, _}, emqx_access_control:authenticate(clientinfo())), + emqx_zone:set_env(zone, allow_anonymous, true), + ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())). + t_check_acl(_) -> emqx_zone:set_env(zone, acl_nomatch, deny), application:set_env(emqx, enable_acl_cache, false), From 85b0ce3843a289fc6846925fa83125f6498cb7d3 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 29 Jun 2022 15:57:50 +0800 Subject: [PATCH 12/34] fix: change the type of enable_pipelining from bool to integer --- apps/emqx_auth_http/priv/emqx_auth_http.schema | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_http/priv/emqx_auth_http.schema b/apps/emqx_auth_http/priv/emqx_auth_http.schema index 0f56390e2..59a0c09b9 100644 --- a/apps/emqx_auth_http/priv/emqx_auth_http.schema +++ b/apps/emqx_auth_http/priv/emqx_auth_http.schema @@ -110,8 +110,8 @@ end}. ]}. {mapping, "auth.http.enable_pipelining", "emqx_auth_http.enable_pipelining", [ - {default, true}, - {datatype, {enum, [true, false]}} + {default, 100}, + {datatype, integer} ]}. {mapping, "auth.http.ssl.cacertfile", "emqx_auth_http.cacertfile", [ From 6cd79f92e016256b19df8de74bff99697dc00c36 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 6 Jul 2022 17:04:12 +0800 Subject: [PATCH 13/34] fix(emqx_auth_http): make configure backward compatible --- apps/emqx_auth_http/etc/emqx_auth_http.conf | 2 +- .../emqx_auth_http/priv/emqx_auth_http.schema | 23 +++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index 584448046..458a1d0a0 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -139,7 +139,7 @@ auth.http.pool_size = 32 ## Whether to enable HTTP Pipelining ## ## See: https://en.wikipedia.org/wiki/HTTP_pipelining -auth.http.enable_pipelining = true +auth.http.enable_pipelining = 100 ##------------------------------------------------------------------------------ ## SSL options diff --git a/apps/emqx_auth_http/priv/emqx_auth_http.schema b/apps/emqx_auth_http/priv/emqx_auth_http.schema index 59a0c09b9..627870b58 100644 --- a/apps/emqx_auth_http/priv/emqx_auth_http.schema +++ b/apps/emqx_auth_http/priv/emqx_auth_http.schema @@ -110,10 +110,29 @@ end}. ]}. {mapping, "auth.http.enable_pipelining", "emqx_auth_http.enable_pipelining", [ - {default, 100}, - {datatype, integer} + {default, "100"}, + {datatype, string} ]}. +{translation, "emqx_auth_http.enable_pipelining", fun(Conf) -> + case cuttlefish:conf_get("auth.http.enable_pipelining", Conf, undefined) of + undefined -> 100; + Str -> + try + erlang:list_to_integer(Str) + catch _:_ -> + case erlang:list_to_atom(Str) of + true -> + 100; + false -> + 1; + _ -> + 100 + end + end + end +end}. + {mapping, "auth.http.ssl.cacertfile", "emqx_auth_http.cacertfile", [ {datatype, string} ]}. From 5c2797e1ca03928a3496ded883e481f18016b16f Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 6 Jul 2022 18:13:26 +0800 Subject: [PATCH 14/34] chore: Add compatibility note --- apps/emqx_web_hook/src/emqx_web_hook_actions.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index ae68e79e1..67826fc9c 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -67,6 +67,8 @@ description => #{en => <<"Connection Pool">>, zh => <<"连接池大小"/utf8>>} }, + %% NOTE: In the new version `enable_pipelining` is changed to integer type + %% but it needs to be compatible with the old version, so here keep it as boolean enable_pipelining => #{order => 5, type => boolean, default => true, From e1346148ec1e61773e2d09f91b346fe02c7aa3fc Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 6 Jul 2022 18:28:08 +0800 Subject: [PATCH 15/34] chore: bump version && update appup --- apps/emqx_auth_http/src/emqx_auth_http.app.src | 2 +- apps/emqx_auth_http/src/emqx_auth_http.appup.src | 12 ++++++++++-- apps/emqx_web_hook/src/emqx_web_hook.app.src | 2 +- apps/emqx_web_hook/src/emqx_web_hook.appup.src | 4 ++++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index fba56740f..e943317f8 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_http, [{description, "EMQ X Authentication/ACL with HTTP API"}, - {vsn, "4.3.6"}, % strict semver, bump manually! + {vsn, "4.3.7"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_http_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.appup.src b/apps/emqx_auth_http/src/emqx_auth_http.appup.src index 519604d24..f5c2bfe42 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.appup.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.appup.src @@ -1,6 +1,10 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.5", + [{"4.3.6", + [ %% There are only changes to the schema file, so we don't need any + %% commands here + ]}, + {"4.3.5", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, {"4.3.4", @@ -19,7 +23,11 @@ {<<"4.3.[0-1]">>, [{restart_application,emqx_auth_http}]}, {<<".*">>,[]}], - [{"4.3.5", + [{"4.3.6", + [ %% There are only changes to the schema file, so we don't need any + %% commands here + ]}, + {"4.3.5", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, {"4.3.4", diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index fd69f9601..43e28fe1a 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.12"}, % strict semver, bump manually! + {vsn, "4.3.13"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index 23136d849..fff4ec08c 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -26,6 +26,8 @@ {"4.3.11", [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, + {"4.3.12", + [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{<<"4\\.3\\.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, @@ -52,4 +54,6 @@ {"4.3.11", [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, + {"4.3.12", + [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. From 820d981b4dbea9eebc46f4476b17ec8908ca147d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Jul 2022 13:33:52 +0200 Subject: [PATCH 16/34] docs: fix typo in code comment Co-authored-by: Ilya Averyanov --- src/emqx_access_control.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index 4675c7f81..929d3dc8a 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -93,7 +93,7 @@ default_auth_result(Username, Zone) -> IsAnonymous = (Username =:= undefined orelse Username =:= <<>>), AllowAnonymous = emqx_zone:get_env(Zone, allow_anonymous, false), Bypass = emqx_zone:get_env(Zone, bypass_auth_plugins, false), - %% the `anonymous` filed in auth result does not mean the client is + %% the `anonymous` field in auth result does not mean the client is %% connected without username, but if the auth result is based on %% allowing anonymous access. IsResultBasedOnAllowAnonymous = From e145fdbef39014ed6f1ceca25639642276648ae2 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Jul 2022 10:46:03 +0200 Subject: [PATCH 17/34] fix(shared_sub): Fix crash on dispatch with subscribers disconnected --- src/emqx.appup.src | 14 ++++++++++---- src/emqx_shared_sub.erl | 4 ++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 2534585eb..db9818c07 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,9 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.17",[{load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, + [{"4.3.17", + [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", - [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -652,9 +655,12 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.17",[{load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, + [{"4.3.17", + [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", - [{load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index c7fb1a3cb..1e77b6014 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -242,7 +242,7 @@ maybe_nack_dropped(Msg) -> %% For fresh Ref we send a nack and return true, to note that the inflight is full {Sender, {fresh, _Group, Ref}} -> nack(Sender, Ref, dropped), drop; - + %% For retry Ref we can't reject a message if inflight is full, so we mark it as %% acknowledged and put it into mqueue {_Sender, {retry, _Group, _Ref}} -> maybe_ack(Msg), store; @@ -311,7 +311,7 @@ do_pick(Strategy, ClientId, SourceTopic, Group, Topic, FailedSubs) -> false; [] -> %% We try redispatch to subs who dropped the message because inflight was full. - Found = maps_find_by(FailedSubs, fun({SubPid, FailReason}) -> + Found = maps_find_by(FailedSubs, fun(SubPid, FailReason) -> FailReason == dropped andalso is_alive_sub(SubPid) end), case Found of From 77e974971ed9a9f4c80c10eff2a9500330890d99 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Jul 2022 14:02:02 +0200 Subject: [PATCH 18/34] chore: Update changelog --- CHANGES-4.3.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index c32cd77b3..a4cac8965 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -17,6 +17,8 @@ File format: - Fixed issue where the dashboard APIs were being exposed under the management listener. [#8411] +- Fixed crash when shared persistent subscription [#8441] + ## v4.3.16 ### Enhancements From 2abeb53eb756f54024d1894a31fc2f59d931ef6d Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 5 Jul 2022 10:15:58 +0800 Subject: [PATCH 19/34] chore: update ekka version --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 7c5e01ff9..8782d1560 100644 --- a/rebar.config +++ b/rebar.config @@ -46,7 +46,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.6"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.10"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.11"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.2"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} From d1254faf6f1316eff3c592e006530e5288465e23 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 8 Jul 2022 15:38:28 +0800 Subject: [PATCH 20/34] feat: list rules support for pagination and fuzzy filtering --- CHANGES-4.3.md | 3 + .../src/emqx_rule_engine.appup.src | 12 ++-- .../src/emqx_rule_engine_api.erl | 58 ++++++++++++++- .../test/emqx_rule_engine_SUITE.erl | 72 ++++++++++++++++++- 4 files changed, 138 insertions(+), 7 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index a4cac8965..f23e5aeaa 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -19,6 +19,9 @@ File format: - Fixed crash when shared persistent subscription [#8441] +### Enhancements +- HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450] + ## v4.3.16 ### Enhancements diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index be070305b..fd764bae2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,14 +2,16 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.11", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, @@ -162,14 +164,16 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.11", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, - {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 6587f9c5f..fb6b6f3ca 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -184,6 +184,17 @@ ]). -export([list_events/2]). +-export([query/3]). + +-define(RULE_QS_SCHEMA, {?RULE_TAB, + [ + {<<"enabled">>, atom}, + {<<"for">>, binary}, + {<<"_like_id">>, binary}, + {<<"_like_for">>, binary}, + {<<"_match_for">>, binary}, + {<<"_like_description">>, binary} + ]}). -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~s Not Found", [(ID)]))). -define(ERR_NO_ACTION(NAME), list_to_binary(io_lib:format("Action ~s Not Found", [(NAME)]))). @@ -261,8 +272,11 @@ update_rule(#{id := Id}, Params) -> return({error, 400, ?ERR_BADARGS(Reason)}) end. -list_rules(_Bindings, _Params) -> - return_all(emqx_rule_registry:get_rules_ordered_by_ts()). +list_rules(_Bindings, []) -> + return_all(emqx_rule_registry:get_rules_ordered_by_ts()); +list_rules(_Bindings, Params) -> + SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, + return({ok, emqx_mgmt_api:node_query(node(), Params, ?RULE_QS_SCHEMA, {?MODULE, query}, SortFun)}). show_rule(#{id := Id}, _Params) -> reply_with(fun emqx_rule_registry:get_rule/1, Id). @@ -454,6 +468,7 @@ record_to_map(#rule{id = Id, actions = Actions, on_action_failed = OnFailed, enabled = Enabled, + created_at = CreatedAt, description = Descr}) -> #{id => Id, for => Hook, @@ -462,6 +477,7 @@ record_to_map(#rule{id = Id, on_action_failed => OnFailed, metrics => get_rule_metrics(Id), enabled => Enabled, + created_at => CreatedAt, description => Descr }; @@ -599,3 +615,41 @@ get_action_metrics(Id) -> Res -> [maps:put(node, Node, Res)] end || Node <- ekka_mnesia:running_nodes()]). + +query({Qs, []}, Start, Limit) -> + Ms = qs2ms(Qs), + emqx_mgmt_api:select_table(?RULE_TAB, Ms, Start, Limit, fun record_to_map/1); + +query({Qs, Fuzzy}, Start, Limit) -> + Ms = qs2ms(Qs), + MatchFun = match_fun(Ms, Fuzzy), + emqx_mgmt_api:traverse_table(?RULE_TAB, MatchFun, Start, Limit, fun record_to_map/1). + +qs2ms(Qs) -> + Init = #rule{for = '_', enabled = '_', _ = '_'}, + MatchHead = lists:foldl(fun(Q, Acc) -> match_ms(Q, Acc) end, Init, Qs), + [{MatchHead, [], ['$_']}]. + +match_ms({for, '=:=', Value}, MatchHead) -> MatchHead#rule{for = Value}; +match_ms({enabled, '=:=', Value}, MatchHead) -> MatchHead#rule{enabled = Value}; +match_ms(_, MatchHead) -> MatchHead. + +match_fun(Ms, Fuzzy) -> + MsC = ets:match_spec_compile(Ms), + fun(Rows) -> + Ls = ets:match_spec_run(Rows, MsC), + lists:filter(fun(E) -> run_fuzzy_match(E, Fuzzy) end, Ls) + end. + +run_fuzzy_match(_, []) -> true; +run_fuzzy_match(E = #rule{id = Id}, [{id, like, Pattern}|Fuzzy]) -> + binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = #rule{description = Desc}, [{description, like, Pattern}|Fuzzy]) -> + binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = #rule{for = Topics}, [{for, match, Pattern}|Fuzzy]) -> + lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) + andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = #rule{for = Topics}, [{for, like, Pattern}|Fuzzy]) -> + lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) + andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(_E, [{_Key, like, _SubStr}| _Fuzzy]) -> false. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 663eddd5a..7368b5911 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -62,7 +62,8 @@ groups() -> t_show_action_api, t_crud_resources_api, t_list_resource_types_api, - t_show_resource_type_api + t_show_resource_type_api, + t_list_rule_api ]}, {cli, [], [t_rules_cli, @@ -513,6 +514,75 @@ t_crud_rule_api(_Config) -> ?assertMatch({ok, #{code := 404, message := _Message}}, NotFound), ok. +t_list_rule_api(_Config) -> + AddIds = + lists:map(fun(Seq) -> + SeqBin = integer_to_binary(Seq), + {ok, #{code := 0, data := #{id := Id}}} = + emqx_rule_engine_api:create_rule(#{}, + [{<<"name">>, <<"debug-rule-", SeqBin/binary>>}, + {<<"rawsql">>, <<"select * from \"t/a/", SeqBin/binary, "\"">>}, + {<<"actions">>, [[{<<"name">>,<<"inspect">>}, {<<"params">>,[{<<"arg1">>,1}]}]]}, + {<<"description">>, <<"debug rule desc ", SeqBin/binary>>}]), + Id + end, lists:seq(1, 20)), + + {ok, #{code := 0, data := Rules11}} = emqx_rule_engine_api:list_rules(#{}, + [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"1">>}]), + ?assertEqual(10, length(Rules11)), + {ok, #{code := 0, data := Rules12}} = emqx_rule_engine_api:list_rules(#{}, + [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"2">>}]), + ?assertEqual(10, length(Rules12)), + Rules1 = Rules11 ++ Rules12, + + [RuleID | _] = AddIds, + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"enabled">>, false}]), + Params1 = [{<<"enabled">>,<<"true">>}], + {ok, #{code := 0, data := Rules2}} = emqx_rule_engine_api:list_rules(#{}, Params1), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules2)), + + Params2 = [{<<"for">>, RuleID}], + {ok, #{code := 0, data := Rules3}} = emqx_rule_engine_api:list_rules(#{}, Params2), + ?assert(lists:all(fun(#{id := ID}) -> ID =:= RuleID end, Rules3)), + + Params3 = [{<<"_like_id">>,<<"rule:">>}], + {ok, #{code := 0, data := Rules4}} = emqx_rule_engine_api:list_rules(#{}, Params3), + ?assertEqual(length(Rules1), length(Rules4)), + + Params4 = [{<<"_like_for">>,<<"t/a/">>}], + {ok, #{code := 0, data := Rules5}} = emqx_rule_engine_api:list_rules(#{}, Params4), + ?assertEqual(length(Rules1), length(Rules5)), + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"rawsql">>, <<"select * from \"t/b/c\"">>}]), + {ok, #{code := 0, data := Rules6}} = emqx_rule_engine_api:list_rules(#{}, Params4), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules6)), + ?assertEqual(1, length(Rules1) - length(Rules6)), + + Params5 = [{<<"_match_for">>,<<"t/+/+">>}], + {ok, #{code := 0, data := Rules7}} = emqx_rule_engine_api:list_rules(#{}, Params5), + ?assertEqual(length(Rules1), length(Rules7)), + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"rawsql">>, <<"select * from \"t1/b\"">>}]), + {ok, #{code := 0, data := Rules8}} = emqx_rule_engine_api:list_rules(#{}, Params5), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules8)), + ?assertEqual(1, length(Rules1) - length(Rules8)), + + Params6 = [{<<"_like_description">>,<<"rule">>}], + {ok, #{code := 0, data := Rules9}} = emqx_rule_engine_api:list_rules(#{}, Params6), + ?assertEqual(length(Rules1), length(Rules9)), + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"description">>, <<"not me">>}]), + {ok, #{code := 0, data := Rules10}} = emqx_rule_engine_api:list_rules(#{}, Params6), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules10)), + ?assertEqual(1, length(Rules1) - length(Rules10)), + + lists:foreach(fun(ID) -> + ?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_rule(#{id => ID}, [])) + end, AddIds), + ok. + + t_list_actions_api(_Config) -> {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:list_actions(#{}, []), %ct:pal("RList : ~p", [Actions]), From bc132a42755d39264ae89ce54bc41f7dd3a52190 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 11 Jul 2022 11:50:20 -0300 Subject: [PATCH 21/34] fix: default config should specify `cacert` file (4.3) The default configuration had the `cacert` option for the `ssl.external` listener commented out. If one is to set `verify = verify_peer` in that listener without noticing that `cacert` is unspecified, all connections will fail with: ``` 2022-07-11T11:48:28.540807-03:00 [error] supervisor: 'esockd_connection_sup - <0.3971.0>', errorContext: connection_shutdown, reason: {ssl_error,{options,{cacertfile,[]}}}, offender: [{pid,<0.8225.13>},{name,connection},{mfargs,{emqx_connection,start_link,[[{deflate_options,[]},{max_conn_rate,10000},{active_n,100},{zone,external},{proxy_address_header,<<>>},{proxy_port_header,<<>>},{supported_subprotocols,[]}]]}}] ``` --- etc/emqx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 94d5cbecb..098c908d0 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1472,7 +1472,7 @@ listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## are used during server authentication and when building the client certificate chain. ## ## Value: File -## listener.ssl.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +listener.ssl.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem ## The Ephemeral Diffie-Helman key exchange is a very effective way of ## ensuring Forward Secrecy by exchanging a set of keys that never hit From 13e9e9de5182f352aa3e18171cf1cc63f0a9fcff Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 12 Jul 2022 10:30:54 +0800 Subject: [PATCH 22/34] chore(rewrite): add some log --- lib-ce/emqx_modules/src/emqx_mod_rewrite.erl | 26 ++++++++++++------- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- .../emqx_modules/src/emqx_modules.appup.src | 6 +++-- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl b/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl index 73f4be5be..c8bd3a967 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl @@ -20,6 +20,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). -ifdef(TEST). -export([ compile/1 @@ -45,6 +46,7 @@ load(RawRules) -> {PubRules, SubRules} = compile(RawRules), + ?LOG(info, "[Rewrite] Load rule pub ~0p sub ~0p", [PubRules, SubRules]), emqx_hooks:put('client.subscribe', {?MODULE, rewrite_subscribe, [SubRules]}, 1000), emqx_hooks:put('client.unsubscribe', {?MODULE, rewrite_unsubscribe, [SubRules]}, 1000), emqx_hooks:put('message.publish', {?MODULE, rewrite_publish, [PubRules]}, 1000). @@ -62,6 +64,7 @@ rewrite_publish(Message = #message{topic = Topic}, Rules) -> {ok, Message#message{topic = match_and_rewrite(Topic, Rules, Binds)}}. unload(_) -> + ?LOG(info, "[Rewrite] Unload"), emqx_hooks:del('client.subscribe', {?MODULE, rewrite_subscribe}), emqx_hooks:del('client.unsubscribe', {?MODULE, rewrite_unsubscribe}), emqx_hooks:del('message.publish', {?MODULE, rewrite_publish}). @@ -93,16 +96,19 @@ match_and_rewrite(Topic, [{rewrite, Filter, MP, Dest} | Rules], Binds) -> end. rewrite(Topic, MP, Dest, Binds) -> - case re:run(Topic, MP, [{capture, all_but_first, list}]) of - {match, Captured} -> - Vars = lists:zip(["\\$" ++ integer_to_list(I) - || I <- lists:seq(1, length(Captured))], Captured), - iolist_to_binary(lists:foldl( - fun({Var, Val}, Acc) -> - re:replace(Acc, Var, Val, [global]) - end, Dest, Binds ++ Vars)); - nomatch -> Topic - end. + NewTopic = + case re:run(Topic, MP, [{capture, all_but_first, list}]) of + {match, Captured} -> + Vars = lists:zip(["\\$" ++ integer_to_list(I) + || I <- lists:seq(1, length(Captured))], Captured), + iolist_to_binary(lists:foldl( + fun({Var, Val}, Acc) -> + re:replace(Acc, Var, Val, [global]) + end, Dest, Binds ++ Vars)); + nomatch -> Topic + end, + ?LOG(debug, "[Rewrite] topic ~0p, params: ~0p dest topic: ~p", [Topic, Binds, NewTopic]), + NewTopic. fill_client_binds(#{clientid := ClientId, username := Username}) -> filter_client_binds([{"%c", ClientId}, {"%u", Username}]); diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index 49af5d3ea..9db13dbc8 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.7"}, + {vsn, "4.3.8"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index d32cc7286..5b8fb4434 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.6",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + [{"4.3.7",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + {"4.3.6",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, @@ -31,7 +32,8 @@ {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.6",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + [{"4.3.7",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + {"4.3.6",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, From fe56985eeca14bd59542e6f5a93a9fcbddf49bff Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 12 Jul 2022 14:30:50 +0800 Subject: [PATCH 23/34] chore(relup): download relup base version packages from s3 --- scripts/relup-base-packages.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 1c99f0c9c..60a301f7d 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -14,15 +14,15 @@ fi case $PROFILE in "emqx") - DIR='broker' + DIR='emqx-ce' EDITION='community' ;; "emqx-ee") - DIR='enterprise' + DIR='emqx-ee' EDITION='enterprise' ;; "emqx-edge") - DIR='edge' + DIR='emqx-edge' EDITION='edge' ;; esac @@ -55,7 +55,7 @@ pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" - url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename" + url="https://packages.emqx.io/$DIR/${tag#[e|v]}/$filename" echo "downloading base package from ${url} ..." if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then curl -L -o "${filename}" "${url}" From d2774935dfbb228a0b808961ea63579702acc960 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 12 Jul 2022 14:50:59 +0800 Subject: [PATCH 24/34] chore(relup): fix bad url --- scripts/relup-base-packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 60a301f7d..01d2fa96b 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -55,7 +55,7 @@ pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" - url="https://packages.emqx.io/$DIR/${tag#[e|v]}/$filename" + url="https://packages.emqx.io/$DIR/$tag/$filename" echo "downloading base package from ${url} ..." if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then curl -L -o "${filename}" "${url}" From 790f97c8533dad5ab93c6439fc69ff337b223a2e Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 12 Jul 2022 14:53:25 +0800 Subject: [PATCH 25/34] fix: better zh param name --- apps/emqx_web_hook/src/emqx_web_hook_actions.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index 67826fc9c..00094d46d 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -72,7 +72,7 @@ enable_pipelining => #{order => 5, type => boolean, default => true, - title => #{en => <<"Enable Pipelining">>, zh => <<"Enable Pipelining"/utf8>>}, + title => #{en => <<"Enable Pipelining">>, zh => <<"开启 Pipelining"/utf8>>}, description => #{en => <<"Whether to enable HTTP Pipelining">>, zh => <<"是否开启 HTTP Pipelining"/utf8>>} }, From 9b433eb0027212ab9540e0816d17d09e0be31629 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 13 Jul 2022 10:06:18 +0800 Subject: [PATCH 26/34] fix: list_rules support enable_paging --- .../src/emqx_rule_engine_api.erl | 11 +++++++---- .../test/emqx_rule_engine_SUITE.erl | 17 ++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index fb6b6f3ca..223c672e6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -272,11 +272,14 @@ update_rule(#{id := Id}, Params) -> return({error, 400, ?ERR_BADARGS(Reason)}) end. -list_rules(_Bindings, []) -> - return_all(emqx_rule_registry:get_rules_ordered_by_ts()); list_rules(_Bindings, Params) -> - SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, - return({ok, emqx_mgmt_api:node_query(node(), Params, ?RULE_QS_SCHEMA, {?MODULE, query}, SortFun)}). + case proplists:get_value(<<"enable_paging">>, Params, true) of + true -> + SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, + return({ok, emqx_mgmt_api:node_query(node(), Params, ?RULE_QS_SCHEMA, {?MODULE, query}, SortFun)}); + false -> + return_all(emqx_rule_registry:get_rules_ordered_by_ts()) + end. show_rule(#{id := Id}, _Params) -> reply_with(fun emqx_rule_registry:get_rule/1, Id). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 7368b5911..9031c3df8 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -528,29 +528,29 @@ t_list_rule_api(_Config) -> end, lists:seq(1, 20)), {ok, #{code := 0, data := Rules11}} = emqx_rule_engine_api:list_rules(#{}, - [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"1">>}]), + [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"1">>}, {<<"enable_paging">>, true}]), ?assertEqual(10, length(Rules11)), {ok, #{code := 0, data := Rules12}} = emqx_rule_engine_api:list_rules(#{}, - [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"2">>}]), + [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"2">>}, {<<"enable_paging">>, true}]), ?assertEqual(10, length(Rules12)), Rules1 = Rules11 ++ Rules12, [RuleID | _] = AddIds, {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, [{<<"enabled">>, false}]), - Params1 = [{<<"enabled">>,<<"true">>}], + Params1 = [{<<"enabled">>,<<"true">>}, {<<"enable_paging">>, true}], {ok, #{code := 0, data := Rules2}} = emqx_rule_engine_api:list_rules(#{}, Params1), ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules2)), - Params2 = [{<<"for">>, RuleID}], + Params2 = [{<<"for">>, RuleID}, {<<"enable_paging">>, true}], {ok, #{code := 0, data := Rules3}} = emqx_rule_engine_api:list_rules(#{}, Params2), ?assert(lists:all(fun(#{id := ID}) -> ID =:= RuleID end, Rules3)), - Params3 = [{<<"_like_id">>,<<"rule:">>}], + Params3 = [{<<"_like_id">>,<<"rule:">>}, {<<"enable_paging">>, true}], {ok, #{code := 0, data := Rules4}} = emqx_rule_engine_api:list_rules(#{}, Params3), ?assertEqual(length(Rules1), length(Rules4)), - Params4 = [{<<"_like_for">>,<<"t/a/">>}], + Params4 = [{<<"_like_for">>,<<"t/a/">>}, {<<"enable_paging">>, true}], {ok, #{code := 0, data := Rules5}} = emqx_rule_engine_api:list_rules(#{}, Params4), ?assertEqual(length(Rules1), length(Rules5)), {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, @@ -559,7 +559,7 @@ t_list_rule_api(_Config) -> ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules6)), ?assertEqual(1, length(Rules1) - length(Rules6)), - Params5 = [{<<"_match_for">>,<<"t/+/+">>}], + Params5 = [{<<"_match_for">>,<<"t/+/+">>}, {<<"enable_paging">>, true}], {ok, #{code := 0, data := Rules7}} = emqx_rule_engine_api:list_rules(#{}, Params5), ?assertEqual(length(Rules1), length(Rules7)), {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, @@ -568,7 +568,7 @@ t_list_rule_api(_Config) -> ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules8)), ?assertEqual(1, length(Rules1) - length(Rules8)), - Params6 = [{<<"_like_description">>,<<"rule">>}], + Params6 = [{<<"_like_description">>,<<"rule">>}, {<<"enable_paging">>, true}], {ok, #{code := 0, data := Rules9}} = emqx_rule_engine_api:list_rules(#{}, Params6), ?assertEqual(length(Rules1), length(Rules9)), {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, @@ -582,7 +582,6 @@ t_list_rule_api(_Config) -> end, AddIds), ok. - t_list_actions_api(_Config) -> {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:list_actions(#{}, []), %ct:pal("RList : ~p", [Actions]), From 4b6f275a438210ac2b1fab41fb41b551cde959c0 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 13 Jul 2022 20:14:20 +0800 Subject: [PATCH 27/34] fix: emqx start failed after uninstalling old version --- bin/emqx | 4 ++++ data/emqx_vars | 12 +++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bin/emqx b/bin/emqx index 7eed9ab3c..d6ce86e10 100755 --- a/bin/emqx +++ b/bin/emqx @@ -8,6 +8,10 @@ RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 . "$RUNNER_ROOT_DIR"/releases/emqx_vars +EMQX_LICENSE_CONF='' +REL_NAME="emqx" +ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin" + RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME" CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}" REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN" diff --git a/data/emqx_vars b/data/emqx_vars index a872da03a..8ca6bf22d 100644 --- a/data/emqx_vars +++ b/data/emqx_vars @@ -12,12 +12,10 @@ RUNNER_LIB_DIR="{{ runner_lib_dir }}" RUNNER_ETC_DIR="{{ runner_etc_dir }}" RUNNER_DATA_DIR="{{ runner_data_dir }}" RUNNER_USER="{{ runner_user }}" +EMQX_DESCRIPTION='{{ emqx_description }}' -EMQX_LICENSE_CONF='' -export EMQX_DESCRIPTION='{{ emqx_description }}' +## Warning: DO NOT create new variables using the above vars in this file, +## as the vars above can be overwritten by the relup scripts later, like: +## REL_VSN="new_version" -## computed vars -REL_NAME="emqx" -ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin" - -## updated vars here +## overwritten vars here From 87aeead65356eaeed99f4680558e4b3d41a9090f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 14 Jul 2022 17:11:15 +0800 Subject: [PATCH 28/34] feat: add check_conf cli to check config format --- CHANGES-4.3.md | 1 + bin/emqx | 28 +++++++++++++++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index f23e5aeaa..ecca0dc8b 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -21,6 +21,7 @@ File format: ### Enhancements - HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450] +- Add check_conf cli to check config format. [#8486] ## v4.3.16 diff --git a/bin/emqx b/bin/emqx index d6ce86e10..b70ce2e2f 100755 --- a/bin/emqx +++ b/bin/emqx @@ -196,6 +196,7 @@ usage() { echo " Up/Down-grade: upgrade | downgrade | install | uninstall" echo " Install info: ertspath | root_dir | versions" echo " Runtime info: pid | ping | versions" + echo " Configure check: check_conf" echo " Advanced: console_clean | escript | rpc | rpcterms | eval" echo '' echo "Execute '$REL_NAME COMMAND help' for more information" @@ -338,9 +339,12 @@ trim() { # Function to generate app.config and vm.args generate_config() { - ## Delete the *.siz files first or it cann't start after - ## changing the config 'log.rotation.size' - rm -rf "${RUNNER_LOG_DIR}"/*.siz + check_only="$1" + if [ "$check_only" != "check_only" ]; then + ## Delete the *.siz files first or it cann't start after + ## changing the config 'log.rotation.size' + rm -rf "${RUNNER_LOG_DIR}"/*.siz + fi set +e if [ "${EMQX_LICENSE_CONF:-}" = "" ]; then @@ -392,15 +396,22 @@ generate_config() { fi fi done - mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" if ! relx_nodetool chkconfig -config "$CONFIG_FILE"; then echoerr "Error reading $CONFIG_FILE" exit 1 fi + + if [ "$check_only" = "check_only" ]; then + rm -rf "$TMP_ARG_FILE" + rm -rf "$CUTTLE_GEN_ARG_FILE" + rm -rf "$CONFIG_FILE" + else + mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" + fi } -# Call bootstrapd for daemon commands like start/stop/console +# Call bootstrap for daemon commands like start/stop/console bootstrapd() { if [ -e "$RUNNER_DATA_DIR/.erlang.cookie" ]; then chown "$RUNNER_USER" "$RUNNER_DATA_DIR"/.erlang.cookie @@ -451,6 +462,9 @@ case "$1" in foreground) IS_BOOT_COMMAND='yes' ;; + check_conf) + IS_BOOT_COMMAND='yes' + ;; esac @@ -819,6 +833,10 @@ case "$1" in ertspath) echo "$ERTS_PATH" ;; + check_conf) + generate_config "check_only" + echo "$RUNNER_ETC_DIR/emqx.conf is ok" + ;; ctl) assert_node_alive From 5aca96afa316fe3b64c97eb1e2cfb2ba08ba87f3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 15 Jul 2022 16:07:56 +0800 Subject: [PATCH 29/34] chore: missing version update by wolff wolff 1.5.6 requires replayq 0.3.4 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 8782d1560..cf46d87c9 100644 --- a/rebar.config +++ b/rebar.config @@ -51,7 +51,7 @@ , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} - , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} + , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.4"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.2"}}} From 2366794c2f590fc5be0763ae43ce494c84bbb4a2 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 18 Jul 2022 14:09:27 +0800 Subject: [PATCH 30/34] chore: update dashboard version --- scripts/get-dashboard.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index 10c393fb4..98b09921e 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -8,8 +8,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" case "${PKG_VSN}" in 4.3*) - EMQX_CE_DASHBOARD_VERSION='v4.3.8' - EMQX_EE_DASHBOARD_VERSION='v4.3.21' + EMQX_CE_DASHBOARD_VERSION='v4.3.9' + EMQX_EE_DASHBOARD_VERSION='v4.3.22' ;; *) echo "Unsupported version $PKG_VSN" >&2 From 0834bc52d4554de92977d1416697641d2e36b41b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 18 Jul 2022 15:09:59 +0800 Subject: [PATCH 31/34] chore: update bin/emqx Co-authored-by: Zaiming (Stone) Shi --- bin/emqx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index b70ce2e2f..28ef379f2 100755 --- a/bin/emqx +++ b/bin/emqx @@ -411,7 +411,7 @@ generate_config() { fi } -# Call bootstrap for daemon commands like start/stop/console +# Call bootstrapd for daemon commands like start/stop/console bootstrapd() { if [ -e "$RUNNER_DATA_DIR/.erlang.cookie" ]; then chown "$RUNNER_USER" "$RUNNER_DATA_DIR"/.erlang.cookie From 53c3f487559c2f2228a3d6467333038fa05c8f19 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 18 Jul 2022 15:10:08 +0800 Subject: [PATCH 32/34] chore: update bin/emqx Co-authored-by: Zaiming (Stone) Shi --- bin/emqx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index 28ef379f2..b7b1b35b3 100755 --- a/bin/emqx +++ b/bin/emqx @@ -196,7 +196,7 @@ usage() { echo " Up/Down-grade: upgrade | downgrade | install | uninstall" echo " Install info: ertspath | root_dir | versions" echo " Runtime info: pid | ping | versions" - echo " Configure check: check_conf" + echo " Config check: check_conf" echo " Advanced: console_clean | escript | rpc | rpcterms | eval" echo '' echo "Execute '$REL_NAME COMMAND help' for more information" From 5f6259f6f4da6809e40954b340f6ca295d4df62b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 18 Jul 2022 15:10:15 +0800 Subject: [PATCH 33/34] chore: update bin/emqx Co-authored-by: Zaiming (Stone) Shi --- bin/emqx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/emqx b/bin/emqx index b7b1b35b3..c96dd2cd9 100755 --- a/bin/emqx +++ b/bin/emqx @@ -403,9 +403,9 @@ generate_config() { fi if [ "$check_only" = "check_only" ]; then - rm -rf "$TMP_ARG_FILE" - rm -rf "$CUTTLE_GEN_ARG_FILE" - rm -rf "$CONFIG_FILE" + rm -f "$TMP_ARG_FILE" + rm -f "$CUTTLE_GEN_ARG_FILE" + rm -f "$CONFIG_FILE" else mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" fi From c6ba22a0b487af002ca169bd09b3ffff7b54206e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 18 Jul 2022 09:47:02 +0200 Subject: [PATCH 34/34] chore: indentation fix in bin/emqx --- bin/emqx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index c96dd2cd9..219a99883 100755 --- a/bin/emqx +++ b/bin/emqx @@ -196,7 +196,7 @@ usage() { echo " Up/Down-grade: upgrade | downgrade | install | uninstall" echo " Install info: ertspath | root_dir | versions" echo " Runtime info: pid | ping | versions" - echo " Config check: check_conf" + echo " Config check: check_conf" echo " Advanced: console_clean | escript | rpc | rpcterms | eval" echo '' echo "Execute '$REL_NAME COMMAND help' for more information"