Merge pull request #9435 from zmstone/1128-sync-main-v4.3-to-4.4

1128 sync main v4.3 to 4.4

.github/workflows/build_packages.yaml

@@ -98,7 +98,7 @@ jobs:
         echo "EQMX uninstaled"
     - uses: actions/upload-artifact@v3
       with:
-        name: ${{ matrix.profile }}-windows
+        name: ${{ matrix.profile }}
         path: source/_packages/${{ matrix.profile }}/
 
   mac:
@@ -133,7 +133,7 @@ jobs:
         apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
     - uses: actions/upload-artifact@v3
       with:
-        name: ${{ env.EMQX_NAME }}-${{ matrix.otp }}
+        name: ${{ env.EMQX_NAME }}
         path: _packages/${{ env.EMQX_NAME }}/
 
   linux:
@@ -209,7 +209,7 @@ jobs:
           --builder "ghcr.io/emqx/emqx-builder/4.4-20:${OTP}-${SYSTEM}"
     - uses: actions/upload-artifact@v1
       with:
-        name: ${{ matrix.profile }}-${{ matrix.otp }}
+        name: ${{ matrix.profile }}
         path: source/_packages/${{ matrix.profile }}/
 
   docker:
@@ -315,16 +315,11 @@ jobs:
       fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-        otp:
-          - 24.3.4.2-1
-        include:
-          - profile: emqx
-            otp: windows # otp version on windows is rather fixed
 
     steps:
     - uses: actions/download-artifact@v2
       with:
-        name: ${{ matrix.profile }}-${{ matrix.otp }}
+        name: ${{ matrix.profile }}
         path: packages/${{ matrix.profile }}
     - name: install dos2unix
       run: sudo apt-get update && sudo apt install -y dos2unix

.github/workflows/build_slim_packages.yaml

@@ -36,9 +36,7 @@ jobs:
 
     steps:
     - uses: AutoModality/action-clean@v1
-      # keep using v1 for now as the otp-23 image has an old version git
+    - uses: actions/checkout@v3
-      # TODO: change to v3 after OTP is upgraded to 23.3.4.18-1
-    - uses: actions/checkout@v1
       with:
           fetch-depth: 0 # clone full git history
     - name: fix-git-unsafe-repository

apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_jwt,
  [{description, "EMQ X Authentication with JWT"},
-  {vsn, "4.4.7"}, % strict semver, bump manually!
+  {vsn, "4.4.8"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_jwt_sup]},
   {applications, [kernel,stdlib,jose]},

apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src

@@ -1,7 +1,8 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.6",
+  [{"4.4.7",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]},
+   {"4.4.6",
     [{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]},
    {"4.4.5",
@@ -18,7 +19,8 @@
      {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]},
    {<<"4\\.4\\.[0-1]">>,[{restart_application,emqx_auth_jwt}]},
    {<<".*">>,[]}],
-  [{"4.4.6",
+  [{"4.4.7",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]},
+   {"4.4.6",
     [{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]},
    {"4.4.5",

apps/emqx_auth_jwt/src/emqx_auth_jwt.erl

@@ -67,7 +67,7 @@ check_acl(ClientInfo = #{jwt_claims := Claims},
             case is_expired(Exp) of
                 true ->
                     ?DEBUG("acl_deny_due_to_jwt_expired", []),
-                    deny;
+                    {stop, deny};
                 false ->
                     verify_acl(ClientInfo, Acl, PubSub, Topic)
             end;

apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl

@@ -462,6 +462,16 @@ t_check_jwt_acl_expire(_Config) ->
        {ok, #{}, [?RC_NOT_AUTHORIZED]},
        emqtt:subscribe(C, <<"a/b">>, 0)),
 
+    Default = emqx_zone:get_env(external, acl_nomatch, deny),
+    emqx_zone:set_env(external, acl_nomatch, allow),
+    try
+        ?assertMatch(
+           {ok, #{}, [?RC_NOT_AUTHORIZED]},
+           emqtt:subscribe(C, <<"a/b">>, 0))
+    after
+        emqx_zone:set_env(external, acl_nomatch, Default)
+    end,
+
     ok = emqtt:disconnect(C).
 
 t_check_jwt_acl_no_exp(init, _Config) ->

apps/emqx_coap/src/emqx_coap.app.src

@@ -1,6 +1,6 @@
 {application, emqx_coap,
  [{description, "EMQ X CoAP Gateway"},
-  {vsn, "4.3.2"}, % strict semver, bump manually!
+  {vsn, "4.3.3"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,gen_coap]},

apps/emqx_coap/src/emqx_coap.appup.src

@@ -1,15 +1,19 @@
-%% -*-: erlang -*-
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{<<"4\\.3\\.[0-1]">>,[
+  [{"4.3.2",
-    {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []},
+    [{load_module,emqx_coap_resource,brutal_purge,soft_purge,[]},
-    {load_module, emqx_coap_pubsub_resource, brutal_purge, soft_purge, []},
+     {load_module,emqx_coap_mqtt_adapter,brutal_purge,soft_purge,[]}]},
-    {load_module, emqx_coap_resource, brutal_purge, soft_purge, []}
+   {<<"4\\.3\\.[0-1]">>,
-   ]},
+    [{load_module,emqx_coap_mqtt_adapter,brutal_purge,soft_purge,[]},
-   {<<".*">>, []}],
+     {load_module,emqx_coap_pubsub_resource,brutal_purge,soft_purge,[]},
-  [{<<"4\\.3\\.[0-1]">>,[
+     {load_module,emqx_coap_resource,brutal_purge,soft_purge,[]}]},
-    {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []},
+   {<<".*">>,[]}],
-    {load_module, emqx_coap_pubsub_resource, brutal_purge, soft_purge, []},
+  [{"4.3.2",
-    {load_module, emqx_coap_resource, brutal_purge, soft_purge, []}
+    [{load_module,emqx_coap_resource,brutal_purge,soft_purge,[]},
-   ]},
+     {load_module,emqx_coap_mqtt_adapter,brutal_purge,soft_purge,[]}]},
-   {<<".*">>, []}]
+   {<<"4\\.3\\.[0-1]">>,
-}.
+    [{load_module,emqx_coap_mqtt_adapter,brutal_purge,soft_purge,[]},
+     {load_module,emqx_coap_pubsub_resource,brutal_purge,soft_purge,[]},
+     {load_module,emqx_coap_resource,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl

@@ -120,7 +120,7 @@ call(Pid, Msg, _) ->
 
 init({ClientId, Username, Password, Channel}) ->
     ?LOG(debug, "try to start adapter ClientId=~p, Username=~p, Password=~p, "
-                "Channel=~0p", [ClientId, Username, Password, Channel]),
+                "Channel=~0p", [ClientId, Username, "******", Channel]),
     State0 = #state{peername = Channel,
                     clientid = ClientId,
                     username = Username,

apps/emqx_coap/src/emqx_coap_resource.erl

@@ -48,7 +48,7 @@ coap_discover(_Prefix, _Args) ->
     [{absolute, [<<"mqtt">>], []}].
 
 coap_get(ChId, ?MQTT_PREFIX, Path, Query, _Content) ->
-    ?LOG(debug, "coap_get() Path=~p, Query=~p~n", [Path, Query]),
+    ?LOG(debug, "coap_get() Path=~p, Query=~p~n", [Path, redact_query(Query)]),
     #coap_mqtt_auth{clientid = Clientid, username = Usr, password = Passwd} = get_auth(Query),
     case emqx_coap_mqtt_adapter:client_pid(Clientid, Usr, Passwd, ChId) of
         {ok, Pid} ->
@@ -65,7 +65,8 @@ coap_get(ChId, ?MQTT_PREFIX, Path, Query, _Content) ->
             {error, internal_server_error}
     end;
 coap_get(ChId, Prefix, Path, Query, _Content) ->
-    ?LOG(error, "ignore bad get request ChId=~p, Prefix=~p, Path=~p, Query=~p", [ChId, Prefix, Path, Query]),
+    ?LOG(error, "ignore bad get request ChId=~p, Prefix=~p, Path=~p, Query=~p",
+         [ChId, Prefix, Path, redact_query(Query)]),
     {error, bad_request}.
 
 coap_post(_ChId, _Prefix, _Topic, _Content) ->
@@ -149,3 +150,10 @@ topic([Path | TopicPath]) ->
             <<Path/binary, $/, RemTopic/binary>>
     end.
 
+redact_query(Auths) ->
+    lists:map(fun(<<$p, $=, _Rest/binary>>) ->
+                      <<$p, $=, "******">>;
+                 (E) ->
+                      E
+              end,
+              Auths).

apps/emqx_exhook/src/emqx_exhook.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_exhook,
  [{description, "EMQ X Extension for Hook"},
-  {vsn, "4.4.4"},
+  {vsn, "4.4.5"},
   {modules, []},
   {registered, []},
   {mod, {emqx_exhook_app, []}},

apps/emqx_exhook/src/emqx_exhook.appup.src

@@ -1,9 +1,13 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.3",[{load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]},
+  [{"4.4.4",[{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}]},
+   {"4.4.3",
+    [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]},
    {<<"4\\.4\\.[1-2]">>,
-    [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}]},
@@ -15,9 +19,13 @@
      {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
      {update,emqx_exhook_mngr,{advanced,["4.4.0"]}}]},
    {<<".*">>,[]}],
-  [{"4.4.3",[{load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]},
+  [{"4.4.4",[{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}]},
+   {"4.4.3",
+    [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]},
    {<<"4\\.4\\.[1-2]">>,
-    [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook,brutal_purge,soft_purge,[]},
      {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}]},

apps/emqx_exproto/src/emqx_exproto.app.src

@@ -1,6 +1,6 @@
 {application, emqx_exproto,
  [{description, "EMQ X Extension for Protocol"},
-  {vsn, "4.3.13"}, %% 4.3.3 is used by ee
+  {vsn, "4.3.14"}, %% 4.3.3 is used by ee
   {modules, []},
   {registered, []},
   {mod, {emqx_exproto_app, []}},

apps/emqx_exproto/src/emqx_exproto.appup.src

@@ -1,17 +1,25 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.12",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]},
+  [{"4.3.13",[{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}]},
-   {"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]},
+   {"4.3.12",
+    [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]},
+   {"4.3.11",
+    [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
-    [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[2-8]">>,
-    [{load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[0-1]">>,
@@ -20,17 +28,25 @@
      {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.12",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]},
+  [{"4.3.13",[{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}]},
-   {"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]},
+   {"4.3.12",
+    [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]},
+   {"4.3.11",
+    [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
-    [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
-    [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[2-8]">>,
-    [{load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[0-1]">>,

apps/emqx_exproto/src/emqx_exproto_gsvr.erl

@@ -60,7 +60,7 @@ close(Req = #{conn := Conn}, Md) ->
 authenticate(Req = #{conn := Conn,
                      password := Password,
                      clientinfo := ClientInfo}, Md) ->
-    ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]),
+    ?LOG_SENSITIVE(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]),
     case validate(clientinfo, ClientInfo) of
         false ->
             {ok, response({error, ?RESP_REQUIRED_PARAMS_MISSED}), Md};

apps/emqx_rule_engine/src/emqx_rule_engine.app.src

@@ -1,6 +1,6 @@
 {application, emqx_rule_engine,
  [{description, "EMQ X Rule Engine"},
-  {vsn, "4.4.11"}, % strict semver, bump manually!
+  {vsn, "4.4.12"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_rule_engine_sup, emqx_rule_registry, emqx_rule_engine_jwt_sup]},
   {applications, [kernel,stdlib,rulesql,getopt,jose]},

apps/emqx_rule_engine/src/emqx_rule_engine.appup.src

@@ -1,7 +1,8 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.10",
+  [{"4.4.11",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.4.10",
     [{add_module,emqx_rule_engine_jwt},
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
@@ -196,7 +197,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.10",
+  [{"4.4.11",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.4.10",
     [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},

bin/emqx

@@ -324,7 +324,7 @@ relx_rem_sh() {
 
 # Generate a random id
 relx_gen_id() {
-    od -t x -N 4 /dev/urandom | head -n1 | awk '{print $2}'
+    od -t u -N 4 /dev/urandom | head -n1 | awk '{print $2 % 1000}'
 }
 
 # Control a node
@@ -579,9 +579,13 @@ if [ -z "$COOKIE" ]; then
     fi
 fi
 
-if [ -z "$COOKIE" ]; then
+[ -z "$COOKIE" ] && COOKIE="$EMQX_DEFAULT_ERLANG_COOKIE"
+if [ $IS_BOOT_COMMAND = 'yes' ] && [ "$COOKIE" = "$EMQX_DEFAULT_ERLANG_COOKIE" ]; then
+    echoerr "!!!!!!"
+    echoerr "WARNING: Default (insecure) Erlang cookie is in use."
     echoerr "Please set node.cookie in $RUNNER_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE_COOKIE"
-    exit 1
+    echoerr "NOTE: Use the same config value for all nodes in the cluster."
+    echoerr "!!!!!!"
 fi
 
 cd "$ROOTDIR"

bin/nodetool

@@ -202,7 +202,12 @@ nodename(Name) ->
 
 this_node_name(Name) ->
     [Node, Host] = re:split(Name, "@", [{return, list}, unicode]),
-    list_to_atom(lists:concat(["remsh_maint_", Node, os:getpid(), "@", Host])).
+    list_to_atom(lists:concat(["remsh_maint_", Node, node_name_suffix_id(), "@", Host])).
+
+%% use the reversed value that from pid mod 1000 as the node name suffix
+node_name_suffix_id() ->
+    Pid = os:getpid(),
+    string:slice(string:reverse(Pid), 0, 3).
 
 %% For windows???
 create_mnesia_dir(DataDir, NodeName) ->

changes/v4.3.23-en.md

@@ -0,0 +1,13 @@
+# v4.3.23
+
+## Enhancements
+
+- Added topic validation for `emqx_mod_rewrite`. The dest topics contains wildcards are not allowed to publish [#9359](https://github.com/emqx/emqx/issues/9359).
+
+- Print a warning message when boot with the default (insecure) Erlang cookie [#9340](https://github.com/emqx/emqx/pull/9340).
+
+- Improve node name generation rules to avoid potential atom table overflow risk [#9391](https://github.com/emqx/emqx/pull/9391).
+
+## Bug fixes
+
+- Fix a bug where the JWT ACL would not short-circuit with a deny response when the token is expired [#9338](https://github.com/emqx/emqx/pull/9338).

changes/v4.3.23-zh.md

@@ -0,0 +1,13 @@
+# v4.3.23
+
+## 增强
+
+- 为主题重写模块增加主题合法性检查，带有通配符的目标主题不允许被发布 [#9359](https://github.com/emqx/emqx/issues/9359)。
+
+- 使用默认的(不安全的) Erlang cookie 进行启动时，将会打印一条警告信息 [#9340](https://github.com/emqx/emqx/pull/9340)。
+
+- 改进了节点名称生成规则，以避免潜在的原子表溢出风险 [#9391](https://github.com/emqx/emqx/pull/9391)。
+
+## 修复
+
+- 修复 JWT ACL 在令牌超期后授权检查不生效的问题 [#9338](https://github.com/emqx/emqx/pull/9338)。

changes/v4.4.11-en.md

@@ -27,3 +27,5 @@
 - Fix get trace list crash when trace not initialize. [#9156](https://github.com/emqx/emqx/pull/9156)
 
 - Fix create trace sometime failed by end_at time has already passed. [#9156](https://github.com/emqx/emqx/pull/9156)
+
+- Fix the `Discovery error: no such service` error that occurred during helm chart deployment, resulting in an abnormal discovery of cluster nodes. [#9388](https://github.com/emqx/emqx/pull/9388)

changes/v4.4.11-zh.md

@@ -24,3 +24,5 @@
 - 修复日志追踪模块没开启时，GET Trace 列表接口报错的问题。[#9156](https://github.com/emqx/emqx/pull/9156)
 
 - 修复创建追踪日志时偶尔会报`end_at time has already passed`错误，导致创建失败。[#9156](https://github.com/emqx/emqx/pull/9156)
+
+- 修复 helm chart 部署时出现`Discovery error: no such service`错误，导致集群节点发现异常。[#9388](https://github.com/emqx/emqx/pull/9388)

data/emqx_vars

@@ -14,6 +14,10 @@ RUNNER_DATA_DIR="{{ runner_data_dir }}"
 RUNNER_USER="{{ runner_user }}"
 EMQX_DESCRIPTION='{{ emqx_description }}'
 
+## Do not change EMQX_DEFAULT_ERLANG_COOKIE.
+## Configure EMQX_NODE_COOKIE instead
+EMQX_DEFAULT_ERLANG_COOKIE='{{ emqx_default_erlang_cookie }}'
+
 ## Warning: DO NOT create new variables using the above vars in this file,
 ## as the vars above can be overwritten by the relup scripts later, like:
 ## REL_VSN="new_version"

data/relup-paths.eterm

@@ -45,6 +45,12 @@
         <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>,<<"4.4.8">>,<<"4.4.9">>,
         <<"4.4.10">>],
    otp => <<"24.3.4.2-1">>}}.
+{<<"4.4.12">>,
+ #{from_versions =>
+       [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>,
+        <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>,<<"4.4.8">>,<<"4.4.9">>,
+        <<"4.4.10">>,<<"4.4.11">>],
+   otp => <<"24.3.4.2-1">>}}.
 {<<"4.5.0">>,
  #{from_versions => [<<"4.4.8">>,<<"4.4.9">>,<<"4.4.10">>,
                      <<"4.4.11">>],

deploy/charts/emqx/Chart.yaml

@@ -13,8 +13,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 4.4.11
+version: 4.4.12
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 4.4.11
+appVersion: 4.4.12

deploy/charts/emqx/templates/configmap.env.yaml

@@ -10,6 +10,19 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 data:
+  EMQX_NAME:  {{ .Release.Name }}
+  {{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY)  "k8s"  }}
+  EMQX_CLUSTER__K8S__APP_NAME: {{ .Release.Name }}
+  EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
+  EMQX_CLUSTER__K8S__SERVICE_NAME:  {{ include "emqx.fullname" . }}-headless
+  EMQX_CLUSTER__K8S__NAMESPACE: {{ .Release.Namespace }}
+  EMQX_CLUSTER__K8S__ADDRESS_TYPE: "hostname"
+  EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local"  
+  {{- else if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY)  "dns"  }}
+  EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
+  EMQX_CLUSTER__DNS__APP: {{ .Release.Name }}
+  EMQX_CLUSTER__DNS__TYPE: "srv"
+  {{- end -}}
   {{- range $index, $value := .Values.emqxConfig }}
   {{- if $value }}
   {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}

deploy/charts/emqx/values.yaml

@@ -72,27 +72,8 @@ initContainers: {}
 
 ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
 emqxConfig:
-  EMQX_NAME:  "{{ .Release.Name }}"
-
-  ## Cluster discovery by dns
-  # EMQX_CLUSTER__DISCOVERY: "dns"
-  # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
-  # EMQX_CLUSTER__DNS__APP: "{{ .Release.Name }}"
-  # EMQX_CLUSTER__DNS__TYPE: "srv"
-
   ## Cluster discovery by k8s
   EMQX_CLUSTER__DISCOVERY: "k8s"
-  EMQX_CLUSTER__K8S__APP_NAME: "{{ .Release.Name }}"
-  EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
-  EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless"
-  EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}"
-  ## The address type is used to extract host from k8s service.
-  ## Value: ip | dns | hostname
-  ## Note：Hostname is only supported after v4.0-rc.2
-  EMQX_CLUSTER__K8S__ADDRESS_TYPE: "hostname"
-  EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local"
-  ## if EMQX_CLUSTER__K8S__ADDRESS_TYPE eq dns
-  # EMQX_CLUSTER__K8S__SUFFIX: "pod.cluster.local"
   EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID: "admin"
   EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET: "public"
 

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.4.11"}).
+-define(EMQX_RELEASE, {opensource, "4.4.12-alpha.1"}).
 
 -else.
 

lib-ce/emqx_dashboard/src/emqx_dashboard.app.src

@@ -1,6 +1,6 @@
 {application, emqx_dashboard,
  [{description, "EMQX Web Dashboard"},
-  {vsn, "4.4.10"}, % strict semver, bump manually!
+  {vsn, "4.4.11"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_dashboard_sup]},
   {applications, [kernel,stdlib,mnesia,minirest]},

lib-ce/emqx_modules/src/emqx_mod_rewrite.erl

@@ -23,7 +23,7 @@
 -include_lib("emqx/include/logger.hrl").
 
 -ifdef(TEST).
--export([ compile/1
+-export([ compile_rules/1
         , match_and_rewrite/3
         ]).
 -endif.
@@ -40,13 +40,14 @@
         , description/0
         ]).
 
+-type(topic() :: binary()).
+
 %%--------------------------------------------------------------------
 %% Load/Unload
 %%--------------------------------------------------------------------
 
 load(RawRules) ->
-    {PubRules, SubRules} = compile(RawRules),
+    {PubRules, SubRules} = compile_rules(RawRules),
-    log_start(RawRules),
     emqx_hooks:put('client.subscribe',   {?MODULE, rewrite_subscribe, [SubRules]}, 1000),
     emqx_hooks:put('client.unsubscribe', {?MODULE, rewrite_unsubscribe, [SubRules]}, 1000),
     emqx_hooks:put('message.publish',    {?MODULE, rewrite_publish, [PubRules]}, 1000).
@@ -75,30 +76,59 @@ description() ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
-log_start(Rules) ->
+compile_rules(RawRules) ->
-    PubRules = [{pub, Topic, Re, Dest} || {rewrite, pub, Topic, Re, Dest} <- Rules],
+    compile(validate_rules(RawRules)).
-    SubRules = [{sub, Topic, Re, Dest} || {rewrite, sub, Topic, Re, Dest} <- Rules],
+
+compile({PubRules, SubRules}) ->
+    CompileRE =
+        fun({rewrite, RewriteFrom, Re, RewriteTo}) ->
+                {ok, MP} = re:compile(Re),
+                {rewrite, RewriteFrom, MP, RewriteTo}
+        end,
+    {lists:map(CompileRE, PubRules), lists:map(CompileRE, SubRules)}.
+
+validate_rules(Rules) ->
+    PubRules = [{rewrite, RewriteFrom, Re, RewriteTo} ||
+                   {rewrite, pub, RewriteFrom, Re, RewriteTo} <- Rules,
+                   validate_rule(pub, RewriteFrom, RewriteTo)
+               ],
+    SubRules = [{rewrite, RewriteFrom, Re, RewriteTo} ||
+                   {rewrite, sub, RewriteFrom, Re, RewriteTo} <- Rules,
+                   validate_rule(sub, RewriteFrom, RewriteTo)
+               ],
     ?LOG(info, "[Rewrite] Load: pub rules count ~p sub rules count ~p",
         [erlang:length(PubRules), erlang:length(SubRules)]),
-    log_rule(PubRules, 1),
+    log_rules(pub, PubRules),
-    log_rule(SubRules, 1).
+    log_rules(sub, SubRules),
+    {PubRules, SubRules}.
 
-log_rule([], _Index) -> ok;
+validate_rule(Type, RewriteFrom, RewriteTo) ->
-log_rule([{Type, Topic, Re, Dest} | Rules], Index) ->
+    case validate_topic(filter, RewriteFrom) of
+        ok ->
+            case validate_topic(dest_topic_type(Type), RewriteTo) of
+                ok ->
+                    true;
+                {error, Reason} ->
+                    log_invalid_rule(to, Type, RewriteTo, Reason),
+                    false
+            end;
+        {error, Reason} ->
+            log_invalid_rule(from, Type, RewriteFrom, Reason),
+            false
+    end.
+
+log_invalid_rule(Direction, Type, Topic, Reason) ->
+    ?LOG(warning, "Invalid rewrite ~p rule for rewrite ~p topic '~ts' discarded. Reason: ~p",
+         [type_to_name(Type), Direction, Topic, Reason]).
+
+log_rules(Type, Rules) ->
+    do_log_rules(Type, Rules, 1).
+
+do_log_rules(_Type, [], _Index) -> ok;
+do_log_rules(Type, [{_, Topic, Re, Dest} | Rules], Index) ->
     ?LOG(info, "[Rewrite] Load ~p rule[~p]: source: ~ts, re: ~ts, dest: ~ts",
         [Type, Index, Topic, Re, Dest]),
-    log_rule(Rules, Index + 1).
+    do_log_rules(Type, Rules, Index + 1).
-
-compile(Rules) ->
-    PubRules = [ begin
-                     {ok, MP} = re:compile(Re),
-                     {rewrite, Topic, MP, Dest}
-                 end || {rewrite, pub, Topic, Re, Dest}<- Rules ],
-    SubRules = [ begin
-                     {ok, MP} = re:compile(Re),
-                     {rewrite, Topic, MP, Dest}
-                 end || {rewrite, sub, Topic, Re, Dest}<- Rules ],
-    {PubRules, SubRules}.
 
 match_and_rewrite(Topic, [], _) ->
     Topic;
@@ -138,3 +168,19 @@ filter_client_binds(Binds) ->
                     (_) -> true
                    end,
                  Binds).
+
+type_to_name(pub) -> 'PUBLISH';
+type_to_name(sub) -> 'SUBSCRIBE'.
+
+dest_topic_type(pub) -> name;
+dest_topic_type(sub) -> filter.
+
+-spec(validate_topic(name | filter, topic()) -> ok | {error, term()}).
+validate_topic(Type, Topic) ->
+    try
+        true = emqx_topic:validate(Type, Topic),
+        ok
+    catch
+        error:Reason ->
+            {error, Reason}
+    end.

lib-ce/emqx_modules/src/emqx_modules.app.src

@@ -1,6 +1,6 @@
 {application, emqx_modules,
  [{description, "EMQ X Module Management"},
-  {vsn, "4.4.7"},
+  {vsn, "4.4.8"},
   {modules, []},
   {applications, [kernel,stdlib]},
   {mod, {emqx_modules_app, []}},

lib-ce/emqx_modules/src/emqx_modules.appup.src

@@ -1,9 +1,13 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.6",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
+  [{"4.4.7",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
+   {"4.4.6",
+    [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
    {"4.4.5",
-    [{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
    {"4.4.4",
@@ -36,9 +40,13 @@
      {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.6",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
+  [{"4.4.7",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
+   {"4.4.6",
+    [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
    {"4.4.5",
-    [{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
    {"4.4.4",

lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl

@@ -30,6 +30,13 @@
                 {rewrite, sub, <<"c/#">>,<<"^c/(.+)$">>,<<"sub/%c/$1">>}
                ]).
 
+-define(BAD_RULES_1, [{rewrite, pub, <<"x/#">>,<<"^x/y/(.+)$">>,<<"z/y/+">>}]).
+
+%% empty topic filter/name won't be ranched cased `emqx.conf` will be checked before emqx started
+%% but we need this check for emqx-ee modules api
+-define(BAD_RULES_2, [{rewrite, pub, <<"">>,<<"^x/y/(.+)$">>,<<"z/y/+">>}]).
+-define(BAD_RULES_3, [{rewrite, pub, <<"name/#">>,<<"^name/(.+)$">>,<<"">>}]).
+
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
@@ -82,12 +89,30 @@ t_mod_rewrite(_Config) ->
     ok = emqx_mod_rewrite:unload(?RULES).
 
 t_rewrite_rule(_Config) ->
-    {PubRules, SubRules} = emqx_mod_rewrite:compile(?RULES),
+    {PubRules, SubRules} = emqx_mod_rewrite:compile_rules(?RULES),
+    %% assert ordering
+    ?assertMatch([{rewrite, <<"x/#">>, _, <<"z/y/$1">>},
+                  {rewrite, <<"name/#">>, _, <<"pub/%u/$1">>},
+                  {rewrite, <<"c/#">>, _, <<"pub/%c/$1">>}],
+                 PubRules),
+    ?assertMatch([{rewrite, <<"y/+/z/#">>, _, <<"y/z/$2">>},
+                  {rewrite, <<"name/#">>, _, <<"sub/%u/$1">>},
+                  {rewrite, <<"c/#">>, _, <<"sub/%c/$1">>}],
+                 SubRules),
+
     ?assertEqual(<<"z/y/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/y/2">>, PubRules, [])),
     ?assertEqual(<<"x/1/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/1/2">>, PubRules, [])),
     ?assertEqual(<<"y/z/b">>, emqx_mod_rewrite:match_and_rewrite(<<"y/a/z/b">>, SubRules, [])),
     ?assertEqual(<<"y/def">>, emqx_mod_rewrite:match_and_rewrite(<<"y/def">>, SubRules, [])).
 
+t_rewrite_bad_rule_1(_Config) ->
+    ?assertEqual({[], []}, emqx_mod_rewrite:compile_rules(?BAD_RULES_1)).
+
+t_rewrite_bad_rule_2(_Config) ->
+    ?assertEqual({[], []}, emqx_mod_rewrite:compile_rules(?BAD_RULES_2)).
+
+t_rewrite_bad_rule_3(_Config) ->
+    ?assertEqual({[], []}, emqx_mod_rewrite:compile_rules(?BAD_RULES_3)).
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------

rebar.config.erl

@@ -196,18 +196,23 @@ overlay_vars(RelType, PkgType, false) ->
     overlay_vars_rel(RelType) ++ overlay_vars_pkg(PkgType).
 
 %% vars per release type, cloud or edge
-overlay_vars_rel(RelType) ->
+overlay_vars_rel(cloud) ->
-    VmArgs = case RelType of
+    [ {vm_args_file, "vm.args"}
-                 cloud -> "vm.args";
+    | overlay_vars_rel_common(cloud)
-                 edge -> "vm.args.edge"
+    ];
-             end,
+overlay_vars_rel(edge) ->
-    [ {enable_plugin_emqx_rule_engine, RelType =:= cloud}
+    [ {vm_args_file, "vm.args.edge"}
+    | overlay_vars_rel_common(edge)
+    ].
+
+overlay_vars_rel_common(RelType) ->
+    [ {emqx_default_erlang_cookie, "emqxsecretcookie"}
+    , {enable_plugin_emqx_rule_engine, RelType =:= cloud}
     , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge}
     , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce
     , {enable_plugin_emqx_recon, true}
     , {enable_plugin_emqx_retainer, true}
     , {enable_plugin_emqx_telemetry, true}
-    , {vm_args_file, VmArgs}
     ].
 
 %% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb)

src/emqx.app.src

@@ -6,7 +6,7 @@
   %% the emqx `release' version, which in turn is comprised of several
   %% apps, one of which is this.  See `emqx_release.hrl' for more
   %% info.
-  {vsn, "4.4.11"}, % strict semver, bump manually!
+  {vsn, "4.4.12"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [ kernel

src/emqx.appup.src

@@ -1,7 +1,12 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.10",
+  [{"4.4.11",
+    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
+   {"4.4.10",
     [{add_module,emqx_ocsp_cache},
      {add_module,emqx_crl_cache},
      {add_module,emqx_const_v1},
@@ -347,7 +352,12 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.10",
+  [{"4.4.11",
+    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
+   {"4.4.10",
     [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
      {load_module,emqx_kernel_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_rule,brutal_purge,soft_purge,[]},

src/emqx_channel.erl

@@ -669,7 +669,6 @@ after_message_acked(ClientInfo, Msg, PubAckProps) ->
 %% Process Subscribe
 %%--------------------------------------------------------------------
 
--compile({inline, [process_subscribe/3]}).
 process_subscribe(TopicFilters, SubProps, Channel) ->
     process_subscribe(TopicFilters, SubProps, Channel, []).
 

test/emqx_channel_SUITE.erl

@@ -324,11 +324,17 @@ t_handle_in_pubcomp_not_found_error(_) ->
 t_handle_in_subscribe(_) ->
     ok = meck:expect(emqx_session, subscribe,
                      fun(_, _, _, Session) -> {ok, Session} end),
-    Channel = channel(#{conn_state => connected}),
+    meck:new(emqx_mqtt_caps),
-    TopicFilters = [{<<"+">>, ?DEFAULT_SUBOPTS}],
+    ok = meck:expect(emqx_mqtt_caps, check_sub, fun(_, _, _) -> ok end),
-    Subscribe = ?SUBSCRIBE_PACKET(1, #{}, TopicFilters),
+    try
-    Replies = [{outgoing, ?SUBACK_PACKET(1, [?QOS_0])}, {event, updated}],
+        Channel = channel(#{conn_state => connected}),
-    {ok, Replies, _Chan} = emqx_channel:handle_in(Subscribe, Channel).
+        TopicFilters = [{<<"+">>, ?DEFAULT_SUBOPTS}],
+        Subscribe = ?SUBSCRIBE_PACKET(1, #{}, TopicFilters),
+        Replies = [{outgoing, ?SUBACK_PACKET(1, [?QOS_0])}, {event, updated}],
+        {ok, Replies, _Chan} = emqx_channel:handle_in(Subscribe, Channel)
+    after
+        meck:unload(emqx_mqtt_caps)
+    end.
 
 t_handle_in_unsubscribe(_) ->
     ok = meck:expect(emqx_session, unsubscribe,
@@ -397,9 +403,15 @@ t_process_publish_qos1(_) ->
 
 t_process_subscribe(_) ->
     ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end),
-    TopicFilters = [ TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}],
+    meck:new(emqx_mqtt_caps),
-    {[{TopicFilter, ?RC_SUCCESS}], _Channel} =
+    ok = meck:expect(emqx_mqtt_caps, check_sub, fun(_, _, _) -> ok end),
-        emqx_channel:process_subscribe(TopicFilters, #{}, channel()).
+    try
+        TopicFilters = [ TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}],
+        {[{TopicFilter, ?RC_SUCCESS}], _Channel} =
+            emqx_channel:process_subscribe(TopicFilters, #{}, channel())
+    after
+        meck:unload(emqx_mqtt_caps)
+    end.
 
 t_process_unsubscribe(_) ->
     ok = meck:expect(emqx_session, unsubscribe, fun(_, _, _, Session) -> {ok, Session} end),