Merge pull request #7587 from HJianBo/merge-main-v4.3-into-v4.4

Pre-release v4.4.3-rc2: Merge main-v4.3 into main-v4.4

CHANGES-4.3.md

@@ -26,16 +26,24 @@ File format:
   - prints check points of mnesia internal stats
   - prints check points of per table loading stats
   Help to locate the problem of long table loading time.
+* Add `local` strategy for Shared Subscription.
+  That will preferentially dispatch messages to a shared subscriber at the same
+  node. It will improves the efficiency of shared messages dispatching in certain
+  scenarios, especially when the emqx-bridge-mqtt plugin is configured as shared
+  subscription. [#7462]
 
 ### Bug fixes
 
 * Prohibit empty topics in strict mode
 * Make sure ehttpc delete useless pool always succeed.
 * Update mongodb driver to fix potential process leak.
-* Dashboard admin password persists after leaving/joining the cluster
+* Fix a potential security issue #3155 with emqx-dashboard plugin.
+  In the earlier implementation, the Dashboard password is reset back to the
+  default value of emqx_dashboard.conf after the node left cluster.
+  Now we persist changed password to protect against reset. [#7518]
 * Silence grep/sed warnings in docker-entrypoint.sh. [#7520]
-* Generate `loaded_modules` and `loaded_plugins` files with default
+* Generate `loaded_modules` and `loaded_plugins` files with default values when no such files exists. [#7520]
-  values when no such files exists. [#7520]
+* Fix the configuration `server_name_indication` set to disable does not take effect.
 
 ## v4.3.13
 

apps/emqx_auth_http/src/emqx_auth_http.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_http,
  [{description, "EMQ X Authentication/ACL with HTTP API"},
-  {vsn, "4.3.4"}, % strict semver, bump manually!
+  {vsn, "4.3.5"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_http_sup]},
   {applications, [kernel,stdlib,ehttpc]},

apps/emqx_auth_http/src/emqx_auth_http.appup.src

@@ -1,6 +1,9 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.3",
+  [{"4.3.4",
+    [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.3",
     [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
    {"4.3.2",
@@ -11,7 +14,9 @@
    {<<"4.3.[0-1]">>,
     [{restart_application,emqx_auth_http}]},
    {<<".*">>,[]}],
-  [{"4.3.3",
+  [{"4.3.4",
+    [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}]},
+   {"4.3.3",
     [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
    {"4.3.2",

apps/emqx_auth_http/src/emqx_auth_http_app.erl

@@ -71,6 +71,7 @@ translate_env(EnvName) ->
                                      end,
                             SNI = case application:get_env(?APP, server_name_indication, undefined) of
                                     "disable" -> disable;
+                                    "" -> undefined;
                                     SNI0 -> SNI0
                                   end,
                             TLSOpts = lists:filter(

apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema

@@ -73,6 +73,7 @@
                  {verify, cuttlefish:conf_get("auth.ldap.ssl.verify", Conf, undefined)},
                  {server_name_indication, case cuttlefish:conf_get("auth.ldap.ssl.server_name_indication", Conf, undefined) of
                                             "disable" -> disable;
+                                            "" -> undefined;
                                             SNI -> SNI
                                           end}]
               end,

apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_ldap,
  [{description, "EMQ X Authentication/ACL with LDAP"},
-  {vsn, "4.3.3"}, % strict semver, bump manually!
+  {vsn, "4.3.4"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_ldap_sup]},
   {applications, [kernel,stdlib,eldap2,ecpool]},

apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src

@@ -1,6 +1,10 @@
 %% -*-: erlang -*-
 {VSN,
- [ {"4.3.0",
+ [ {"4.3.3", [
+      %% There are only changes to the schema file, so we don't need
+      %% any commands here.
+   ]},
+   {"4.3.0",
     [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
     , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []}
     , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
@@ -16,7 +20,7 @@
     ]},
    {<<".*">>, []}
  ],
- [
+ [ {"4.3.3", []},
    {"4.3.0",
     [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
     , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []}

apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema

@@ -147,6 +147,7 @@
                 Filter([{verify, Verify},
                         {server_name_indication, case cuttlefish:conf_get(Prefix ++ ".server_name_indication", Conf, undefined) of
                                                    "disable" -> disable;
+                                                   "" -> undefined;
                                                    SNI -> SNI
                                                  end},
                         {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)},

apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema

@@ -109,6 +109,7 @@
                          end,
                 SNI = case cuttlefish:conf_get("auth.mysql.ssl.server_name_indication", Conf, undefined) of
                         "disable" -> disable;
+                        "" -> undefined;
                         SNI0 -> SNI0
                       end,
                 Options ++ [{ssl, Filter([{server_name_indication, SNI},

apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_mysql,
  [{description, "EMQ X Authentication/ACL with MySQL"},
-  {vsn, "4.3.1"}, % strict semver, bump manually!
+  {vsn, "4.3.2"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_mysql_sup]},
   {applications, [kernel,stdlib,mysql,ecpool]},

apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src

@@ -1,10 +1,15 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.0",
+  [{"4.3.1", [
+      %% There are only changes to the schema file, so we don't need
+      %% any commands here.
+   ]},
+   {"4.3.0",
     [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.0",
+  [{"4.3.1", []},
+   {"4.3.0",
     [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]

apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema

@@ -109,6 +109,7 @@
                         {verify,     Verify},
                         {server_name_indication, case cuttlefish:conf_get(Prefix ++ ".server_name_indication", Conf, undefined) of
                                                    "disable" -> disable;
+                                                   "" -> undefined;
                                                    SNI -> SNI
                                                  end},
                         {versions, [list_to_existing_atom(Value)

apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_pgsql,
  [{description, "EMQ X Authentication/ACL with PostgreSQL"},
-  {vsn, "4.4.1"}, % strict semver, bump manually!
+  {vsn, "4.4.2"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_pgsql_sup]},
   {applications, [kernel,stdlib,epgsql,ecpool]},

apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src

@@ -1,10 +1,18 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.4.0",
+  [{"4.4.1", [
+      %% There are only changes to the schema file, so we don't need
+      %% any commands here.
+   ]},
+   {"4.4.0",
     [{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.0",
+  [{"4.4.1", [
+      %% There are only changes to the schema file, so we don't need
+      %% any commands here.
+   ]},
+   {"4.4.0",
     [{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]

apps/emqx_auth_redis/priv/emqx_auth_redis.schema

@@ -98,6 +98,7 @@
                      end,
            SNI = case cuttlefish:conf_get("auth.redis.ssl.server_name_indication", Conf, undefined) of
                    "disable" -> disable;
+                   "" -> undefined;
                    SNI0 -> SNI0
                  end,
            [{options, [{ssl_options,

apps/emqx_auth_redis/src/emqx_auth_redis.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_redis,
  [{description, "EMQ X Authentication/ACL with Redis"},
-  {vsn, "4.3.1"}, % strict semver, bump manually!
+  {vsn, "4.3.2"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_redis_sup]},
   {applications, [kernel,stdlib,eredis,eredis_cluster,ecpool]},

apps/emqx_auth_redis/src/emqx_auth_redis.appup.src

@@ -1,10 +1,15 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.0",
+  [{"4.3.1", [
+      %% There are only changes to the schema file, so we don't need
+      %% any commands here.
+   ]},
+   {"4.3.0",
     [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.0",
+  [{"4.3.1", []},
+   {"4.3.0",
     [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]

apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema

@@ -183,8 +183,12 @@
                     [{verify, verify_peer}];
                (verify, false) ->
                     [{verify, verify_none}];
-               (server_name_indication, "disabled") ->
+               (server_name_indication, "disable") ->
-                    [{server_name_indication, disabled}];
+                    [{server_name_indication, disable}];
+               (server_name_indication, undefined) ->
+                    [];
+               (server_name_indication, "") ->
+                    [];
                (server_name_indication, Hostname) ->
                     [{server_name_indication, Hostname}];
                (Opt, Val) ->

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src

@@ -1,39 +1,29 @@
-%% -*-: erlang -*-
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [
+  [{"4.3.4",
-    {"4.3.4", [
+    [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]},
-      %% There are only changes to the schema file, so we don't need
+   {"4.3.3",
-      %% any commands here.
+    [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]},
-    ]},
+     {load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]}]},
-    {"4.3.3", [
+   {<<"4\\.3\\.[1-2]">>,
-      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}
+    [{load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]},
-    ]},
+     {load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]},
-    {<<"4\\.3\\.[1-2]">>, [
+   {"4.3.0",
-      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []},
+    [{load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]},
-      {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []}
-    ]},
-    {"4.3.0", [
-      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []},
      {load_module,emqx_bridge_worker,brutal_purge,soft_purge,[]},
-      {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []}
+     {load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]},
-    ]},
+   {<<".*">>,[]}],
-    {<<".*">>, []}
+  [{"4.3.4",
-  ],
+    [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]},
-  [
+   {"4.3.3",
-    {"4.3.4", [
+    [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]},
-    ]},
+     {load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]}]},
-    {"4.3.3", [
+   {<<"4\\.3\\.[1-2]">>,
-      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}
+    [{load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]},
-    ]},
+     {load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]},
-    {<<"4\\.3\\.[1-2]">>, [
+   {"4.3.0",
-      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []},
+    [{load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]},
-      {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []}
-    ]},
-    {"4.3.0", [
-      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []},
      {load_module,emqx_bridge_worker,brutal_purge,soft_purge,[]},
-      {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []}
+     {load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]},
-    ]},
+   {<<".*">>,[]}]}.
-    {<<".*">>, []}
-  ]
-}.

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl

@@ -240,6 +240,23 @@
                        zh => <<"SSL 加密算法"/utf8>>},
             description => #{en => <<"SSL Ciphers">>,
                              zh => <<"SSL 加密算法"/utf8>>}
+        },
+        verify => #{
+            order => 19,
+            type => boolean,
+            default => false,
+            title => #{en => <<"Verify Server Certfile">>,
+                       zh => <<"校验服务器证书"/utf8>>},
+            description => #{en => <<"Whether to verify the server certificate. By default, the client will not verify the server's certificate. If verification is required, please set it to true.">>,
+                             zh => <<"是否校验服务器证书。 默认客户端不会去校验服务器的证书，如果需要校验，请设置成true。"/utf8>>}
+        },
+        server_name_indication => #{
+            order => 20,
+            type => string,
+            title => #{en => <<"Server Name Indication">>,
+                    zh => <<"服务器名称指示"/utf8>>},
+            description => #{en => <<"Specify the hostname used for peer certificate verification, or set to disable to turn off this verification.">>,
+                            zh => <<"指定用于对端证书验证时使用的主机名，或者设置为 disable 以关闭此项验证。"/utf8>>}
         }
     }).
 

apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl

@@ -66,13 +66,23 @@ save_files_return_opts(Options, Dir) ->
                   _ -> verify_peer
              end,
     SNI = case Get(<<"server_name_indication">>) of
+              <<"disable">> -> disable;
+              "disable" -> disable;
+              "" -> undefined;
+              <<>> -> undefined;
               undefined -> undefined;
               SNI0 -> ensure_str(SNI0)
           end,
     Versions = emqx_tls_lib:integral_versions(Get(<<"tls_versions">>)),
     Ciphers = emqx_tls_lib:integral_ciphers(Versions, Get(<<"ciphers">>)),
-    filter([{keyfile, Key}, {certfile, Cert}, {cacertfile, CA},
+    filter([ {keyfile, Key}
-            {verify, Verify}, {server_name_indication, SNI}, {versions, Versions}, {ciphers, Ciphers}]).
+           , {certfile, Cert}
+           , {cacertfile, CA}
+           , {verify, Verify}
+           , {server_name_indication, SNI}
+           , {versions, Versions}
+           , {ciphers, Ciphers}
+           ]).
 
 %% @doc Save a key or certificate file in data dir,
 %% and return path of the saved file.

apps/emqx_rule_engine/src/emqx_rule_engine.erl

@@ -332,7 +332,9 @@ start_resource(ResId) ->
 test_resource(#{type := Type} = Params) ->
     case emqx_rule_registry:find_resource_type(Type) of
         {ok, #resource_type{}} ->
-            ResId = maps:get(id, Params, resource_id()),
+            %% Resource will be deleted after test.
+            %% Use random resource id, ensure test func will not delete the resource in used.
+            ResId = resource_id(),
             try
                 case create_resource(maps:put(id, ResId, Params), no_retry) of
                     {ok, _} ->

apps/emqx_web_hook/src/emqx_web_hook.app.src

@@ -1,6 +1,6 @@
 {application, emqx_web_hook,
  [{description, "EMQ X WebHook Plugin"},
-  {vsn, "4.3.10"}, % strict semver, bump manually!
+  {vsn, "4.3.11"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_web_hook_sup]},
   {applications, [kernel,stdlib,ehttpc]},

apps/emqx_web_hook/src/emqx_web_hook.appup.src

@@ -1,4 +1,5 @@
 %% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{<<"4\\.3\\.[0-2]">>,
     [{apply,{application,stop,[emqx_web_hook]}},
@@ -11,11 +12,10 @@
      {load_module,emqx_web_hook,brutal_purge,soft_purge,[]},
      {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
-    [{load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]},
-   {"4.3.9",
+     {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]},
-    [ %% nothing so far
+   {"4.3.9",[{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]},
-      %% 4.3.9 is taken by release 4.3.12
+   {"4.3.10",[{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]},
-    ]},
    {<<".*">>,[]}],
   [{<<"4\\.3\\.[0-2]">>,
     [{apply,{application,stop,[emqx_web_hook]}},
@@ -28,9 +28,8 @@
      {load_module,emqx_web_hook,brutal_purge,soft_purge,[]},
      {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
-    [{load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]},
-   {"4.3.9",
+     {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]},
-    [ %% nothing so far
+   {"4.3.9",[{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]},
-      %% 4.3.9 is taken by release 4.3.12
+   {"4.3.10",[{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]},
-    ]},
    {<<".*">>,[]}]}.

apps/emqx_web_hook/src/emqx_web_hook_app.erl

@@ -61,6 +61,7 @@ translate_env() ->
                                    end,
                        SNI = case application:get_env(?APP, server_name_indication, undefined) of
                                  "disable" -> disable;
+                                 "" -> undefined;
                                  SNI0 -> SNI0
                              end,
                        TLSOpts = lists:filter(fun({_K, V}) ->

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.4.3-rc.1"}).
+-define(EMQX_RELEASE, {opensource, "4.4.3-rc.2"}).
 
 -else.
 

rebar.config

@@ -39,7 +39,7 @@
 
 {deps,
     [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
-    , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.15"}}}
+    , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.2.0"}}}
     , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.1"}}}
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}

scripts/get-distro.sh

@@ -17,6 +17,9 @@ case "$UNAME" in
         if grep -q -i 'rhel' /etc/*-release; then
             DIST='el'
             VERSION_ID="$(rpm --eval '%{rhel}')"
+        elif grep -q -i 'centos' /etc/*-release; then
+            DIST='centos'
+            VERSION_ID="$(rpm --eval '%{centos_ver}')"
         else
             DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')"
             VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"