Merge pull request #8648 from zmstone/0803-sync-v4.3-to-v4.4

sync v4.3 to v4.4

.github/workflows/build_packages.yaml

@@ -117,7 +117,6 @@ jobs:
           - 24.1.5-3
         macos:
           - macos-11
-          - macos-10.15
         exclude:
           - profile: emqx-edge
     runs-on: ${{ matrix.macos }}

.github/workflows/build_slim_packages.yaml

@@ -104,7 +104,6 @@ jobs:
         - 24.1.5-3
         macos:
         - macos-11
-        - macos-10.15
 
     runs-on: ${{  matrix.macos }}
 

CHANGES-4.3.md

@@ -10,6 +10,13 @@ File format:
 - One list item per change topic
   Change log ends with a list of github PRs
 
+## v4.3.18
+
+### Enhancements
+
+- Upgrade Erlang/OTP from 23.2.7.2-emqx-3 to 23.3.4.9-3 [#8511](https://github.com/emqx/emqx/pull/8511)
+- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553)
+
 ## v4.3.17
 
 ### Bug fixes
@@ -24,6 +31,7 @@ File format:
   This causes the clients to not expire as long as a new UDP packet arrives [#8575]
 
 ### Enhancements
+
 - HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450]
 - Add check_conf cli to check config format. [#8486]
 - Optimize performance of shared subscription

apps/emqx_rule_engine/src/emqx_rule_engine.app.src

@@ -1,6 +1,6 @@
 {application, emqx_rule_engine,
  [{description, "EMQ X Rule Engine"},
-  {vsn, "4.4.6"}, % strict semver, bump manually!
+  {vsn, "4.4.7"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_rule_engine_sup, emqx_rule_registry]},
   {applications, [kernel,stdlib,rulesql,getopt]},

apps/emqx_rule_engine/src/emqx_rule_engine.appup.src

@@ -1,12 +1,13 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-   [{"4.4.5",
+  [{"4.4.6",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+   {"4.4.5",
-      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
-    {"4.4.4",
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
-     [{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+   {"4.4.4",
+    [{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
@@ -71,7 +72,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.5",
+  [{"4.4.6",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.4.5",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},

bin/emqx

@@ -48,6 +48,8 @@ assert_node_alive() {
 }
 
 check_erlang_start() {
+    # Fix bin permission
+    find "$BINDIR" ! -executable -exec chmod a+x {} \;
     "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()"
 }
 

build

@@ -96,7 +96,37 @@ make_relup() {
     fi
     RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
     export RELX_BASE_VERSIONS
+    if [[ ${PKG_VSN} == 4.3* ]]; then
+        echo "EMQX 4.3 specific, overwrite OTP app versions"
+        local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel"
+        if [ ! -f "${emqx_rel_file}" ]; then
+            ./rebar3 as "${PROFILE}" release
+        fi
+        scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}"
+    fi
     ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"
+
+    # assert that there is no 'restart_emulator' in relup
+    # EMQX wants hot upgrade without VM restart
+    if grep restart_emulator "${releases_dir}/${PKG_VSN}/relup"; then
+        echo "Error: restart_emulator instruction found in relup";
+        exit 1
+    fi
+
+    # rollback rel file per releases
+    #
+    if [[ ${PKG_VSN} == 4.3* ]]; then
+        echo "restore upgrade base rel files... "
+        for rel in ${releases[*]};
+        do
+            bakfile="${releases_dir}/${rel}/${PROFILE}.rel.bak"
+            echo "restore $bakfile ..."
+            if [ -f "$bakfile" ]; then
+                echo "restore from $bakfile"
+                mv "${bakfile}" "${bakfile%.bak}"
+            fi
+        done
+    fi
 }
 
 ## try to be portable for zip packages.

deploy/charts/emqx/Chart.yaml

@@ -13,8 +13,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 4.4.6
+version: 4.4.7
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 4.4.6
+appVersion: 4.4.7

etc/emqx.conf

@@ -1580,6 +1580,14 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL
 ## Value: cn | dn | crt | pem | md5
 ## listener.ssl.external.peer_cert_as_clientid = cn
 
+## Default is 'notice', set 'debug' to inspect TLS handshake messaes.
+## This log level is not related to EMQX's log level.
+##
+## NOTE: never set to 'debug' in production environemnts.
+##
+## Value: emergency | alert | critical | error | warning | notice | info | debug
+## listener.ssl.external.log_level = notice
+#
 ## TCP backlog for the SSL connection.
 ##
 ## See listener.tcp.$name.backlog
@@ -2091,6 +2099,10 @@ listener.wss.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL
 ## Value: cn | dn | crt | pem | md5
 ## listener.wss.external.peer_cert_as_clientid = cn
 
+## See: listener.ssl.$name.log_level
+## Value: emergency | alert | critical | error | warning | notice | info | debug
+## listener.wss.external.log_level = notice
+
 ## TCP backlog for the WebSocket/SSL connection.
 ##
 ## See: listener.tcp.$name.backlog

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.4.6"}).
+-define(EMQX_RELEASE, {opensource, "4.4.7"}).
 
 -else.
 

lib-ce/emqx_dashboard/src/emqx_dashboard.app.src

@@ -1,5 +1,4 @@
 {application, emqx_dashboard,
-
  [{description, "EMQX Web Dashboard"},
   {vsn, "4.4.6"}, % strict semver, bump manually!
   {modules, []},

priv/emqx.schema

@@ -1637,6 +1637,10 @@ end}.
   {datatype, {enum, [cn, dn, crt, pem, md5]}}
 ]}.
 
+{mapping, "listener.ssl.$name.log_level", "emqx.listeners", [
+  {datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug, none, all]}}
+]}.
+
 %%--------------------------------------------------------------------
 %% MQTT/WebSocket Listeners
 
@@ -2089,6 +2093,10 @@ end}.
   hidden
 ]}.
 
+{mapping, "listener.wss.$name.log_level", "emqx.listeners", [
+  {datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug, none, all]}}
+]}.
+
 {translation, "emqx.listeners", fun(Conf) ->
 
     Filter  = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end,
@@ -2159,7 +2167,9 @@ end}.
                           {mqtt_piggyback, cuttlefish:conf_get(Prefix ++ ".mqtt_piggyback", Conf, undefined)},
                           {check_origin_enable, cuttlefish:conf_get(Prefix ++ ".check_origin_enable", Conf, undefined)},
                           {allow_origin_absence, cuttlefish:conf_get(Prefix ++ ".allow_origin_absence", Conf, undefined)},
-                          {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)])
+                          {check_origins, WsOpts(Prefix)}
+                         | AccOpts(Prefix)
+                         ])
               end,
     DeflateOpts = fun(Prefix) ->
                       Filter([{level, cuttlefish:conf_get(Prefix ++ ".deflate_opts.level", Conf, undefined)},
@@ -2252,7 +2262,9 @@ end}.
                           {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)},
                           {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)},
                           {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)},
-                          {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}])
+                          {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)},
+                          {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)}
+                         ])
               end,
 
     Listen_fix = fun({Ip, Port}) -> case inet:parse_address(Ip) of

rebar.config.erl

@@ -239,21 +239,10 @@ overlay_vars_pkg(pkg) ->
     ].
 
 relx_apps(ReleaseType) ->
-    [ kernel
+    relx_otp_apps() ++
-    , sasl
+    [ redbug
-    , crypto
-    , public_key
-    , asn1
-    , syntax_tools
-    , ssl
-    , os_mon
-    , inets
-    , compiler
-    , runtime_tools
-    , redbug
     , cuttlefish
     , emqx
-    , {mnesia, load}
     , {ekka, load}
     , {emqx_plugin_libs, load}
     , observer_cli
@@ -264,9 +253,13 @@ relx_apps(ReleaseType) ->
     ++ relx_apps_per_rel(ReleaseType)
     ++ [{N, load} || N <- relx_plugin_apps(ReleaseType)].
 
+relx_otp_apps() ->
+    {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"),
+    true = is_list(Apps),
+    Apps.
+
 relx_apps_per_rel(cloud) ->
     [ luerl
-    , xmerl
     | [{observer, load} || is_app(observer)]
     ];
 relx_apps_per_rel(edge) ->

scripts/emqx_rel_otp_app_overwrite.escript

@@ -0,0 +1,51 @@
+#!/usr/bin/env -S escript -c
+%% This script is part of 'relup' process to overwrite the OTP app versions (incl. ERTS) in rel files from upgrade base
+%% so that 'rebar relup' call will not generate instructions for restarting OTP apps or restarting the emulator.
+%%
+%% It simply read OTP app version (incl. ERTS) from the rel file of *NEW* Release ($RelVsn) and write back to the ones
+%% in *OLD* versions ($BASE_VERSIONS)
+%%
+%% note, we use NEW to overwrite OLD is because the modified NEW rel file will be overwritten by next 'rebar relup'
+%%
+
+main([Dir, RelVsn, BASE_VERSIONS]) ->
+    {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Dir, RelVsn), RelVsn),
+    lists:foreach(fun(BaseVer) ->
+                          base_rel_overwrites(BaseVer, Dir, ErtsVsn, Overwrites)
+                  end, string:tokens(BASE_VERSIONS, ",")).
+
+get_otp_apps(RelFile, RelVsn) ->
+    case file:consult(RelFile) of
+        {ok, [{release, {"emqx", RelVsn}, {erts, ErtsVsn}, AppList}]} ->
+            Apps = lists:filter(fun(X) -> lists:member(element(1, X), otp_apps()) end, AppList),
+            {ErtsVsn, Apps};
+        {error, Reason} ->
+            io:format(standard_error, "failed_to_read_file ~p for release ~p~nreason=~p~n", [RelFile, RelVsn, Reason]),
+            halt(1)
+    end.
+
+base_rel_overwrites(RelVsn, Dir, ErtsVsn, Overwrites) ->
+    RelFile = rel_file(Dir, RelVsn),
+    file:copy(RelFile, RelFile++".bak"),
+    {ok, [{release, {"emqx", RelVsn}, {erts, _BaseErtsVsn}, BaseAppList}]} = file:consult(RelFile),
+    NewData = [ {release, {"emqx", RelVsn}, {erts, ErtsVsn},
+                 lists:map(fun(X) ->
+                                   Name = element(1, X),
+                                   case lists:keyfind(Name, 1, Overwrites) of
+                                       false -> X;
+                                       Y when is_tuple(Y) -> Y
+                                   end
+                           end, BaseAppList)
+                }
+              ],
+    ok = file:write_file(RelFile, io_lib:format("~p.", NewData)).
+
+rel_file(Dir, RelVsn)->
+    filename:join([Dir, RelVsn, "emqx.rel"]).
+
+otp_apps() ->
+    {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"),
+    true = is_list(Apps),
+    lists:map(fun(App) when is_atom(App) -> App;
+                 ({App, _}) -> App %% handle like {mnesia, load}
+              end, Apps).

scripts/fail-on-old-otp-version.escript

@@ -1,4 +1,4 @@
-#!/usr/bin/env escript
+#!/usr/bin/env -S escript -c
 
 main(_) ->
     OtpRelease = list_to_integer(erlang:system_info(otp_release)),

scripts/rel_otp_apps.eterm

@@ -0,0 +1,16 @@
+%% Single source of truth of list otp apps that we use
+[ kernel
+, stdlib
+, sasl
+, crypto
+, public_key
+, asn1
+, syntax_tools
+, ssl
+, os_mon
+, inets
+, compiler
+, runtime_tools
+, {mnesia, load}
+, xmerl
+].

src/emqx.app.src

@@ -1,12 +1,12 @@
 {application, emqx,
  [{id, "emqx"},
-  {description, "EMQ X"},
+  {description, "EMQX"},
   %% Note: this version is not the same as the release version!  This
   %% is simply the emqx `application' version, which is separate from
   %% the emqx `release' version, which in turn is comprised of several
   %% apps, one of which is this.  See `emqx_release.hrl' for more
   %% info.
-  {vsn, "4.4.6"}, % strict semver, bump manually!
+  {vsn, "4.4.7"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [ kernel

src/emqx.appup.src

@@ -1,9 +1,11 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.5",
+  [{"4.4.6",
-    [
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_relup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_relup,brutal_purge,soft_purge,[]}]},
+   {"4.4.5",
+    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {update,emqx_broker_sup,supervisor},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]},
@@ -160,7 +162,10 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.5",
+  [{"4.4.6",
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_relup,brutal_purge,soft_purge,[]}]},
+   {"4.4.5",
     [{update,emqx_broker_sup,supervisor},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},