Merge pull request #8665 from zmstone/chore-sync-dev/v4.3.18-todev/v4.4.7

Chore sync dev/v4.3.18 to dev/v4.4.7

.ci/build_packages/tests.sh

@@ -177,11 +177,21 @@ relup_test(){
         fi
         ./emqx/bin/emqx_ctl status
         ./emqx/bin/emqx versions
+        OldVsn="$(./emqx/bin/emqx eval 'Versions=[{S, V} ||  {_,V,_, S} <- release_handler:which_releases()],
+                    Current = proplists:get_value(current, Versions, proplists:get_value(permanent, Versions)),
+                    io:format("~s", [Current])')"
         cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*.zip ./emqx/releases/
         ./emqx/bin/emqx install "${TARGET_VERSION}"
         [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
         export EMQX_WAIT_FOR_STOP=300
         ./emqx/bin/emqx_ctl status
+
+        # also test remove old rel
+        ./emqx/bin/emqx uninstall "$OldVsn"
+
+        # check emqx still runs
+        ./emqx/bin/emqx ping
+
         if ! ./emqx/bin/emqx stop; then
             cat emqx/log/erlang.log.1 || true
             cat emqx/log/emqx.log.1 || true

.github/workflows/release.yaml

@@ -67,11 +67,16 @@ jobs:
                -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
                ${{ secrets.EMQX_IO_RELEASE_API }}
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: get version
+        id: version
+        run: echo "::set-output name=version::$(./pkg-vsn.sh)"
       - uses: emqx/push-helm-action@v1
         if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
         with:
           charts_dir: "${{ github.workspace }}/deploy/charts/emqx"
-          version: ${{ github.ref_name }}
+          version: ${{ steps.version.outputs.version }}
           aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws_region: "us-west-2"
@@ -80,7 +85,7 @@ jobs:
         if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
         with:
           charts_dir: "${{ github.workspace }}/deploy/charts/emqx-ee"
-          version: ${{ github.ref_name }}
+          version: ${{ steps.version.outputs.version }}
           aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws_region: "us-west-2"

CHANGES-4.3.md

@@ -16,6 +16,8 @@ File format:
 
 - Upgrade Erlang/OTP from 23.2.7.2-emqx-3 to 23.3.4.9-3 [#8511](https://github.com/emqx/emqx/pull/8511)
 - Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553)
+- Add option to perform GC on connection process after TLS/SSL handshake is performed. [#8649](https://github.com/emqx/emqx/pull/8649)
+  Expected to reduce around 35% memory consumption for each SSL connection. See [#8637](https://github.com/emqx/emqx/pull/8637) for more details.
 
 ## v4.3.17
 

apps/emqx_sn/rebar.config

@@ -2,7 +2,7 @@
 {plugins, [rebar3_proper]}.
 
 {deps,
- [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}},
+ [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.7"}}},
   {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}}
  ]}.
 

bin/install_upgrade.escript

@@ -111,6 +111,7 @@ uninstall({_RelName, NameTypeArg, NodeName, Cookie}, Opts) ->
     TargetNode = start_distribution(NodeName, NameTypeArg, Cookie),
     WhichReleases = which_releases(TargetNode),
     Version = proplists:get_value(version, Opts),
+    ensure_latest_boot_file(current_release_version(TargetNode)),
     case proplists:get_value(Version, WhichReleases) of
         undefined ->
             ?INFO("Release ~s is already uninstalled.", [Version]);
@@ -362,6 +363,9 @@ permafy(TargetNode, RelName, Vsn) ->
     [{ok, _} = file:copy(filename:join(["bin", File++"-"++Vsn]),
                          filename:join(["bin", File]))
      || File <- Scripts],
+
+    ensure_latest_boot_file(Vsn),
+
     %% update the vars
     UpdatedVars = io_lib:format("REL_VSN=\"~s\"~nERTS_VSN=\"~s\"~n", [Vsn, erts_vsn()]),
     file:write_file(filename:absname(filename:join(["releases", "emqx_vars"])), UpdatedVars, [append]).
@@ -458,3 +462,9 @@ str(A) when is_binary(A) ->
     binary_to_list(A);
 str(A) when is_list(A) ->
     (A).
+
+%%% For otp upgrade, the boot file need match the otp version
+%%% This is for emqx cli tool
+ensure_latest_boot_file(Vsn) ->
+    {ok, _} = file:copy(filename:join(["releases", Vsn, "no_dot_erlang.boot"]),
+                        filename:join(["bin", "no_dot_erlang.boot"])).

etc/emqx.conf

@@ -1649,6 +1649,13 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL
 ## Value: true | false
 listener.ssl.external.reuseaddr = true
 
+## Whether to perform GC after TLS/SSL handshake.
+##
+## Default: false
+##
+## Value: true | false
+## listener.ssl.external.gc_after_handshake = false
+
 ##--------------------------------------------------------------------
 ## External WebSocket listener for MQTT protocol
 

priv/emqx.schema

@@ -1641,6 +1641,11 @@ end}.
   {datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug, none, all]}}
 ]}.
 
+{mapping, "listener.ssl.$name.gc_after_handshake", "emqx.listeners", [
+  {default, false},
+  {datatype, {enum, [true, false]}}
+]}.
+
 %%--------------------------------------------------------------------
 %% MQTT/WebSocket Listeners
 
@@ -2263,7 +2268,8 @@ end}.
                           {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)},
                           {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)},
                           {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)},
-                          {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)}
+                          {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)},
+                          {gc_after_handshake, cuttlefish:conf_get(Prefix ++ ".gc_after_handshake", Conf, undefined)}
                          ])
               end,
 

rebar.config

@@ -45,7 +45,7 @@
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
-    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.6"}}}
+    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.7"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.11"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.1"}}}
     , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}}

scripts/emqx_rel_otp_app_overwrite.escript

@@ -1,4 +1,5 @@
-#!/usr/bin/env -S escript -c
+#!/usr/bin/env escript
+
 %% This script is part of 'relup' process to overwrite the OTP app versions (incl. ERTS) in rel files from upgrade base
 %% so that 'rebar relup' call will not generate instructions for restarting OTP apps or restarting the emulator.
 %%
@@ -8,6 +9,8 @@
 %% note, we use NEW to overwrite OLD is because the modified NEW rel file will be overwritten by next 'rebar relup'
 %%
 
+-mode(compile).
+
 main([Dir, RelVsn, BASE_VERSIONS]) ->
     {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Dir, RelVsn), RelVsn),
     lists:foreach(fun(BaseVer) ->

scripts/fail-on-old-otp-version.escript

@@ -1,5 +1,7 @@
 #!/usr/bin/env -S escript -c
 
+-mode(compile).
+
 main(_) ->
     OtpRelease = list_to_integer(erlang:system_info(otp_release)),
     case OtpRelease < 21 of
@@ -9,4 +11,3 @@ main(_) ->
         false ->
             ok
     end.
-

scripts/update_appup.escript

@@ -1,6 +1,8 @@
-#!/usr/bin/env -S escript -c
+#!/usr/bin/env escript
 %% -*- erlang-indent-level:4 -*-
 
+-mode(compile).
+
 usage() ->
 "A script that fills in boilerplate for appup files.