diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 04d8018ac..a51885c1e 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -177,11 +177,21 @@ relup_test(){ fi ./emqx/bin/emqx_ctl status ./emqx/bin/emqx versions + OldVsn="$(./emqx/bin/emqx eval 'Versions=[{S, V} || {_,V,_, S} <- release_handler:which_releases()], + Current = proplists:get_value(current, Versions, proplists:get_value(permanent, Versions)), + io:format("~s", [Current])')" cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*.zip ./emqx/releases/ ./emqx/bin/emqx install "${TARGET_VERSION}" [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 export EMQX_WAIT_FOR_STOP=300 ./emqx/bin/emqx_ctl status + + # also test remove old rel + ./emqx/bin/emqx uninstall "$OldVsn" + + # check emqx still runs + ./emqx/bin/emqx ping + if ! ./emqx/bin/emqx stop; then cat emqx/log/erlang.log.1 || true cat emqx/log/emqx.log.1 || true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b1c892dfc..560ce0ed1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -67,11 +67,16 @@ jobs: -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: get version + id: version + run: echo "::set-output name=version::$(./pkg-vsn.sh)" - uses: emqx/push-helm-action@v1 if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' with: charts_dir: "${{ github.workspace }}/deploy/charts/emqx" - version: ${{ github.ref_name }} + version: ${{ steps.version.outputs.version }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-west-2" @@ -80,7 +85,7 @@ jobs: if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' with: charts_dir: "${{ github.workspace }}/deploy/charts/emqx-ee" - version: ${{ github.ref_name }} + version: ${{ steps.version.outputs.version }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-west-2" diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 5a3fee5ae..282ecdd94 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -16,6 +16,8 @@ File format: - Upgrade Erlang/OTP from 23.2.7.2-emqx-3 to 23.3.4.9-3 [#8511](https://github.com/emqx/emqx/pull/8511) - Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553) +- Add option to perform GC on connection process after TLS/SSL handshake is performed. [#8649](https://github.com/emqx/emqx/pull/8649) + Expected to reduce around 35% memory consumption for each SSL connection. See [#8637](https://github.com/emqx/emqx/pull/8637) for more details. ## v4.3.17 diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index f06ac21ce..ff34ad7f5 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,10 +1,16 @@ # EMQX 4.4 Changes + ## v4.4.8 ### Enhancements (synced from v4.3.19) * Support HTTP API `/trace/:name/detail`. + +### Bug fixes +- Fix: Check if emqx_mod_trace is enabled when the trace file is not found. + + ## v4.4.5 ### Enhancements (synced from v4.3.16) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 9315f8c53..4b6b041a2 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.4.4"}, + {vsn, "4.4.5"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index 3d12e8b8a..e4b8e9d1c 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -2,6 +2,9 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [ + {"4.4.4", + [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, {"4.4.3", [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, @@ -21,6 +24,9 @@ {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [ + {"4.4.4", + [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, {"4.4.3", [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index 28ce21d65..95e705337 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -179,7 +179,13 @@ trace_file(File) -> Node = atom_to_list(node()), case file:read_file(FileName) of {ok, Bin} -> {ok, Node, Bin}; - {error, Reason} -> {error, Node, Reason} + {error, enoent} -> + case emqx_trace:is_enable() of + false -> {error, Node, trace_disabled}; + true -> {error, Node, enoent} + end; + {error, Reason} -> + {error, Node, Reason} end. trace_file_detail(File) -> diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 94a0e8bc1..a478b8a63 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -125,6 +125,9 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> ok -> [FileName | Acc]; _ -> Acc end; + {error, Node, trace_disabled} -> + ?LOG(warning, "emqx_mod_trace modules is disabled on ~s ~s", [Node, TraceLog]), + Acc; {error, Node, Reason} -> ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), Acc @@ -169,6 +172,8 @@ stream_log_file(#{name := Name}, Params) -> {eof, Size} -> Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => <<"">>}}; + {error, trace_disabled} -> + {error, io_lib:format("trace_disable_on_~s", [Node0])}; {error, Reason} -> logger:log(error, "read_file_failed ~p", [{Node, Name, Reason, Position, Bytes}]), {error, Reason}; @@ -218,6 +223,11 @@ read_file(Path, Offset, Bytes) -> after file:close(IoDevice) end; + {error, enoent} -> + case emqx_trace:is_enable() of + false -> {error, trace_disabled}; + true -> {error, enoent} + end; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl index f4107863d..23b677572 100644 --- a/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_trace_SUITE.erl @@ -300,7 +300,7 @@ t_trace_file(_Config) -> t_download_log(_Config) -> ClientId = <<"client-test">>, - Now = erlang:system_time(second), + Now = erlang:system_time(second) - 2, Start = to_rfc3339(Now), Name = <<"test_client_id">>, ok = emqx_trace:create([{<<"name">>, Name}, @@ -316,7 +316,7 @@ t_download_log(_Config) -> t_trace_file_detail(_Config) -> ClientId = <<"client-test1">>, - Now = erlang:system_time(second), + Now = erlang:system_time(second) - 10, Start = to_rfc3339(Now), Name = <<"test_client_id1">>, ok = emqx_trace:create([{<<"name">>, Name}, @@ -330,7 +330,7 @@ t_trace_file_detail(_Config) -> = emqx_trace_api:trace_file_detail(#{name => Name}, []), ct:pal("~p detail:~p~n", [{Name, Now}, Detail]), ?assertEqual(atom_to_binary(node()), Node), - ?assert(Size > 0), + ?assert(Size >= 0), ?assert(Mtime >= Now), ok = emqtt:disconnect(Client), ok. diff --git a/apps/emqx_sn/rebar.config b/apps/emqx_sn/rebar.config index 120d4767d..295751b23 100644 --- a/apps/emqx_sn/rebar.config +++ b/apps/emqx_sn/rebar.config @@ -2,7 +2,7 @@ {plugins, [rebar3_proper]}. {deps, - [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}}, + [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.7"}}}, {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} ]}. diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 14d90156c..88268d893 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -111,6 +111,7 @@ uninstall({_RelName, NameTypeArg, NodeName, Cookie}, Opts) -> TargetNode = start_distribution(NodeName, NameTypeArg, Cookie), WhichReleases = which_releases(TargetNode), Version = proplists:get_value(version, Opts), + ensure_latest_boot_file(current_release_version(TargetNode)), case proplists:get_value(Version, WhichReleases) of undefined -> ?INFO("Release ~s is already uninstalled.", [Version]); @@ -362,6 +363,9 @@ permafy(TargetNode, RelName, Vsn) -> [{ok, _} = file:copy(filename:join(["bin", File++"-"++Vsn]), filename:join(["bin", File])) || File <- Scripts], + + ensure_latest_boot_file(Vsn), + %% update the vars UpdatedVars = io_lib:format("REL_VSN=\"~s\"~nERTS_VSN=\"~s\"~n", [Vsn, erts_vsn()]), file:write_file(filename:absname(filename:join(["releases", "emqx_vars"])), UpdatedVars, [append]). @@ -458,3 +462,9 @@ str(A) when is_binary(A) -> binary_to_list(A); str(A) when is_list(A) -> (A). + +%%% For otp upgrade, the boot file need match the otp version +%%% This is for emqx cli tool +ensure_latest_boot_file(Vsn) -> + {ok, _} = file:copy(filename:join(["releases", Vsn, "no_dot_erlang.boot"]), + filename:join(["bin", "no_dot_erlang.boot"])). diff --git a/etc/emqx.conf b/etc/emqx.conf index 69320f9de..850d85723 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1649,6 +1649,13 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL ## Value: true | false listener.ssl.external.reuseaddr = true +## Whether to perform GC after TLS/SSL handshake. +## +## Default: false +## +## Value: true | false +## listener.ssl.external.gc_after_handshake = false + ##-------------------------------------------------------------------- ## External WebSocket listener for MQTT protocol diff --git a/priv/emqx.schema b/priv/emqx.schema index 4f81fccf1..d2daf3c6b 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1641,6 +1641,11 @@ end}. {datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug, none, all]}} ]}. +{mapping, "listener.ssl.$name.gc_after_handshake", "emqx.listeners", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + %%-------------------------------------------------------------------- %% MQTT/WebSocket Listeners @@ -2263,7 +2268,8 @@ end}. {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}, - {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)} + {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)}, + {gc_after_handshake, cuttlefish:conf_get(Prefix ++ ".gc_after_handshake", Conf, undefined)} ]) end, diff --git a/rebar.config b/rebar.config index 4d6dd0d89..5cc31077d 100644 --- a/rebar.config +++ b/rebar.config @@ -45,7 +45,7 @@ , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.6"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.7"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.11"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript index da1ad7ad6..35afe8f63 100755 --- a/scripts/emqx_rel_otp_app_overwrite.escript +++ b/scripts/emqx_rel_otp_app_overwrite.escript @@ -1,4 +1,5 @@ -#!/usr/bin/env -S escript -c +#!/usr/bin/env escript + %% This script is part of 'relup' process to overwrite the OTP app versions (incl. ERTS) in rel files from upgrade base %% so that 'rebar relup' call will not generate instructions for restarting OTP apps or restarting the emulator. %% @@ -8,6 +9,8 @@ %% note, we use NEW to overwrite OLD is because the modified NEW rel file will be overwritten by next 'rebar relup' %% +-mode(compile). + main([Dir, RelVsn, BASE_VERSIONS]) -> {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Dir, RelVsn), RelVsn), lists:foreach(fun(BaseVer) -> diff --git a/scripts/fail-on-old-otp-version.escript b/scripts/fail-on-old-otp-version.escript index 8399028cb..c85e120ef 100755 --- a/scripts/fail-on-old-otp-version.escript +++ b/scripts/fail-on-old-otp-version.escript @@ -1,5 +1,7 @@ #!/usr/bin/env -S escript -c +-mode(compile). + main(_) -> OtpRelease = list_to_integer(erlang:system_info(otp_release)), case OtpRelease < 21 of @@ -9,4 +11,3 @@ main(_) -> false -> ok end. - diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 0ac5084ab..141fd5833 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -1,6 +1,8 @@ -#!/usr/bin/env -S escript -c +#!/usr/bin/env escript %% -*- erlang-indent-level:4 -*- +-mode(compile). + usage() -> "A script that fills in boilerplate for appup files.