From 80727065ba79821986b21c1d8713b67412827ce0 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 19 Jul 2022 17:07:22 +0200 Subject: [PATCH 01/26] ci: [4.3 only] release with OTP 23.3 rpm/deb: build packages with two tracks: 23.2 and 23.3 - Track 1: Add 23.3, bring in fix of https://github.com/erlang/otp/issues/5346 - Track 2: Keep 23.2 for hot upgrading docker: Default to 23.3 --- .github/workflows/build_packages.yaml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 8735e0d04..51f63971f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -227,6 +227,9 @@ jobs: - centos6 # - raspbian10 #armv6l is too slow to emulate # - raspbian9 + otp_version: + - 23.2.7.2-emqx-3 + - 23.3.4.9-3 exclude: - os: centos6 arch: arm64 @@ -265,7 +268,7 @@ jobs: run: unzip -q source.zip - name: build emqx packages env: - ERL_OTP: erl23.2.7.2-emqx-3 + ERL_OTP: erl${{ matrix.otp_version }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} @@ -289,11 +292,21 @@ jobs: docker volume prune -f - name: create sha256 env: - PROFILE: ${{ matrix.profile}} + PROFILE: ${{ matrix.profile }} + ERL_OTP: erl${{ matrix.otp_version }} + ARCH: ${{ matrix.arch }} run: | if [ -d /tmp/packages/$PROFILE ]; then cd /tmp/packages/$PROFILE for var in $(ls emqx-* ); do + if [[ $ERL_OTP == erl23.2* ]]; then + # Keep package with new OTP as default + # But move package with old otp to track 2 + echo "rename track 2 package" + oldfile="$var" + var="${var/${ARCH}/2-${ARCH}}" + mv "$oldfile" "$var" + fi bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" done cd - @@ -372,7 +385,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine + BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine RUN_FROM=alpine:3.12 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -388,7 +401,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine + BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine RUN_FROM=alpine:3.12 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise From abbb2e2ec7fbf0b2868b471a4991c5597d24b528 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 20 Jul 2022 13:31:51 +0200 Subject: [PATCH 02/26] build: otp23.2 is marked in package name --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 51f63971f..be76d5525 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -304,7 +304,7 @@ jobs: # But move package with old otp to track 2 echo "rename track 2 package" oldfile="$var" - var="${var/${ARCH}/2-${ARCH}}" + var="${var/${ARCH}/otp23.2-${ARCH}}" mv "$oldfile" "$var" fi bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" From 935e6e2f36ef7b912c49999dee3615cc846b1bac Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 22 Jul 2022 14:54:41 +0200 Subject: [PATCH 03/26] feat: make possible to debug ssl handshake --- CHANGES-4.3.md | 2 ++ etc/emqx.conf | 12 ++++++++++++ priv/emqx.schema | 16 ++++++++++++++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 299900da9..d47968096 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -20,9 +20,11 @@ File format: - Fixed crash when shared persistent subscription [#8441] ### Enhancements + - HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450] - Add check_conf cli to check config format. [#8486] - Optimize performance of shared subscription +- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553) ## v4.3.16 diff --git a/etc/emqx.conf b/etc/emqx.conf index 098c908d0..05e537e86 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1572,6 +1572,14 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL ## Value: cn | dn | crt | pem | md5 ## listener.ssl.external.peer_cert_as_clientid = cn +## Default is 'notice', set 'debug' to inspect TLS handshake messaes. +## This log level is not related to EMQX's log level. +## +## NOTE: never set to 'debug' in production environemnts. +## +## Value: emergency | alert | critical | error | warning | notice | info | debug +## listener.ssl.external.log_level = notice +# ## TCP backlog for the SSL connection. ## ## See listener.tcp.$name.backlog @@ -2083,6 +2091,10 @@ listener.wss.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL ## Value: cn | dn | crt | pem | md5 ## listener.wss.external.peer_cert_as_clientid = cn +## See: listener.ssl.$name.log_level +## Value: emergency | alert | critical | error | warning | notice | info | debug +## listener.wss.external.log_level = notice + ## TCP backlog for the WebSocket/SSL connection. ## ## See: listener.tcp.$name.backlog diff --git a/priv/emqx.schema b/priv/emqx.schema index a18df0a0a..48d31bfbd 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1595,6 +1595,10 @@ end}. {datatype, {enum, [cn, dn, crt, pem, md5]}} ]}. +{mapping, "listener.ssl.$name.log_level", "emqx.listeners", [ + {datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug, none, all]}} +]}. + %%-------------------------------------------------------------------- %% MQTT/WebSocket Listeners @@ -2047,6 +2051,10 @@ end}. hidden ]}. +{mapping, "listener.wss.$name.log_level", "emqx.listeners", [ + {datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug, none, all]}} +]}. + {translation, "emqx.listeners", fun(Conf) -> Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, @@ -2117,7 +2125,9 @@ end}. {mqtt_piggyback, cuttlefish:conf_get(Prefix ++ ".mqtt_piggyback", Conf, undefined)}, {check_origin_enable, cuttlefish:conf_get(Prefix ++ ".check_origin_enable", Conf, undefined)}, {allow_origin_absence, cuttlefish:conf_get(Prefix ++ ".allow_origin_absence", Conf, undefined)}, - {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)]) + {check_origins, WsOpts(Prefix)} + | AccOpts(Prefix) + ]) end, DeflateOpts = fun(Prefix) -> Filter([{level, cuttlefish:conf_get(Prefix ++ ".deflate_opts.level", Conf, undefined)}, @@ -2210,7 +2220,9 @@ end}. {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, - {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) + {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}, + {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)} + ]) end, Listen_fix = fun({Ip, Port}) -> case inet:parse_address(Ip) of From 3655ea050c167e5daeff15c58e9d0520b3cc1882 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 21 Jul 2022 10:14:51 +0200 Subject: [PATCH 04/26] build(relup): overwrite OTP vers of upgrade bases --- build | 8 +++ scripts/emqx_rel_otp_app_overwrite.escript | 57 ++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100755 scripts/emqx_rel_otp_app_overwrite.escript diff --git a/build b/build index 44287bff5..b028aae75 100755 --- a/build +++ b/build @@ -94,6 +94,14 @@ make_relup() { fi RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")" export RELX_BASE_VERSIONS + if [[ ${PKG_VSN} == 4.3* ]]; then + echo "EMQX 4.3 specific, overwrite OTP app versions" + local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel" + if [ ! -f "${emqx_rel_file}" ]; then + ./rebar3 as "${PROFILE}" release + fi + scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PROFILE}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}" + fi ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" } diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript new file mode 100755 index 000000000..843628e24 --- /dev/null +++ b/scripts/emqx_rel_otp_app_overwrite.escript @@ -0,0 +1,57 @@ +#!/usr/bin/env escript +%% This script is part of 'relup' process to overwrite the OTP app versions (incl. ERTS) in rel files from upgrade base +%% so that 'rebar relup' call will not generate instructions for restarting OTP apps or restarting the emulator. +%% +%% It simply read OTP app version (incl. ERTS) from the rel file of *NEW* Release ($RelVsn) and write back to the ones +%% in *OLD* versions ($BASE_VERSIONS) +%% +%% note, we use NEW to overwrite OLD is because the modified NEW rel file will be overwritten by next 'rebar relup' +%% +main([Dir, Profile, RelVsn, BASE_VERSIONS]) -> + {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Profile, Dir, RelVsn), RelVsn), + lists:foreach(fun(BaseVer) -> + base_rel_overwrites(BaseVer, Profile, Dir, ErtsVsn, Overwrites) + end, string:tokens(BASE_VERSIONS, ",")). + +get_otp_apps(RelFile, RelVsn) -> + {ok, [{release, {"emqx", RelVsn}, {erts, ErtsVsn}, AppList}]} = file:consult(RelFile), + Apps = lists:filter(fun(X) -> lists:member(element(1, X), otp_apps()) end, AppList), + {ErtsVsn, Apps}. + +base_rel_overwrites(RelVsn, Profile, Dir, ErtsVsn, Overwrites) -> + RelFile = rel_file(Profile, Dir, RelVsn), + file:copy(RelFile, RelFile++".bak"), + {ok, [{release, {"emqx", RelVsn}, {erts, _BaseErtsVsn}, BaseAppList}]} = file:consult(RelFile), + NewData = [ {release, {"emqx", RelVsn}, {erts, ErtsVsn}, + lists:map(fun(X) -> + Name = element(1, X), + case lists:keyfind(Name, 1, Overwrites) of + false -> X; + Y when is_tuple(Y) -> Y + end + end, BaseAppList) + } + ], + ok = file:write_file(RelFile, io_lib:format("~p.", NewData)). + +rel_file(Profile, Dir, RelVsn)-> + filename:join([Dir, RelVsn, Profile++".rel"]). + + +%% Couldn't find a good way to get this list dynamicly. +otp_apps() -> + [ kernel + , stdlib + , sasl + , crypto + , public_key + , asn1 + , syntax_tools + , ssl + , os_mon + , inets + , compiler + , runtime_tools + , mnesia + , xmerl + ]. From 08033a82e80511b41e1802708a87fa8862cf470d Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 22 Jul 2022 17:46:12 +0200 Subject: [PATCH 05/26] build(relup): restore emqx.rel backups after rebar relup --- build | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/build b/build index b028aae75..6672685be 100755 --- a/build +++ b/build @@ -103,6 +103,21 @@ make_relup() { scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PROFILE}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}" fi ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" + + # rollback relup + # + if [[ ${PKG_VSN} == 4.3* ]]; then + echo "restore upgrade base rel files... " + for rel in ${releases[*]}; + do + bakfile="${releases_dir}/${rel}/${PROFILE}.rel.bak" + echo "restore $bakfile ..." + if [ -f "$bakfile" ]; then + echo "restore from $bakfile" + mv "${bakfile}" "${bakfile%.bak}" + fi + done + fi } cp_dyn_libs() { From c9c1dfb8236e302680e72f00bc99a561c5c0d709 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 25 Jul 2022 23:09:17 +0200 Subject: [PATCH 06/26] fix(upgrade): ensure files in BINDIR executable During upgrade procedure, emqx zip file is repacked to .tar.gz but does not preserve permission of files. This commit ensure files under BINDIR is executable --- bin/emqx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/emqx b/bin/emqx index d6ce86e10..af3751ad7 100755 --- a/bin/emqx +++ b/bin/emqx @@ -42,6 +42,8 @@ assert_node_alive() { } check_erlang_start() { + # Fix bin permission + find "$BINDIR" ! -executable -exec chmod a+x {} \; "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()" } From b17078b93f0fd7edac25c6defa5b86d474463efb Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 26 Jul 2022 10:41:01 +0200 Subject: [PATCH 07/26] fix(otp23.3): OTP app overwrites support emqx-edge --- scripts/emqx_rel_otp_app_overwrite.escript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript index 843628e24..52c6bb70e 100755 --- a/scripts/emqx_rel_otp_app_overwrite.escript +++ b/scripts/emqx_rel_otp_app_overwrite.escript @@ -34,6 +34,8 @@ base_rel_overwrites(RelVsn, Profile, Dir, ErtsVsn, Overwrites) -> ], ok = file:write_file(RelFile, io_lib:format("~p.", NewData)). +rel_file("emqx-edge", Dir, RelVsn)-> + rel_file("emqx", Dir, RelVsn); rel_file(Profile, Dir, RelVsn)-> filename:join([Dir, RelVsn, Profile++".rel"]). From 8835296c7afed5c4abff86f00c71fe3363f8ca6d Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 1 Aug 2022 15:09:41 +0200 Subject: [PATCH 08/26] ci(rel): intro. rel_otp_apps.eterm --- rebar.config.erl | 21 +++++++-------------- scripts/emqx_rel_otp_app_overwrite.escript | 22 +++++----------------- scripts/rel_otp_apps.eterm | 16 ++++++++++++++++ 3 files changed, 28 insertions(+), 31 deletions(-) create mode 100644 scripts/rel_otp_apps.eterm diff --git a/rebar.config.erl b/rebar.config.erl index c94f80c18..e0ee9b891 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -246,21 +246,10 @@ overlay_vars_pkg(pkg) -> ]. relx_apps(ReleaseType) -> - [ kernel - , sasl - , crypto - , public_key - , asn1 - , syntax_tools - , ssl - , os_mon - , inets - , compiler - , runtime_tools - , redbug + relx_otp_apps() ++ + [ redbug , cuttlefish , emqx - , {mnesia, load} , {ekka, load} , {emqx_plugin_libs, load} , observer_cli @@ -271,9 +260,13 @@ relx_apps(ReleaseType) -> ++ relx_apps_per_rel(ReleaseType) ++ [{N, load} || N <- relx_plugin_apps(ReleaseType)]. +relx_otp_apps() -> + {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"), + true = is_list(Apps), + Apps. + relx_apps_per_rel(cloud) -> [ luerl - , xmerl | [{observer, load} || is_app(observer)] ]; relx_apps_per_rel(edge) -> diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript index 52c6bb70e..5eace01ff 100755 --- a/scripts/emqx_rel_otp_app_overwrite.escript +++ b/scripts/emqx_rel_otp_app_overwrite.escript @@ -39,21 +39,9 @@ rel_file("emqx-edge", Dir, RelVsn)-> rel_file(Profile, Dir, RelVsn)-> filename:join([Dir, RelVsn, Profile++".rel"]). - -%% Couldn't find a good way to get this list dynamicly. otp_apps() -> - [ kernel - , stdlib - , sasl - , crypto - , public_key - , asn1 - , syntax_tools - , ssl - , os_mon - , inets - , compiler - , runtime_tools - , mnesia - , xmerl - ]. + {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"), + true = is_list(Apps), + lists:map(fun(App) when is_atom(App) -> App; + ({App, _}) -> App %% handle like {mnesia, load} + end, Apps). diff --git a/scripts/rel_otp_apps.eterm b/scripts/rel_otp_apps.eterm new file mode 100644 index 000000000..4999bbffa --- /dev/null +++ b/scripts/rel_otp_apps.eterm @@ -0,0 +1,16 @@ +%% Single source of truth of list otp apps that we use +[ kernel +, stdlib +, sasl +, crypto +, public_key +, asn1 +, syntax_tools +, ssl +, os_mon +, inets +, compiler +, runtime_tools +, {mnesia, load} +, xmerl +]. From 19d36fbf144517faacd752518f37684cd73d7f00 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 1 Aug 2022 15:27:51 +0200 Subject: [PATCH 09/26] ci: assert no restart_emulator in relup --- build | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build b/build index 6672685be..2f773d295 100755 --- a/build +++ b/build @@ -104,7 +104,14 @@ make_relup() { fi ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" - # rollback relup + # assert that there is no 'restart_emulator' in relup + # EMQX wants hot upgrade without VM restart + if grep restart_emulator "${releases_dir}/${PKG_VSN}/relup"; then + echo "Error: restart_emulator instruction found in relup"; + exit 1 + fi + + # rollback rel file per releases # if [[ ${PKG_VSN} == 4.3* ]]; then echo "restore upgrade base rel files... " From 874d30833422acff365b5d8e67279efc1cdbf8cb Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 2 Aug 2022 09:59:59 +0200 Subject: [PATCH 10/26] build: docker build from otp-23.3 --- deploy/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index f24b4a348..346d33630 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine +ARG BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine ARG RUN_FROM=alpine:3.12 FROM ${BUILD_FROM} AS builder From 0d1b26cabfc652c98a83ab8e5661fbdf0465dfab Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 2 Aug 2022 10:01:54 +0200 Subject: [PATCH 11/26] ci(linux): disable 23.2.7.2-emqx-3 --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index be76d5525..84e02bf89 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -228,7 +228,7 @@ jobs: # - raspbian10 #armv6l is too slow to emulate # - raspbian9 otp_version: - - 23.2.7.2-emqx-3 + #- 23.2.7.2-emqx-3 - 23.3.4.9-3 exclude: - os: centos6 From a3c6c9125e6972b55faad3249cf8a07e5c09678c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 2 Aug 2022 15:26:49 +0200 Subject: [PATCH 12/26] docs: update changes made for v4.3.18 --- CHANGES-4.3.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 0c99fb57d..db82c8eb1 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,6 +10,13 @@ File format: - One list item per change topic Change log ends with a list of github PRs +## v4.3.18 + +### Enhancements + +- Upgrade Erlang/OTP from 23.2.7.2-emqx-3 to 23.3.4.9-3 [#8511](https://github.com/emqx/emqx/pull/8511) +- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553) + ## v4.3.17 ### Bug fixes @@ -28,7 +35,6 @@ File format: - HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450] - Add check_conf cli to check config format. [#8486] - Optimize performance of shared subscription -- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553) ## v4.3.16 From 1b9ce77d7d2c48e50e5afc5efdccb62b9844ded8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 2 Aug 2022 15:50:49 +0200 Subject: [PATCH 13/26] chore: bump release version to 4.3.18 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 03e167b6d..ca8163462 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.17"}). +-define(EMQX_RELEASE, {opensource, "4.3.18"}). -else. From 744af708f1c146190169e55942b552bdf13ffc40 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 2 Aug 2022 15:58:04 +0200 Subject: [PATCH 14/26] chore: update appup files for 4.3.18 --- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 6 ++++-- src/emqx.app.src | 2 +- src/emqx.appup.src | 6 ++++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 6c2fb96c8..259efa08d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.12"}, % strict semver, bump manually! + {vsn, "4.3.13"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index caed4d0a8..ba39c4ec9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.11", + [{"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.11", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, @@ -164,7 +165,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.11", + [{"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.11", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, diff --git a/src/emqx.app.src b/src/emqx.app.src index 5faeb3caf..14a6f4fa4 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.18"}, % strict semver, bump manually! + {vsn, "4.3.19"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 2a744201a..c86661891 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.17", + [{"4.3.18",[{load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {"4.3.17", [{load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -677,7 +678,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.17", + [{"4.3.18",[{load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {"4.3.17", [{load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, From 901d6909f5956077f69c305444cead4226cec617 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 2 Aug 2022 19:49:01 +0200 Subject: [PATCH 15/26] chore: ensure enqx_dashboard app version bump --- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 7b7b76ede..1281145f0 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.13"}, % strict semver, bump manually! + {vsn, "4.3.14"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, From 29df428052fd5d19b81258a3975eadcb78e0c3b2 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 3 Aug 2022 10:01:08 +0800 Subject: [PATCH 16/26] chore: update vsn for helm chart --- deploy/charts/emqx/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 0a1d90daa..0d00f8948 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -13,8 +13,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 4.3.17 +version: 4.3.18 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 4.3.17 +appVersion: 4.3.18 From 9d71aff5d305facc85a31ad787ab8cd5a8ada817 Mon Sep 17 00:00:00 2001 From: Rory Z Date: Wed, 3 Aug 2022 10:16:16 +0800 Subject: [PATCH 17/26] ci: fix helm version error --- .github/workflows/release.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c4ffa81e2..8e6b02c21 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -67,11 +67,16 @@ jobs: -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: get version + id: version + run: echo "::set-output name=version::$(./pkg-vsn.sh)" - uses: emqx/push-helm-action@v1 if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' with: charts_dir: "${{ github.workspace }}/deploy/charts/emqx" - version: ${{ github.ref_name }} + version: ${{ steps.version.outputs.version }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-west-2" @@ -80,7 +85,7 @@ jobs: if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' with: charts_dir: "${{ github.workspace }}/deploy/charts/emqx-ee" - version: ${{ github.ref_name }} + version: ${{ steps.version.outputs.version }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-west-2" From e74a97013eb068dd1513769d9103b4da7625b1e6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 3 Aug 2022 10:01:20 +0200 Subject: [PATCH 18/26] build: fix relup app overwrite escript the rel file name is always emqx.rel, has nothing to do with profile --- build | 2 +- scripts/emqx_rel_otp_app_overwrite.escript | 27 ++++++++++++---------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/build b/build index 2f773d295..ace072b79 100755 --- a/build +++ b/build @@ -100,7 +100,7 @@ make_relup() { if [ ! -f "${emqx_rel_file}" ]; then ./rebar3 as "${PROFILE}" release fi - scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PROFILE}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}" + scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}" fi ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript index 5eace01ff..85ccca33c 100755 --- a/scripts/emqx_rel_otp_app_overwrite.escript +++ b/scripts/emqx_rel_otp_app_overwrite.escript @@ -7,19 +7,24 @@ %% %% note, we use NEW to overwrite OLD is because the modified NEW rel file will be overwritten by next 'rebar relup' %% -main([Dir, Profile, RelVsn, BASE_VERSIONS]) -> - {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Profile, Dir, RelVsn), RelVsn), +main([Dir, RelVsn, BASE_VERSIONS]) -> + {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Dir, RelVsn), RelVsn), lists:foreach(fun(BaseVer) -> - base_rel_overwrites(BaseVer, Profile, Dir, ErtsVsn, Overwrites) + base_rel_overwrites(BaseVer, Dir, ErtsVsn, Overwrites) end, string:tokens(BASE_VERSIONS, ",")). get_otp_apps(RelFile, RelVsn) -> - {ok, [{release, {"emqx", RelVsn}, {erts, ErtsVsn}, AppList}]} = file:consult(RelFile), - Apps = lists:filter(fun(X) -> lists:member(element(1, X), otp_apps()) end, AppList), - {ErtsVsn, Apps}. + case file:consult(RelFile) of + {ok, [{release, {"emqx", RelVsn}, {erts, ErtsVsn}, AppList}]} -> + Apps = lists:filter(fun(X) -> lists:member(element(1, X), otp_apps()) end, AppList), + {ErtsVsn, Apps}; + {error, Reason} -> + io:format(standard_error, "failed_to_read_file ~p for release ~p~nreason=~p~n", [RelFile, RelVsn, Reason]), + halt(1) + end. -base_rel_overwrites(RelVsn, Profile, Dir, ErtsVsn, Overwrites) -> - RelFile = rel_file(Profile, Dir, RelVsn), +base_rel_overwrites(RelVsn, Dir, ErtsVsn, Overwrites) -> + RelFile = rel_file(Dir, RelVsn), file:copy(RelFile, RelFile++".bak"), {ok, [{release, {"emqx", RelVsn}, {erts, _BaseErtsVsn}, BaseAppList}]} = file:consult(RelFile), NewData = [ {release, {"emqx", RelVsn}, {erts, ErtsVsn}, @@ -34,10 +39,8 @@ base_rel_overwrites(RelVsn, Profile, Dir, ErtsVsn, Overwrites) -> ], ok = file:write_file(RelFile, io_lib:format("~p.", NewData)). -rel_file("emqx-edge", Dir, RelVsn)-> - rel_file("emqx", Dir, RelVsn); -rel_file(Profile, Dir, RelVsn)-> - filename:join([Dir, RelVsn, Profile++".rel"]). +rel_file(Dir, RelVsn)-> + filename:join([Dir, RelVsn, "emqx.rel"]). otp_apps() -> {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"), From ddd6f066e49f3b0cf6f291d3898d712b1f91911c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 3 Aug 2022 10:41:45 +0200 Subject: [PATCH 19/26] chore: run escript in compile mode --- scripts/emqx_rel_otp_app_overwrite.escript | 3 ++- scripts/fail-on-old-otp-version.escript | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript index 85ccca33c..da1ad7ad6 100755 --- a/scripts/emqx_rel_otp_app_overwrite.escript +++ b/scripts/emqx_rel_otp_app_overwrite.escript @@ -1,4 +1,4 @@ -#!/usr/bin/env escript +#!/usr/bin/env -S escript -c %% This script is part of 'relup' process to overwrite the OTP app versions (incl. ERTS) in rel files from upgrade base %% so that 'rebar relup' call will not generate instructions for restarting OTP apps or restarting the emulator. %% @@ -7,6 +7,7 @@ %% %% note, we use NEW to overwrite OLD is because the modified NEW rel file will be overwritten by next 'rebar relup' %% + main([Dir, RelVsn, BASE_VERSIONS]) -> {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Dir, RelVsn), RelVsn), lists:foreach(fun(BaseVer) -> diff --git a/scripts/fail-on-old-otp-version.escript b/scripts/fail-on-old-otp-version.escript index 0e4cd2a1b..8399028cb 100755 --- a/scripts/fail-on-old-otp-version.escript +++ b/scripts/fail-on-old-otp-version.escript @@ -1,4 +1,4 @@ -#!/usr/bin/env escript +#!/usr/bin/env -S escript -c main(_) -> OtpRelease = list_to_integer(erlang:system_info(otp_release)), From c533f58ee598c4f31da47495d3bca4cc600c1669 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 3 Aug 2022 17:47:11 -0300 Subject: [PATCH 20/26] fix: rm non-portable `env` option On some systems, like `centos7`, `env` does not have a `-S` option. --- scripts/emqx_rel_otp_app_overwrite.escript | 3 ++- scripts/fail-on-old-otp-version.escript | 5 +++-- scripts/update_appup.escript | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript index da1ad7ad6..073caa560 100755 --- a/scripts/emqx_rel_otp_app_overwrite.escript +++ b/scripts/emqx_rel_otp_app_overwrite.escript @@ -1,4 +1,4 @@ -#!/usr/bin/env -S escript -c +#!/usr/bin/env escript %% This script is part of 'relup' process to overwrite the OTP app versions (incl. ERTS) in rel files from upgrade base %% so that 'rebar relup' call will not generate instructions for restarting OTP apps or restarting the emulator. %% @@ -7,6 +7,7 @@ %% %% note, we use NEW to overwrite OLD is because the modified NEW rel file will be overwritten by next 'rebar relup' %% +-mode(compile). main([Dir, RelVsn, BASE_VERSIONS]) -> {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Dir, RelVsn), RelVsn), diff --git a/scripts/fail-on-old-otp-version.escript b/scripts/fail-on-old-otp-version.escript index 8399028cb..3081c0d29 100755 --- a/scripts/fail-on-old-otp-version.escript +++ b/scripts/fail-on-old-otp-version.escript @@ -1,4 +1,6 @@ -#!/usr/bin/env -S escript -c +#!/usr/bin/env escript + +-mode(compile). main(_) -> OtpRelease = list_to_integer(erlang:system_info(otp_release)), @@ -9,4 +11,3 @@ main(_) -> false -> ok end. - diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 0ac5084ab..141fd5833 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -1,6 +1,8 @@ -#!/usr/bin/env -S escript -c +#!/usr/bin/env escript %% -*- erlang-indent-level:4 -*- +-mode(compile). + usage() -> "A script that fills in boilerplate for appup files. From 5e9462afc8377e01dedee184b4364d013dba88f9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 3 Aug 2022 10:46:15 -0300 Subject: [PATCH 21/26] feat: add option to gc after TLS/SSL handshake (4.3) Port of https://github.com/emqx/emqx/pull/8637 --- CHANGES-4.3.md | 1 + apps/emqx_sn/rebar.config | 2 +- etc/emqx.conf | 7 +++++++ priv/emqx.schema | 9 ++++++++- rebar.config | 2 +- 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index db82c8eb1..a9d89473e 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -16,6 +16,7 @@ File format: - Upgrade Erlang/OTP from 23.2.7.2-emqx-3 to 23.3.4.9-3 [#8511](https://github.com/emqx/emqx/pull/8511) - Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553) +- Add option to perform GC on connection process after TLS/SSL handshake is performed. [#8649](https://github.com/emqx/emqx/pull/8649) ## v4.3.17 diff --git a/apps/emqx_sn/rebar.config b/apps/emqx_sn/rebar.config index 120d4767d..295751b23 100644 --- a/apps/emqx_sn/rebar.config +++ b/apps/emqx_sn/rebar.config @@ -2,7 +2,7 @@ {plugins, [rebar3_proper]}. {deps, - [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.5"}}}, + [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.7"}}}, {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} ]}. diff --git a/etc/emqx.conf b/etc/emqx.conf index a31412ea8..f7dccfd27 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1644,6 +1644,13 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL ## Value: true | false listener.ssl.external.reuseaddr = true +## Whether to perform GC after TLS/SSL handshake. +## +## Default: false +## +## Value: true | false +## listener.ssl.external.gc_after_handshake = false + ##-------------------------------------------------------------------- ## External WebSocket listener for MQTT protocol diff --git a/priv/emqx.schema b/priv/emqx.schema index 48d31bfbd..b6b267abf 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1599,6 +1599,11 @@ end}. {datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug, none, all]}} ]}. +{mapping, "listener.ssl.$name.gc_after_handshake", "emqx.listeners", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + %%-------------------------------------------------------------------- %% MQTT/WebSocket Listeners @@ -2055,6 +2060,7 @@ end}. {datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug, none, all]}} ]}. + {translation, "emqx.listeners", fun(Conf) -> Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, @@ -2221,7 +2227,8 @@ end}. {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}, - {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)} + {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)}, + {gc_after_handshake, cuttlefish:conf_get(Prefix ++ ".gc_after_handshake", Conf, undefined)} ]) end, diff --git a/rebar.config b/rebar.config index cf46d87c9..daddc8b97 100644 --- a/rebar.config +++ b/rebar.config @@ -45,7 +45,7 @@ , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.6"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.7"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.11"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.2"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} From a88c42cf82db1851c361c3e73aa1415e498e9b7c Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 4 Aug 2022 14:54:39 +0200 Subject: [PATCH 22/26] fix(upgrade): ensure use correct no_dot_erlang.boot --- .ci/build_packages/tests.sh | 11 +++++++++++ bin/install_upgrade.escript | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index b74d61ff2..d4620e2d3 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -144,17 +144,28 @@ relup_test(){ ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 ) ./emqx/bin/emqx_ctl status ./emqx/bin/emqx versions + OldVsn="$(./emqx/bin/emqx eval 'Versions=[{S, V} || {_,V,_, S} <- release_handler:which_releases()], + Current = proplists:get_value(current, Versions, proplists:get_value(permanent, Versions)), + io:format("~s", [Current])')" cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases ./emqx/bin/emqx install "${TARGET_VERSION}" [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 export EMQX_WAIT_FOR_STOP=300 ./emqx/bin/emqx_ctl status + + # also test remove old rel + ./emqx/bin/emqx uninstall "$OldVsn" + + # check emqx still runs + ./emqx/bin/emqx ping + if ! ./emqx/bin/emqx stop; then cat emqx/log/erlang.log.1 || true cat emqx/log/emqx.log.1 || true echo "failed to stop emqx" exit 1 fi + rm -rf emqx done fi diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 14d90156c..88268d893 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -111,6 +111,7 @@ uninstall({_RelName, NameTypeArg, NodeName, Cookie}, Opts) -> TargetNode = start_distribution(NodeName, NameTypeArg, Cookie), WhichReleases = which_releases(TargetNode), Version = proplists:get_value(version, Opts), + ensure_latest_boot_file(current_release_version(TargetNode)), case proplists:get_value(Version, WhichReleases) of undefined -> ?INFO("Release ~s is already uninstalled.", [Version]); @@ -362,6 +363,9 @@ permafy(TargetNode, RelName, Vsn) -> [{ok, _} = file:copy(filename:join(["bin", File++"-"++Vsn]), filename:join(["bin", File])) || File <- Scripts], + + ensure_latest_boot_file(Vsn), + %% update the vars UpdatedVars = io_lib:format("REL_VSN=\"~s\"~nERTS_VSN=\"~s\"~n", [Vsn, erts_vsn()]), file:write_file(filename:absname(filename:join(["releases", "emqx_vars"])), UpdatedVars, [append]). @@ -458,3 +462,9 @@ str(A) when is_binary(A) -> binary_to_list(A); str(A) when is_list(A) -> (A). + +%%% For otp upgrade, the boot file need match the otp version +%%% This is for emqx cli tool +ensure_latest_boot_file(Vsn) -> + {ok, _} = file:copy(filename:join(["releases", Vsn, "no_dot_erlang.boot"]), + filename:join(["bin", "no_dot_erlang.boot"])). From f943892fa8a468f59cd8e0bed66492e630fbcacd Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 5 Aug 2022 11:41:00 +0200 Subject: [PATCH 23/26] docs: update changelog for SSL gc --- CHANGES-4.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index a9d89473e..1ce9dcbe2 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -17,6 +17,7 @@ File format: - Upgrade Erlang/OTP from 23.2.7.2-emqx-3 to 23.3.4.9-3 [#8511](https://github.com/emqx/emqx/pull/8511) - Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553) - Add option to perform GC on connection process after TLS/SSL handshake is performed. [#8649](https://github.com/emqx/emqx/pull/8649) + Expected to reduce around 35% memory consumption for each SSL connection. See [#8637](https://github.com/emqx/emqx/pull/8637) for more details. ## v4.3.17 From ccb51265b829901cee9159bb5500b50eafad45b0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 5 Aug 2022 11:41:21 +0200 Subject: [PATCH 24/26] chore: prepare for 4.3.18-beta.1 release --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index ca8163462..77a02177b 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.18"}). +-define(EMQX_RELEASE, {opensource, "4.3.18-beta.1"}). -else. From 2740c80b20f9b3d6eaeed43a09f4adec6c7087e7 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 8 Aug 2022 11:59:26 +0800 Subject: [PATCH 25/26] fix: check if trace is enabled when the file is not found --- apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 49321086b..4fbec0091 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -145,6 +145,8 @@ stream_log_file(#{name := Name}, Params) -> {eof, Size} -> Meta = #{<<"position">> => Size, <<"bytes">> => Bytes}, {ok, #{meta => Meta, items => <<"">>}}; + {error, trace_disabled} -> + {error, io_lib:format("trace_disable_on_~s", [Node0])}; {error, Reason} -> logger:log(error, "read_file_failed ~p", [{Node, Name, Reason, Position, Bytes}]), {error, Reason}; @@ -194,6 +196,11 @@ read_file(Path, Offset, Bytes) -> after file:close(IoDevice) end; + {error, enoent} -> + case emqx_trace:is_enable() of + false -> {error, trace_disabled}; + true -> {error, enoent} + end; {error, Reason} -> {error, Reason} end. From 9b10aefd80929f7d4692a14f902528d6fb6b64e8 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 8 Aug 2022 14:12:15 +0800 Subject: [PATCH 26/26] chore: update changelog and appup.src --- CHANGES-4.4.md | 7 +++++++ apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src | 2 +- apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src | 6 ++++++ apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl | 8 +++++++- apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl | 3 +++ 5 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index b2b331240..4c4f65ba1 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,5 +1,12 @@ # EMQX 4.4 Changes +## v4.4.7 + +### Enhancements (synced from v4.3.18) + +### Bug fixes +- Fix: Check if emqx_mod_trace is enabled when the trace file is not found. + ## v4.4.5 ### Enhancements (synced from v4.3.16) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 9315f8c53..4b6b041a2 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.4.4"}, + {vsn, "4.4.5"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index 3d12e8b8a..e4b8e9d1c 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -2,6 +2,9 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [ + {"4.4.4", + [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, {"4.4.3", [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, @@ -21,6 +24,9 @@ {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [ + {"4.4.4", + [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, {"4.4.3", [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl index d2c02d06d..1df2e4bf0 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace.erl @@ -177,7 +177,13 @@ trace_file(File) -> Node = atom_to_list(node()), case file:read_file(FileName) of {ok, Bin} -> {ok, Node, Bin}; - {error, Reason} -> {error, Node, Reason} + {error, enoent} -> + case emqx_trace:is_enable() of + false -> {error, Node, trace_disabled}; + true -> {error, Node, enoent} + end; + {error, Reason} -> + {error, Node, Reason} end. delete_files_after_send(TraceLog, Zips) -> diff --git a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl index 4fbec0091..7836aacb0 100644 --- a/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl +++ b/apps/emqx_plugin_libs/src/emqx_trace/emqx_trace_api.erl @@ -115,6 +115,9 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> ok -> [FileName | Acc]; _ -> Acc end; + {error, Node, trace_disabled} -> + ?LOG(warning, "emqx_mod_trace modules is disabled on ~s ~s", [Node, TraceLog]), + Acc; {error, Node, Reason} -> ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]), Acc