diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 592e282c7..5a3fee5ae 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,6 +10,13 @@ File format: - One list item per change topic Change log ends with a list of github PRs +## v4.3.18 + +### Enhancements + +- Upgrade Erlang/OTP from 23.2.7.2-emqx-3 to 23.3.4.9-3 [#8511](https://github.com/emqx/emqx/pull/8511) +- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553) + ## v4.3.17 ### Bug fixes @@ -24,6 +31,7 @@ File format: This causes the clients to not expire as long as a new UDP packet arrives [#8575] ### Enhancements + - HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450] - Add check_conf cli to check config format. [#8486] - Optimize performance of shared subscription diff --git a/bin/emqx b/bin/emqx index 6ad57a931..189708ecc 100755 --- a/bin/emqx +++ b/bin/emqx @@ -48,6 +48,8 @@ assert_node_alive() { } check_erlang_start() { + # Fix bin permission + find "$BINDIR" ! -executable -exec chmod a+x {} \; "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()" } diff --git a/build b/build index fa6a41615..979af4510 100755 --- a/build +++ b/build @@ -96,7 +96,37 @@ make_relup() { fi RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")" export RELX_BASE_VERSIONS + if [[ ${PKG_VSN} == 4.3* ]]; then + echo "EMQX 4.3 specific, overwrite OTP app versions" + local emqx_rel_file="${releases_dir}/${PKG_VSN}/emqx.rel" + if [ ! -f "${emqx_rel_file}" ]; then + ./rebar3 as "${PROFILE}" release + fi + scripts/emqx_rel_otp_app_overwrite.escript "${releases_dir}" "${PKG_VSN}" "${RELX_BASE_VERSIONS}" + fi ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" + + # assert that there is no 'restart_emulator' in relup + # EMQX wants hot upgrade without VM restart + if grep restart_emulator "${releases_dir}/${PKG_VSN}/relup"; then + echo "Error: restart_emulator instruction found in relup"; + exit 1 + fi + + # rollback rel file per releases + # + if [[ ${PKG_VSN} == 4.3* ]]; then + echo "restore upgrade base rel files... " + for rel in ${releases[*]}; + do + bakfile="${releases_dir}/${rel}/${PROFILE}.rel.bak" + echo "restore $bakfile ..." + if [ -f "$bakfile" ]; then + echo "restore from $bakfile" + mv "${bakfile}" "${bakfile%.bak}" + fi + done + fi } ## try to be portable for zip packages. diff --git a/etc/emqx.conf b/etc/emqx.conf index 25f88bfc6..69320f9de 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1580,6 +1580,14 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL ## Value: cn | dn | crt | pem | md5 ## listener.ssl.external.peer_cert_as_clientid = cn +## Default is 'notice', set 'debug' to inspect TLS handshake messaes. +## This log level is not related to EMQX's log level. +## +## NOTE: never set to 'debug' in production environemnts. +## +## Value: emergency | alert | critical | error | warning | notice | info | debug +## listener.ssl.external.log_level = notice +# ## TCP backlog for the SSL connection. ## ## See listener.tcp.$name.backlog @@ -2091,6 +2099,10 @@ listener.wss.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL ## Value: cn | dn | crt | pem | md5 ## listener.wss.external.peer_cert_as_clientid = cn +## See: listener.ssl.$name.log_level +## Value: emergency | alert | critical | error | warning | notice | info | debug +## listener.wss.external.log_level = notice + ## TCP backlog for the WebSocket/SSL connection. ## ## See: listener.tcp.$name.backlog diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 29067c712..4e9770dab 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,5 +1,4 @@ {application, emqx_dashboard, - [{description, "EMQX Web Dashboard"}, {vsn, "4.4.6"}, % strict semver, bump manually! {modules, []}, diff --git a/priv/emqx.schema b/priv/emqx.schema index 94f124361..4f81fccf1 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1637,6 +1637,10 @@ end}. {datatype, {enum, [cn, dn, crt, pem, md5]}} ]}. +{mapping, "listener.ssl.$name.log_level", "emqx.listeners", [ + {datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug, none, all]}} +]}. + %%-------------------------------------------------------------------- %% MQTT/WebSocket Listeners @@ -2089,6 +2093,10 @@ end}. hidden ]}. +{mapping, "listener.wss.$name.log_level", "emqx.listeners", [ + {datatype, {enum, [emergency, alert, critical, error, warning, notice, info, debug, none, all]}} +]}. + {translation, "emqx.listeners", fun(Conf) -> Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, @@ -2159,7 +2167,9 @@ end}. {mqtt_piggyback, cuttlefish:conf_get(Prefix ++ ".mqtt_piggyback", Conf, undefined)}, {check_origin_enable, cuttlefish:conf_get(Prefix ++ ".check_origin_enable", Conf, undefined)}, {allow_origin_absence, cuttlefish:conf_get(Prefix ++ ".allow_origin_absence", Conf, undefined)}, - {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)]) + {check_origins, WsOpts(Prefix)} + | AccOpts(Prefix) + ]) end, DeflateOpts = fun(Prefix) -> Filter([{level, cuttlefish:conf_get(Prefix ++ ".deflate_opts.level", Conf, undefined)}, @@ -2252,7 +2262,9 @@ end}. {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, - {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) + {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}, + {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)} + ]) end, Listen_fix = fun({Ip, Port}) -> case inet:parse_address(Ip) of diff --git a/rebar.config.erl b/rebar.config.erl index 097fdf56c..c23b32717 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -239,21 +239,10 @@ overlay_vars_pkg(pkg) -> ]. relx_apps(ReleaseType) -> - [ kernel - , sasl - , crypto - , public_key - , asn1 - , syntax_tools - , ssl - , os_mon - , inets - , compiler - , runtime_tools - , redbug + relx_otp_apps() ++ + [ redbug , cuttlefish , emqx - , {mnesia, load} , {ekka, load} , {emqx_plugin_libs, load} , observer_cli @@ -264,9 +253,13 @@ relx_apps(ReleaseType) -> ++ relx_apps_per_rel(ReleaseType) ++ [{N, load} || N <- relx_plugin_apps(ReleaseType)]. +relx_otp_apps() -> + {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"), + true = is_list(Apps), + Apps. + relx_apps_per_rel(cloud) -> [ luerl - , xmerl | [{observer, load} || is_app(observer)] ]; relx_apps_per_rel(edge) -> diff --git a/scripts/emqx_rel_otp_app_overwrite.escript b/scripts/emqx_rel_otp_app_overwrite.escript new file mode 100755 index 000000000..da1ad7ad6 --- /dev/null +++ b/scripts/emqx_rel_otp_app_overwrite.escript @@ -0,0 +1,51 @@ +#!/usr/bin/env -S escript -c +%% This script is part of 'relup' process to overwrite the OTP app versions (incl. ERTS) in rel files from upgrade base +%% so that 'rebar relup' call will not generate instructions for restarting OTP apps or restarting the emulator. +%% +%% It simply read OTP app version (incl. ERTS) from the rel file of *NEW* Release ($RelVsn) and write back to the ones +%% in *OLD* versions ($BASE_VERSIONS) +%% +%% note, we use NEW to overwrite OLD is because the modified NEW rel file will be overwritten by next 'rebar relup' +%% + +main([Dir, RelVsn, BASE_VERSIONS]) -> + {ErtsVsn, Overwrites} = get_otp_apps(rel_file(Dir, RelVsn), RelVsn), + lists:foreach(fun(BaseVer) -> + base_rel_overwrites(BaseVer, Dir, ErtsVsn, Overwrites) + end, string:tokens(BASE_VERSIONS, ",")). + +get_otp_apps(RelFile, RelVsn) -> + case file:consult(RelFile) of + {ok, [{release, {"emqx", RelVsn}, {erts, ErtsVsn}, AppList}]} -> + Apps = lists:filter(fun(X) -> lists:member(element(1, X), otp_apps()) end, AppList), + {ErtsVsn, Apps}; + {error, Reason} -> + io:format(standard_error, "failed_to_read_file ~p for release ~p~nreason=~p~n", [RelFile, RelVsn, Reason]), + halt(1) + end. + +base_rel_overwrites(RelVsn, Dir, ErtsVsn, Overwrites) -> + RelFile = rel_file(Dir, RelVsn), + file:copy(RelFile, RelFile++".bak"), + {ok, [{release, {"emqx", RelVsn}, {erts, _BaseErtsVsn}, BaseAppList}]} = file:consult(RelFile), + NewData = [ {release, {"emqx", RelVsn}, {erts, ErtsVsn}, + lists:map(fun(X) -> + Name = element(1, X), + case lists:keyfind(Name, 1, Overwrites) of + false -> X; + Y when is_tuple(Y) -> Y + end + end, BaseAppList) + } + ], + ok = file:write_file(RelFile, io_lib:format("~p.", NewData)). + +rel_file(Dir, RelVsn)-> + filename:join([Dir, RelVsn, "emqx.rel"]). + +otp_apps() -> + {ok, [Apps]} = file:consult("scripts/rel_otp_apps.eterm"), + true = is_list(Apps), + lists:map(fun(App) when is_atom(App) -> App; + ({App, _}) -> App %% handle like {mnesia, load} + end, Apps). diff --git a/scripts/fail-on-old-otp-version.escript b/scripts/fail-on-old-otp-version.escript index 0e4cd2a1b..8399028cb 100755 --- a/scripts/fail-on-old-otp-version.escript +++ b/scripts/fail-on-old-otp-version.escript @@ -1,4 +1,4 @@ -#!/usr/bin/env escript +#!/usr/bin/env -S escript -c main(_) -> OtpRelease = list_to_integer(erlang:system_info(otp_release)), diff --git a/scripts/rel_otp_apps.eterm b/scripts/rel_otp_apps.eterm new file mode 100644 index 000000000..4999bbffa --- /dev/null +++ b/scripts/rel_otp_apps.eterm @@ -0,0 +1,16 @@ +%% Single source of truth of list otp apps that we use +[ kernel +, stdlib +, sasl +, crypto +, public_key +, asn1 +, syntax_tools +, ssl +, os_mon +, inets +, compiler +, runtime_tools +, {mnesia, load} +, xmerl +].