diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 5c0649b17..3efe0ef81 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -1,8 +1,5 @@ name: 'Create MacOS package' inputs: - profile: # emqx, emqx-enterprise - required: true - type: string otp: # 24.2.1-1, 23.3.4.9-3 required: true type: string @@ -49,7 +46,7 @@ runs: kerl update releases kerl build ${{ inputs.otp }} kerl install ${{ inputs.otp }} $HOME/.kerl/${{ inputs.otp }} - - name: build ${{ inputs.profile }} + - name: build env: AUTO_INSTALL_BUILD_DEPS: 1 APPLE_SIGN_BINARIES: 1 @@ -64,14 +61,17 @@ runs: . $HOME/.kerl/${{ inputs.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 - make ${{ inputs.profile }}-zip - - name: test ${{ inputs.profile }} + make ${EMQX_NAME}-zip + - name: test shell: bash run: | - pkg_name=$(basename _packages/${{ inputs.profile }}/${{ inputs.profile }}-*.zip) - unzip -q _packages/${{ inputs.profile }}/$pkg_name - gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins - ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 + pkg_name=$(basename _packages/${EMQX_NAME}/${EMQX_NAME}-*.zip) + unzip -q _packages/${EMQX_NAME}/$pkg_name + # test with a spaces in path + mv ./emqx "./emqx home/" + cd "./emqx home/" + gsed -i '/emqx_telemetry/d' data/loaded_plugins + ./bin/emqx start || cat log/erlang.log.1 ready='no' for i in {1..10}; do if curl -fs 127.0.0.1:18083 > /dev/null; then @@ -82,14 +82,15 @@ runs: done if [ "$ready" != "yes" ]; then echo "Timed out waiting for emqx to be ready" - cat emqx/log/erlang.log.1 + cat log/erlang.log.1 exit 1 fi - ./emqx/bin/emqx_ctl status - if ! ./emqx/bin/emqx stop; then - cat emqx/log/erlang.log.1 || true - cat emqx/log/emqx.log.1 || true + ./bin/emqx_ctl status + if ! ./bin/emqx stop; then + cat log/erlang.log.1 || true + cat log/emqx.log.1 || true echo "failed to stop emqx" exit 1 fi - rm -rf emqx + cd .. + rm -rf "emqx home" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 78b1bf14d..68e8a49be 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -108,11 +108,8 @@ jobs: strategy: fail-fast: false matrix: - profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - 24.1.5-3 - exclude: - - profile: emqx-edge os: - macos-11 runs-on: ${{ matrix.os }} @@ -127,9 +124,12 @@ jobs: ln -s . source unzip -q source.zip rm source source.zip + - id: detect-profiles + uses: ./.github/actions/detect-profiles + with: + ci_git_token: ${{ secrets.CI_GIT_TOKEN }} - uses: ./.github/actions/package-macos with: - profile: ${{ matrix.profile }} otp: ${{ matrix.otp }} os: ${{ matrix.os }} apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} @@ -138,8 +138,8 @@ jobs: apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - uses: actions/upload-artifact@v1 with: - name: ${{ matrix.profile }}-${{ matrix.otp }} - path: _packages/${{ matrix.profile }}/. + name: ${EMQX_NAME}-${{ matrix.otp }} + path: _packages/${EMQX_NAME}/. linux: runs-on: ubuntu-20.04 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 8f7e172ae..9ef69ff70 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -17,10 +17,10 @@ jobs: fail-fast: false matrix: otp: - - 24.1.5-3 + - 24.1.5-3 os: - - ubuntu20.04 - - el8 + - ubuntu20.04 + - el8 runs-on: - aws-amd64 - ubuntu-20.04 @@ -110,8 +110,6 @@ jobs: strategy: fail-fast: false matrix: - profile: - - emqx otp: - 24.1.5-3 os: @@ -124,35 +122,18 @@ jobs: ci_git_token: ${{ secrets.CI_GIT_TOKEN }} - uses: ./.github/actions/package-macos with: - profile: ${{ matrix.profile }} otp: ${{ matrix.otp }} os: ${{ matrix.os }} apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - - name: test - run: | - pkg_name=$(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip) - unzip -q $pkg_name - gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins - ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 - ready='no' - for i in {1..10}; do - if curl -fs 127.0.0.1:18083 > /dev/null; then - ready='yes' - break - fi - sleep 1 - done - if [ "$ready" != "yes" ]; then - echo "Timed out waiting for emqx to be ready" - cat emqx/log/erlang.log.1 - exit 1 - fi - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx stop - rm -rf emqx + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: rebar3.crashdump + path: ./rebar3.crashdump + - uses: actions/upload-artifact@v2 with: name: macos diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 54a87dd2a..6b0d0c4e9 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -12,6 +12,8 @@ File format: ## v4.3.22 +### Minor changes + ## v4.3.21 ### Enhancements @@ -25,13 +27,27 @@ File format: - TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007) Eliminate uncertainty that the buffer size is set by OS default. -- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) - - Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955) - Added a test to prevent a last will testament message to be published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894) +### Bug fixes + +- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) + +- Hide redis password in error logs [#9071](https://github.com/emqx/emqx/pull/9071) + In this change, it also included more changes in redis client: + - Improve redis connection error logging [eredis:19](https://github.com/emqx/eredis/pull/19). + Also added support for eredis to accept an anonymous function as password instead of + passing around plaintext args which may get dumpped to crash logs (hard to predict where). + This change also added `format_status` callback for `gen_server` states which hold plaintext + password so the process termination log and `sys:get_status` will print '******' instead of + the password to console. + - Avoid pool name clashing [eredis_cluster#22](https://github.com/emqx/eredis_cluster/pull/22) + Same `format_status` callback is added here too for `gen_server`s which hold password in + their state. + ## v4.3.20 ### Bug fixes diff --git a/CHANGES-4.4.md b/CHANGES-4.4.md index 6cfa9f74a..16d57a37c 100644 --- a/CHANGES-4.4.md +++ b/CHANGES-4.4.md @@ -1,5 +1,11 @@ # EMQX 4.4 Changes +## v4.4.10 + +### Bug fixes + +- Fix the latency statistics error of the slow subscription module when `stats_type` is `internal` or `response`. [#8981](https://github.com/emqx/emqx/pull/8981) + ## v4.4.9 ### Bug fixes (synced from v4.3.20) diff --git a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl index 96e66d3c0..7fd18a62b 100644 --- a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl @@ -20,6 +20,7 @@ -compile(export_all). -include("emqx_auth_mnesia.hrl"). +-include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -78,15 +79,37 @@ init_per_testcase_migration(_, Config) -> emqx_acl_mnesia_migrator:migrate_records(), Config. +init_per_testcase_other(t_last_will_testament_message_check_acl, Config) -> + OriginalACLNoMatch = application:get_env(emqx, acl_nomatch), + application:set_env(emqx, acl_nomatch, deny), + emqx_mod_acl_internal:unload([]), + %% deny all for this client + ClientID = <<"lwt_client">>, + ok = emqx_acl_mnesia_db:add_acl({clientid, ClientID}, <<"#">>, pubsub, deny), + [ {original_acl_nomatch, OriginalACLNoMatch} + , {clientid, ClientID} + | Config]; +init_per_testcase_other(_TestCase, Config) -> + Config. + init_per_testcase(Case, Config) -> PerTestInitializers = [ fun init_per_testcase_clean/2, fun init_per_testcase_migration/2, - fun init_per_testcase_emqx_hook/2 + fun init_per_testcase_emqx_hook/2, + fun init_per_testcase_other/2 ], lists:foldl(fun(Init, Conf) -> Init(Case, Conf) end, Config, PerTestInitializers). -end_per_testcase(_, Config) -> +end_per_testcase(t_last_will_testament_message_check_acl, Config) -> + emqx:unhook('client.check_acl', fun emqx_acl_mnesia:check_acl/5), + case ?config(original_acl_nomatch, Config) of + {ok, Original} -> application:set_env(emqx, acl_nomatch, Original); + _ -> ok + end, + emqx_mod_acl_internal:load([]), + ok; +end_per_testcase(_TestCase, Config) -> emqx:unhook('client.check_acl', fun emqx_acl_mnesia:check_acl/5), Config. @@ -465,6 +488,35 @@ t_rest_api(_Config) -> {ok, Res3} = request_http_rest_list(["$all"]), ?assertMatch([], get_http_data(Res3)). +%% asserts that we check ACL for the LWT topic before publishing the +%% LWT. +t_last_will_testament_message_check_acl(Config) -> + ClientID = ?config(clientid, Config), + {ok, C} = emqtt:start_link([ + {clientid, ClientID}, + {will_topic, <<"$SYS/lwt">>}, + {will_payload, <<"should not be published">>} + ]), + {ok, _} = emqtt:connect(C), + ok = emqx:subscribe(<<"$SYS/lwt">>), + unlink(C), + ok = snabbkaffe:start_trace(), + {true, {ok, _}} = + ?wait_async_action( + exit(C, kill), + #{?snk_kind := last_will_testament_publish_denied}, + 1_000 + ), + ok = snabbkaffe:stop(), + + receive + {deliver, <<"$SYS/lwt">>, #message{payload = <<"should not be published">>}} -> + error(lwt_should_not_be_published_to_forbidden_topic) + after 1_000 -> + ok + end, + + ok. create_conflicting_records() -> Records = [ diff --git a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl index 1765b3821..0031bc8c4 100644 --- a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl +++ b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl @@ -115,7 +115,7 @@ init_per_suite(Config) -> end_per_suite(_Cfg) -> deinit_mongo_data(), %% avoid inter-suite flakiness - ok = emqx_mod_acl_internal:load([]), + emqx_mod_acl_internal:load([]), emqx_ct_helpers:stop_apps([emqx_auth_mongo]). set_special_confs(emqx) -> @@ -186,6 +186,8 @@ end_per_testcase(TestCase, Config) when TestCase =:= t_available_acl_query_timeout; TestCase =:= t_acl_superuser_timeout; TestCase =:= t_authn_no_connection; + TestCase =:= t_available_authn_query_timeout; + TestCase =:= t_authn_timeout; TestCase =:= t_available_acl_query_no_connection -> ProxyHost = ?config(proxy_host, Config), ProxyPort = ?config(proxy_port, Config), diff --git a/apps/emqx_psk_file/include/emqx_psk_file.hrl b/apps/emqx_psk_file/include/emqx_psk_file.hrl new file mode 100644 index 000000000..fe8ed1f94 --- /dev/null +++ b/apps/emqx_psk_file/include/emqx_psk_file.hrl @@ -0,0 +1,25 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_PSK_FILE). +-define(EMQX_PSK_FILE, true). + +-define(PSK_FILE_TAB, emqx_psk_file). + +-record(psk_entry, {psk_id :: binary(), + psk_str :: binary()}). + +-endif. diff --git a/apps/emqx_psk_file/src/emqx_psk_file.app.src b/apps/emqx_psk_file/src/emqx_psk_file.app.src index b8a6f08a0..ef18c8b69 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file.app.src +++ b/apps/emqx_psk_file/src/emqx_psk_file.app.src @@ -1,6 +1,6 @@ {application, emqx_psk_file, [{description,"EMQX PSK Plugin from File"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules,[]}, {registered,[emqx_psk_file_sup]}, {applications,[kernel,stdlib]}, diff --git a/apps/emqx_psk_file/src/emqx_psk_file.appup.src b/apps/emqx_psk_file/src/emqx_psk_file.appup.src new file mode 100644 index 000000000..c782000b5 --- /dev/null +++ b/apps/emqx_psk_file/src/emqx_psk_file.appup.src @@ -0,0 +1,10 @@ +%% -*- mode: erlang -*- +{VSN, + [{"4.3.0", + [{load_module,emqx_psk_file,brutal_purge,soft_purge,[]}, + {load_module,emqx_psk_file_sup,brutal_purge,soft_purge,[]}]} + ], + [{"4.3.0", + [{load_module,emqx_psk_file,brutal_purge,soft_purge,[]}, + {load_module,emqx_psk_file_sup,brutal_purge,soft_purge,[]}]} + ]}. diff --git a/apps/emqx_psk_file/src/emqx_psk_file.erl b/apps/emqx_psk_file/src/emqx_psk_file.erl index b4daee370..e252393c4 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file.erl +++ b/apps/emqx_psk_file/src/emqx_psk_file.erl @@ -16,6 +16,7 @@ -module(emqx_psk_file). +-include("emqx_psk_file.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -26,15 +27,10 @@ %% Hooks functions -export([on_psk_lookup/2]). --define(TAB, ?MODULE). -define(LF, 10). --record(psk_entry, {psk_id :: binary(), - psk_str :: binary()}). - %% Called when the plugin application start load(Env) -> - _ = ets:new(?TAB, [set, named_table, {keypos, #psk_entry.psk_id}]), {ok, PskFile} = file:open(get_value(path, Env), [read, raw, binary, read_ahead]), preload_psks(PskFile, bin(get_value(delimiter, Env))), _ = file:close(PskFile), @@ -45,7 +41,7 @@ unload() -> emqx:unhook('tls_handshake.psk_lookup', fun ?MODULE:on_psk_lookup/2). on_psk_lookup(ClientPSKID, UserState) -> - case ets:lookup(?TAB, ClientPSKID) of + case ets:lookup(?PSK_FILE_TAB, ClientPSKID) of [#psk_entry{psk_str = PskStr}] -> {stop, PskStr}; [] -> @@ -57,7 +53,9 @@ preload_psks(FileHandler, Delimiter) -> {ok, Line} -> case binary:split(Line, Delimiter) of [Key, Rem] -> - ets:insert(?TAB, #psk_entry{psk_id = Key, psk_str = trim_lf(Rem)}), + ets:insert( + ?PSK_FILE_TAB, + #psk_entry{psk_id = Key, psk_str = trim_lf(Rem)}), preload_psks(FileHandler, Delimiter); [Line] -> ?LOG(warning, "[~p] - Invalid line: ~p, delimiter: ~p", [?MODULE, Line, Delimiter]) diff --git a/apps/emqx_psk_file/src/emqx_psk_file_sup.erl b/apps/emqx_psk_file/src/emqx_psk_file_sup.erl index 2e739519c..e643c000a 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file_sup.erl +++ b/apps/emqx_psk_file/src/emqx_psk_file_sup.erl @@ -16,6 +16,8 @@ -module(emqx_psk_file_sup). +-include("emqx_psk_file.hrl"). + -behaviour(supervisor). %% API @@ -25,8 +27,11 @@ -export([init/1]). start_link() -> + _ = ets:new( + ?PSK_FILE_TAB, + [set, named_table, public, {keypos, #psk_entry.psk_id}] + ), supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> {ok, { {one_for_one, 0, 1}, []} }. - diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index eb5613661..27baf5e88 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.4.2"}, % strict semver, bump manually! + {vsn, "4.4.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl index 2028affb6..ca16a98bb 100644 --- a/apps/emqx_retainer/src/emqx_retainer_sup.erl +++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl @@ -34,6 +34,23 @@ init([Env]) -> type => worker, modules => [emqx_retainer]} || not is_managed_by_modules()]}}. +-ifdef(EMQX_ENTERPRISE). + +is_managed_by_modules() -> + try + case supervisor:get_childspec(emqx_modules_sup, emqx_retainer) of + {ok, _} -> true; + _ -> false + end + catch + exit : {noproc, _} -> + false + end. + +-else. + is_managed_by_modules() -> %% always false for opensource edition false. + +-endif. diff --git a/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl b/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl index fd12a4e01..67f58ec26 100644 --- a/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl +++ b/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl @@ -20,7 +20,9 @@ -export([ensure_start/0, ensure_stop/0]). -ifdef(EMQX_ENTERPRISE). ensure_start() -> + %% for enterprise edition, retainer is started by modules application:stop(emqx_modules), + ensure_stop(), init_conf(), emqx_ct_helpers:start_apps([emqx_retainer]), ok. @@ -29,6 +31,7 @@ ensure_start() -> ensure_start() -> init_conf(), + ensure_stop(), emqx_ct_helpers:start_apps([emqx_retainer]), ok. diff --git a/bin/emqx b/bin/emqx index a27db494d..01f2802cf 100755 --- a/bin/emqx +++ b/bin/emqx @@ -523,6 +523,16 @@ case "$1" in ;; esac +if [ "$IS_BOOT_COMMAND" = 'no' ]; then + # for non-boot commands, inspect vm.