From 3f758ca36f3c4201d04ced0b71a67361c3242838 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 6 Sep 2022 16:42:02 +0800 Subject: [PATCH 1/9] fix(tls): fix ciphers cache return `ok` at the first time call --- apps/emqx/src/emqx_tls_lib.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 28bda5cfa..4bc18f1e0 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -145,7 +145,8 @@ all_ciphers_set_cached() -> case persistent_term:get(?FUNCTION_NAME, false) of false -> S = sets:from_list(all_ciphers()), - persistent_term:put(?FUNCTION_NAME, S); + persistent_term:put(?FUNCTION_NAME, S), + S; Set -> Set end. From ac840e318ef061264bab227ae75d297cbdbfb1a4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 6 Sep 2022 18:22:56 +0200 Subject: [PATCH 2/9] refactor(cookie): Warning message when boot with default Erlang cookie --- CHANGES-5.0.md | 1 + apps/emqx_conf/etc/emqx_conf.conf | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- bin/emqx | 15 ++++++++++----- mix.exs | 6 ++++++ rebar.config.erl | 9 ++++----- rel/emqx_vars | 10 ++++------ scripts/get-dashboard.sh | 2 +- 8 files changed, 28 insertions(+), 19 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 82d907458..e5225af3e 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -9,6 +9,7 @@ ## Enhancements +* Print a warning message when boot with the default (insecure) Erlang cookie. [#8905](https://github.com/emqx/emqx/pull/8905) * Change the `/gateway` API path to plural form. [#8823](https://github.com/emqx/emqx/pull/8823) * Remove `node.etc_dir` from emqx.conf, because it is never used. Also allow user to customize the logging directory [#8892](https://github.com/emqx/emqx/pull/8892) diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index fe1b7ab91..86147bf25 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -10,7 +10,7 @@ node { name = "emqx@127.0.0.1" - cookie = emqxsecretcookie + cookie = "{{ emqx_default_erlang_cookie }}" data_dir = "{{ platform_data_dir }}" } diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 8c4cddbc3..a00bfe6f3 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -400,7 +400,7 @@ fields("node") -> string(), #{ mapping => "vm_args.-setcookie", - default => "emqxsecretcookie", + required => true, 'readOnly' => true, sensitive => true, desc => ?DESC(node_cookie) diff --git a/bin/emqx b/bin/emqx index e73240d5d..20394b96f 100755 --- a/bin/emqx +++ b/bin/emqx @@ -600,7 +600,7 @@ is_down() { if ps -p "$PID" | grep -q 'defunct'; then # zombie state, print parent pid parent="$(ps -o ppid= -p "$PID" | tr -d ' ')" - echo "WARN: $PID is marked , parent:" + echo "WARNING: $PID is marked , parent:" ps -p "$parent" return 0 fi @@ -748,8 +748,9 @@ export ESCRIPT_NAME="$SHORT_NAME" PIPE_DIR="${PIPE_DIR:-/$DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}" -## make EMQX_NODE_COOKIE right +## Resolve Erlang cookie. if [ -n "${EMQX_NODE_COOKIE:-}" ]; then + ## To be backward compatible, read EMQX_NODE_COOKIE export EMQX_NODE__COOKIE="${EMQX_NODE_COOKIE}" unset EMQX_NODE_COOKIE fi @@ -762,9 +763,13 @@ if [ -z "$COOKIE" ]; then COOKIE="$(grep -E '^-setcookie' "${vm_args_file}" | awk '{print $2}')" fi fi - -if [ -z "$COOKIE" ]; then - die "Please set node.cookie in $EMQX_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE" +[ -z "$COOKIE" ] && COOKIE="$EMQX_DEFAULT_ERLANG_COOKIE" +if [ $IS_BOOT_COMMAND = 'yes' ] && [ "$COOKIE" = "$EMQX_DEFAULT_ERLANG_COOKIE" ]; then + echoerr "!!!!!!" + echoerr "WARNING: Default (insecure) Erlang cookie is in use." + echoerr "WARNING: Configure node.cookie in $EMQX_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE" + echoerr "NOTE: Use the same config value for all nodes in the cluster." + echoerr "!!!!!!" fi ## check if OTP version has mnesia_hook feature; if not, fallback to diff --git a/mix.exs b/mix.exs index 7cac420ac..3f55de64b 100644 --- a/mix.exs +++ b/mix.exs @@ -547,6 +547,7 @@ defmodule EMQXUmbrella.MixProject do defp template_vars(release, release_type, :bin = _package_type, edition_type) do [ + emqx_default_erlang_cookie: default_cookie(), platform_data_dir: "data", platform_etc_dir: "etc", platform_log_dir: "log", @@ -569,6 +570,7 @@ defmodule EMQXUmbrella.MixProject do defp template_vars(release, release_type, :pkg = _package_type, edition_type) do [ + emqx_default_erlang_cookie: default_cookie(), platform_data_dir: "/var/lib/emqx", platform_etc_dir: "/etc/emqx", platform_log_dir: "/var/log/emqx", @@ -589,6 +591,10 @@ defmodule EMQXUmbrella.MixProject do ] ++ build_info() end + defp default_cookie() do + "emqx50elixir" + end + defp emqx_description(release_type, edition_type) do case {release_type, edition_type} do {:cloud, :enterprise} -> diff --git a/rebar.config.erl b/rebar.config.erl index 17c94374c..c6ab19818 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -298,14 +298,13 @@ relform() -> emqx_description(cloud, ee) -> "EMQX Enterprise"; emqx_description(cloud, ce) -> "EMQX". -overlay_vars(RelType, PkgType, Edition) -> - overlay_vars_rel(RelType) ++ +overlay_vars(cloud, PkgType, Edition) -> + [ + {emqx_default_erlang_cookie, "emqxsecretcookie"} + ] ++ overlay_vars_pkg(PkgType) ++ overlay_vars_edition(Edition). -overlay_vars_rel(cloud) -> - [{vm_args_file, "vm.args"}]. - overlay_vars_edition(ce) -> [ {emqx_schema_mod, emqx_conf_schema}, diff --git a/rel/emqx_vars b/rel/emqx_vars index 27de79e0d..1ec95b4de 100644 --- a/rel/emqx_vars +++ b/rel/emqx_vars @@ -9,19 +9,17 @@ ERL_OPTS="{{ erl_opts }}" RUNNER_BIN_DIR="{{ runner_bin_dir }}" RUNNER_LIB_DIR="{{ runner_lib_dir }}" IS_ELIXIR="${IS_ELIXIR:-{{ is_elixir }}}" - ## Allow users to pre-set `RUNNER_LOG_DIR` because it only affects boot commands like `start` and `console`, ## but not other commands such as `ping` and `ctl`. RUNNER_LOG_DIR="${RUNNER_LOG_DIR:-{{ runner_log_dir }}}" - EMQX_ETC_DIR="{{ emqx_etc_dir }}" RUNNER_USER="{{ runner_user }}" SCHEMA_MOD="{{ emqx_schema_mod }}" IS_ENTERPRISE="{{ is_enterprise }}" - +## Do not change EMQX_DEFAULT_ERLANG_COOKIE. +## Configure EMQX_NODE_COOKIE instead +EMQX_DEFAULT_ERLANG_COOKIE='{{ emqx_default_erlang_cookie }}' +REL_NAME="emqx" export EMQX_DESCRIPTION='{{ emqx_description }}' -## computed vars -REL_NAME="emqx" - ## updated vars here diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index 0069f3cc2..c3559865f 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -42,7 +42,7 @@ curl -L --silent --show-error \ --output "${RELEASE_ASSET_FILE}" \ "$DIRECT_DOWNLOAD_URL" -unzip -q "$RELEASE_ASSET_FILE" -d "$DASHBOARD_PATH" +unzip -o -q "$RELEASE_ASSET_FILE" -d "$DASHBOARD_PATH" rm -rf "$DASHBOARD_PATH/www" mv "$DASHBOARD_PATH/dist" "$DASHBOARD_PATH/www" rm -f "$RELEASE_ASSET_FILE" From 85cb552e5351c1afb03fe5d514416628c160354f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 6 Sep 2022 18:48:59 +0200 Subject: [PATCH 3/9] chore: ensure no 'find' command error the _upgrade_base dir must exist --- build | 1 + 1 file changed, 1 insertion(+) diff --git a/build b/build index 92f033be6..879251ede 100755 --- a/build +++ b/build @@ -157,6 +157,7 @@ make_relup() { local name_pattern name_pattern="${PROFILE}-$(./pkg-vsn.sh "$PROFILE" --vsn_matcher --long)" local releases=() + mkdir -p _upgrade_base while read -r tgzfile ; do local base_vsn base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9])?(-[0-9a-f]{8})?" | head -1)" From f785da075b38be82a7e1a05e2a4f4eb495ba3b63 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 4 Sep 2022 22:01:42 +0200 Subject: [PATCH 4/9] ci: only start required docker for integration tests --- .github/workflows/run_test_cases.yaml | 21 +---- Makefile | 1 - apps/emqx_authn/docker-ct | 4 + apps/emqx_authz/docker-ct | 4 + apps/emqx_connector/docker-ct | 4 + scripts/apps-version-check.sh | 5 +- scripts/ct/run.sh | 112 ++++++++++++++++++++++++++ scripts/docker-ct-apps | 4 - scripts/find-apps.sh | 31 +++---- 9 files changed, 144 insertions(+), 42 deletions(-) create mode 100644 apps/emqx_authn/docker-ct create mode 100644 apps/emqx_authz/docker-ct create mode 100644 apps/emqx_connector/docker-ct create mode 100755 scripts/ct/run.sh delete mode 100644 scripts/docker-ct-apps diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index b4b797621..540383ed6 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -121,27 +121,8 @@ jobs: PGSQL_TAG: 13 REDIS_TAG: 6 run: | - docker-compose \ - -f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-single-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml \ - -f .ci/docker-compose-file/docker-compose.yaml \ - up -d --build - - # produces .coverdata - - name: run common test - working-directory: source - run: | - docker exec -i ${{ matrix.otp_release }} bash -c "git config --global --add safe.directory \"$GITHUB_WORKSPACE\" && make ${{ matrix.app_name }}-ct" + ./scripts/ct/run.sh --app ${{ matrix.app_name }} - uses: actions/upload-artifact@v1 - if: matrix.otp_release == 'erlang24' with: name: coverdata path: source/_build/test/cover diff --git a/Makefile b/Makefile index 87404325e..ead7847f8 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,6 @@ static_checks: APPS=$(shell $(SCRIPTS)/find-apps.sh) -## app/name-ct targets are intended for local tests hence cover is not enabled .PHONY: $(APPS:%=%-ct) define gen-app-ct-target $1-ct: $(REBAR) diff --git a/apps/emqx_authn/docker-ct b/apps/emqx_authn/docker-ct new file mode 100644 index 000000000..7b9a4c068 --- /dev/null +++ b/apps/emqx_authn/docker-ct @@ -0,0 +1,4 @@ +mongo +redis +mysql +pgsql diff --git a/apps/emqx_authz/docker-ct b/apps/emqx_authz/docker-ct new file mode 100644 index 000000000..7b9a4c068 --- /dev/null +++ b/apps/emqx_authz/docker-ct @@ -0,0 +1,4 @@ +mongo +redis +mysql +pgsql diff --git a/apps/emqx_connector/docker-ct b/apps/emqx_connector/docker-ct new file mode 100644 index 000000000..7b9a4c068 --- /dev/null +++ b/apps/emqx_connector/docker-ct @@ -0,0 +1,4 @@ +mongo +redis +mysql +pgsql diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 72ddd5dc3..f760c18b3 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -12,7 +12,8 @@ parse_semver() { echo "$1" | tr '.|-' ' ' } -while read -r app; do +APPS="$(./scripts/find-apps.sh)" +for app in ${APPS}; do if [ "$app" != "emqx" ]; then app_path="$app" else @@ -46,7 +47,7 @@ while read -r app; do bad_app_count=$(( bad_app_count + 1)) fi fi -done < <(./scripts/find-apps.sh) +done if [ $bad_app_count -gt 0 ]; then exit 1 diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh new file mode 100755 index 000000000..879efc434 --- /dev/null +++ b/scripts/ct/run.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +## This script runs CT (and necessary dependencies) in docker container(s) + +set -euo pipefail + +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." + +help() { + echo + echo "-h|--help: To display this usage info" + echo "--app lib_dir/app_name: Print apps in json" + echo "--console: Start EMQX in console mode" +} + +WHICH_APP='novalue' +CONSOLE='no' +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + --app) + WHICH_APP="$2" + shift 2 + ;; + --console) + CONSOLE='yes' + shift 1 + ;; + *) + echo "unknown option $1" + exit 1 + ;; + esac +done + +if [ "${WHICH_APP}" = 'novalue' ]; then + echo "must provide --app arg" + exit 1 +fi + +ERLANG_CONTAINER='erlang24' +DOCKER_CT_ENVS_FILE="${WHICH_APP}/docker-ct" + +if [ -f "$DOCKER_CT_ENVS_FILE" ]; then + # shellcheck disable=SC2002 + CT_DEPS="$(cat "$DOCKER_CT_ENVS_FILE" | xargs)" +fi +CT_DEPS="${ERLANG_CONTAINER} ${CT_DEPS}" + +FILES=( ) + +for dep in ${CT_DEPS}; do + case "${dep}" in + erlang24) + FILES+=( '.ci/docker-compose-file/docker-compose.yaml' ) + ;; + mongo) + FILES+=( '.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml' + '.ci/docker-compose-file/docker-compose-mongo-single-tls.yaml' ) + ;; + redis) + FILES+=( '.ci/docker-compose-file/docker-compose-redis-single-tcp.yaml' + '.ci/docker-compose-file/docker-compose-redis-single-tls.yaml' + '.ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml' + '.ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml' ) + ;; + mysql) + FILES+=( '.ci/docker-compose-file/docker-compose-mysql-tcp.yaml' + '.ci/docker-compose-file/docker-compose-mysql-tls.yaml' ) + ;; + pgsql) + FILES+=( '.ci/docker-compose-file/docker-compose-pgsql-tcp.yaml' + '.ci/docker-compose-file/docker-compose-pgsql-tls.yaml' ) + ;; + *) + echo "unknown_ct_dependency $dep" + exit 1 + ;; + esac +done + +F_OPTIONS="" + +for file in "${FILES[@]}"; do + F_OPTIONS="$F_OPTIONS -f $file" +done + +# shellcheck disable=2086 # no quotes for F_OPTIONS +docker-compose $F_OPTIONS up -d --build + +# /emqx is where the source dir is mounted to the Erlang container +# in .ci/docker-compose-file/docker-compose.yaml +TTY='' +if [[ -t 1 ]]; then + TTY='-t' +fi +docker exec -i $TTY "$ERLANG_CONTAINER" bash -c 'git config --global --add safe.directory /emqx' + +if [ "$CONSOLE" = 'yes' ]; then + docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make run" +else + set +e + docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make ${WHICH_APP}-ct" + RESULT=$? + # shellcheck disable=2086 # no quotes for F_OPTIONS + docker-compose $F_OPTIONS down + exit $RESULT +fi diff --git a/scripts/docker-ct-apps b/scripts/docker-ct-apps deleted file mode 100644 index 882a68285..000000000 --- a/scripts/docker-ct-apps +++ /dev/null @@ -1,4 +0,0 @@ -# apps need docker-compose to run CT -apps/emqx_authn -apps/emqx_authz -apps/emqx_connector diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index f888eaa22..8c41fecbf 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -50,24 +50,25 @@ find_app() { CE="$(find_app 'apps')" EE="$(find_app 'lib-ee')" +APPS_ALL="$(echo -e "${CE}\n${EE}")" if [ "$CT" = 'novalue' ]; then - echo -e "${CE}\n${EE}" - exit 0 -fi - -APPS_ALL="$(echo -e "${CE}\n${EE}")" -APPS_DOCKER_CT="$(grep -v -E '^#.*' scripts/docker-ct-apps)" - -# shellcheck disable=SC2068 -for app in ${APPS_DOCKER_CT[@]}; do - APPS_ALL=("${APPS_ALL[@]/$app}") -done - -if [ "$CT" = 'docker' ]; then - RESULT="${APPS_DOCKER_CT}" + RESULT="${APPS_ALL}" else - RESULT="${APPS_ALL[*]}" + APPS_NORMAL_CT=( ) + APPS_DOCKER_CT=( ) + for app in ${APPS_ALL}; do + if [ -f "${app}/docker-ct" ]; then + APPS_DOCKER_CT+=("$app") + else + APPS_NORMAL_CT+=("$app") + fi + done + if [ "$CT" = 'docker' ]; then + RESULT="${APPS_DOCKER_CT[*]}" + else + RESULT="${APPS_NORMAL_CT[*]}" + fi fi if [ "$WANT_JSON" = 'yes' ]; then From 1175008a747201785247040363dc4e4a9052aabb Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 1 Sep 2022 10:56:12 +0800 Subject: [PATCH 5/9] feat: cluster-rpc failed fast when some nodes is down --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 40 ++++++++++++++++--------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index ddc4eccc5..ed6f32508 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -72,6 +72,7 @@ -define(TIMEOUT, timer:minutes(1)). -define(APPLY_KIND_REPLICATE, replicate). -define(APPLY_KIND_INITIATE, initiate). +-define(IS_ACTION(_A_), (_A_ =:= peers_lagging orelse _A_ =:= stopped_nodes)). -type tnx_id() :: pos_integer(). @@ -123,13 +124,13 @@ start_link(Node, Name, RetryMs) -> %% the result is expected to be `ok | {ok, _}' to indicate success, %% and `{error, _}' to indicate failure. %% -%% The excpetion of the MFA evaluation is captured and translated +%% The exception of the MFA evaluation is captured and translated %% into an `{error, _}' tuple. %% This call tries to wait for all peer nodes to be in-sync before %% returning the result. %% %% In case of partial success, an `error' level log is emitted -%% but the initial localy apply result is returned. +%% but the initial locally apply result is returned. -spec multicall(module(), atom(), list()) -> term(). multicall(M, F, A) -> multicall(M, F, A, all, timer:minutes(2)). @@ -141,11 +142,12 @@ multicall(M, F, A, RequiredSyncs, Timeout) when RequiredSyncs =:= all orelse Req Result; {init_failure, Error} -> Error; - {peers_lagging, TnxId, Res, Nodes} -> + {Action, TnxId, Res, Nodes} when ?IS_ACTION(Action) -> %% The init MFA return ok, but some other nodes failed. ?SLOG(error, #{ msg => "cluster_rpc_peers_lagging", - lagging_nodes => Nodes, + action => Action, + nodes => Nodes, tnx_id => TnxId }), Res @@ -193,9 +195,9 @@ do_multicall(M, F, A, RequiredSyncs, Timeout) -> InitRes; {init_failure, Error0} -> {init_failure, Error0}; - {peers_lagging, Nodes} -> + {Action, Nodes} when ?IS_ACTION(Action) -> {ok, TnxId0, MFARes} = InitRes, - {peers_lagging, TnxId0, MFARes, Nodes} + {Action, TnxId0, MFARes, Nodes} end. -spec query(pos_integer()) -> {'atomic', map()} | {'aborted', Reason :: term()}. @@ -509,14 +511,18 @@ do_alarm(Fun, Res, #{tnx_id := Id} = Meta) -> emqx_alarm:Fun(cluster_rpc_apply_failed, Meta#{result => ?TO_BIN(Res)}, AlarmMsg). wait_for_all_nodes_commit(TnxId, Delay, Remain) -> - case lagging_node(TnxId) of + Lagging = lagging_node(TnxId), + Stopped = stopped_nodes(), + case Lagging -- Stopped of + [] when Stopped =:= [] -> + ok; + [] -> + {stopped_nodes, Stopped}; [_ | _] when Remain > 0 -> ok = timer:sleep(Delay), wait_for_all_nodes_commit(TnxId, Delay, Remain - Delay); - [] -> - ok; - Nodes -> - {peers_lagging, Nodes} + [_ | _] -> + {peers_lagging, Lagging} end. wait_for_nodes_commit(RequiredSyncs, TnxId, Delay, Remain) -> @@ -527,10 +533,13 @@ wait_for_nodes_commit(RequiredSyncs, TnxId, Delay, Remain) -> false when Remain > 0 -> wait_for_nodes_commit(RequiredSyncs, TnxId, Delay, Remain - Delay); false -> - case lagging_node(TnxId) of + Lagging = lagging_node(TnxId), + Stopped = stopped_nodes(), + case Lagging -- Stopped of %% All commit but The succeedNum > length(nodes()). - [] -> ok; - Nodes -> {peers_lagging, Nodes} + [] when Stopped =:= [] -> ok; + [] -> {stopped_nodes, Stopped}; + [_ | _] -> {peers_lagging, Lagging} end end. @@ -548,6 +557,9 @@ commit_status_trans(Operator, TnxId) -> Result = '$2', mnesia:select(?CLUSTER_COMMIT, [{MatchHead, [Guard], [Result]}]). +stopped_nodes() -> + ekka_cluster:info(stopped_nodes). + get_retry_ms() -> emqx_conf:get([node, cluster_call, retry_interval], timer:minutes(1)). From 758b1979ab7912cc205c65926dfcd09c30265fc5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 6 Sep 2022 16:09:08 +0800 Subject: [PATCH 6/9] chore: Apply suggestions from code review Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index ed6f32508..5de2932f9 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -72,7 +72,7 @@ -define(TIMEOUT, timer:minutes(1)). -define(APPLY_KIND_REPLICATE, replicate). -define(APPLY_KIND_INITIATE, initiate). --define(IS_ACTION(_A_), (_A_ =:= peers_lagging orelse _A_ =:= stopped_nodes)). +-define(IS_STATUS(_A_), (_A_ =:= peers_lagging orelse _A_ =:= stopped_nodes)). -type tnx_id() :: pos_integer(). @@ -130,7 +130,7 @@ start_link(Node, Name, RetryMs) -> %% returning the result. %% %% In case of partial success, an `error' level log is emitted -%% but the initial locally apply result is returned. +%% but the initial local apply result is returned. -spec multicall(module(), atom(), list()) -> term(). multicall(M, F, A) -> multicall(M, F, A, all, timer:minutes(2)). @@ -142,11 +142,11 @@ multicall(M, F, A, RequiredSyncs, Timeout) when RequiredSyncs =:= all orelse Req Result; {init_failure, Error} -> Error; - {Action, TnxId, Res, Nodes} when ?IS_ACTION(Action) -> + {Status, TnxId, Res, Nodes} when ?IS_STATUS(Status) -> %% The init MFA return ok, but some other nodes failed. ?SLOG(error, #{ msg => "cluster_rpc_peers_lagging", - action => Action, + status=> Status, nodes => Nodes, tnx_id => TnxId }), @@ -195,9 +195,9 @@ do_multicall(M, F, A, RequiredSyncs, Timeout) -> InitRes; {init_failure, Error0} -> {init_failure, Error0}; - {Action, Nodes} when ?IS_ACTION(Action) -> + {Status, Nodes} when ?IS_STATUS(Status) -> {ok, TnxId0, MFARes} = InitRes, - {Action, TnxId0, MFARes, Nodes} + {Status, TnxId0, MFARes, Nodes} end. -spec query(pos_integer()) -> {'atomic', map()} | {'aborted', Reason :: term()}. From 33341011d8bdbb2c3ee9373e6eb3c3b646ebf6ee Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 6 Sep 2022 16:17:51 +0800 Subject: [PATCH 7/9] chore: improve wait_for_nodes_commit/4 function --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 5de2932f9..4b3886798 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -146,7 +146,7 @@ multicall(M, F, A, RequiredSyncs, Timeout) when RequiredSyncs =:= all orelse Req %% The init MFA return ok, but some other nodes failed. ?SLOG(error, #{ msg => "cluster_rpc_peers_lagging", - status=> Status, + status => Status, nodes => Nodes, tnx_id => TnxId }), @@ -511,7 +511,7 @@ do_alarm(Fun, Res, #{tnx_id := Id} = Meta) -> emqx_alarm:Fun(cluster_rpc_apply_failed, Meta#{result => ?TO_BIN(Res)}, AlarmMsg). wait_for_all_nodes_commit(TnxId, Delay, Remain) -> - Lagging = lagging_node(TnxId), + Lagging = lagging_nodes(TnxId), Stopped = stopped_nodes(), case Lagging -- Stopped of [] when Stopped =:= [] -> @@ -533,17 +533,18 @@ wait_for_nodes_commit(RequiredSyncs, TnxId, Delay, Remain) -> false when Remain > 0 -> wait_for_nodes_commit(RequiredSyncs, TnxId, Delay, Remain - Delay); false -> - Lagging = lagging_node(TnxId), - Stopped = stopped_nodes(), - case Lagging -- Stopped of - %% All commit but The succeedNum > length(nodes()). - [] when Stopped =:= [] -> ok; - [] -> {stopped_nodes, Stopped}; - [_ | _] -> {peers_lagging, Lagging} + case lagging_nodes(TnxId) of + [] -> + ok; + Lagging -> + case stopped_nodes() of + [] -> {peers_lagging, Lagging}; + Stopped -> {stopped_nodes, Stopped} + end end end. -lagging_node(TnxId) -> +lagging_nodes(TnxId) -> {atomic, Nodes} = transaction(fun ?MODULE:commit_status_trans/2, ['<', TnxId]), Nodes. From fdbf8c1c27d8ac358eb84f39b20ee328d9cafbad Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 7 Sep 2022 10:26:26 +0800 Subject: [PATCH 8/9] chore: update changelog --- CHANGES-5.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 82d907458..a9e694081 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -6,6 +6,7 @@ * Fix JWT plugin don't support non-integer timestamp claims. [#8867](https://github.com/emqx/emqx/pull/8867) * Avoid publishing will message when client fails to auhtenticate. [#8887](https://github.com/emqx/emqx/pull/8887) * Speed up dispatching of shared subscription messages in a cluster [#8893](https://github.com/emqx/emqx/pull/8893) +* Speed up updating the configuration, When some nodes in the cluster are down. [#8857](https://github.com/emqx/emqx/pull/8857) ## Enhancements From 351046abf9d1c8085609cfb721d1b79e955b749b Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 7 Sep 2022 09:27:36 +0200 Subject: [PATCH 9/9] docs: finish sync of README-RU --- README-RU.md | 87 ++++++++++------------------------------------------ 1 file changed, 16 insertions(+), 71 deletions(-) diff --git a/README-RU.md b/README-RU.md index 2217313fe..cd8943795 100644 --- a/README-RU.md +++ b/README-RU.md @@ -88,9 +88,9 @@ docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 ## Сборка из исходного кода -Начиная с релиза 3.0, для сборки требуется Erlang/OTP R21 или выше. +Ветка `master` предназначена для последней версии 5, переключитесь на ветку `main-v4.3` для версии 4.3 и `main-v4.4` для версии 4.4. -Инструкция для сборки версии 4.3 и выше: +EMQX требует OTP 22 или 23 для версии 4.3 и OTP 24 для версий 4.4 и 5.0. ```bash git clone https://github.com/emqx/emqx.git @@ -99,7 +99,7 @@ make _build/emqx/rel/emqx/bin/emqx console ``` -Более ранние релизы могут быть собраны с помощью другого репозитория: +Версии до 4.2 (включительно) нужно собирать из другого репозитория: ```bash git clone https://github.com/emqx/emqx-rel.git @@ -108,79 +108,24 @@ make _build/emqx/rel/emqx/bin/emqx console ``` -## Первый запуск +### Сборка на Apple silicon (M1, M2) -Если emqx был собран из исходников: `cd _build/emqx/rel/emqx`. -Или перейдите в директорию, куда emqx был установлен из бинарного пакета. +Пакетный менеджер Homebrew, когда установлен на Apple silicon, [стал использовать другую домашнюю папку по умолчанию](https://github.com/Homebrew/brew/issues/9177), `/opt/homebrew` вместо `/usr/local`. В результате некоторые библиотеки перестали собираться автоматически. + +Касательно EMQX, сборка Erlang из исходного кода не найдёт библиотеку `unixodbc`, установленную с homebrew, без дополнительных действий: ```bash -# Запуск: -./bin/emqx start - -# Проверка статуса: -./bin/emqx_ctl status - -# Остановка: -./bin/emqx stop +brew install unixodbc kerl +sudo ln -s $(realpath $(brew --prefix unixodbc)) /usr/local/odbc +export CC="/usr/bin/gcc -I$(brew --prefix unixodbc)/include" +export LDFLAGS="-L$(brew --prefix unixodbc)/lib" +kerl build 24.3 +mkdir ~/.kerl/installations +kerl install 24.3 ~/.kerl/installations/24.3 +. ~/.kerl/installations/24.3/activate ``` -Веб-интерфейс брокера будет доступен по ссылке: http://localhost:18083 - -## Тесты - -### Полное тестирование - -``` -make eunit ct -``` - -### Запуск части тестов - -Пример: - -```bash -make apps/emqx_retainer-ct -``` - -### Dialyzer -##### Статический анализ всех приложений -``` -make dialyzer -``` - -##### Статический анализ части приложений (список через запятую) -``` -DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_authz make dialyzer -``` - -## Сообщество - -### FAQ - -Наиболее частые проблемы разобраны в [EMQX FAQ](https://www.emqx.io/docs/en/latest/faq/faq.html). - - -### Вопросы - -Задать вопрос или поделиться идеей можно в [GitHub Discussions](https://github.com/emqx/emqx/discussions). - -### Предложения - -Более масштабные предложения можно присылать в виде pull request в репозиторий [EIP](https://github.com/emqx/eip). - -### Разработка плагинов - -Инструкция по разработке собственных плагинов доступна по ссылке: [PLUGIN.md](./PLUGIN.md) - -## Спецификации стандарта MQTT - -Следующие ссылки содержат спецификации стандартов: - -[MQTT Version 3.1.1](https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html) - -[MQTT Version 5.0](https://docs.oasis-open.org/mqtt/mqtt/v5.0/cs02/mqtt-v5.0-cs02.html) - -[MQTT SN](https://www.oasis-open.org/committees/download.php/66091/MQTT-SN_spec_v1.2.pdf) +Дальше можно собирать emqx как обычно, с помощью `make`. ## Лицензия