From 30a02c74dc208238d513e8dd39edde8af9545cf2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 14 Sep 2022 20:17:47 +0200 Subject: [PATCH 01/10] build: allow passing ssh agent to docker run in buildx.sh --- scripts/buildx.sh | 48 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 98386efa0..5f4599765 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -20,10 +20,13 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian11" echo "--system : The target OS system the package is being built for, ex: debian11" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian10" + echo "--ssh: Pass ssh agent to the builder." + echo " Also configures git in container to use ssh instead of https to clone deps" } +USE_SSH='no' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -54,6 +57,10 @@ while [ "$#" -gt 0 ]; do SYSTEM="$2" shift 2 ;; + --ssh) + USE_SSH='yes' + shift + ;; *) echo "WARN: Unknown arg (ignored): $1" shift @@ -72,18 +79,45 @@ if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then exit 1 fi +## Although we have 'deterministic' set in 'erl_opts', and foced overriding at project level, +## still, some of the beams might be compiled (e.g. by erlang.mk) without this flag +## longer file path means larger beam files +## i.e. Keep the path to work dir short! +DOCKER_WORKDIR='/emqx' + cd "${SRC_DIR:-.}" -set -x -# $SYSTEM below is used by the `relup-base-vsns.escript` to correctly -# output the list of relup base versions. +cat <.gitconfig.tmp +[core] + sshCommand = ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no +[safe] + directory = $DOCKER_WORKDIR +EOF + +if [ "$USE_SSH" = 'yes' ]; then + cat <>.gitconfig.tmp +[url "ssh://git@github.com/"] + insteadOf = https://github.com/ +EOF + # when passing ssh agent, we assume this command is executed locally not in ci, so add '-t' option + SSH_AGENT_OPTION="-t -e SSH_AUTH_SOCK=/ssh-agent -v ${SSH_AUTH_SOCK}:/ssh-agent" +else + SSH_AGENT_OPTION='' +fi + docker info docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" + +# $SYSTEM below is used by the `relup-base-vsns.escript` to correctly +# output the list of relup base versions. +# shellcheck disable=SC2086 docker run -i --rm \ - -v "$(pwd)":/emqx \ - --workdir /emqx \ + -v "$(pwd)":$DOCKER_WORKDIR \ + -v "$(pwd)/.gitconfig.tmp":/root/.gitconfig \ + --workdir $DOCKER_WORKDIR \ --platform="linux/$ARCH" \ --user root \ -e SYSTEM="$SYSTEM" \ + $SSH_AGENT_OPTION \ "$BUILDER" \ - bash -euc "git config --global --add safe.directory /emqx && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" + bash -euc "mkdir -p _build && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" From 20ddd4557f7f767da0484be63fd5e4b2de985b64 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 14 Sep 2022 20:43:29 +0200 Subject: [PATCH 02/10] refactor: no need for the --system flag for buildx.sh --- .github/workflows/build_packages.yaml | 1 - scripts/buildx.sh | 8 -------- scripts/relup-base-vsns.escript | 10 +++++++++- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index b82fe03c5..9392735b5 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -262,7 +262,6 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --system "${SYSTEM}" \ --builder "ghcr.io/emqx/emqx-builder/4.4-19:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 with: diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 5f4599765..87e548e7e 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -21,7 +21,6 @@ help() { echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" echo " E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian11" - echo "--system : The target OS system the package is being built for, ex: debian11" echo "--ssh: Pass ssh agent to the builder." echo " Also configures git in container to use ssh instead of https to clone deps" } @@ -53,10 +52,6 @@ while [ "$#" -gt 0 ]; do ARCH="$2" shift 2 ;; - --system) - SYSTEM="$2" - shift 2 - ;; --ssh) USE_SSH='yes' shift @@ -108,8 +103,6 @@ fi docker info docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" -# $SYSTEM below is used by the `relup-base-vsns.escript` to correctly -# output the list of relup base versions. # shellcheck disable=SC2086 docker run -i --rm \ -v "$(pwd)":$DOCKER_WORKDIR \ @@ -117,7 +110,6 @@ docker run -i --rm \ --workdir $DOCKER_WORKDIR \ --platform="linux/$ARCH" \ --user root \ - -e SYSTEM="$SYSTEM" \ $SSH_AGENT_OPTION \ "$BUILDER" \ bash -euc "mkdir -p _build && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript index 6c618ca45..157f604cb 100755 --- a/scripts/relup-base-vsns.escript +++ b/scripts/relup-base-vsns.escript @@ -160,7 +160,7 @@ fetch_version(Vsn, VsnMap) -> filter_froms(Froms0, AvailableVersionsIndex) -> Froms1 = - case os:getenv("SYSTEM") of + case get_system() of %% we do not support relup for windows "windows" -> []; @@ -178,6 +178,14 @@ filter_froms(Froms0, AvailableVersionsIndex) -> fun(V) -> maps:get(V, AvailableVersionsIndex, false) end, Froms1). +get_system() -> + case os:getenv("SYSTEM") of + false -> + string:trim(os:cmd("./scripts/get-distro.sh")); + System -> + System + end. + %% assumes that's X.Y.Z, without pre-releases parse_vsn(VsnBin) -> {match, [Major0, Minor0, Patch0]} = re:run(VsnBin, "([0-9]+)\\.([0-9]+)\\.([0-9]+)", From bceafe509442b05dd757aac2f28a8410122eb5b0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 14 Sep 2022 20:49:21 +0200 Subject: [PATCH 03/10] chore: gitignore .gitconfig.tmp --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 13217dc00..c3f1ea6d1 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ erlang_ls.config # For direnv .envrc mix.lock +.gitconfig.tmp From 2fbb79f84f62b527a3554c1fb26a527944239012 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 14 Sep 2022 19:43:00 +0200 Subject: [PATCH 04/10] build: do not skip downloading relup base package do not skip downloading only because of curl error for known missing base packages, we should list them in the script --- scripts/relup-base-packages.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 01d2fa96b..b45e72cc3 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -57,15 +57,17 @@ for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" url="https://packages.emqx.io/$DIR/$tag/$filename" echo "downloading base package from ${url} ..." - if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then - curl -L -o "${filename}" "${url}" - if [ "$SYSTEM" != "centos6" ]; then - curl -L -o "${filename}.sha256" "${url}.sha256" - SUMSTR=$(cat "${filename}.sha256") - echo "got sha265sum: ${SUMSTR}" - ## https://askubuntu.com/questions/1202208/checking-sha256-checksum - echo "${SUMSTR} ${filename}" | $SHASUM -c || exit 1 - fi + if [ -f "$filename" ]; then + echo "file $filename already downloaded; skikpped" + continue + fi + curl -L -o "${filename}" "${url}" + if [ "$SYSTEM" != "centos6" ]; then + curl -L -o "${filename}.sha256" "${url}.sha256" + SUMSTR=$(cat "${filename}.sha256") + echo "got sha265sum: ${SUMSTR}" + ## https://askubuntu.com/questions/1202208/checking-sha256-checksum + echo "${SUMSTR} ${filename}" | $SHASUM -c || exit 1 fi done From ab21801449853d35ebd0e58bcaa6e255aab2a677 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 14 Sep 2022 20:36:15 +0200 Subject: [PATCH 05/10] ci: exclude v4.3.13 for centos8 --- scripts/relup-base-vsns.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 8f391b01b..8e1ff4b78 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -58,7 +58,13 @@ case "${EDITION}" in ;; esac +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" + while read -r git_tag; do + if [ "$SYSTEM" = 'centos8' ] && [ "$git_tag" = 'v4.3.13' ]; then + # This version for centos8 was broken and deleted from archive + continue + fi # shellcheck disable=SC2207 semver=($(parse_semver "$git_tag")) if [ "${#semver[@]}" -eq 3 ] && [ "${semver[2]}" -le "${CUR_SEMVER[2]}" ]; then From a90d289480c49e9eb1af5a969442f08ae3ac6aff Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 15 Sep 2022 09:35:48 +0200 Subject: [PATCH 06/10] chore: update release version to v4.3.20-alpha.2 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index ce07c1cdb..b51281a54 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.20-alpha.1"}). +-define(EMQX_RELEASE, {opensource, "4.3.20-alpha.2"}). -else. From b5594aea8a1ed5130bbffbbfc3977386e37ea30c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 15 Sep 2022 09:43:50 +0200 Subject: [PATCH 07/10] chore: bump to v4.4.9-alpha.3 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 3626ca4ef..a482aac49 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.9-alpha.2"}). +-define(EMQX_RELEASE, {opensource, "4.4.9-alpha.3"}). -else. From e4c2715842e098e78a59c9ee32d8d89c47fda7e4 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 16 Sep 2022 15:36:17 +0800 Subject: [PATCH 08/10] fix: add ip to dashboard listener_on --- apps/emqx_management/src/emqx_mgmt.erl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 948088f8e..c5062d477 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -458,7 +458,7 @@ list_listeners(Node) when Node =:= node() -> end, esockd:listeners()), Http = lists:map(fun({Protocol, Opts}) -> #{protocol => Protocol, - listen_on => proplists:get_value(port, Opts), + listen_on => format_http_bind(Opts), acceptors => maps:get( num_acceptors , proplists:get_value(transport_options, Opts, #{}), 0), max_conns => proplists:get_value(max_connections, Opts), @@ -528,7 +528,7 @@ delete_banned(Who) -> %%-------------------------------------------------------------------- -%% Telemtry API +%% Telemetry API %%-------------------------------------------------------------------- -ifndef(EMQX_ENTERPRISE). @@ -570,7 +570,7 @@ item(route, {Topic, Node}) -> #{topic => Topic, node => Node}. %%-------------------------------------------------------------------- -%% Internel Functions. +%% Internal Functions. %%-------------------------------------------------------------------- rpc_call(Node, Fun, Args) -> @@ -603,3 +603,10 @@ max_row_limit() -> application:get_env(?APP, max_row_limit, ?MAX_ROW_LIMIT). table_size(Tab) -> ets:info(Tab, size). + +format_http_bind(Opts) -> + Port = proplists:get_value(port, Opts), + case proplists:get_value(ip, Opts) of + undefined -> Port; + IP -> {IP, Port} + end. From 89ed4a350fd2efe5d9678075d59979acd9d0e4e4 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 16 Sep 2022 17:41:25 +0800 Subject: [PATCH 09/10] chore: avoid generating crash logs for hook execution --- apps/emqx_auth_jwt/src/emqx_auth_jwt.erl | 6 ++- .../test/emqx_auth_jwt_SUITE.erl | 41 ++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl index 1259bd263..26fd34365 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl @@ -18,6 +18,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[JWT]"). @@ -70,7 +71,10 @@ check_acl(ClientInfo = #{jwt_claims := Claims}, _ -> ?DEBUG("no_acl_jwt_claim", []), ignore - end. + end; +check_acl(_ClientInfo, _PubSub, _Topic, _NoMatchAction, _Env) -> + ?tp(debug, no_jwt_claim, #{}), + ignore. is_expired(Exp) when is_binary(Exp) -> case string_to_number(Exp) of diff --git a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl index 596b829a6..eb6d3c195 100644 --- a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl @@ -31,7 +31,8 @@ init_per_testcase(TestCase, Config) -> emqx_ct_helpers:start_apps([emqx_auth_jwt], fun set_special_configs/1), Config. -end_per_testcase(_Case, _Config) -> +end_per_testcase(TestCase, Config) -> + try ?MODULE:TestCase('end', Config) catch _:_ -> ok end, emqx_ct_helpers:stop_apps([emqx_auth_jwt]). set_special_configs(emqx) -> @@ -377,6 +378,44 @@ t_check_jwt_acl_no_acl_claim(_Config) -> ok = emqtt:disconnect(C). +t_check_jwt_acl_no_jwt_claims_helper(_ClientInfo, _LastAuthResult) -> + {stop, #{auth_result => success, anonymous => false}}. +t_check_jwt_acl_no_jwt_claims(init, _Config) -> + ok; +t_check_jwt_acl_no_jwt_claims('end', _Config) -> + ok = emqx_hooks:del( + 'client.authenticate', + {?MODULE, t_check_jwt_acl_no_jwt_claims_helper, []} + ). +t_check_jwt_acl_no_jwt_claims(_Config) -> + %% bypass the jwt authentication checking + ok = emqx_hooks:add( + 'client.authenticate', + {?MODULE, t_check_jwt_acl_no_jwt_claims_helper, []}, + _Priority = 99999 + ), + + {ok, C} = emqtt:start_link( + [{clean_start, true}, + {proto_ver, v5}, + {client_id, <<"client1">>}, + {username, <<"client1">>}, + {password, <<"password">>}]), + {ok, _} = emqtt:connect(C), + + ok = snabbkaffe:start_trace(), + + ?assertMatch( + {ok, #{}, [?RC_NOT_AUTHORIZED]}, + emqtt:subscribe(C, <<"a/b">>, 0)), + + {ok, _} = ?block_until(#{?snk_kind := no_jwt_claim}, 1000), + Trace = snabbkaffe:collect_trace(), + ?assertEqual(1, length(?of_kind(no_jwt_claim, Trace))), + + snabbkaffe:stop(), + ok = emqtt:disconnect(C). + t_check_jwt_acl_expire(init, _Config) -> application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]). t_check_jwt_acl_expire(_Config) -> From 889380191073d6021cb5f8fc71468a6b455f5f16 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 16 Sep 2022 13:33:17 +0200 Subject: [PATCH 10/10] chore: bump release version to v4.3.20 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index b51281a54..007c81aaf 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.20-alpha.2"}). +-define(EMQX_RELEASE, {opensource, "4.3.20"}). -else.