diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index b82fe03c5..9392735b5 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -262,7 +262,6 @@ jobs: --profile "${PROFILE}" \ --pkgtype "${PACKAGE}" \ --arch "${ARCH}" \ - --system "${SYSTEM}" \ --builder "ghcr.io/emqx/emqx-builder/4.4-19:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 with: diff --git a/.gitignore b/.gitignore index 13217dc00..c3f1ea6d1 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ erlang_ls.config # For direnv .envrc mix.lock +.gitconfig.tmp diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl index 54716df1c..33002a2f5 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl @@ -18,6 +18,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[JWT]"). @@ -76,6 +77,7 @@ check_acl(_ClientInfo, _Topic, _NoMatchAction, _AclEnv) -> + ?tp(debug, no_jwt_claim, #{}), ignore. is_expired(Exp) when is_binary(Exp) -> diff --git a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl index 596b829a6..eb6d3c195 100644 --- a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl @@ -31,7 +31,8 @@ init_per_testcase(TestCase, Config) -> emqx_ct_helpers:start_apps([emqx_auth_jwt], fun set_special_configs/1), Config. -end_per_testcase(_Case, _Config) -> +end_per_testcase(TestCase, Config) -> + try ?MODULE:TestCase('end', Config) catch _:_ -> ok end, emqx_ct_helpers:stop_apps([emqx_auth_jwt]). set_special_configs(emqx) -> @@ -377,6 +378,44 @@ t_check_jwt_acl_no_acl_claim(_Config) -> ok = emqtt:disconnect(C). +t_check_jwt_acl_no_jwt_claims_helper(_ClientInfo, _LastAuthResult) -> + {stop, #{auth_result => success, anonymous => false}}. +t_check_jwt_acl_no_jwt_claims(init, _Config) -> + ok; +t_check_jwt_acl_no_jwt_claims('end', _Config) -> + ok = emqx_hooks:del( + 'client.authenticate', + {?MODULE, t_check_jwt_acl_no_jwt_claims_helper, []} + ). +t_check_jwt_acl_no_jwt_claims(_Config) -> + %% bypass the jwt authentication checking + ok = emqx_hooks:add( + 'client.authenticate', + {?MODULE, t_check_jwt_acl_no_jwt_claims_helper, []}, + _Priority = 99999 + ), + + {ok, C} = emqtt:start_link( + [{clean_start, true}, + {proto_ver, v5}, + {client_id, <<"client1">>}, + {username, <<"client1">>}, + {password, <<"password">>}]), + {ok, _} = emqtt:connect(C), + + ok = snabbkaffe:start_trace(), + + ?assertMatch( + {ok, #{}, [?RC_NOT_AUTHORIZED]}, + emqtt:subscribe(C, <<"a/b">>, 0)), + + {ok, _} = ?block_until(#{?snk_kind := no_jwt_claim}, 1000), + Trace = snabbkaffe:collect_trace(), + ?assertEqual(1, length(?of_kind(no_jwt_claim, Trace))), + + snabbkaffe:stop(), + ok = emqtt:disconnect(C). + t_check_jwt_acl_expire(init, _Config) -> application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]). t_check_jwt_acl_expire(_Config) -> diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index fbfc70d9c..fe38d84c0 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -467,9 +467,9 @@ list_listeners(Node) when Node =:= node() -> end, esockd:listeners()), Http = lists:map(fun({Protocol, Opts}) -> #{protocol => Protocol, - listen_on => proplists:get_value(port, Opts), - acceptors => maps:get(num_acceptors, - proplists:get_value(transport_options, Opts, #{}), 0), + listen_on => format_http_bind(Opts), + acceptors => maps:get( num_acceptors + , proplists:get_value(transport_options, Opts, #{}), 0), max_conns => proplists:get_value(max_connections, Opts), current_conns => proplists:get_value(all_connections, Opts), shutdown_count => []} @@ -535,7 +535,7 @@ delete_banned(Who) -> %%-------------------------------------------------------------------- -%% Telemtry API +%% Telemetry API %%-------------------------------------------------------------------- -ifndef(EMQX_ENTERPRISE). @@ -577,7 +577,7 @@ item(route, {Topic, Node}) -> #{topic => Topic, node => Node}. %%-------------------------------------------------------------------- -%% Internel Functions. +%% Internal Functions. %%-------------------------------------------------------------------- rpc_call(Node, Fun, Args) -> @@ -610,3 +610,10 @@ has_any_ok(Results) -> true -> ok; false -> lists:last(Results) end. + +format_http_bind(Opts) -> + Port = proplists:get_value(port, Opts), + case proplists:get_value(ip, Opts) of + undefined -> Port; + IP -> {IP, Port} + end. diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 3626ca4ef..d6c9c9078 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.9-alpha.2"}). +-define(EMQX_RELEASE, {opensource, "4.4.9"}). -else. diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 98386efa0..87e548e7e 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -20,10 +20,12 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo "--system : The target OS system the package is being built for, ex: debian11" - echo " E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian11" + echo "--ssh: Pass ssh agent to the builder." + echo " Also configures git in container to use ssh instead of https to clone deps" } +USE_SSH='no' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -50,9 +52,9 @@ while [ "$#" -gt 0 ]; do ARCH="$2" shift 2 ;; - --system) - SYSTEM="$2" - shift 2 + --ssh) + USE_SSH='yes' + shift ;; *) echo "WARN: Unknown arg (ignored): $1" @@ -72,18 +74,42 @@ if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then exit 1 fi +## Although we have 'deterministic' set in 'erl_opts', and foced overriding at project level, +## still, some of the beams might be compiled (e.g. by erlang.mk) without this flag +## longer file path means larger beam files +## i.e. Keep the path to work dir short! +DOCKER_WORKDIR='/emqx' + cd "${SRC_DIR:-.}" -set -x -# $SYSTEM below is used by the `relup-base-vsns.escript` to correctly -# output the list of relup base versions. +cat <.gitconfig.tmp +[core] + sshCommand = ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no +[safe] + directory = $DOCKER_WORKDIR +EOF + +if [ "$USE_SSH" = 'yes' ]; then + cat <>.gitconfig.tmp +[url "ssh://git@github.com/"] + insteadOf = https://github.com/ +EOF + # when passing ssh agent, we assume this command is executed locally not in ci, so add '-t' option + SSH_AGENT_OPTION="-t -e SSH_AUTH_SOCK=/ssh-agent -v ${SSH_AUTH_SOCK}:/ssh-agent" +else + SSH_AGENT_OPTION='' +fi + docker info docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" + +# shellcheck disable=SC2086 docker run -i --rm \ - -v "$(pwd)":/emqx \ - --workdir /emqx \ + -v "$(pwd)":$DOCKER_WORKDIR \ + -v "$(pwd)/.gitconfig.tmp":/root/.gitconfig \ + --workdir $DOCKER_WORKDIR \ --platform="linux/$ARCH" \ --user root \ - -e SYSTEM="$SYSTEM" \ + $SSH_AGENT_OPTION \ "$BUILDER" \ - bash -euc "git config --global --add safe.directory /emqx && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" + bash -euc "mkdir -p _build && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 60785adc8..cb23dcabe 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -70,19 +70,20 @@ otp_vsn_for() { for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-${tag#[e|v]}-otp$(otp_vsn_for "$tag")-$SYSTEM-$ARCH.zip" url="https://packages.emqx.io/$DIR/$tag/$filename" - if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then - echo "downloading base package from ${url} ..." - curl -L -o "${filename}" "${url}" - if [ "$SYSTEM" != "centos6" ]; then - echo "downloading sha256 sum from ${url}.sha256 ..." - curl -L -o "${filename}.sha256" "${url}.sha256" - SUMSTR=$(cat "${filename}.sha256") - echo "got sha265sum: ${SUMSTR}" - ## https://askubuntu.com/questions/1202208/checking-sha256-checksum - echo "${SUMSTR} ${filename}" | $SHASUM -c || exit 1 - fi - else - echo "file $filename already downloaded or doesn't exist in the archives; skipping it" + echo "downloading base package from ${url} ..." + if [ -f "$filename" ]; then + echo "file $filename already downloaded; skikpped" + continue + fi + echo "downloading base package from ${url} ..." + curl -L -o "${filename}" "${url}" + if [ "$SYSTEM" != "centos6" ]; then + echo "downloading sha256 sum from ${url}.sha256 ..." + curl -L -o "${filename}.sha256" "${url}.sha256" + SUMSTR=$(cat "${filename}.sha256") + echo "got sha265sum: ${SUMSTR}" + ## https://askubuntu.com/questions/1202208/checking-sha256-checksum + echo "${SUMSTR} ${filename}" | $SHASUM -c || exit 1 fi done diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript index 6c618ca45..157f604cb 100755 --- a/scripts/relup-base-vsns.escript +++ b/scripts/relup-base-vsns.escript @@ -160,7 +160,7 @@ fetch_version(Vsn, VsnMap) -> filter_froms(Froms0, AvailableVersionsIndex) -> Froms1 = - case os:getenv("SYSTEM") of + case get_system() of %% we do not support relup for windows "windows" -> []; @@ -178,6 +178,14 @@ filter_froms(Froms0, AvailableVersionsIndex) -> fun(V) -> maps:get(V, AvailableVersionsIndex, false) end, Froms1). +get_system() -> + case os:getenv("SYSTEM") of + false -> + string:trim(os:cmd("./scripts/get-distro.sh")); + System -> + System + end. + %% assumes that's X.Y.Z, without pre-releases parse_vsn(VsnBin) -> {match, [Major0, Minor0, Patch0]} = re:run(VsnBin, "([0-9]+)\\.([0-9]+)\\.([0-9]+)",