Merge remote-tracking branch 'origin/dev/v4.4.9' into main-v4.4

.github/workflows/build_packages.yaml

@@ -262,7 +262,6 @@ jobs:
           --profile "${PROFILE}" \
           --pkgtype "${PACKAGE}" \
           --arch "${ARCH}" \
-          --system "${SYSTEM}" \
           --builder "ghcr.io/emqx/emqx-builder/4.4-19:${OTP}-${SYSTEM}"
     - uses: actions/upload-artifact@v1
       with:

.gitignore

@@ -60,3 +60,4 @@ erlang_ls.config
 # For direnv
 .envrc
 mix.lock
+.gitconfig.tmp

apps/emqx_auth_jwt/src/emqx_auth_jwt.erl

@@ -18,6 +18,7 @@
 
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -logger_header("[JWT]").
 
@@ -76,6 +77,7 @@ check_acl(_ClientInfo,
           _Topic,
           _NoMatchAction,
           _AclEnv) ->
+    ?tp(debug, no_jwt_claim, #{}),
     ignore.
 
 is_expired(Exp) when is_binary(Exp)  ->

apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl

@@ -31,7 +31,8 @@ init_per_testcase(TestCase, Config) ->
     emqx_ct_helpers:start_apps([emqx_auth_jwt], fun set_special_configs/1),
     Config.
 
-end_per_testcase(_Case, _Config) ->
+end_per_testcase(TestCase, Config) ->
+    try ?MODULE:TestCase('end', Config) catch _:_ -> ok end,
     emqx_ct_helpers:stop_apps([emqx_auth_jwt]).
 
 set_special_configs(emqx) ->
@@ -377,6 +378,44 @@ t_check_jwt_acl_no_acl_claim(_Config) ->
 
     ok = emqtt:disconnect(C).
 
+t_check_jwt_acl_no_jwt_claims_helper(_ClientInfo, _LastAuthResult) ->
+    {stop, #{auth_result => success, anonymous => false}}.
+t_check_jwt_acl_no_jwt_claims(init, _Config) ->
+    ok;
+t_check_jwt_acl_no_jwt_claims('end', _Config) ->
+    ok = emqx_hooks:del(
+            'client.authenticate',
+            {?MODULE, t_check_jwt_acl_no_jwt_claims_helper, []}
+          ).
+t_check_jwt_acl_no_jwt_claims(_Config) ->
+    %% bypass the jwt authentication checking
+    ok = emqx_hooks:add(
+            'client.authenticate',
+            {?MODULE, t_check_jwt_acl_no_jwt_claims_helper, []},
+            _Priority = 99999
+          ),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {username, <<"client1">>},
+                 {password, <<"password">>}]),
+    {ok, _} = emqtt:connect(C),
+
+    ok = snabbkaffe:start_trace(),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    {ok, _} = ?block_until(#{?snk_kind := no_jwt_claim}, 1000),
+    Trace = snabbkaffe:collect_trace(),
+    ?assertEqual(1, length(?of_kind(no_jwt_claim, Trace))),
+
+    snabbkaffe:stop(),
+    ok = emqtt:disconnect(C).
+
 t_check_jwt_acl_expire(init, _Config) ->
     application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
 t_check_jwt_acl_expire(_Config) ->

apps/emqx_management/src/emqx_mgmt.erl

@@ -467,9 +467,9 @@ list_listeners(Node) when Node =:= node() ->
     end, esockd:listeners()),
     Http = lists:map(fun({Protocol, Opts}) ->
         #{protocol        => Protocol,
-          listen_on       => proplists:get_value(port, Opts),
+          listen_on       => format_http_bind(Opts),
-          acceptors       => maps:get(num_acceptors,
+          acceptors       => maps:get( num_acceptors
-              proplists:get_value(transport_options, Opts, #{}), 0),
+                                     , proplists:get_value(transport_options, Opts, #{}), 0),
           max_conns       => proplists:get_value(max_connections, Opts),
           current_conns   => proplists:get_value(all_connections, Opts),
           shutdown_count  => []}
@@ -535,7 +535,7 @@ delete_banned(Who) ->
 
 
 %%--------------------------------------------------------------------
-%% Telemtry API
+%% Telemetry API
 %%--------------------------------------------------------------------
 
 -ifndef(EMQX_ENTERPRISE).
@@ -577,7 +577,7 @@ item(route, {Topic, Node}) ->
     #{topic => Topic, node => Node}.
 
 %%--------------------------------------------------------------------
-%% Internel Functions.
+%% Internal Functions.
 %%--------------------------------------------------------------------
 
 rpc_call(Node, Fun, Args) ->
@@ -610,3 +610,10 @@ has_any_ok(Results) ->
         true -> ok;
         false -> lists:last(Results)
     end.
+
+format_http_bind(Opts) ->
+    Port = proplists:get_value(port, Opts),
+    case proplists:get_value(ip, Opts) of
+        undefined -> Port;
+        IP -> {IP, Port}
+    end.

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.4.9-alpha.2"}).
+-define(EMQX_RELEASE, {opensource, "4.4.9"}).
 
 -else.
 

scripts/buildx.sh

@@ -20,10 +20,12 @@ help() {
     echo "--arch amd64|arm64:  Target arch to build the EMQ X package for"
     echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD"
     echo "--builder <BUILDER>: Builder image to pull"
-    echo "--system <SYSTEM>:   The target OS system the package is being built for, ex: debian11"
+    echo "                     E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian11"
-    echo "                     E.g. ghcr.io/emqx/emqx-builder/4.4-19:24.1.5-3-debian10"
+    echo "--ssh:               Pass ssh agent to the builder."
+    echo "                     Also configures git in container to use ssh instead of https to clone deps"
 }
 
+USE_SSH='no'
 while [ "$#" -gt 0 ]; do
     case $1 in
     -h|--help)
@@ -50,9 +52,9 @@ while [ "$#" -gt 0 ]; do
         ARCH="$2"
         shift 2
         ;;
-    --system)
+    --ssh)
-        SYSTEM="$2"
+        USE_SSH='yes'
-        shift 2
+        shift
         ;;
     *)
       echo "WARN: Unknown arg (ignored): $1"
@@ -72,18 +74,42 @@ if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then
     exit 1
 fi
 
+## Although we have 'deterministic' set in 'erl_opts', and foced overriding at project level,
+## still, some of the beams might be compiled (e.g. by erlang.mk) without this flag
+## longer file path means larger beam files
+## i.e. Keep the path to work dir short!
+DOCKER_WORKDIR='/emqx'
+
 cd "${SRC_DIR:-.}"
 
-set -x
+cat <<EOF >.gitconfig.tmp
-# $SYSTEM below is used by the `relup-base-vsns.escript` to correctly
+[core]
-# output the list of relup base versions.
+    sshCommand = ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
+[safe]
+    directory = $DOCKER_WORKDIR
+EOF
+
+if [ "$USE_SSH" = 'yes' ]; then
+    cat <<EOF >>.gitconfig.tmp
+[url "ssh://git@github.com/"]
+    insteadOf = https://github.com/
+EOF
+    # when passing ssh agent, we assume this command is executed locally not in ci, so add '-t' option
+    SSH_AGENT_OPTION="-t -e SSH_AUTH_SOCK=/ssh-agent -v ${SSH_AUTH_SOCK}:/ssh-agent"
+else
+    SSH_AGENT_OPTION=''
+fi
+
 docker info
 docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}"
+
+# shellcheck disable=SC2086
 docker run -i --rm \
-    -v "$(pwd)":/emqx \
+    -v "$(pwd)":$DOCKER_WORKDIR \
-    --workdir /emqx \
+    -v "$(pwd)/.gitconfig.tmp":/root/.gitconfig \
+    --workdir $DOCKER_WORKDIR \
     --platform="linux/$ARCH" \
     --user root \
-    -e SYSTEM="$SYSTEM" \
+    $SSH_AGENT_OPTION \
     "$BUILDER" \
-    bash -euc "git config --global --add safe.directory /emqx && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE"
+    bash -euc "mkdir -p _build && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE"

scripts/relup-base-packages.sh

@@ -70,7 +70,11 @@ otp_vsn_for() {
 for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
     filename="$PROFILE-${tag#[e|v]}-otp$(otp_vsn_for "$tag")-$SYSTEM-$ARCH.zip"
     url="https://packages.emqx.io/$DIR/$tag/$filename"
-    if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then
+    echo "downloading base package from ${url} ..."
+    if [ -f "$filename" ]; then
+        echo "file $filename already downloaded; skikpped"
+        continue
+    fi
     echo "downloading base package from ${url} ..."
     curl -L -o "${filename}" "${url}"
     if [ "$SYSTEM" != "centos6" ]; then
@@ -81,9 +85,6 @@ for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
         ## https://askubuntu.com/questions/1202208/checking-sha256-checksum
         echo "${SUMSTR}  ${filename}" | $SHASUM -c || exit 1
     fi
-    else
-        echo "file $filename already downloaded or doesn't exist in the archives; skipping it"
-    fi
 done
 
 popd

scripts/relup-base-vsns.escript

@@ -160,7 +160,7 @@ fetch_version(Vsn, VsnMap) ->
 
 filter_froms(Froms0, AvailableVersionsIndex) ->
     Froms1 =
-        case os:getenv("SYSTEM") of
+        case get_system() of
             %% we do not support relup for windows
             "windows" ->
                 [];
@@ -178,6 +178,14 @@ filter_froms(Froms0, AvailableVersionsIndex) ->
       fun(V) -> maps:get(V, AvailableVersionsIndex, false) end,
       Froms1).
 
+get_system() ->
+    case os:getenv("SYSTEM") of
+        false ->
+            string:trim(os:cmd("./scripts/get-distro.sh"));
+        System ->
+            System
+    end.
+
 %% assumes that's X.Y.Z, without pre-releases
 parse_vsn(VsnBin) ->
     {match, [Major0, Minor0, Patch0]} = re:run(VsnBin, "([0-9]+)\\.([0-9]+)\\.([0-9]+)",