Merge remote-tracking branch 'ce/main-v4.3' into merge-main-v4.3-into-v4.4

.github/workflows/build_packages.yaml

@@ -5,6 +5,9 @@ concurrency:
   cancel-in-progress: true
 
 on:
+  push:
+    branches:
+      - 'main-v4.**'
   schedule:
     - cron:  '0 */6 * * *'
   release:
@@ -20,7 +23,6 @@ jobs:
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
-      old_vsns: ${{ steps.set_profile.outputs.old_vsns }}
 
     steps:
       - uses: actions/checkout@v2
@@ -33,12 +35,8 @@ jobs:
         working-directory: source
         run: |
           if make emqx-ee --dry-run > /dev/null 2>&1; then
-            old_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)"
-            echo "::set-output name=old_vsns::$old_vsns"
             echo "::set-output name=profiles::[\"emqx-ee\"]"
           else
-            old_vsns="$(./scripts/relup-base-vsns.sh community | xargs)"
-            echo "::set-output name=old_vsns::$old_vsns"
             echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
           fi
       - name: get_all_deps
@@ -254,42 +252,6 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: downloads old emqx zip packages
-      if: matrix.package == 'zip'
-      env:
-        OTP_VSN: ${{ matrix.otp }}
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch }}
-        SYSTEM: ${{ matrix.os }}
-        OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
-      working-directory: source
-      run: |
-        set -e -x -u
-        broker=$PROFILE
-        if [ $PROFILE = "emqx" ];then
-            broker="emqx-ce"
-        fi
-        if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then
-            export ARCH="arm"
-        fi
-        if [ "$SYSTEM" = 'centos7' ]; then
-          DISTRO='el7'
-        elif [ "$SYSTEM" = 'rockylinux8' ]; then
-          DISTRO='el8'
-        else
-          DISTRO=${SYSTEM}
-        fi
-        mkdir -p _upgrade_base
-        cd _upgrade_base
-        old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-        for tag in ${old_vsns[@]}; do
-          package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${DISTRO}-${ARCH}"
-          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256
-            echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1
-          fi
-        done
     - name: build emqx packages
       env:
         OTP: ${{ matrix.otp }}
@@ -427,6 +389,7 @@ jobs:
     needs: [prepare, mac, linux, docker]
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         otp:

.github/workflows/code_style_check.yaml

@@ -1,24 +0,0 @@
-name: Code style check
-
-on: [pull_request]
-
-jobs:
-  build:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 1000
-      - name: Set git token
-        if: endsWith(github.repository, 'enterprise')
-        run: |
-          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-          git config --global credential.helper store
-      - name: Run elvis check
-        run: |
-          set -e
-          if [ -f EMQX_ENTERPRISE ]; then
-            ./scripts/elvis-check.sh $GITHUB_BASE_REF emqx-enterprise
-          else
-            ./scripts/elvis-check.sh $GITHUB_BASE_REF emqx
-          fi

.github/workflows/run_fvt_tests.yaml

@@ -290,6 +290,7 @@ jobs:
           name: Checkout
           with:
             path: emqx
+            fetch-depth: 0
         - name: Prepare credentials
           run: |
             if [ "$PROFILE" = "emqx-ee" ]; then
@@ -297,15 +298,6 @@ jobs:
               git config --global credential.helper store
               echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
             fi
-        - name: Download bases
-          run: |
-            set -e -x -u
-            mkdir -p emqx/_upgrade_base
-            cd emqx/_upgrade_base
-            old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-            for old_vsn in ${old_vsns[@]}; do
-              wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${{ matrix.otp }}-ubuntu20.04-amd64.zip
-            done
         - name: Build emqx
           run: make -C emqx ${PROFILE}-zip
         - uses: actions/upload-artifact@v2

CHANGES-4.3.md

@@ -10,6 +10,18 @@ File format:
 - One list item per change topic
   Change log ends with a list of github PRs
 
+## v4.3.13
+
+### Enhancements
+
+* CLI `emqx_ctl pem_cache clean` to force purge x509 certificate cache,
+  to force an immediate reload of all certificates after the files are updated on disk.
+
+### Bug fixes
+
+* Fix case where publishing to a non-existent topic alias would crash the connection [#6979]
+* Fix HTTP-API 500 error on querying the lwm2m client list on the another node [#7009]
+
 ## v4.3.12
 ### Important changes
 

Makefile

@@ -2,6 +2,7 @@ $(shell $(CURDIR)/scripts/git-hooks-init.sh)
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
+export EMQX_RELUP ?= true
 export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-5:24.1.5-3-alpine3.14
 export EMQX_DEFAULT_RUNNER = alpine:3.14
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
@@ -130,10 +131,19 @@ COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS)
 $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
 	@$(BUILD) $(subst -rel,,$(@)) rel
 
+## download relup base packages
+.PHONY: $(REL_PROFILES:%=%-relup-downloads)
+define download-relup-packages
+$1-relup-downloads:
+	@if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-base-packages.sh $1; fi
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt))))
+
 ## relup target is to create relup instructions
 .PHONY: $(REL_PROFILES:%=%-relup)
 define gen-relup-target
-$1-relup: $(COMMON_DEPS)
+$1-relup: $1-relup-downloads $(COMMON_DEPS)
 	@$(BUILD) $1 relup
 endef
 ALL_ZIPS = $(REL_PROFILES)

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_mqtt,
  [{description, "EMQ X Bridge to MQTT Broker"},
-  {vsn, "4.3.3"}, % strict semver, bump manually!
+  {vsn, "4.3.4"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,replayq,emqtt]},

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src

@@ -1,21 +1,31 @@
 %% -*-: erlang -*-
 
-{"4.3.3",
+{"4.3.4",
   [
+    {<<"4.3.3">>, [
+      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}
+    ]},
     {<<"4.3.[1-2]">>, [
+      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []},
       {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []}
     ]},
     {"4.3.0", [
+      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []},
       {load_module, emqx_bridge_worker, brutal_purge, soft_purge, []},
       {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []}
     ]},
     {<<".*">>, []}
   ],
   [
+    {<<"4.3.3">>, [
+      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}
+    ]},
     {<<"4.3.[1-2]">>, [
+      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []},
       {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []}
     ]},
     {"4.3.0", [
+      {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []},
       {load_module, emqx_bridge_worker, brutal_purge, soft_purge, []},
       {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []}
     ]},

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl

@@ -144,7 +144,7 @@ send(#{client_pid := ClientPid} = Conn, [Msg | Rest], PktIds) ->
         {ok, PktId} ->
             send(Conn, Rest, [PktId | PktIds]);
         {error, Reason} ->
-            %% NOTE: There is no partial sucess of a batch and recover from the middle
+            %% NOTE: There is no partial success of a batch and recover from the middle
             %% only to retry all messages in one batch
             {error, Reason}
     end.
@@ -154,7 +154,7 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent)
        RC =:= ?RC_NO_MATCHING_SUBSCRIBERS ->
     Parent ! {batch_ack, PktId}, ok;
 handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) ->
-    ?LOG(warning, "Publish ~p to remote node falied, reason_code: ~p", [PktId, RC]).
+    ?LOG(warning, "Publish ~p to remote node failed, reason_code: ~p", [PktId, RC]).
 
 handle_publish(Msg, Mountpoint) ->
     emqx_broker:publish(emqx_bridge_msg:to_broker_msg(Msg, Mountpoint)).

apps/emqx_lwm2m/src/emqx_lwm2m.app.src

@@ -1,7 +1,8 @@
 {application,emqx_lwm2m,
  [{description,"EMQ X LwM2M Gateway"},
-              {vsn, "4.3.5"}, % strict semver, bump manually!
+  {vsn, "4.3.6"}, % strict semver, bump manually!
   {modules,[]},
   {registered,[emqx_lwm2m_sup]},
   {applications,[kernel,stdlib,lwm2m_coap]},
-              {mod,{emqx_lwm2m_app,[]}}]}.
+  {mod,{emqx_lwm2m_app,[]}}]
+}.

apps/emqx_lwm2m/src/emqx_lwm2m.appup.src

@@ -4,13 +4,17 @@
     [{restart_application,emqx_lwm2m}]},
    {"4.3.2",
     [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
-     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
-   {"4.3.3",[{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]},
-   {"4.3.4",[{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}]}],
+   {<<"4\\.3\\.[3-5]">>,
+    [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]}],
   [{<<"4\\.3\\.[0-1]">>,
     [{restart_application,emqx_lwm2m}]},
    {"4.3.2",
     [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
-     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
-   {"4.3.3",[{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]},
-   {"4.3.4",[{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]}]}]}.
+   {<<"4\\.3\\.[3-5]">>,
+    [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]}]}.

apps/emqx_lwm2m/src/emqx_lwm2m_api.erl

@@ -51,7 +51,7 @@
         ]).
 
 list(#{node := Node }, Params) ->
-    case Node = node() of
+    case Node =:= node() of
         true -> list(#{}, Params);
         _ -> rpc_call(Node, list, [#{}, Params])
     end;
@@ -61,7 +61,7 @@ list(#{}, _Params) ->
     return({ok, format(Channels)}).
 
 lookup_cmd(#{ep := Ep, node := Node}, Params) ->
-    case Node = node() of
+    case Node =:= node() of
         true -> lookup_cmd(#{ep => Ep}, Params);
         _ -> rpc_call(Node, lookup_cmd, [#{ep => Ep}, Params])
     end;

apps/emqx_management/src/emqx_management.app.src

@@ -1,6 +1,6 @@
 {application, emqx_management,
  [{description, "EMQ X Management API and CLI"},
-  {vsn, "4.4.1"}, % strict semver, bump manually!
+  {vsn, "4.4.2"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_management_sup]},
   {applications, [kernel,stdlib,minirest]},

apps/emqx_management/src/emqx_mgmt.erl

@@ -55,6 +55,10 @@
         , set_keepalive/2
         ]).
 
+-export([ clean_pem_cache/0
+        , clean_pem_cache/1
+        ]).
+
 %% Internal funcs
 -export([call_client/3]).
 
@@ -123,8 +127,6 @@
 -define(MAX_ROW_LIMIT, 10000).
 
 -define(APP, emqx_management).
-
--elvis([{elvis_style, god_modules, disable}]).
 %%--------------------------------------------------------------------
 %% Node Info
 %%--------------------------------------------------------------------
@@ -257,15 +259,17 @@ clean_acl_cache(Node, ClientId) ->
     rpc_call(Node, clean_acl_cache, [Node, ClientId]).
 
 clean_acl_cache_all() ->
-    Results = [{Node, clean_acl_cache_all(Node)} || Node <- ekka_mnesia:running_nodes()],
+    for_nodes(fun clean_acl_cache_all/1).
-    case lists:filter(fun({_Node, Item}) -> Item =/= ok end, Results) of
+
+for_nodes(F) ->
+    Results = [{Node, F(Node)} || Node <- ekka_mnesia:running_nodes()],
+    case lists:filter(fun({_Node, Res}) -> Res =/= ok end, Results) of
         []  -> ok;
         BadNodes -> {error, BadNodes}
     end.
 
 clean_acl_cache_all(Node) when Node =:= node() ->
     emqx_acl_cache:drain_cache();
-
 clean_acl_cache_all(Node) ->
     rpc_call(Node, clean_acl_cache_all, [Node]).
 
@@ -280,6 +284,15 @@ set_keepalive(ClientId, Interval)when Interval >= 0 andalso Interval =< 65535 ->
 set_keepalive(_ClientId, _Interval) ->
     {error, ?ERROR2, <<"mqtt3.1.1 specification: keepalive must between 0~65535">>}.
 
+clean_pem_cache() ->
+    for_nodes(fun clean_pem_cache/1).
+
+clean_pem_cache(Node) when Node =:= node() ->
+    _ = ssl_pem_cache:clear(),
+    ok;
+clean_pem_cache(Node) ->
+    rpc_call(Node, ?FUNCTION_NAME, [Node]).
+
 %% @private
 call_client(ClientId, Req) ->
     Results = [call_client(Node, ClientId, Req) || Node <- ekka_mnesia:running_nodes()],

apps/emqx_management/src/emqx_mgmt_cli.erl

@@ -43,6 +43,7 @@
         , mgmt/1
         , data/1
         , acl/1
+        , pem_cache/1
         ]).
 
 -define(PROC_INFOKEYS, [status,
@@ -676,21 +677,11 @@ data(_) ->
 %% @doc acl Command
 
 acl(["cache-clean", "node", Node]) ->
-    case emqx_mgmt:clean_acl_cache_all(erlang:list_to_existing_atom(Node)) of
+    with_log(fun() -> for_node(fun emqx_mgmt:clean_acl_cache_all/1, Node) end,
-        ok ->
+             "ACL cache drain start");
-            emqx_ctl:print("ACL cache drain started on node ~s.~n", [Node]);
-        {error, Reason} ->
-            emqx_ctl:print("ACL drain failed on node ~s: ~0p.~n", [Node, Reason])
-    end;
-
 acl(["cache-clean", "all"]) ->
-    case emqx_mgmt:clean_acl_cache_all() of
+    with_log(fun emqx_mgmt:clean_acl_cache_all/1,
-        ok ->
+             "ACL cache drain start");
-            emqx_ctl:print("Started ACL cache drain in all nodes~n");
-        {error, Reason} ->
-            emqx_ctl:print("ACL cache-clean failed: ~p.~n", [Reason])
-    end;
-
 acl(["cache-clean", ClientId]) ->
     emqx_mgmt:clean_acl_cache(ClientId);
 
@@ -700,6 +691,15 @@ acl(_) ->
                     {"acl cache-clean <ClientId>",      "Clears acl cache for given client"}
                    ]).
 
+pem_cache(["clean", "all"]) ->
+    with_log(fun emqx_mgmt:clean_pem_cache/0, "PEM cache clean");
+pem_cache(["clean", "node", Node]) ->
+    with_log(fun() -> for_node(fun emqx_mgmt:clean_pem_cache/1, Node) end, "PEM cache clean");
+pem_cache(_) ->
+    emqx_ctl:usage([{"pem_cache clean all",         "Clears x509 certificate cache on all nodes"},
+                    {"pem_cache clean node <Node>", "Clears x509 certificate cache on given node"}
+                   ]).
+
 %%--------------------------------------------------------------------
 %% Dump ETS
 %%--------------------------------------------------------------------
@@ -829,3 +829,20 @@ http_mod_name(Name) -> Name.
 print_app_info({AppId, AppSecret, Name, Desc, Status, Expired}) ->
     emqx_ctl:print("app_id: ~s, secret: ~s, name: ~s, desc: ~s, status: ~s, expired: ~p~n",
         [AppId, AppSecret, Name, Desc, Status, Expired]).
+
+for_node(Fun, Node) ->
+    try list_to_existing_atom(Node) of
+        NodeAtom ->
+            Fun(NodeAtom)
+    catch
+        error : badarg ->
+            {error, unknown_node}
+    end.
+
+with_log(Fun, Msg) ->
+    case Fun() of
+        ok ->
+            emqx_ctl:print("~s OK~n", [Msg]);
+        {error, Reason} ->
+            emqx_ctl:print("~s FAILED~n~p~n", [Msg, Reason])
+    end.

apps/emqx_stomp/src/emqx_stomp.app.src

@@ -1,6 +1,6 @@
 {application, emqx_stomp,
  [{description, "EMQ X Stomp Protocol Plugin"},
-  {vsn, "4.3.4"}, % strict semver, bump manually!
+  {vsn, "4.3.5"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_stomp_sup]},
   {applications, [kernel,stdlib]},

apps/emqx_stomp/src/emqx_stomp.appup.src

@@ -1,8 +1,15 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.3.3",[{load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
+  [{"4.3.4",
-   {"4.3.2",
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
+   {"4.3.3",
+    [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
@@ -12,14 +19,20 @@
     [{restart_application,emqx_stomp},
      {apply,{emqx_stomp,force_clear_after_app_stoped,[]}}]},
    {<<".*">>,[]}],
-  [{"4.3.3",[{load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
+  [{"4.3.4",
-   {"4.3.2",
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
+   {"4.3.3",
+    [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
     [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]},
      {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]},
-   {"4.3.0",
+   {"4.3.0",[{restart_application,emqx_stomp}]},
-    [{restart_application,emqx_stomp}]},
    {<<".*">>,[]}]}.

apps/emqx_stomp/src/emqx_stomp_connection.erl

@@ -91,8 +91,6 @@
 
 -define(ENABLED(X), (X =/= undefined)).
 
--elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_stomp_connection]}}]).
-
 -dialyzer({nowarn_function, [ ensure_stats_timer/2
                             ]}).
 

apps/emqx_stomp/src/emqx_stomp_protocol.erl

@@ -108,8 +108,6 @@
                             , init/2
                             ]}).
 
--elvis([{elvis_style, dont_repeat_yourself, disable}]).
-
 -type(pstate() :: #pstate{}).
 
 %% @doc Init protocol

build

@@ -4,6 +4,9 @@
 # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
 # arg2: artifact, e.g. rel | relup | zip | pkg
 
+if [[ -n "$DEBUG" ]]; then
+    set -x
+fi
 set -euo pipefail
 
 PROFILE="$1"
@@ -50,7 +53,7 @@ case "$UNAME" in
         ARCH=arm
         ;;
 esac
-# used in -pkg Makefile:s
+# used in -pkg Makefile
 export ARCH
 
 log() {

deploy/docker/Dockerfile

@@ -23,6 +23,8 @@ COPY . /emqx
 ARG PKG_VSN
 ARG EMQX_NAME=emqx
 
+ENV EMQX_RELUP=false
+
 RUN cd /emqx \
     && rm -rf _build/$EMQX_NAME/lib \
     && make $EMQX_NAME

elvis.config

@@ -1,47 +0,0 @@
-%% -*-: erlang -*-
-[
- {
-   elvis,
-   [
-    {config,
-     [
-      #{dirs => ["src", "apps/**/src", "lib-ce/**/src", "lib-ee/**/src"],
-        filter => "*.erl",
-        ruleset => erl_files,
-        rules => [
-            {elvis_style, state_record_and_type, disable},
-            {elvis_style, no_common_caveats_call, #{}},
-            {elvis_style, no_debug_call, #{ debug_functions => [ {ct, pal}
-                                                               , {ct, print}
-                                                               ]}},
-            {elvis_style, operator_spaces, #{rules => [{right, "|"},
-                                                       {left, "|"},
-                                                       {right, "||"},
-                                                       {left, "||"}]}}
-           ]
-        },
-      #{dirs => ["test", "apps/**/test", "lib-ce/**/src"],
-        filter => "*.erl",
-        rules => [
-           {elvis_text_style, line_length, #{ limit => 100
-                                             , skip_comments => false }},
-           {elvis_style, dont_repeat_yourself, #{ min_complexity => 100 }}
-          ]
-        },
-      #{dirs => ["."],
-        filter => "Makefile",
-        ruleset => makefiles
-       },
-      #{dirs => ["."],
-        filter => "rebar.config",
-        ruleset => rebar_config
-       },
-      #{dirs => ["."],
-        filter => "elvis.config",
-        ruleset => elvis_config
-       }
-     ]
-    }
-   ]
- }
-].

etc/emqx_cloud/vm.args

@@ -1,5 +1,5 @@
 ######################################################################
-## Erlang VM Args for EMQ X Broker
+## Erlang VM Args
 ######################################################################
 
 ## NOTE:

etc/emqx_edge/vm.args

@@ -1,5 +1,5 @@
 ######################################################################
-## Erlang VM Args for EMQ X Edge
+## Erlang VM Args
 ######################################################################
 
 ## NOTE:

pkg-vsn.sh

@@ -18,7 +18,7 @@ RELEASE="$(grep -E "define.+EMQX_RELEASE.+${EDITION}" include/emqx_release.hrl |
 git_exact_vsn() {
     local tag
     tag="$(git describe --tags --match "[e|v]*" --exact 2>/dev/null)"
-    echo "$tag" | sed 's/^[v|e]//g'
+    echo "${tag#[e|v]}"
 }
 
 GIT_EXACT_VSN="$(git_exact_vsn)"

scripts/elvis-check.sh

@@ -1,84 +0,0 @@
-#!/usr/bin/env bash
-
-## This script checks style of changed files.
-## Expect argument 1 to be the git compare base.
-
-set -euo pipefail
-
-elvis_version='1.0.0-emqx-2'
-
-base="${1:-}"
-repo="${2:-emqx/emqx}"
-REPO="${GITHUB_REPOSITORY:-${repo}}"
-if [ "${base}" = "" ]; then
-    echo "Usage $0 <git-compare-base-ref>"
-    exit 1
-fi
-
-echo "elvis -v: $elvis_version"
-echo "git diff base: $base"
-
-if [ ! -f ./elvis ] || [ "$(./elvis -v | grep -oE '[1-9]+\.[0-9]+\.[0-9]+\-emqx-[0-9]+')" != "$elvis_version" ]; then
-    curl  --silent --show-error -fLO "https://github.com/emqx/elvis/releases/download/$elvis_version/elvis"
-    chmod +x ./elvis
-fi
-
-if [[ "$base" =~ [0-9a-f]{8,40} ]]; then
-    # base is a commit sha1
-    compare_base="$base"
-else
-    remote="$(git remote -v | grep -E "github\.com(:|/)$REPO((\.git)|(\s))" | grep fetch | awk '{print $1}')"
-    git fetch "$remote" "$base"
-    compare_base="$remote/$base"
-fi
-
-git_diff() {
-    git diff --name-only --diff-filter=ACMRTUXB "$compare_base"...HEAD
-}
-
-bad_file_count=0
-for file in $(git_diff); do
-    if [ ! -f "$file" ]; then
-        # file is deleted, skip
-        continue
-    fi
-    if [[ $file != *.erl ]]; then
-        # not .erl file
-        continue
-    fi
-    if ! ./elvis rock "$file" -c elvis.config; then
-        bad_file_count=$(( bad_file_count + 1))
-    fi
-done
-if [ $bad_file_count -gt 0 ]; then
-    echo "elvis: $bad_file_count errors"
-    exit 1
-fi
-
-### now check new-line at EOF for changed files
-
-nl_at_eof() {
-    local file="$1"
-    if ! [ -f "$file" ]; then
-        return
-    fi
-    case "$file" in
-        *.png|*rebar3)
-            return
-            ;;
-    esac
-    local lastbyte
-    lastbyte="$(tail -c 1 "$file" 2>&1)"
-    if [ "$lastbyte" != '' ]; then
-        echo "$file"
-        return 1
-    fi
-}
-
-for file in $(git_diff); do
-    if ! nl_at_eof "$file"; then
-        bad_file_count=$(( bad_file_count  + 1 ))
-    fi
-done
-
-exit $bad_file_count

scripts/find-props.sh

@@ -13,4 +13,5 @@ if [ "$1" != "emqx" ]; then
     BASEDIR="$1"
 fi
 
+# shellcheck disable=SC2038
 find "${BASEDIR}/test/props" -name "prop_*.erl" 2>/dev/null | xargs -I{} basename {} .erl | xargs | tr ' ' ','

scripts/find-suites.sh

@@ -12,4 +12,5 @@ TESTDIR="test"
 if [ "$1" != "emqx" ]; then
     TESTDIR="$1/test"
 fi
+# shellcheck disable=SC2038
 find "${TESTDIR}" -name "*_SUITE.erl" 2>/dev/null | xargs | tr ' ' ','

scripts/relup-base-packages.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+## This script helps to download relup base version packages
+
+if [[ -n "$DEBUG" ]]; then
+    set -x
+fi
+set -euo pipefail
+
+PROFILE="${1}"
+if [ "$PROFILE" = "" ]; then
+    PROFILE="emqx"
+fi
+
+case $PROFILE in
+    "emqx")
+        DIR='broker'
+        EDITION='community'
+        ;;
+    "emqx-ee")
+        DIR='enterprise'
+        EDITION='enterprise'
+        ;;
+    "emqx-edge")
+        DIR='edge'
+        EDITION='edge'
+        ;;
+esac
+
+SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}"
+
+ARCH="${ARCH:-$(uname -m)}"
+case "$ARCH" in
+    x86_64)
+        ARCH='amd64'
+        ;;
+    aarch64)
+        ARCH='arm64'
+        ;;
+    arm*)
+        ARCH=arm
+        ;;
+esac
+
+SHASUM="sha256sum"
+if [ "$SYSTEM" = "macos" ]; then
+    SHASUM="shasum -a 256"
+fi
+
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
+mkdir -p _upgrade_base
+pushd _upgrade_base
+
+for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do
+    filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip"
+    url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename"
+    echo "downloading base package from ${url} ..."
+    if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then
+        curl -L -o "${filename}" "${url}"
+        if [ "$SYSTEM" != "centos6" ]; then
+            curl -L -o "${filename}.sha256" "${url}.sha256"
+            SUMSTR=$(cat "${filename}.sha256")
+            echo "got sha265sum: ${SUMSTR}"
+            ## https://askubuntu.com/questions/1202208/checking-sha256-checksum
+            echo "${SUMSTR}  ${filename}" | $SHASUM -c || exit 1
+        fi
+    fi
+done
+
+popd

scripts/relup-base-vsns.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+
 ## This script prints the relup upgrade base versions
 ## for the given EMQX edition (specified as first arg)
 ##
@@ -17,8 +20,8 @@ parse_semver() {
     echo "$1" | tr '.|-' ' '
 }
 
-PROFILE="${1:-}"
+EDITION="${1:-}"
-[ -z "${PROFILE}" ] && usage
+[ -z "${EDITION}" ] && usage
 
 ## Get the current release version
 ## e.g.
@@ -46,7 +49,7 @@ else
     IS_RELEASE=false
 fi
 
-case "${PROFILE}" in
+case "${EDITION}" in
     *enterprise*)
         GIT_TAG_PREFIX="e"
         ;;

scripts/shellcheck.sh

@@ -3,9 +3,10 @@
 set -euo pipefail
 
 target_files=()
-while IFS='' read -r line; do
+while IFS='' read -r line;
+do
   target_files+=("$line");
-done < <(grep -r -l --exclude-dir=.git --exclude-dir=_build "#!/bin/" .)
+done < <(git grep -r -l '^#!/\(bin/\|usr/bin/env bash\)' .)
 
 return_code=0
 for i in "${target_files[@]}"; do

src/emqx.app.src

@@ -1,7 +1,12 @@
 {application, emqx,
  [{id, "emqx"},
   {description, "EMQ X"},
-  {vsn, "4.4.1"}, % strict semver, bump manually!
+  %% Note: this version is not the same as the release version!  This
+  %% is simply the emqx `application' version, which is separate from
+  %% the emqx `release' version, which in turn is comprised of several
+  %% apps, one of which is this.  See `emqx_release.hrl' for more
+  %% info.
+  {vsn, "4.4.2"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]},

src/emqx.appup.src

@@ -1,7 +1,14 @@
 %% -*- mode: erlang -*-
 {VSN,
-  [{"4.4.0",
+  [{"4.4.1",
+    [ {load_module,emqx_connection,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_banned,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_ctl,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_channel,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.4.0",
     [ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}
+     {load_module,emqx_banned,brutal_purge,soft_purge,[]},
     , {load_module,emqx_ctl,brutal_purge,soft_purge,[]}
     , {load_module,emqx_channel,brutal_purge,soft_purge,[]}
     , {load_module,emqx_connection,brutal_purge,soft_purge,[]}
@@ -19,8 +26,15 @@
     ]},
    {<<".*">>,[]}
   ],
-  [{"4.4.0",
+  [{"4.4.1",
+    [ {load_module,emqx_connection,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_banned,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_ctl,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_channel,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.4.0",
     [ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}
+    , {load_module,emqx_banned,brutal_purge,soft_purge,[]}
     , {load_module,emqx_ctl,brutal_purge,soft_purge,[]}
     , {load_module,emqx_channel,brutal_purge,soft_purge,[]}
     , {load_module,emqx_connection,brutal_purge,soft_purge,[]}

src/emqx_banned.erl

@@ -47,8 +47,6 @@
         , code_change/3
         ]).
 
--elvis([{elvis_style, state_record_and_type, disable}]).
-
 -define(BANNED_TAB, ?MODULE).
 
 %%--------------------------------------------------------------------

src/emqx_channel.erl

@@ -1377,7 +1377,7 @@ process_alias(Packet = #mqtt_packet{
         {ok, Topic} ->
             NPublish = Publish#mqtt_packet_publish{topic_name = Topic},
             {ok, Packet#mqtt_packet{variable = NPublish}, Channel};
-        false -> {error, ?RC_PROTOCOL_ERROR}
+        error -> {error, ?RC_PROTOCOL_ERROR}
     end;
 
 process_alias(#mqtt_packet{
@@ -1551,11 +1551,13 @@ enrich_connack_caps(AckProps, _Channel) -> AckProps.
 %%--------------------------------------------------------------------
 %% Enrich server keepalive
 
-enrich_server_keepalive(AckProps, #channel{clientinfo = #{zone := Zone}}) ->
+enrich_server_keepalive(AckProps, ?IS_MQTT_V5 = #channel{clientinfo = #{zone := Zone}}) ->
     case emqx_zone:server_keepalive(Zone) of
         undefined -> AckProps;
         Keepalive -> AckProps#{'Server-Keep-Alive' => Keepalive}
-    end.
+    end;
+
+enrich_server_keepalive(AckProps, _Channel) -> AckProps.
 
 %%--------------------------------------------------------------------
 %% Enrich response information
@@ -1601,7 +1603,7 @@ init_alias_maximum(#mqtt_packet_connect{proto_ver  = ?MQTT_PROTO_V5,
 init_alias_maximum(_ConnPkt, _ClientInfo) -> undefined.
 
 %%--------------------------------------------------------------------
-%% Enrich Keepalive
+%% Ensure Keepalive
 
 %% MQTT 5
 ensure_keepalive(#{'Server-Keep-Alive' := Interval}, Channel = #channel{conninfo = ConnInfo}) ->
@@ -1723,7 +1725,7 @@ run_hooks(Name, Args, Acc) ->
 
 -compile({inline, [find_alias/3, save_alias/4]}).
 
-find_alias(_, _, undefined) -> false;
+find_alias(_, _, undefined) -> error;
 find_alias(inbound, AliasId, _TopicAliases = #{inbound := Aliases}) ->
     maps:find(AliasId, Aliases);
 find_alias(outbound, Topic, _TopicAliases = #{outbound := Aliases}) ->

src/emqx_connection.erl

@@ -30,8 +30,6 @@
 -compile(nowarn_export_all).
 -endif.
 
--elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_connection]}}]).
-
 %% API
 -export([ start_link/3
         , stop/1

src/emqx_ctl.erl

@@ -57,8 +57,6 @@
         , code_change/3
         ]).
 
--elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_ctl]}}]).
-
 -record(state, {seq = 0}).
 
 -type(cmd() :: atom()).

src/emqx_logger_jsonfmt.erl

@@ -37,8 +37,6 @@
 
 -export_type([config/0]).
 
--elvis([{elvis_style, no_nested_try_catch, #{ ignore => [emqx_logger_jsonfmt]}}]).
-
 -type config() :: #{depth       => pos_integer() | unlimited,
                     report_cb   => logger:report_cb(),
                     single_line => boolean()}.

src/emqx_message.erl

@@ -84,8 +84,6 @@
 
 -export([format/1]).
 
--elvis([{elvis_style, god_modules, disable}]).
-
 -spec(make(emqx_topic:topic(), emqx_types:payload()) -> emqx_types:message()).
 make(Topic, Payload) ->
     make(undefined, Topic, Payload).

test/emqx_channel_SUITE.erl

@@ -676,6 +676,13 @@ t_process_alias(_) ->
     {ok, #mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"t">>}}, _Chan} =
         emqx_channel:process_alias(#mqtt_packet{variable = Publish}, Channel).
 
+t_process_alias_inexistent_alias(_) ->
+    Publish = #mqtt_packet_publish{topic_name = <<>>, properties = #{'Topic-Alias' => 1}},
+    Channel = channel(),
+    ?assertEqual(
+      {error, ?RC_PROTOCOL_ERROR},
+      emqx_channel:process_alias(#mqtt_packet{variable = Publish}, Channel)).
+
 t_packing_alias(_) ->
     Packet1 = #mqtt_packet{variable = #mqtt_packet_publish{
                                          topic_name = <<"x">>,
@@ -712,6 +719,20 @@ t_packing_alias(_) ->
                    #mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"z">>}},
                    channel())).
 
+t_packing_alias_inexistent_alias(_) ->
+    Publish = #mqtt_packet_publish{topic_name = <<>>, properties = #{'Topic-Alias' => 1}},
+    Channel = channel(),
+    Packet = #mqtt_packet{variable = Publish},
+    ExpectedChannel = emqx_channel:set_field(
+                        topic_aliases,
+                        #{ inbound => #{}
+                         , outbound => #{<<>> => 1}
+                         },
+                        Channel),
+    ?assertEqual(
+      {Packet, ExpectedChannel},
+      emqx_channel:packing_alias(Packet, Channel)).
+
 t_check_pub_acl(_) ->
     ok = meck:expect(emqx_zone, enable_acl, fun(_) -> true end),
     Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>),