Merge tag 'v4.3-alpha.1' into merge-4.3-alpha.1-to-5.0

.ci/compatibility_tests/docker-compose-mysql-tls.yaml

@@ -28,14 +28,13 @@ services:
       - emqx_bridge
     command:
       --bind-address "::"
-      --default-authentication-plugin=mysql_native_password
       --character-set-server=utf8mb4
       --collation-server=utf8mb4_general_ci
       --explicit_defaults_for_timestamp=true
       --lower_case_table_names=1
       --max_allowed_packet=128M
       --skip-symbolic-links
-      --ssl-ca=/etc/certs/ca.pem
+      --ssl-ca=/etc/certs/ca-cert.pem
       --ssl-cert=/etc/certs/server-cert.pem
       --ssl-key=/etc/certs/server-key.pem
 

.github/workflows/build_packages.yaml

@@ -16,6 +16,8 @@ jobs:
   windows:
     runs-on: windows-2019
 
+    if: startsWith(github.ref, 'refs/tags/')
+
     steps:
     - uses: actions/checkout@v1
     - uses: ilammy/msvc-dev-cmd@v1
@@ -64,7 +66,6 @@ jobs:
         ./_build/emqx/rel/emqx/bin/emqx install
         ./_build/emqx/rel/emqx/bin/emqx uninstall
     - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
         name: emqx
         path: ./_packages/emqx/.
@@ -72,6 +73,8 @@ jobs:
   mac:
     runs-on: macos-10.15
 
+    if: startsWith(github.ref, 'refs/tags/')
+
     steps:
     - uses: actions/checkout@v1
     - name: prepare
@@ -80,6 +83,7 @@ jobs:
         echo "/usr/local/bin" >> $GITHUB_PATH
         git config --global credential.helper store
     - name: build erlang
+      timeout-minutes: 60
       run: |
         kerl build 23.2.2
         kerl install 23.2.2 $HOME/.kerl/23.2.2
@@ -111,7 +115,6 @@ jobs:
         rm -rf emqx
         openssl dgst -sha256 ./_packages/emqx/$pkg_name | awk '{print $2}'  > ./_packages/emqx/$pkg_name.sha256
     - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
         name: emqx
         path: ./_packages/emqx/.
@@ -245,6 +248,8 @@ jobs:
   docker:
     runs-on: ubuntu-20.04
 
+    if: startsWith(github.ref, 'refs/tags/')
+
     strategy:
       matrix:
         arch:
@@ -280,12 +285,10 @@ jobs:
         sudo TARGET=emqx/emqx-edge ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
         cd _packages/emqx-edge && for var in $(ls emqx-edge-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
     - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
         name: emqx
         path: ./_packages/emqx/.
     - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
         name: emqx-edge
         path: ./_packages/emqx-edge/.

.github/workflows/run_cts_tests.yaml

@@ -121,7 +121,7 @@ jobs:
         - ipv4
         - ipv6
         connect_type:
-        # - tls
+        - tls
         - tcp
 
     steps:

apps/emqx_auth_mysql/rebar.config

@@ -1,6 +1,6 @@
 {deps,
  [
-  {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.6.1"}}}
+  {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}
  ]}.
 
 {edoc_opts, [{preprocess, true}]}.

apps/emqx_management/src/emqx_mgmt.erl

@@ -72,8 +72,7 @@
         ]).
 
 %% Routes
--export([ list_routes/0
+-export([ lookup_routes/1
-        , lookup_routes/1
         ]).
 
 %% PubSub
@@ -421,12 +420,6 @@ lookup_subscriptions(Node, ClientId) ->
 %% Routes
 %%--------------------------------------------------------------------
 
-list_routes() ->
-    case check_row_limit([emqx_route]) of
-        false -> throw(max_row_limit);
-        ok    -> ets:tab2list(emqx_route)
-    end.
-
 lookup_routes(Topic) ->
     emqx_router:lookup_routes(Topic).
 

apps/emqx_rule_engine/src/emqx_rule_engine_api.erl

@@ -341,20 +341,19 @@ update_resource(#{id := Id}, NewParams) ->
         ok ->
             return(ok);
         {error, not_found} ->
-            ?LOG(error, "resource not found: ~0p", [Id]),
+            ?LOG(error, "Resource not found: ~0p", [Id]),
-            return({error, 400, <<"resource not found:", Id/binary>>});
+            return({error, 400, <<"Resource not found:", Id/binary>>});
         {error, {init_resource_failure, _}} ->
-            ?LOG(error, "init resource failure: ~0p", [Id]),
+            ?LOG(error, "Init resource failure: ~0p", [Id]),
-            return({error, 500, <<"init resource failure:", Id/binary>>});
+            return({error, 500, <<"Init resource failure:", Id/binary>>});
         {error, {dependency_exists, RuleId}} ->
-            ?LOG(error, "dependency exists: ~0p", [RuleId]),
+            ?LOG(error, "Dependency exists: ~0p", [RuleId]),
-            return({error, 500, <<"resource dependency by rule:", RuleId/binary>>});
+            return({error, 500, <<"Dependency exists:", RuleId/binary>>});
         {error, Reason} ->
-            ?LOG(error, "update resource failed: ~0p", [Reason]),
+            ?LOG(error, "Resource update failed: ~0p", [Reason]),
-            return({error, 500, <<"update resource failed,error info have been written to logfile!">>})
+            return({error, 500, <<"Resource update failed!">>})
     end.
 
-
 delete_resource(#{id := Id}, _Params) ->
     case emqx_rule_engine:delete_resource(Id) of
         ok -> return(ok);
@@ -531,14 +530,7 @@ parse_resource_params([_ | Params], Res) ->
     parse_resource_params(Params, Res).
 
 json_term_to_map(List) ->
-    Data = lists:map(fun({K, V}) ->
+    emqx_json:decode(emqx_json:encode(List), [return_maps]).
-                    case V of
-                        {} ->{K, [{}]};
-                        _ -> {K, V}
-                    end
-                 end,
-            List),
-    emqx_json:decode(emqx_json:encode(Data), [return_maps]).
 
 sort_by_title(action, Actions) ->
     sort_by(#action.title, Actions);

apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl

@@ -173,7 +173,7 @@ resources(["update" | Params]) ->
             ok ->
                 emqx_ctl:print("Resource update successfully~n");
             {error, Reason} ->
-                emqx_ctl:print("update resource failed, reason: ~p!~n", [Reason])
+                emqx_ctl:print("Resource update failed: ~0p~n", [Reason])
             end
         end, Params, ?OPTSPEC_RESOURCES_UPDATE, {?FUNCTION_NAME, update});
 

apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl

@@ -382,7 +382,7 @@ t_crud_rule_api(_Config) ->
     RuleID = maps:get(id, Rule),
     %ct:pal("RCreated : ~p", [Rule]),
 
-    {ok, #{code := 0, data := Rules}} = emqx_rule_engine_api:list_rules(#{},[]),
+    {ok, #{code := 0, data := Rules}} = emqx_rule_engine_api:list_rules(#{}, []),
     %ct:pal("RList : ~p", [Rules]),
     ?assert(length(Rules) > 0),
 
@@ -424,13 +424,13 @@ t_crud_rule_api(_Config) ->
     ok.
 
 t_list_actions_api(_Config) ->
-    {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:list_actions(#{},[]),
+    {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:list_actions(#{}, []),
     %ct:pal("RList : ~p", [Actions]),
     ?assert(length(Actions) > 0),
     ok.
 
 t_show_action_api(_Config) ->
-    {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:show_action(#{name => 'inspect'},[]),
+    {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:show_action(#{name => 'inspect'}, []),
     ?assertEqual('inspect', maps:get(name, Actions)),
     ok.
 
@@ -442,24 +442,22 @@ t_crud_resources_api(_Config) ->
              {<<"config">>, [{<<"a">>, 1}]},
              {<<"description">>, <<"Simple Resource">>}]),
     ResId = maps:get(id, Resources1),
-    {ok, #{code := 0, data := Resources}} = emqx_rule_engine_api:list_resources(#{},[]),
+    {ok, #{code := 0, data := Resources}} = emqx_rule_engine_api:list_resources(#{}, []),
     ?assert(length(Resources) > 0),
-    {ok, #{code := 0, data := Resources2}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]),
+    {ok, #{code := 0, data := Resources2}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
     ?assertEqual(ResId, maps:get(id, Resources2)),
     %
     {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId},
                                                               [{<<"config">>, [{<<"a">>, 2}]},
                                                                {<<"description">>, <<"2">>}]),
-    {ok, #{code := 0, data := Resources3}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]),
+    {ok, #{code := 0, data := Resources3}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
     ?assertEqual(ResId, maps:get(id, Resources3)),
     ?assertEqual(#{<<"a">> => 2}, maps:get(config, Resources3)),
     ?assertEqual(<<"2">>, maps:get(description, Resources3)),
-    {ok, #{code := 0, data := Resources3}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]),
-    ?assertEqual(ResId, maps:get(id, Resources3)),
     %
     {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId},
                                                               [{<<"config">>, [{<<"a">>, 3}]}]),
-    {ok, #{code := 0, data := Resources4}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]),
+    {ok, #{code := 0, data := Resources4}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
     ?assertEqual(ResId, maps:get(id, Resources4)),
     ?assertEqual(#{<<"a">> => 3}, maps:get(config, Resources4)),
     ?assertEqual(<<"2">>, maps:get(description, Resources4)),
@@ -468,36 +466,35 @@ t_crud_resources_api(_Config) ->
                                                               [{<<"config">>, [{<<"a">>, 1},
                                                                                {<<"b">>, 2},
                                                                                {<<"c">>, 3}]}]),
-    {ok, #{code := 0, data := Resources5}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]),
+    {ok, #{code := 0, data := Resources5}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
     ?assertEqual(ResId, maps:get(id, Resources5)),
     ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources5)),
     ?assertEqual(<<"2">>, maps:get(description, Resources5)),
     % Only description
     {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId},
                                                               [{<<"description">>, <<"new5">>}]),
-    {ok, #{code := 0, data := Resources6}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]),
+    {ok, #{code := 0, data := Resources6}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
     ?assertEqual(ResId, maps:get(id, Resources6)),
     ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources6)),
     ?assertEqual(<<"new5">>, maps:get(description, Resources6)),
     % None
-    {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId},[]),
+    {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, []),
-    {ok, #{code := 0, data := Resources7}} = emqx_rule_engine_api:show_resource(#{id => ResId},[]),
+    {ok, #{code := 0, data := Resources7}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []),
     ?assertEqual(ResId, maps:get(id, Resources7)),
     ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources7)),
     ?assertEqual(<<"new5">>, maps:get(description, Resources7)),
     %
     ?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_resource(#{id => ResId},#{})),
-    ?assertMatch({ok, #{code := 404}}, emqx_rule_engine_api:show_resource(#{id => ResId},[])),
+    ?assertMatch({ok, #{code := 404}}, emqx_rule_engine_api:show_resource(#{id => ResId}, [])),
     ok.
 
-
 t_list_resource_types_api(_Config) ->
-    {ok, #{code := 0, data := ResourceTypes}} = emqx_rule_engine_api:list_resource_types(#{},[]),
+    {ok, #{code := 0, data := ResourceTypes}} = emqx_rule_engine_api:list_resource_types(#{}, []),
     ?assert(length(ResourceTypes) > 0),
     ok.
 
 t_show_resource_type_api(_Config) ->
-    {ok, #{code := 0, data := RShow}} = emqx_rule_engine_api:show_resource_type(#{name => 'built_in'},[]),
+    {ok, #{code := 0, data := RShow}} = emqx_rule_engine_api:show_resource_type(#{name => 'built_in'}, []),
     %ct:pal("RShow : ~p", [RShow]),
     ?assertEqual(built_in, maps:get(name, RShow)),
     ok.

pkg-vsn.sh

@@ -1,12 +1,20 @@
 #!/bin/bash
+set -e -u
 
 # This script prints the release version for emqx
 
 # ensure dir
 cd -P -- "$(dirname -- "$0")"
 
+case $(uname) in
+    *Darwin*) SED="sed -E";;
+    *) SED="sed -r";;
+esac
+
 # comment SUFFIX out when finalising RELEASE
-RELEASE="4.3.0"
+RELEASE="$(grep -oE '\{vsn, (.*)\}' src/emqx.app.src | $SED 's/\{vsn, (.*)\}/\1/g' | $SED 's/\"//g')"
-SUFFIX="-pre-$(git rev-parse HEAD | cut -b1-8)"
+if [ -d .git ] && ! git describe --tags --match "v${RELEASE}" --exact >/dev/null 2>&1; then
+  SUFFIX="-$(git rev-parse HEAD | cut -b1-8)"
+fi
 
 echo "${RELEASE}${SUFFIX:-}"

src/emqx.app.src

@@ -1,7 +1,7 @@
 {application, emqx,
  [{description, "EMQ X Broker"},
   {id, "emqx"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3-alpha.1"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]},