Merge pull request #11082 from emqx/release-51

sync release-51 to master

.github/workflows/build_and_push_docker_images.yaml

@@ -167,9 +167,14 @@ jobs:
           img_suffix="-elixir"
           extra_labels="org.opencontainers.image.elixir.version=${{ matrix.elixir }}"
         fi
+        extra_deps=
+        if [[ "${{ matrix.profile }}" = *enterprise* ]]; then
+          extra_deps='libsasl2-2'
+        fi
 
         echo "img_suffix=$img_suffix" >> $GITHUB_OUTPUT
         echo "extra_labels=$extra_labels" >> $GITHUB_OUTPUT
+        echo "extra_deps=$extra_deps" >> $GITHUB_OUTPUT
 
     - uses: docker/metadata-action@v4
       id: meta
@@ -196,5 +201,6 @@ jobs:
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
           EMQX_NAME=${{ matrix.profile }}${{ steps.pre-meta.outputs.img_suffix }}
+          EXTRA_DEPS=${{ steps.pre-meta.outputs.extra_deps }}
         file: source/${{ matrix.os[2] }}
         context: source

Makefile

@@ -13,10 +13,10 @@ else
 	FIND=find
 endif
 
-# Dashbord version
+# Dashboard version
 # from https://github.com/emqx/emqx-dashboard5
-export EMQX_DASHBOARD_VERSION ?= v1.2.6-beta.1
-export EMQX_EE_DASHBOARD_VERSION ?= e1.1.0-beta.5
+export EMQX_DASHBOARD_VERSION ?= v1.3.0
+export EMQX_EE_DASHBOARD_VERSION ?= e1.1.0-beta.7
 
 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used
 # In make 4.4+, for backward-compatibility the value from the original environment is used.

apps/emqx/include/emqx_release.hrl

@@ -35,7 +35,7 @@
 -define(EMQX_RELEASE_CE, "5.1.0-alpha.5").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.1.0-alpha.6").
+-define(EMQX_RELEASE_EE, "5.1.0-alpha.7").
 
 %% The HTTP API version
 -define(EMQX_API_VERSION, "5.0").

apps/emqx/src/config/emqx_config_logger.erl

@@ -151,7 +151,7 @@ tr_file_handler({HandlerName, SubConf}) ->
         level => conf_get("level", SubConf),
         config => (log_handler_conf(SubConf))#{
             type => wrap,
-            file => conf_get("to", SubConf),
+            file => conf_get("path", SubConf),
             max_no_files => conf_get("rotation_count", SubConf),
             max_no_bytes => conf_get("rotation_size", SubConf)
         },

apps/emqx/src/emqx_config_handler.erl

@@ -277,12 +277,37 @@ check_and_save_configs(
     OldConf = emqx_config:get_root(ConfKeyPath),
     case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of
         {ok, Result0} ->
+            post_update_ok(
+                AppEnvs,
+                NewConf,
+                NewRawConf,
+                OverrideConf,
+                Opts,
+                ConfKeyPath,
+                UpdateArgs,
+                Result0
+            );
+        {error, {post_config_update, HandlerName, Reason}} ->
+            HandlePostFailureFun =
+                fun() ->
+                    post_update_ok(
+                        AppEnvs,
+                        NewConf,
+                        NewRawConf,
+                        OverrideConf,
+                        Opts,
+                        ConfKeyPath,
+                        UpdateArgs,
+                        #{}
+                    )
+                end,
+            {error, {post_config_update, HandlerName, {Reason, HandlePostFailureFun}}}
+    end.
+
+post_update_ok(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts, ConfKeyPath, UpdateArgs, Result0) ->
     ok = emqx_config:save_configs(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts),
     Result1 = return_change_result(ConfKeyPath, UpdateArgs),
-            {ok, Result1#{post_config_update => Result0}};
-        Error ->
-            Error
-    end.
+    {ok, Result1#{post_config_update => Result0}}.
 
 do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, Result) ->
     do_post_config_update(

apps/emqx/src/emqx_listeners.erl

@@ -492,7 +492,9 @@ pre_config_update([?ROOT_KEY], RawConf, RawConf) ->
 pre_config_update([?ROOT_KEY], NewConf, _RawConf) ->
     {ok, convert_certs(NewConf)}.
 
-post_config_update([?ROOT_KEY, Type, Name], {create, _Request}, NewConf, undefined, _AppEnvs) ->
+post_config_update([?ROOT_KEY, Type, Name], {create, _Request}, NewConf, OldConf, _AppEnvs) when
+    OldConf =:= undefined orelse OldConf =:= ?TOMBSTONE_TYPE
+->
     create_listener(Type, Name, NewConf);
 post_config_update([?ROOT_KEY, Type, Name], {update, _Request}, NewConf, OldConf, _AppEnvs) ->
     update_listener(Type, Name, {OldConf, NewConf});

apps/emqx/test/emqx_config_handler_SUITE.erl

@@ -317,11 +317,19 @@ wait_for_new_pid() ->
             Pid
     end.
 
-callback_error(FailedPath, Update, Error) ->
+callback_error(FailedPath, Update, ExpectError) ->
     Opts = #{rawconf_with_defaults => true},
     ok = emqx_config_handler:add_handler(FailedPath, ?MODULE),
     Old = emqx:get_raw_config(FailedPath, undefined),
-    ?assertEqual(Error, emqx:update_config(FailedPath, Update, Opts)),
+    Error = emqx:update_config(FailedPath, Update, Opts),
+    case ExpectError of
+        {error, {post_config_update, ?MODULE, post_config_update_error}} ->
+            ?assertMatch(
+                {error, {post_config_update, ?MODULE, {post_config_update_error, _}}}, Error
+            );
+        _ ->
+            ?assertEqual(ExpectError, Error)
+    end,
     New = emqx:get_raw_config(FailedPath, undefined),
     ?assertEqual(Old, New),
     ok = emqx_config_handler:remove_handler(FailedPath),

apps/emqx_bridge_influxdb/rebar.config

@@ -1,7 +1,7 @@
 {erl_opts, [debug_info]}.
 
 {deps, [
-    {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.10"}}},
+    {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.11"}}},
     {emqx_connector, {path, "../../apps/emqx_connector"}},
     {emqx_resource, {path, "../../apps/emqx_resource"}},
     {emqx_bridge, {path, "../../apps/emqx_bridge"}}

apps/emqx_conf/src/emqx_cluster_rpc.erl

@@ -476,7 +476,23 @@ trans_query(TnxId) ->
 apply_mfa(TnxId, {M, F, A}, Kind) ->
     Res =
         try
-            erlang:apply(M, F, A)
+            case erlang:apply(M, F, A) of
+                {error, {post_config_update, HandlerName, {Reason0, PostFailureFun}}} when
+                    Kind =/= ?APPLY_KIND_INITIATE
+                ->
+                    ?SLOG(error, #{
+                        msg => "post_config_update_failed",
+                        handler => HandlerName,
+                        reason => Reason0
+                    }),
+                    PostFailureFun();
+                {error, {post_config_update, HandlerName, {Reason0, _Fun}}} when
+                    Kind =:= ?APPLY_KIND_INITIATE
+                ->
+                    {error, {post_config_update, HandlerName, Reason0}};
+                Result ->
+                    Result
+            end
         catch
             throw:Reason ->
                 {error, #{reason => Reason}};

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -164,7 +164,7 @@ fields("cluster") ->
             sc(
                 emqx_schema:duration(),
                 #{
-                    mapping => "ekka.cluster_autoclean",
+                    mapping => "mria.cluster_autoclean",
                     default => <<"5m">>,
                     desc => ?DESC(cluster_autoclean),
                     'readOnly' => true
@@ -174,7 +174,7 @@ fields("cluster") ->
             sc(
                 boolean(),
                 #{
-                    mapping => "ekka.cluster_autoheal",
+                    mapping => "mria.cluster_autoheal",
                     default => true,
                     desc => ?DESC(cluster_autoheal),
                     'readOnly' => true
@@ -909,13 +909,13 @@ fields("console_handler") ->
     log_handler_common_confs(console);
 fields("log_file_handler") ->
     [
-        {"to",
+        {"path",
             sc(
                 file(),
                 #{
                     desc => ?DESC("log_file_handler_file"),
                     default => <<"${EMQX_LOG_DIR}/emqx.log">>,
-                    aliases => [file],
+                    aliases => [file, to],
                     importance => ?IMPORTANCE_HIGH,
                     converter => fun(Path, Opts) ->
                         emqx_schema:naive_env_interpolation(ensure_unicode_path(Path, Opts))

apps/emqx_conf/test/emqx_conf_logger_SUITE.erl

@@ -43,7 +43,7 @@
                 file {
                 enable = true
                 level = info
-                to = \"log/emqx.log\"
+                path = \"log/emqx.log\"
                 }
              }
     """).
@@ -67,7 +67,7 @@ t_log_conf(_Conf) ->
         <<"rotation_count">> => 10,
         <<"rotation_size">> => <<"50MB">>,
         <<"time_offset">> => <<"system">>,
-        <<"to">> => <<"log/emqx.log">>
+        <<"path">> => <<"log/emqx.log">>
     },
     ExpectLog1 = #{
         <<"console">> =>

apps/emqx_conf/test/emqx_conf_schema_tests.erl

@@ -206,7 +206,7 @@ log_rotation_count_limit_test() ->
     """
     log.file {
     enable = true
-    to = \"log/emqx.log\"
+    path = \"log/emqx.log\"
     formatter = text
     level = debug
     rotation = {count = ~w}
@@ -434,7 +434,7 @@ log_path_test_() ->
         #{<<"log">> => #{<<"file_handlers">> => #{<<"name1">> => #{<<"file">> => Path}}}}
     end,
     Assert = fun(Name, Path, Conf) ->
-        ?assertMatch(#{log := #{file := #{Name := #{to := Path}}}}, Conf)
+        ?assertMatch(#{log := #{file := #{Name := #{path := Path}}}}, Conf)
     end,
 
     [
@@ -452,7 +452,7 @@ log_path_test_() ->
                                 "handler_name" :=
                                     #{
                                         kind := validation_error,
-                                        path := "log.file.name1.to",
+                                        path := "log.file.name1.path",
                                         reason := {"bad_file_path_string", _}
                                     }
                             }
@@ -471,7 +471,7 @@ log_path_test_() ->
                                 "handler_name" :=
                                     #{
                                         kind := validation_error,
-                                        path := "log.file.name1.to",
+                                        path := "log.file.name1.path",
                                         reason := {"not_string", _}
                                     }
                             }

apps/emqx_dashboard/src/emqx_dashboard.erl

@@ -68,19 +68,13 @@ start_listeners(Listeners) ->
             }
         }
     },
-    Dispatch = [
-        {"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}},
-        {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}},
-        {emqx_mgmt_api_status:path(), emqx_mgmt_api_status, []},
-        {'_', emqx_dashboard_not_found, []}
-    ],
     BaseMinirest = #{
         base_path => emqx_dashboard_swagger:base_path(),
         modules => minirest_api:find_api_modules(apps()),
         authorization => Authorization,
         security => [#{'basicAuth' => []}, #{'bearerAuth' => []}],
         swagger_global_spec => GlobalSpec,
-        dispatch => Dispatch,
+        dispatch => dispatch(),
         middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler]
     },
     {OkListeners, ErrListeners} =
@@ -259,3 +253,20 @@ ensure_ssl_cert(Listeners = #{https := Https0}) ->
     Listeners#{https => maps:from_list(Https1)};
 ensure_ssl_cert(Listeners) ->
     Listeners.
+
+dispatch() ->
+    static_dispatch() ++ dynamic_dispatch().
+
+static_dispatch() ->
+    StaticFiles = ["/editor.worker.js", "/json.worker.js", "/version"],
+    [
+        {"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}},
+        {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}
+    ] ++
+        [{Path, cowboy_static, {priv_file, emqx_dashboard, "www" ++ Path}} || Path <- StaticFiles].
+
+dynamic_dispatch() ->
+    [
+        {emqx_mgmt_api_status:path(), emqx_mgmt_api_status, []},
+        {'_', emqx_dashboard_not_found, []}
+    ].

apps/emqx_management/src/emqx_mgmt.erl

@@ -118,6 +118,7 @@
 
 list_nodes() ->
     Running = emqx:cluster_nodes(running),
+    %% all stopped core nodes
     Stopped = emqx:cluster_nodes(stopped),
     DownNodes = lists:map(fun stopped_node_info/1, Stopped),
     [{Node, Info} || #{node := Node} = Info <- node_info(Running)] ++ DownNodes.
@@ -181,7 +182,7 @@ node_info(Nodes) ->
     emqx_rpc:unwrap_erpc(emqx_management_proto_v3:node_info(Nodes)).
 
 stopped_node_info(Node) ->
-    {Node, #{node => Node, node_status => 'stopped'}}.
+    {Node, #{node => Node, node_status => 'stopped', role => core}}.
 
 vm_stats() ->
     Idle =

apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl

@@ -100,7 +100,7 @@ t_log(_Config) ->
     File = "log/emqx-test.log",
     %% update handler
     Log1 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"enable">>], Log, true),
-    Log2 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"to">>], Log1, File),
+    Log2 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"path">>], Log1, File),
     {ok, #{}} = update_config(<<"log">>, Log2),
     {ok, Log3} = logger:get_handler_config(default),
     ?assertMatch(#{config := #{file := File}}, Log3),
@@ -116,7 +116,7 @@ t_log(_Config) ->
     Handler = emqx_utils_maps:deep_get([<<"file">>, <<"default">>], Log2),
     NewLog1 = emqx_utils_maps:deep_put([<<"file">>, <<"new">>], Log2, Handler),
     NewLog2 = emqx_utils_maps:deep_put(
-        [<<"file">>, <<"new">>, <<"to">>], NewLog1, File1
+        [<<"file">>, <<"new">>, <<"path">>], NewLog1, File1
     ),
     {ok, #{}} = update_config(<<"log">>, NewLog2),
     {ok, Log4} = logger:get_handler_config(new),

apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl

@@ -35,7 +35,7 @@ init_per_testcase(t_log_path, Config) ->
     Log = emqx_conf:get_raw([log], #{}),
     File = "log/emqx-test.log",
     Log1 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"enable">>], Log, true),
-    Log2 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"to">>], Log1, File),
+    Log2 = emqx_utils_maps:deep_put([<<"file">>, <<"default">>, <<"path">>], Log1, File),
     {ok, #{}} = emqx_conf:update([log], Log2, #{rawconf_with_defaults => true}),
     Config;
 init_per_testcase(_, Config) ->

build

@@ -380,7 +380,7 @@ make_docker() {
     ## extra_deps is a comma separated list of debian 11 package names
     local extra_deps=''
     if [[ "$PROFILE" = *enterprise* ]]; then
-        extra_deps='libsasl2-2'
+        extra_deps='libsasl2-2,libsasl2-modules-gssapi-mit'
     fi
     echo '_build' >> ./.dockerignore
     set -x

changes/ce/feat-11062.en.md

@@ -0,0 +1 @@
+Rename `log.file.to` to `log.file.path`.

changes/ce/fix-11056.en.md

@@ -0,0 +1,3 @@
+- Fix the issue where newly created listeners do not start properly at times,
+  when you delete a system default listener and add a new one named 'default', it will not start correctly.
+- Fix the bug where configuration failure on certain nodes can cause dashboard unavailability.

changes/ce/fix-11070.en.md

@@ -0,0 +1 @@
+Fix the problem that the cluster.autoclean configuration item does not take effect.

mix.exs

@@ -193,7 +193,7 @@ defmodule EMQXUmbrella.MixProject do
   defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
     [
       {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
-      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.10", override: true},
+      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.11", override: true},
       {:wolff, github: "kafka4beam/wolff", tag: "1.7.5"},
       {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.3", override: true},
       {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0"},

rel/i18n/emqx_connector_api.hocon

@@ -1,46 +0,0 @@
-emqx_connector_api {
-
-conn_get.desc:
-"""List all connectors"""
-
-conn_get.label:
-"""List All Connectors"""
-
-conn_id_delete.desc:
-"""Delete a connector by ID"""
-
-conn_id_delete.label:
-"""Delete Connector"""
-
-conn_id_get.desc:
-"""Get the connector by ID"""
-
-conn_id_get.label:
-"""Get Connector"""
-
-conn_id_put.desc:
-"""Update an existing connector by ID"""
-
-conn_id_put.label:
-"""Update Connector"""
-
-conn_post.desc:
-"""Create a new connector"""
-
-conn_post.label:
-"""Create Connector"""
-
-conn_test_post.desc:
-"""Test creating a new connector by given ID <br/>
-The ID must be of format '{type}:{name}'"""
-
-conn_test_post.label:
-"""Create Test Connector"""
-
-id.desc:
-"""The connector ID. Must be of format {type}:{name}"""
-
-id.label:
-"""Connector ID"""
-
-}