Merge pull request #13133 from zmstone/0527-port-back-diverged-modules

chore: port diverged modules back to oss

apps/emqx/include/emqx_durable_session_metadata.hrl

@@ -0,0 +1,35 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022, 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc This header contains definitions of durable session metadata
+%% keys, that can be consumed by the external code.
+-ifndef(EMQX_DURABLE_SESSION_META_HRL).
+-define(EMQX_DURABLE_SESSION_META_HRL, true).
+
+%% Session metadata keys:
+-define(created_at, created_at).
+-define(last_alive_at, last_alive_at).
+-define(expiry_interval, expiry_interval).
+%% Unique integer used to create unique identities:
+-define(last_id, last_id).
+%% Connection info (relevent for the dashboard):
+-define(peername, peername).
+-define(will_message, will_message).
+-define(clientinfo, clientinfo).
+-define(protocol, protocol).
+-define(offline_info, offline_info).
+
+-endif.

apps/emqx/src/emqx.app.src

@@ -2,7 +2,7 @@
 {application, emqx, [
     {id, "emqx"},
     {description, "EMQX Core"},
-    {vsn, "5.3.1"},
+    {vsn, "5.4.0"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx/src/emqx_persistent_session_ds.erl

@@ -25,7 +25,7 @@
 
 -include("emqx_mqtt.hrl").
 
--include("emqx_persistent_session_ds.hrl").
+-include("emqx_persistent_session_ds/session_internals.hrl").
 
 -ifdef(TEST).
 -include_lib("proper/include/proper.hrl").

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_message_ds_gc_worker.erl

@@ -21,7 +21,7 @@
 -include_lib("stdlib/include/qlc.hrl").
 -include_lib("stdlib/include/ms_transform.hrl").
 
--include("emqx_persistent_session_ds.hrl").
+-include("session_internals.hrl").
 
 %% API
 -export([

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_gc_worker.erl

@@ -21,7 +21,7 @@
 -include_lib("stdlib/include/qlc.hrl").
 -include_lib("stdlib/include/ms_transform.hrl").
 
--include("emqx_persistent_session_ds.hrl").
+-include("session_internals.hrl").
 
 %% API
 -export([

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl

@@ -79,7 +79,7 @@
 ]).
 
 -include("emqx_mqtt.hrl").
--include("emqx_persistent_session_ds.hrl").
+-include("session_internals.hrl").
 -include_lib("snabbkaffe/include/trace.hrl").
 -include_lib("stdlib/include/qlc.hrl").
 

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_stream_scheduler.erl

@@ -29,7 +29,7 @@
 
 -include_lib("emqx/include/logger.hrl").
 -include("emqx_mqtt.hrl").
--include("emqx_persistent_session_ds.hrl").
+-include("session_internals.hrl").
 
 %%================================================================================
 %% Type declarations

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_subs.erl

@@ -41,7 +41,7 @@
 
 -export_type([subscription_state_id/0, subscription/0, subscription_state/0]).
 
--include("emqx_persistent_session_ds.hrl").
+-include("session_internals.hrl").
 -include("emqx_mqtt.hrl").
 -include_lib("snabbkaffe/include/trace.hrl").
 

apps/emqx/src/emqx_persistent_session_ds/session_internals.hrl

@@ -13,10 +13,11 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--ifndef(EMQX_PERSISTENT_SESSION_DS_HRL_HRL).
+-ifndef(EMQX_SESSION_DS_INTERNALS_HRL).
--define(EMQX_PERSISTENT_SESSION_DS_HRL_HRL, true).
+-define(EMQX_SESSION_DS_INTERNALS_HRL, true).
 
 -include("emqx_persistent_message.hrl").
+-include("emqx_durable_session_metadata.hrl").
 
 -define(SESSION_TAB, emqx_ds_session).
 -define(SESSION_SUBSCRIPTIONS_TAB, emqx_ds_session_subscriptions).
@@ -70,17 +71,4 @@
     sub_state_id :: emqx_persistent_session_ds_subs:subscription_state_id()
 }).
 
-%% Session metadata keys:
--define(created_at, created_at).
--define(last_alive_at, last_alive_at).
--define(expiry_interval, expiry_interval).
-%% Unique integer used to create unique identities:
--define(last_id, last_id).
-%% Connection info (relevent for the dashboard):
--define(peername, peername).
--define(will_message, will_message).
--define(clientinfo, clientinfo).
--define(protocol, protocol).
--define(offline_info, offline_info).
-
 -endif.

apps/emqx_auth/src/emqx_auth.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth, [
     {description, "EMQX Authentication and authorization"},
-    {vsn, "0.3.1"},
+    {vsn, "0.4.0"},
     {modules, []},
     {registered, [emqx_auth_sup]},
     {applications, [

apps/emqx_bridge/src/emqx_bridge_api.erl

@@ -55,6 +55,8 @@
 %% only for testing/mocking
 -export([supported_versions/1]).
 
+-export([format_bridge_metrics/1, format_metrics/1]).
+
 -define(BPAPI_NAME, emqx_bridge).
 
 -define(BRIDGE_NOT_ENABLED,

apps/emqx_bridge/src/emqx_bridge_v2_api.erl

@@ -96,7 +96,7 @@
 namespace() -> "actions_and_sources".
 
 api_spec() ->
-    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
+    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun check_api_schema/2}).
 
 paths() ->
     [
@@ -656,6 +656,40 @@ schema("/source_types") ->
         }
     }.
 
+%%------------------------------------------------------------------------------
+
+check_api_schema(Request, ReqMeta = #{path := "/actions/:id", method := put}) ->
+    BridgeId = emqx_utils_maps:deep_get([bindings, id], Request),
+    try emqx_bridge_resource:parse_bridge_id(BridgeId, #{atom_name => false}) of
+        %% NOTE
+        %% Bridge type is known, refine the API schema to get more specific error messages.
+        {BridgeType, _Name} ->
+            Schema = emqx_bridge_v2_schema:action_api_schema("put", BridgeType),
+            emqx_dashboard_swagger:filter_check_request(Request, refine_api_schema(Schema, ReqMeta))
+    catch
+        throw:#{reason := Reason} ->
+            ?NOT_FOUND(<<"Invalid bridge ID, ", Reason/binary>>)
+    end;
+check_api_schema(Request, ReqMeta = #{path := "/sources/:id", method := put}) ->
+    SourceId = emqx_utils_maps:deep_get([bindings, id], Request),
+    try emqx_bridge_resource:parse_bridge_id(SourceId, #{atom_name => false}) of
+        %% NOTE
+        %% Source type is known, refine the API schema to get more specific error messages.
+        {BridgeType, _Name} ->
+            Schema = emqx_bridge_v2_schema:source_api_schema("put", BridgeType),
+            emqx_dashboard_swagger:filter_check_request(Request, refine_api_schema(Schema, ReqMeta))
+    catch
+        throw:#{reason := Reason} ->
+            ?NOT_FOUND(<<"Invalid source ID, ", Reason/binary>>)
+    end;
+check_api_schema(Request, ReqMeta) ->
+    emqx_dashboard_swagger:filter_check_request(Request, ReqMeta).
+
+refine_api_schema(Schema, ReqMeta = #{path := Path, method := Method}) ->
+    Spec = maps:get(Method, schema(Path)),
+    SpecRefined = Spec#{'requestBody' => Schema},
+    ReqMeta#{apispec => SpecRefined}.
+
 %%------------------------------------------------------------------------------
 %% Thin Handlers
 %%------------------------------------------------------------------------------

apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl

@@ -31,6 +31,7 @@
     actions_get_response/0,
     actions_put_request/0,
     actions_post_request/0,
+    action_api_schema/2,
     actions_examples/1,
     action_values/4
 ]).
@@ -39,6 +40,7 @@
     sources_get_response/0,
     sources_put_request/0,
     sources_post_request/0,
+    source_api_schema/2,
     sources_examples/1,
     source_values/4
 ]).
@@ -100,6 +102,15 @@ actions_api_schema(Method) ->
     APISchemas = ?MODULE:registered_actions_api_schemas(Method),
     hoconsc:union(bridge_api_union(APISchemas)).
 
+action_api_schema(Method, BridgeV2Type) ->
+    APISchemas = ?MODULE:registered_actions_api_schemas(Method),
+    case lists:keyfind(atom_to_binary(BridgeV2Type), 1, APISchemas) of
+        {_, SchemaRef} ->
+            hoconsc:mk(SchemaRef);
+        false ->
+            unknown_bridge_schema(BridgeV2Type)
+    end.
+
 registered_actions_api_schemas(Method) ->
     RegisteredSchemas = emqx_action_info:registered_schema_modules_actions(),
     [
@@ -159,6 +170,15 @@ sources_api_schema(Method) ->
     APISchemas = ?MODULE:registered_sources_api_schemas(Method),
     hoconsc:union(bridge_api_union(APISchemas)).
 
+source_api_schema(Method, SourceType) ->
+    APISchemas = ?MODULE:registered_sources_api_schemas(Method),
+    case lists:keyfind(atom_to_binary(SourceType), 1, APISchemas) of
+        {_, SchemaRef} ->
+            hoconsc:mk(SchemaRef);
+        false ->
+            unknown_source_schema(SourceType)
+    end.
+
 registered_sources_api_schemas(Method) ->
     RegisteredSchemas = emqx_action_info:registered_schema_modules_sources(),
     [
@@ -231,6 +251,25 @@ bridge_api_union(Refs) ->
             end
     end.
 
+unknown_bridge_schema(BridgeV2Type) ->
+    erroneous_value_schema(BridgeV2Type, <<"unknown bridge type">>).
+
+unknown_source_schema(SourceType) ->
+    erroneous_value_schema(SourceType, <<"unknown source type">>).
+
+%% @doc Construct a schema that always emits validation error.
+%% We need to silence dialyzer because inner anonymous function always throws.
+-dialyzer({nowarn_function, [erroneous_value_schema/2]}).
+erroneous_value_schema(Value, Reason) ->
+    hoconsc:mk(typerefl:any(), #{
+        validator => fun(_) ->
+            throw(#{
+                value => Value,
+                reason => Reason
+            })
+        end
+    }).
+
 -spec method_values(action | source, http_method(), atom()) -> schema_example_map().
 method_values(Kind, post, Type) ->
     KindBin = atom_to_binary(Kind),

apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl

@@ -156,12 +156,15 @@ t_create_via_http(Config) ->
 t_on_get_status(Config) ->
     emqx_bridge_v2_testlib:t_on_get_status(Config, #{}).
 
-t_invalid_config(Config) ->
+t_create_invalid_config(Config) ->
     ?assertMatch(
         {error,
             {_Status, _, #{
                 <<"code">> := <<"BAD_REQUEST">>,
-                <<"message">> := #{<<"kind">> := <<"validation_error">>}
+                <<"message">> := #{
+                    <<"kind">> := <<"validation_error">>,
+                    <<"reason">> := <<"Inconsistent 'min_part_size'", _/bytes>>
+                }
             }}},
         emqx_bridge_v2_testlib:create_bridge_api(
             Config,
@@ -174,6 +177,28 @@ t_invalid_config(Config) ->
         )
     ).
 
+t_update_invalid_config(Config) ->
+    ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)),
+    ?assertMatch(
+        {error,
+            {_Status, _, #{
+                <<"code">> := <<"BAD_REQUEST">>,
+                <<"message">> := #{
+                    <<"kind">> := <<"validation_error">>,
+                    <<"reason">> := <<"Inconsistent 'min_part_size'", _/bytes>>
+                }
+            }}},
+        emqx_bridge_v2_testlib:update_bridge_api(
+            Config,
+            _Overrides = #{
+                <<"parameters">> => #{
+                    <<"min_part_size">> => <<"5GB">>,
+                    <<"max_part_size">> => <<"100MB">>
+                }
+            }
+        )
+    ).
+
 t_aggreg_upload(Config) ->
     Bucket = ?config(s3_bucket, Config),
     BridgeName = ?config(bridge_name, Config),

apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_sqlserver, [
     {description, "EMQX Enterprise SQL Server Bridge"},
-    {vsn, "0.2.0"},
+    {vsn, "0.2.1"},
     {registered, []},
     {applications, [kernel, stdlib, emqx_resource, odbc]},
     {env, [

apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl

@@ -478,7 +478,7 @@ worker_do_insert(
             {error, {unrecoverable_error, {invalid_request, Reason}}}
     end.
 
--spec execute(pid(), sql()) ->
+-spec execute(connection_reference(), sql()) ->
     updated_tuple()
     | selected_tuple()
     | [updated_tuple()]
@@ -487,7 +487,7 @@ worker_do_insert(
 execute(Conn, SQL) ->
     odbc:sql_query(Conn, str(SQL)).
 
--spec execute(pid(), sql(), time_out()) ->
+-spec execute(connection_reference(), sql(), time_out()) ->
     updated_tuple()
     | selected_tuple()
     | [updated_tuple()]

apps/emqx_conf/include/emqx_conf.hrl

@@ -73,11 +73,21 @@
     (?CE_AUTHN_PROVIDER_SCHEMA_MODS ++ ?EE_AUTHN_PROVIDER_SCHEMA_MODS)
 ).
 
+-define(OTHER_INJECTING_CONFIGS, []).
+
 -else.
 
 -define(AUTHZ_SOURCE_SCHEMA_MODS, ?CE_AUTHZ_SOURCE_SCHEMA_MODS).
 -define(AUTHN_PROVIDER_SCHEMA_MODS, ?CE_AUTHN_PROVIDER_SCHEMA_MODS).
 
--endif.
+-define(OTHER_INJECTING_CONFIGS, []).
+
+-endif.
+
+-define(INJECTING_CONFIGS, [
+    {emqx_authn_schema, ?AUTHN_PROVIDER_SCHEMA_MODS},
+    {emqx_authz_schema, ?AUTHZ_SOURCE_SCHEMA_MODS}
+    | ?OTHER_INJECTING_CONFIGS
+]).
 
 -endif.

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -70,10 +70,6 @@
     emqx_otel_schema,
     emqx_mgmt_api_key_schema
 ]).
--define(INJECTING_CONFIGS, [
-    {emqx_authn_schema, ?AUTHN_PROVIDER_SCHEMA_MODS},
-    {emqx_authz_schema, ?AUTHZ_SOURCE_SCHEMA_MODS}
-]).
 
 %% 1 million default ports counter
 -define(DEFAULT_MAX_PORTS, 1024 * 1024).

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -92,7 +92,14 @@
 -define(DEFAULT_ROW, 100).
 
 -type request() :: #{bindings => map(), query_string => map(), body => map()}.
--type request_meta() :: #{module => module(), path => string(), method => atom()}.
+-type request_meta() :: #{
+    module := module(),
+    path := string(),
+    method := atom(),
+    %% API Operation specification override.
+    %% Takes precedence over the API specification defined in the module.
+    apispec => map()
+}.
 
 %% More exact types are defined in minirest.hrl, but we don't want to include it
 %% because it defines a lot of types and they may clash with the types declared locally.
@@ -360,8 +367,8 @@ filter_check_request_and_translate_body(Request, RequestMeta) ->
 filter_check_request(Request, RequestMeta) ->
     translate_req(Request, RequestMeta, fun check_only/3).
 
-translate_req(Request, #{module := Module, path := Path, method := Method}, CheckFun) ->
+translate_req(Request, ReqMeta = #{module := Module}, CheckFun) ->
-    #{Method := Spec} = apply(Module, schema, [Path]),
+    Spec = find_req_apispec(ReqMeta),
     try
         Params = maps:get(parameters, Spec, []),
         Body = maps:get('requestBody', Spec, []),
@@ -378,6 +385,12 @@ translate_req(Request, #{module := Module, path := Path, method := Method}, Chec
             {400, 'BAD_REQUEST', Msg}
     end.
 
+find_req_apispec(#{apispec := Spec}) ->
+    Spec;
+find_req_apispec(#{module := Module, path := Path, method := Method}) ->
+    #{Method := Spec} = apply(Module, schema, [Path]),
+    Spec.
+
 check_and_translate(Schema, Map, Opts) ->
     hocon_tconf:check_plain(Schema, Map, Opts).
 

apps/emqx_management/src/emqx_management.app.src

@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.2.0"},
+    {vsn, "5.3.0"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [

apps/emqx_management/src/emqx_mgmt_api_clients.erl

@@ -24,6 +24,7 @@
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_utils/include/emqx_utils_api.hrl").
+-include_lib("emqx/include/emqx_durable_session_metadata.hrl").
 
 -include("emqx_mgmt.hrl").
 
@@ -1739,7 +1740,7 @@ format_channel_info(undefined, {ClientId, PSInfo0 = #{}}, _Opts) ->
 format_persistent_session_info(
     _ClientId,
     #{
-        metadata := #{offline_info := #{chan_info := ChanInfo, stats := Stats} = OfflineInfo} =
+        metadata := #{?offline_info := #{chan_info := ChanInfo, stats := Stats} = OfflineInfo} =
             Metadata
     } =
         PSInfo

apps/emqx_management/test/emqx_mgmt_api_configs_2_SUITE.erl

@@ -0,0 +1,154 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_mgmt_api_configs_2_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    _ = application:load(emqx),
+    CertDir = filename:join([code:lib_dir(emqx), "etc", "certs"]),
+    Cert = fun(Name) -> filename:join(CertDir, Name) end,
+    %% keep it the same as default conf in emqx_dashboard.conf
+    Conf =
+        [
+            "dashboard.listeners.http { enable = true, bind = 18083 }",
+            "dashboard.listeners.https {\n",
+            "  bind = 0 # disabled by default\n",
+            "  ssl_options {\n",
+            "    certfile = \"" ++ Cert("cert.pem") ++ "\"\n",
+            "    keyfile = \"" ++ Cert("key.pem") ++ "\"\n",
+            "  }\n"
+            "}\n"
+        ],
+    Apps = emqx_cth_suite:start(
+        [
+            emqx_conf,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard(Conf)
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{suite_apps, Apps} | Config].
+
+end_per_suite(Config) ->
+    ok = emqx_cth_suite:stop(?config(suite_apps, Config)).
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_TestCase, Config) ->
+    Config.
+
+t_dashboard(_Config) ->
+    {ok, Dashboard = #{<<"listeners">> := Listeners}} = get_config("dashboard"),
+    Https1 = #{enable => true, bind => 18084},
+    %% Ensure HTTPS listener can be enabled with just changing bind to a non-zero number
+    %% i.e. the default certs should work
+    ?assertMatch(
+        {ok, _},
+        update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}})
+    ),
+
+    Https2 = #{
+        <<"bind">> => 18084,
+        <<"ssl_options">> =>
+            #{
+                <<"keyfile">> => "etc/certs/badkey.pem",
+                <<"cacertfile">> => "etc/certs/badcacert.pem",
+                <<"certfile">> => "etc/certs/badcert.pem"
+            }
+    },
+    Dashboard2 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https2}},
+    ?assertMatch(
+        {error, {"HTTP/1.1", 400, _}},
+        update_config("dashboard", Dashboard2)
+    ),
+
+    FilePath = fun(Name) ->
+        iolist_to_binary(
+            emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", Name]))
+        )
+    end,
+    KeyFile = FilePath("key.pem"),
+    CertFile = FilePath("cert.pem"),
+    CacertFile = FilePath("cacert.pem"),
+    Https3 = #{
+        <<"bind">> => 18084,
+        <<"ssl_options">> => #{
+            <<"keyfile">> => KeyFile,
+            <<"cacertfile">> => CacertFile,
+            <<"certfile">> => CertFile
+        }
+    },
+    Dashboard3 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https3}},
+    ?assertMatch({ok, _}, update_config("dashboard", Dashboard3)),
+
+    Dashboard4 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => #{<<"bind">> => 0}}},
+    ?assertMatch({ok, _}, update_config("dashboard", Dashboard4)),
+    {ok, Dashboard41} = get_config("dashboard"),
+    ?assertMatch(
+        #{
+            <<"bind">> := 0,
+            <<"ssl_options">> :=
+                #{
+                    <<"keyfile">> := KeyFile,
+                    <<"cacertfile">> := CacertFile,
+                    <<"certfile">> := CertFile
+                }
+        },
+        read_conf([<<"dashboard">>, <<"listeners">>, <<"https">>]),
+        Dashboard41
+    ),
+
+    ?assertMatch({ok, _}, update_config("dashboard", Dashboard)),
+    {ok, Dashboard1} = get_config("dashboard"),
+    ?assertEqual(Dashboard, Dashboard1),
+    timer:sleep(1500),
+    ok.
+
+%% Helpers
+
+get_config(Name) ->
+    Path = emqx_mgmt_api_test_util:api_path(["configs", Name]),
+    case emqx_mgmt_api_test_util:request_api(get, Path) of
+        {ok, Res} ->
+            {ok, emqx_utils_json:decode(Res, [return_maps])};
+        Error ->
+            Error
+    end.
+
+update_config(Name, Change) ->
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    UpdatePath = emqx_mgmt_api_test_util:api_path(["configs", Name]),
+    case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of
+        {ok, Update} -> {ok, emqx_utils_json:decode(Update, [return_maps])};
+        Error -> Error
+    end.
+
+read_conf(RootKeys) when is_list(RootKeys) ->
+    case emqx_config:read_override_conf(#{override_to => cluster}) of
+        undefined -> undefined;
+        Conf -> emqx_utils_maps:deep_get(RootKeys, Conf, undefined)
+    end;
+read_conf(RootKey) ->
+    read_conf([RootKey]).

apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl

@@ -225,71 +225,6 @@ update_global_zone(Change) ->
 %    ?assertEqual(undefined, emqx_config:get_raw([zones, new_zone], undefined)),
 %    ok.
 
-t_dashboard(_Config) ->
-    {ok, Dashboard = #{<<"listeners">> := Listeners}} = get_config("dashboard"),
-    Https1 = #{enable => true, bind => 18084},
-    ?assertMatch(
-        {ok, _},
-        update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}})
-    ),
-
-    Https2 = #{
-        <<"bind">> => 18084,
-        <<"ssl_options">> =>
-            #{
-                <<"keyfile">> => "etc/certs/badkey.pem",
-                <<"cacertfile">> => "etc/certs/badcacert.pem",
-                <<"certfile">> => "etc/certs/badcert.pem"
-            }
-    },
-    Dashboard2 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https2}},
-    ?assertMatch(
-        {error, {"HTTP/1.1", 400, _}},
-        update_config("dashboard", Dashboard2)
-    ),
-
-    FilePath = fun(Name) ->
-        iolist_to_binary(
-            emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", Name]))
-        )
-    end,
-    KeyFile = FilePath("key.pem"),
-    CertFile = FilePath("cert.pem"),
-    CacertFile = FilePath("cacert.pem"),
-    Https3 = #{
-        <<"bind">> => 18084,
-        <<"ssl_options">> => #{
-            <<"keyfile">> => KeyFile,
-            <<"cacertfile">> => CacertFile,
-            <<"certfile">> => CertFile
-        }
-    },
-    Dashboard3 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https3}},
-    ?assertMatch({ok, _}, update_config("dashboard", Dashboard3)),
-
-    Dashboard4 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => #{<<"bind">> => 0}}},
-    ?assertMatch({ok, _}, update_config("dashboard", Dashboard4)),
-    {ok, Dashboard41} = get_config("dashboard"),
-    ?assertMatch(
-        #{
-            <<"bind">> := 0,
-            <<"ssl_options">> :=
-                #{
-                    <<"keyfile">> := KeyFile,
-                    <<"cacertfile">> := CacertFile,
-                    <<"certfile">> := CertFile
-                }
-        },
-        read_conf([<<"dashboard">>, <<"listeners">>, <<"https">>]),
-        Dashboard41
-    ),
-
-    ?assertMatch({ok, _}, update_config("dashboard", Dashboard)),
-    {ok, Dashboard1} = get_config("dashboard"),
-    ?assertEqual(Dashboard, Dashboard1),
-    timer:sleep(1500),
-    ok.
-
 %% v1 version json
 t_configs_node({'init', Config}) ->
     Node = node(),

apps/emqx_retainer/test/emqx_retainer_SUITE.erl

@@ -375,6 +375,7 @@ t_clean(Config) ->
         [{qos, 0}, {retain, true}],
         Config
     ),
+    ct:sleep(100),
     {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/#">>, [{qos, 0}, {rh, 0}]),
     ?assertEqual(3, length(receive_messages(3))),
 

apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl

@@ -284,6 +284,22 @@ t_rule_test_smoke(_Config) ->
                     <<"sql">> => <<"SELECT\n  *\nFROM\n  \"t/#\"">>
                 }
         },
+        #{
+            expected => #{code => 412},
+            input =>
+                #{
+                    <<"context">> =>
+                        #{
+                            <<"clientid">> => <<"c_emqx">>,
+                            <<"event_type">> => <<"client_check_authn_complete">>,
+                            <<"reason_code">> => <<"sucess">>,
+                            <<"is_superuser">> => true,
+                            <<"is_anonymous">> => false,
+                            <<"username">> => <<"u_emqx">>
+                        },
+                    <<"sql">> => <<"SELECT\n  *\nFROM\n  \"t/#\"">>
+                }
+        },
         #{
             expected => #{code => 412},
             input =>