Merge remote-tracking branch 'origin/release-57' into 0527-port-back-diverged-modules

apps/emqx/include/emqx_durable_session_metadata.hrl

@@ -0,0 +1,35 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022, 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc This header contains definitions of durable session metadata
+%% keys, that can be consumed by the external code.
+-ifndef(EMQX_DURABLE_SESSION_META_HRL).
+-define(EMQX_DURABLE_SESSION_META_HRL, true).
+
+%% Session metadata keys:
+-define(created_at, created_at).
+-define(last_alive_at, last_alive_at).
+-define(expiry_interval, expiry_interval).
+%% Unique integer used to create unique identities:
+-define(last_id, last_id).
+%% Connection info (relevent for the dashboard):
+-define(peername, peername).
+-define(will_message, will_message).
+-define(clientinfo, clientinfo).
+-define(protocol, protocol).
+-define(offline_info, offline_info).
+
+-endif.

apps/emqx/src/emqx.app.src

@@ -2,7 +2,7 @@
 {application, emqx, [
     {id, "emqx"},
     {description, "EMQX Core"},
-    {vsn, "5.3.1"},
+    {vsn, "5.4.0"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx/src/emqx_persistent_session_ds.erl

@@ -25,7 +25,7 @@
 
 -include("emqx_mqtt.hrl").
 
--include("emqx_persistent_session_ds.hrl").
+-include("emqx_persistent_session_ds/session_internals.hrl").
 
 -ifdef(TEST).
 -include_lib("proper/include/proper.hrl").

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_message_ds_gc_worker.erl

@@ -21,7 +21,7 @@
 -include_lib("stdlib/include/qlc.hrl").
 -include_lib("stdlib/include/ms_transform.hrl").
 
--include("emqx_persistent_session_ds.hrl").
+-include("session_internals.hrl").
 
 %% API
 -export([

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_gc_worker.erl

@@ -21,7 +21,7 @@
 -include_lib("stdlib/include/qlc.hrl").
 -include_lib("stdlib/include/ms_transform.hrl").
 
--include("emqx_persistent_session_ds.hrl").
+-include("session_internals.hrl").
 
 %% API
 -export([

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl

@@ -79,7 +79,7 @@
 ]).
 
 -include("emqx_mqtt.hrl").
--include("emqx_persistent_session_ds.hrl").
+-include("session_internals.hrl").
 -include_lib("snabbkaffe/include/trace.hrl").
 -include_lib("stdlib/include/qlc.hrl").
 

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_stream_scheduler.erl

@@ -29,7 +29,7 @@
 
 -include_lib("emqx/include/logger.hrl").
 -include("emqx_mqtt.hrl").
--include("emqx_persistent_session_ds.hrl").
+-include("session_internals.hrl").
 
 %%================================================================================
 %% Type declarations

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_subs.erl

@@ -41,7 +41,7 @@
 
 -export_type([subscription_state_id/0, subscription/0, subscription_state/0]).
 
--include("emqx_persistent_session_ds.hrl").
+-include("session_internals.hrl").
 -include("emqx_mqtt.hrl").
 -include_lib("snabbkaffe/include/trace.hrl").
 

apps/emqx/src/emqx_persistent_session_ds/session_internals.hrl

@@ -13,10 +13,11 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--ifndef(EMQX_PERSISTENT_SESSION_DS_HRL_HRL).
+-ifndef(EMQX_SESSION_DS_INTERNALS_HRL).
--define(EMQX_PERSISTENT_SESSION_DS_HRL_HRL, true).
+-define(EMQX_SESSION_DS_INTERNALS_HRL, true).
 
 -include("emqx_persistent_message.hrl").
+-include("emqx_durable_session_metadata.hrl").
 
 -define(SESSION_TAB, emqx_ds_session).
 -define(SESSION_SUBSCRIPTIONS_TAB, emqx_ds_session_subscriptions).
@@ -70,17 +71,4 @@
     sub_state_id :: emqx_persistent_session_ds_subs:subscription_state_id()
 }).
 
-%% Session metadata keys:
--define(created_at, created_at).
--define(last_alive_at, last_alive_at).
--define(expiry_interval, expiry_interval).
-%% Unique integer used to create unique identities:
--define(last_id, last_id).
-%% Connection info (relevent for the dashboard):
--define(peername, peername).
--define(will_message, will_message).
--define(clientinfo, clientinfo).
--define(protocol, protocol).
--define(offline_info, offline_info).
-
 -endif.

apps/emqx_auth/src/emqx_auth.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth, [
     {description, "EMQX Authentication and authorization"},
-    {vsn, "0.3.1"},
+    {vsn, "0.4.0"},
     {modules, []},
     {registered, [emqx_auth_sup]},
     {applications, [

apps/emqx_bridge/src/emqx_bridge_v2_api.erl

@@ -96,7 +96,7 @@
 namespace() -> "actions_and_sources".
 
 api_spec() ->
-    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
+    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun check_api_schema/2}).
 
 paths() ->
     [
@@ -656,6 +656,40 @@ schema("/source_types") ->
         }
     }.
 
+%%------------------------------------------------------------------------------
+
+check_api_schema(Request, ReqMeta = #{path := "/actions/:id", method := put}) ->
+    BridgeId = emqx_utils_maps:deep_get([bindings, id], Request),
+    try emqx_bridge_resource:parse_bridge_id(BridgeId, #{atom_name => false}) of
+        %% NOTE
+        %% Bridge type is known, refine the API schema to get more specific error messages.
+        {BridgeType, _Name} ->
+            Schema = emqx_bridge_v2_schema:action_api_schema("put", BridgeType),
+            emqx_dashboard_swagger:filter_check_request(Request, refine_api_schema(Schema, ReqMeta))
+    catch
+        throw:#{reason := Reason} ->
+            ?NOT_FOUND(<<"Invalid bridge ID, ", Reason/binary>>)
+    end;
+check_api_schema(Request, ReqMeta = #{path := "/sources/:id", method := put}) ->
+    SourceId = emqx_utils_maps:deep_get([bindings, id], Request),
+    try emqx_bridge_resource:parse_bridge_id(SourceId, #{atom_name => false}) of
+        %% NOTE
+        %% Source type is known, refine the API schema to get more specific error messages.
+        {BridgeType, _Name} ->
+            Schema = emqx_bridge_v2_schema:source_api_schema("put", BridgeType),
+            emqx_dashboard_swagger:filter_check_request(Request, refine_api_schema(Schema, ReqMeta))
+    catch
+        throw:#{reason := Reason} ->
+            ?NOT_FOUND(<<"Invalid source ID, ", Reason/binary>>)
+    end;
+check_api_schema(Request, ReqMeta) ->
+    emqx_dashboard_swagger:filter_check_request(Request, ReqMeta).
+
+refine_api_schema(Schema, ReqMeta = #{path := Path, method := Method}) ->
+    Spec = maps:get(Method, schema(Path)),
+    SpecRefined = Spec#{'requestBody' => Schema},
+    ReqMeta#{apispec => SpecRefined}.
+
 %%------------------------------------------------------------------------------
 %% Thin Handlers
 %%------------------------------------------------------------------------------

apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl

@@ -31,6 +31,7 @@
     actions_get_response/0,
     actions_put_request/0,
     actions_post_request/0,
+    action_api_schema/2,
     actions_examples/1,
     action_values/4
 ]).
@@ -39,6 +40,7 @@
     sources_get_response/0,
     sources_put_request/0,
     sources_post_request/0,
+    source_api_schema/2,
     sources_examples/1,
     source_values/4
 ]).
@@ -100,6 +102,15 @@ actions_api_schema(Method) ->
     APISchemas = ?MODULE:registered_actions_api_schemas(Method),
     hoconsc:union(bridge_api_union(APISchemas)).
 
+action_api_schema(Method, BridgeV2Type) ->
+    APISchemas = ?MODULE:registered_actions_api_schemas(Method),
+    case lists:keyfind(atom_to_binary(BridgeV2Type), 1, APISchemas) of
+        {_, SchemaRef} ->
+            hoconsc:mk(SchemaRef);
+        false ->
+            unknown_bridge_schema(BridgeV2Type)
+    end.
+
 registered_actions_api_schemas(Method) ->
     RegisteredSchemas = emqx_action_info:registered_schema_modules_actions(),
     [
@@ -159,6 +170,15 @@ sources_api_schema(Method) ->
     APISchemas = ?MODULE:registered_sources_api_schemas(Method),
     hoconsc:union(bridge_api_union(APISchemas)).
 
+source_api_schema(Method, SourceType) ->
+    APISchemas = ?MODULE:registered_sources_api_schemas(Method),
+    case lists:keyfind(atom_to_binary(SourceType), 1, APISchemas) of
+        {_, SchemaRef} ->
+            hoconsc:mk(SchemaRef);
+        false ->
+            unknown_source_schema(SourceType)
+    end.
+
 registered_sources_api_schemas(Method) ->
     RegisteredSchemas = emqx_action_info:registered_schema_modules_sources(),
     [
@@ -231,6 +251,25 @@ bridge_api_union(Refs) ->
             end
     end.
 
+unknown_bridge_schema(BridgeV2Type) ->
+    erroneous_value_schema(BridgeV2Type, <<"unknown bridge type">>).
+
+unknown_source_schema(SourceType) ->
+    erroneous_value_schema(SourceType, <<"unknown source type">>).
+
+%% @doc Construct a schema that always emits validation error.
+%% We need to silence dialyzer because inner anonymous function always throws.
+-dialyzer({nowarn_function, [erroneous_value_schema/2]}).
+erroneous_value_schema(Value, Reason) ->
+    hoconsc:mk(typerefl:any(), #{
+        validator => fun(_) ->
+            throw(#{
+                value => Value,
+                reason => Reason
+            })
+        end
+    }).
+
 -spec method_values(action | source, http_method(), atom()) -> schema_example_map().
 method_values(Kind, post, Type) ->
     KindBin = atom_to_binary(Kind),

apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl

@@ -156,12 +156,15 @@ t_create_via_http(Config) ->
 t_on_get_status(Config) ->
     emqx_bridge_v2_testlib:t_on_get_status(Config, #{}).
 
-t_invalid_config(Config) ->
+t_create_invalid_config(Config) ->
     ?assertMatch(
         {error,
             {_Status, _, #{
                 <<"code">> := <<"BAD_REQUEST">>,
-                <<"message">> := #{<<"kind">> := <<"validation_error">>}
+                <<"message">> := #{
+                    <<"kind">> := <<"validation_error">>,
+                    <<"reason">> := <<"Inconsistent 'min_part_size'", _/bytes>>
+                }
             }}},
         emqx_bridge_v2_testlib:create_bridge_api(
             Config,
@@ -174,6 +177,28 @@ t_invalid_config(Config) ->
         )
     ).
 
+t_update_invalid_config(Config) ->
+    ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)),
+    ?assertMatch(
+        {error,
+            {_Status, _, #{
+                <<"code">> := <<"BAD_REQUEST">>,
+                <<"message">> := #{
+                    <<"kind">> := <<"validation_error">>,
+                    <<"reason">> := <<"Inconsistent 'min_part_size'", _/bytes>>
+                }
+            }}},
+        emqx_bridge_v2_testlib:update_bridge_api(
+            Config,
+            _Overrides = #{
+                <<"parameters">> => #{
+                    <<"min_part_size">> => <<"5GB">>,
+                    <<"max_part_size">> => <<"100MB">>
+                }
+            }
+        )
+    ).
+
 t_aggreg_upload(Config) ->
     Bucket = ?config(s3_bucket, Config),
     BridgeName = ?config(bridge_name, Config),

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -92,7 +92,14 @@
 -define(DEFAULT_ROW, 100).
 
 -type request() :: #{bindings => map(), query_string => map(), body => map()}.
--type request_meta() :: #{module => module(), path => string(), method => atom()}.
+-type request_meta() :: #{
+    module := module(),
+    path := string(),
+    method := atom(),
+    %% API Operation specification override.
+    %% Takes precedence over the API specification defined in the module.
+    apispec => map()
+}.
 
 %% More exact types are defined in minirest.hrl, but we don't want to include it
 %% because it defines a lot of types and they may clash with the types declared locally.
@@ -360,8 +367,8 @@ filter_check_request_and_translate_body(Request, RequestMeta) ->
 filter_check_request(Request, RequestMeta) ->
     translate_req(Request, RequestMeta, fun check_only/3).
 
-translate_req(Request, #{module := Module, path := Path, method := Method}, CheckFun) ->
+translate_req(Request, ReqMeta = #{module := Module}, CheckFun) ->
-    #{Method := Spec} = apply(Module, schema, [Path]),
+    Spec = find_req_apispec(ReqMeta),
     try
         Params = maps:get(parameters, Spec, []),
         Body = maps:get('requestBody', Spec, []),
@@ -378,6 +385,12 @@ translate_req(Request, #{module := Module, path := Path, method := Method}, Chec
             {400, 'BAD_REQUEST', Msg}
     end.
 
+find_req_apispec(#{apispec := Spec}) ->
+    Spec;
+find_req_apispec(#{module := Module, path := Path, method := Method}) ->
+    #{Method := Spec} = apply(Module, schema, [Path]),
+    Spec.
+
 check_and_translate(Schema, Map, Opts) ->
     hocon_tconf:check_plain(Schema, Map, Opts).
 

apps/emqx_management/src/emqx_management.app.src

@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.2.0"},
+    {vsn, "5.3.0"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [

apps/emqx_management/src/emqx_mgmt_api_clients.erl

@@ -24,6 +24,7 @@
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_utils/include/emqx_utils_api.hrl").
+-include_lib("emqx/include/emqx_durable_session_metadata.hrl").
 
 -include("emqx_mgmt.hrl").
 
@@ -1739,7 +1740,7 @@ format_channel_info(undefined, {ClientId, PSInfo0 = #{}}, _Opts) ->
 format_persistent_session_info(
     _ClientId,
     #{
-        metadata := #{offline_info := #{chan_info := ChanInfo, stats := Stats} = OfflineInfo} =
+        metadata := #{?offline_info := #{chan_info := ChanInfo, stats := Stats} = OfflineInfo} =
             Metadata
     } =
         PSInfo

apps/emqx_management/test/emqx_mgmt_api_configs_2_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -30,16 +30,16 @@ init_per_suite(Config) ->
     Cert = fun(Name) -> filename:join(CertDir, Name) end,
     %% keep it the same as default conf in emqx_dashboard.conf
     Conf =
-        "dashboard.listeners.http { enable = true, bind = 18083 }"
+        [
-        "dashboard.listeners.https {\n"
+            "dashboard.listeners.http { enable = true, bind = 18083 }",
-        "  bind = 0 # disabled by default\n"
+            "dashboard.listeners.https {\n",
-        "  ssl_options {\n"
+            "  bind = 0 # disabled by default\n",
-        "    certfile = \"" ++ Cert("cert.pem") ++
+            "  ssl_options {\n",
-            "\"\n"
+            "    certfile = \"" ++ Cert("cert.pem") ++ "\"\n",
-            "    keyfile = \"" ++ Cert("key.pem") ++
+            "    keyfile = \"" ++ Cert("key.pem") ++ "\"\n",
-            "\"\n"
             "  }\n"
-            "}\n",
+            "}\n"
+        ],
     Apps = emqx_cth_suite:start(
         [
             emqx_conf,
@@ -62,6 +62,8 @@ end_per_testcase(_TestCase, Config) ->
 t_dashboard(_Config) ->
     {ok, Dashboard = #{<<"listeners">> := Listeners}} = get_config("dashboard"),
     Https1 = #{enable => true, bind => 18084},
+    %% Ensure HTTPS listener can be enabled with just changing bind to a non-zero number
+    %% i.e. the default certs should work
     ?assertMatch(
         {ok, _},
         update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}})