diff --git a/apps/emqx/include/emqx_durable_session_metadata.hrl b/apps/emqx/include/emqx_durable_session_metadata.hrl new file mode 100644 index 000000000..74c037dca --- /dev/null +++ b/apps/emqx/include/emqx_durable_session_metadata.hrl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022, 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc This header contains definitions of durable session metadata +%% keys, that can be consumed by the external code. +-ifndef(EMQX_DURABLE_SESSION_META_HRL). +-define(EMQX_DURABLE_SESSION_META_HRL, true). + +%% Session metadata keys: +-define(created_at, created_at). +-define(last_alive_at, last_alive_at). +-define(expiry_interval, expiry_interval). +%% Unique integer used to create unique identities: +-define(last_id, last_id). +%% Connection info (relevent for the dashboard): +-define(peername, peername). +-define(will_message, will_message). +-define(clientinfo, clientinfo). +-define(protocol, protocol). +-define(offline_info, offline_info). + +-endif. diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 97769fe1f..c391bc1c8 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.3.1"}, + {vsn, "5.4.0"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 4bc6b4183..1177cf653 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -25,7 +25,7 @@ -include("emqx_mqtt.hrl"). --include("emqx_persistent_session_ds.hrl"). +-include("emqx_persistent_session_ds/session_internals.hrl"). -ifdef(TEST). -include_lib("proper/include/proper.hrl"). diff --git a/apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_message_ds_gc_worker.erl similarity index 99% rename from apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_message_ds_gc_worker.erl index e59d73db0..e9a00cd70 100644 --- a/apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_message_ds_gc_worker.erl @@ -21,7 +21,7 @@ -include_lib("stdlib/include/qlc.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). --include("emqx_persistent_session_ds.hrl"). +-include("session_internals.hrl"). %% API -export([ diff --git a/apps/emqx/src/emqx_persistent_session_bookkeeper.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_bookkeeper.erl similarity index 100% rename from apps/emqx/src/emqx_persistent_session_bookkeeper.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_bookkeeper.erl diff --git a/apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_gc_worker.erl similarity index 99% rename from apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_gc_worker.erl index 9fe33beea..8b7205dc1 100644 --- a/apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_gc_worker.erl @@ -21,7 +21,7 @@ -include_lib("stdlib/include/qlc.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). --include("emqx_persistent_session_ds.hrl"). +-include("session_internals.hrl"). %% API -export([ diff --git a/apps/emqx/src/emqx_persistent_session_ds_inflight.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_inflight.erl similarity index 100% rename from apps/emqx/src/emqx_persistent_session_ds_inflight.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_inflight.erl diff --git a/apps/emqx/src/emqx_persistent_session_ds_router.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl similarity index 100% rename from apps/emqx/src/emqx_persistent_session_ds_router.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl diff --git a/apps/emqx/src/emqx_persistent_session_ds_state.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl similarity index 99% rename from apps/emqx/src/emqx_persistent_session_ds_state.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl index 40c7bd08b..95f6ee375 100644 --- a/apps/emqx/src/emqx_persistent_session_ds_state.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl @@ -79,7 +79,7 @@ ]). -include("emqx_mqtt.hrl"). --include("emqx_persistent_session_ds.hrl"). +-include("session_internals.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). -include_lib("stdlib/include/qlc.hrl"). diff --git a/apps/emqx/src/emqx_persistent_session_ds_stream_scheduler.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_stream_scheduler.erl similarity index 99% rename from apps/emqx/src/emqx_persistent_session_ds_stream_scheduler.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_stream_scheduler.erl index c6a968a9a..7bf80cc2b 100644 --- a/apps/emqx/src/emqx_persistent_session_ds_stream_scheduler.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_stream_scheduler.erl @@ -29,7 +29,7 @@ -include_lib("emqx/include/logger.hrl"). -include("emqx_mqtt.hrl"). --include("emqx_persistent_session_ds.hrl"). +-include("session_internals.hrl"). %%================================================================================ %% Type declarations diff --git a/apps/emqx/src/emqx_persistent_session_ds_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_subs.erl similarity index 99% rename from apps/emqx/src/emqx_persistent_session_ds_subs.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_subs.erl index 8b4f70a69..3fa65eed5 100644 --- a/apps/emqx/src/emqx_persistent_session_ds_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_subs.erl @@ -41,7 +41,7 @@ -export_type([subscription_state_id/0, subscription/0, subscription_state/0]). --include("emqx_persistent_session_ds.hrl"). +-include("session_internals.hrl"). -include("emqx_mqtt.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). diff --git a/apps/emqx/src/emqx_persistent_session_ds_sup.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_sup.erl similarity index 100% rename from apps/emqx/src/emqx_persistent_session_ds_sup.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_sup.erl diff --git a/apps/emqx/src/emqx_persistent_session_ds.hrl b/apps/emqx/src/emqx_persistent_session_ds/session_internals.hrl similarity index 84% rename from apps/emqx/src/emqx_persistent_session_ds.hrl rename to apps/emqx/src/emqx_persistent_session_ds/session_internals.hrl index fdbf2c6ea..e18e2cb7f 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.hrl +++ b/apps/emqx/src/emqx_persistent_session_ds/session_internals.hrl @@ -13,10 +13,11 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --ifndef(EMQX_PERSISTENT_SESSION_DS_HRL_HRL). --define(EMQX_PERSISTENT_SESSION_DS_HRL_HRL, true). +-ifndef(EMQX_SESSION_DS_INTERNALS_HRL). +-define(EMQX_SESSION_DS_INTERNALS_HRL, true). -include("emqx_persistent_message.hrl"). +-include("emqx_durable_session_metadata.hrl"). -define(SESSION_TAB, emqx_ds_session). -define(SESSION_SUBSCRIPTIONS_TAB, emqx_ds_session_subscriptions). @@ -70,17 +71,4 @@ sub_state_id :: emqx_persistent_session_ds_subs:subscription_state_id() }). -%% Session metadata keys: --define(created_at, created_at). --define(last_alive_at, last_alive_at). --define(expiry_interval, expiry_interval). -%% Unique integer used to create unique identities: --define(last_id, last_id). -%% Connection info (relevent for the dashboard): --define(peername, peername). --define(will_message, will_message). --define(clientinfo, clientinfo). --define(protocol, protocol). --define(offline_info, offline_info). - -endif. diff --git a/apps/emqx_auth/src/emqx_auth.app.src b/apps/emqx_auth/src/emqx_auth.app.src index 6db2d6213..cd77a5ffc 100644 --- a/apps/emqx_auth/src/emqx_auth.app.src +++ b/apps/emqx_auth/src/emqx_auth.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth, [ {description, "EMQX Authentication and authorization"}, - {vsn, "0.3.1"}, + {vsn, "0.4.0"}, {modules, []}, {registered, [emqx_auth_sup]}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index 56b2cb4ed..8ba2ef487 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -96,7 +96,7 @@ namespace() -> "actions_and_sources". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun check_api_schema/2}). paths() -> [ @@ -656,6 +656,40 @@ schema("/source_types") -> } }. +%%------------------------------------------------------------------------------ + +check_api_schema(Request, ReqMeta = #{path := "/actions/:id", method := put}) -> + BridgeId = emqx_utils_maps:deep_get([bindings, id], Request), + try emqx_bridge_resource:parse_bridge_id(BridgeId, #{atom_name => false}) of + %% NOTE + %% Bridge type is known, refine the API schema to get more specific error messages. + {BridgeType, _Name} -> + Schema = emqx_bridge_v2_schema:action_api_schema("put", BridgeType), + emqx_dashboard_swagger:filter_check_request(Request, refine_api_schema(Schema, ReqMeta)) + catch + throw:#{reason := Reason} -> + ?NOT_FOUND(<<"Invalid bridge ID, ", Reason/binary>>) + end; +check_api_schema(Request, ReqMeta = #{path := "/sources/:id", method := put}) -> + SourceId = emqx_utils_maps:deep_get([bindings, id], Request), + try emqx_bridge_resource:parse_bridge_id(SourceId, #{atom_name => false}) of + %% NOTE + %% Source type is known, refine the API schema to get more specific error messages. + {BridgeType, _Name} -> + Schema = emqx_bridge_v2_schema:source_api_schema("put", BridgeType), + emqx_dashboard_swagger:filter_check_request(Request, refine_api_schema(Schema, ReqMeta)) + catch + throw:#{reason := Reason} -> + ?NOT_FOUND(<<"Invalid source ID, ", Reason/binary>>) + end; +check_api_schema(Request, ReqMeta) -> + emqx_dashboard_swagger:filter_check_request(Request, ReqMeta). + +refine_api_schema(Schema, ReqMeta = #{path := Path, method := Method}) -> + Spec = maps:get(Method, schema(Path)), + SpecRefined = Spec#{'requestBody' => Schema}, + ReqMeta#{apispec => SpecRefined}. + %%------------------------------------------------------------------------------ %% Thin Handlers %%------------------------------------------------------------------------------ diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index caec1f53c..6dbad456b 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -31,6 +31,7 @@ actions_get_response/0, actions_put_request/0, actions_post_request/0, + action_api_schema/2, actions_examples/1, action_values/4 ]). @@ -39,6 +40,7 @@ sources_get_response/0, sources_put_request/0, sources_post_request/0, + source_api_schema/2, sources_examples/1, source_values/4 ]). @@ -100,6 +102,15 @@ actions_api_schema(Method) -> APISchemas = ?MODULE:registered_actions_api_schemas(Method), hoconsc:union(bridge_api_union(APISchemas)). +action_api_schema(Method, BridgeV2Type) -> + APISchemas = ?MODULE:registered_actions_api_schemas(Method), + case lists:keyfind(atom_to_binary(BridgeV2Type), 1, APISchemas) of + {_, SchemaRef} -> + hoconsc:mk(SchemaRef); + false -> + unknown_bridge_schema(BridgeV2Type) + end. + registered_actions_api_schemas(Method) -> RegisteredSchemas = emqx_action_info:registered_schema_modules_actions(), [ @@ -159,6 +170,15 @@ sources_api_schema(Method) -> APISchemas = ?MODULE:registered_sources_api_schemas(Method), hoconsc:union(bridge_api_union(APISchemas)). +source_api_schema(Method, SourceType) -> + APISchemas = ?MODULE:registered_sources_api_schemas(Method), + case lists:keyfind(atom_to_binary(SourceType), 1, APISchemas) of + {_, SchemaRef} -> + hoconsc:mk(SchemaRef); + false -> + unknown_source_schema(SourceType) + end. + registered_sources_api_schemas(Method) -> RegisteredSchemas = emqx_action_info:registered_schema_modules_sources(), [ @@ -231,6 +251,25 @@ bridge_api_union(Refs) -> end end. +unknown_bridge_schema(BridgeV2Type) -> + erroneous_value_schema(BridgeV2Type, <<"unknown bridge type">>). + +unknown_source_schema(SourceType) -> + erroneous_value_schema(SourceType, <<"unknown source type">>). + +%% @doc Construct a schema that always emits validation error. +%% We need to silence dialyzer because inner anonymous function always throws. +-dialyzer({nowarn_function, [erroneous_value_schema/2]}). +erroneous_value_schema(Value, Reason) -> + hoconsc:mk(typerefl:any(), #{ + validator => fun(_) -> + throw(#{ + value => Value, + reason => Reason + }) + end + }). + -spec method_values(action | source, http_method(), atom()) -> schema_example_map(). method_values(Kind, post, Type) -> KindBin = atom_to_binary(Kind), diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl index 09cf12329..b7c17bbaa 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl @@ -156,12 +156,15 @@ t_create_via_http(Config) -> t_on_get_status(Config) -> emqx_bridge_v2_testlib:t_on_get_status(Config, #{}). -t_invalid_config(Config) -> +t_create_invalid_config(Config) -> ?assertMatch( {error, {_Status, _, #{ <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := #{<<"kind">> := <<"validation_error">>} + <<"message">> := #{ + <<"kind">> := <<"validation_error">>, + <<"reason">> := <<"Inconsistent 'min_part_size'", _/bytes>> + } }}}, emqx_bridge_v2_testlib:create_bridge_api( Config, @@ -174,6 +177,28 @@ t_invalid_config(Config) -> ) ). +t_update_invalid_config(Config) -> + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + ?assertMatch( + {error, + {_Status, _, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := #{ + <<"kind">> := <<"validation_error">>, + <<"reason">> := <<"Inconsistent 'min_part_size'", _/bytes>> + } + }}}, + emqx_bridge_v2_testlib:update_bridge_api( + Config, + _Overrides = #{ + <<"parameters">> => #{ + <<"min_part_size">> => <<"5GB">>, + <<"max_part_size">> => <<"100MB">> + } + } + ) + ). + t_aggreg_upload(Config) -> Bucket = ?config(s3_bucket, Config), BridgeName = ?config(bridge_name, Config), diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 9112578dc..ae31f4871 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -92,7 +92,14 @@ -define(DEFAULT_ROW, 100). -type request() :: #{bindings => map(), query_string => map(), body => map()}. --type request_meta() :: #{module => module(), path => string(), method => atom()}. +-type request_meta() :: #{ + module := module(), + path := string(), + method := atom(), + %% API Operation specification override. + %% Takes precedence over the API specification defined in the module. + apispec => map() +}. %% More exact types are defined in minirest.hrl, but we don't want to include it %% because it defines a lot of types and they may clash with the types declared locally. @@ -360,8 +367,8 @@ filter_check_request_and_translate_body(Request, RequestMeta) -> filter_check_request(Request, RequestMeta) -> translate_req(Request, RequestMeta, fun check_only/3). -translate_req(Request, #{module := Module, path := Path, method := Method}, CheckFun) -> - #{Method := Spec} = apply(Module, schema, [Path]), +translate_req(Request, ReqMeta = #{module := Module}, CheckFun) -> + Spec = find_req_apispec(ReqMeta), try Params = maps:get(parameters, Spec, []), Body = maps:get('requestBody', Spec, []), @@ -378,6 +385,12 @@ translate_req(Request, #{module := Module, path := Path, method := Method}, Chec {400, 'BAD_REQUEST', Msg} end. +find_req_apispec(#{apispec := Spec}) -> + Spec; +find_req_apispec(#{module := Module, path := Path, method := Method}) -> + #{Method := Spec} = apply(Module, schema, [Path]), + Spec. + check_and_translate(Schema, Map, Opts) -> hocon_tconf:check_plain(Schema, Map, Opts). diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index a1432f367..f01f5c7b9 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.2.0"}, + {vsn, "5.3.0"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [ diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index ecef5b720..754209257 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -24,6 +24,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_utils/include/emqx_utils_api.hrl"). +-include_lib("emqx/include/emqx_durable_session_metadata.hrl"). -include("emqx_mgmt.hrl"). @@ -1739,7 +1740,7 @@ format_channel_info(undefined, {ClientId, PSInfo0 = #{}}, _Opts) -> format_persistent_session_info( _ClientId, #{ - metadata := #{offline_info := #{chan_info := ChanInfo, stats := Stats} = OfflineInfo} = + metadata := #{?offline_info := #{chan_info := ChanInfo, stats := Stats} = OfflineInfo} = Metadata } = PSInfo diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_2_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_2_SUITE.erl index 4bc2d55e7..93ffa3eb8 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_2_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_2_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -30,16 +30,16 @@ init_per_suite(Config) -> Cert = fun(Name) -> filename:join(CertDir, Name) end, %% keep it the same as default conf in emqx_dashboard.conf Conf = - "dashboard.listeners.http { enable = true, bind = 18083 }" - "dashboard.listeners.https {\n" - " bind = 0 # disabled by default\n" - " ssl_options {\n" - " certfile = \"" ++ Cert("cert.pem") ++ - "\"\n" - " keyfile = \"" ++ Cert("key.pem") ++ - "\"\n" + [ + "dashboard.listeners.http { enable = true, bind = 18083 }", + "dashboard.listeners.https {\n", + " bind = 0 # disabled by default\n", + " ssl_options {\n", + " certfile = \"" ++ Cert("cert.pem") ++ "\"\n", + " keyfile = \"" ++ Cert("key.pem") ++ "\"\n", " }\n" - "}\n", + "}\n" + ], Apps = emqx_cth_suite:start( [ emqx_conf, @@ -62,6 +62,8 @@ end_per_testcase(_TestCase, Config) -> t_dashboard(_Config) -> {ok, Dashboard = #{<<"listeners">> := Listeners}} = get_config("dashboard"), Https1 = #{enable => true, bind => 18084}, + %% Ensure HTTPS listener can be enabled with just changing bind to a non-zero number + %% i.e. the default certs should work ?assertMatch( {ok, _}, update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}})