diff --git a/apps/emqx/include/emqx_durable_session_metadata.hrl b/apps/emqx/include/emqx_durable_session_metadata.hrl new file mode 100644 index 000000000..74c037dca --- /dev/null +++ b/apps/emqx/include/emqx_durable_session_metadata.hrl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022, 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc This header contains definitions of durable session metadata +%% keys, that can be consumed by the external code. +-ifndef(EMQX_DURABLE_SESSION_META_HRL). +-define(EMQX_DURABLE_SESSION_META_HRL, true). + +%% Session metadata keys: +-define(created_at, created_at). +-define(last_alive_at, last_alive_at). +-define(expiry_interval, expiry_interval). +%% Unique integer used to create unique identities: +-define(last_id, last_id). +%% Connection info (relevent for the dashboard): +-define(peername, peername). +-define(will_message, will_message). +-define(clientinfo, clientinfo). +-define(protocol, protocol). +-define(offline_info, offline_info). + +-endif. diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 97769fe1f..c391bc1c8 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.3.1"}, + {vsn, "5.4.0"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 4bc6b4183..1177cf653 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -25,7 +25,7 @@ -include("emqx_mqtt.hrl"). --include("emqx_persistent_session_ds.hrl"). +-include("emqx_persistent_session_ds/session_internals.hrl"). -ifdef(TEST). -include_lib("proper/include/proper.hrl"). diff --git a/apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_message_ds_gc_worker.erl similarity index 99% rename from apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_message_ds_gc_worker.erl index e59d73db0..e9a00cd70 100644 --- a/apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_message_ds_gc_worker.erl @@ -21,7 +21,7 @@ -include_lib("stdlib/include/qlc.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). --include("emqx_persistent_session_ds.hrl"). +-include("session_internals.hrl"). %% API -export([ diff --git a/apps/emqx/src/emqx_persistent_session_bookkeeper.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_bookkeeper.erl similarity index 100% rename from apps/emqx/src/emqx_persistent_session_bookkeeper.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_bookkeeper.erl diff --git a/apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_gc_worker.erl similarity index 99% rename from apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_gc_worker.erl index 9fe33beea..8b7205dc1 100644 --- a/apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_gc_worker.erl @@ -21,7 +21,7 @@ -include_lib("stdlib/include/qlc.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). --include("emqx_persistent_session_ds.hrl"). +-include("session_internals.hrl"). %% API -export([ diff --git a/apps/emqx/src/emqx_persistent_session_ds_inflight.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_inflight.erl similarity index 100% rename from apps/emqx/src/emqx_persistent_session_ds_inflight.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_inflight.erl diff --git a/apps/emqx/src/emqx_persistent_session_ds_router.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl similarity index 100% rename from apps/emqx/src/emqx_persistent_session_ds_router.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl diff --git a/apps/emqx/src/emqx_persistent_session_ds_state.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl similarity index 99% rename from apps/emqx/src/emqx_persistent_session_ds_state.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl index 40c7bd08b..95f6ee375 100644 --- a/apps/emqx/src/emqx_persistent_session_ds_state.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl @@ -79,7 +79,7 @@ ]). -include("emqx_mqtt.hrl"). --include("emqx_persistent_session_ds.hrl"). +-include("session_internals.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). -include_lib("stdlib/include/qlc.hrl"). diff --git a/apps/emqx/src/emqx_persistent_session_ds_stream_scheduler.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_stream_scheduler.erl similarity index 99% rename from apps/emqx/src/emqx_persistent_session_ds_stream_scheduler.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_stream_scheduler.erl index c6a968a9a..7bf80cc2b 100644 --- a/apps/emqx/src/emqx_persistent_session_ds_stream_scheduler.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_stream_scheduler.erl @@ -29,7 +29,7 @@ -include_lib("emqx/include/logger.hrl"). -include("emqx_mqtt.hrl"). --include("emqx_persistent_session_ds.hrl"). +-include("session_internals.hrl"). %%================================================================================ %% Type declarations diff --git a/apps/emqx/src/emqx_persistent_session_ds_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_subs.erl similarity index 99% rename from apps/emqx/src/emqx_persistent_session_ds_subs.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_subs.erl index 8b4f70a69..3fa65eed5 100644 --- a/apps/emqx/src/emqx_persistent_session_ds_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_subs.erl @@ -41,7 +41,7 @@ -export_type([subscription_state_id/0, subscription/0, subscription_state/0]). --include("emqx_persistent_session_ds.hrl"). +-include("session_internals.hrl"). -include("emqx_mqtt.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). diff --git a/apps/emqx/src/emqx_persistent_session_ds_sup.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_sup.erl similarity index 100% rename from apps/emqx/src/emqx_persistent_session_ds_sup.erl rename to apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_sup.erl diff --git a/apps/emqx/src/emqx_persistent_session_ds.hrl b/apps/emqx/src/emqx_persistent_session_ds/session_internals.hrl similarity index 84% rename from apps/emqx/src/emqx_persistent_session_ds.hrl rename to apps/emqx/src/emqx_persistent_session_ds/session_internals.hrl index fdbf2c6ea..e18e2cb7f 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.hrl +++ b/apps/emqx/src/emqx_persistent_session_ds/session_internals.hrl @@ -13,10 +13,11 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --ifndef(EMQX_PERSISTENT_SESSION_DS_HRL_HRL). --define(EMQX_PERSISTENT_SESSION_DS_HRL_HRL, true). +-ifndef(EMQX_SESSION_DS_INTERNALS_HRL). +-define(EMQX_SESSION_DS_INTERNALS_HRL, true). -include("emqx_persistent_message.hrl"). +-include("emqx_durable_session_metadata.hrl"). -define(SESSION_TAB, emqx_ds_session). -define(SESSION_SUBSCRIPTIONS_TAB, emqx_ds_session_subscriptions). @@ -70,17 +71,4 @@ sub_state_id :: emqx_persistent_session_ds_subs:subscription_state_id() }). -%% Session metadata keys: --define(created_at, created_at). --define(last_alive_at, last_alive_at). --define(expiry_interval, expiry_interval). -%% Unique integer used to create unique identities: --define(last_id, last_id). -%% Connection info (relevent for the dashboard): --define(peername, peername). --define(will_message, will_message). --define(clientinfo, clientinfo). --define(protocol, protocol). --define(offline_info, offline_info). - -endif. diff --git a/apps/emqx_auth/src/emqx_auth.app.src b/apps/emqx_auth/src/emqx_auth.app.src index 6db2d6213..cd77a5ffc 100644 --- a/apps/emqx_auth/src/emqx_auth.app.src +++ b/apps/emqx_auth/src/emqx_auth.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth, [ {description, "EMQX Authentication and authorization"}, - {vsn, "0.3.1"}, + {vsn, "0.4.0"}, {modules, []}, {registered, [emqx_auth_sup]}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 003ec9a06..3905cf09c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -55,6 +55,8 @@ %% only for testing/mocking -export([supported_versions/1]). +-export([format_bridge_metrics/1, format_metrics/1]). + -define(BPAPI_NAME, emqx_bridge). -define(BRIDGE_NOT_ENABLED, diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index 56b2cb4ed..8ba2ef487 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -96,7 +96,7 @@ namespace() -> "actions_and_sources". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun check_api_schema/2}). paths() -> [ @@ -656,6 +656,40 @@ schema("/source_types") -> } }. +%%------------------------------------------------------------------------------ + +check_api_schema(Request, ReqMeta = #{path := "/actions/:id", method := put}) -> + BridgeId = emqx_utils_maps:deep_get([bindings, id], Request), + try emqx_bridge_resource:parse_bridge_id(BridgeId, #{atom_name => false}) of + %% NOTE + %% Bridge type is known, refine the API schema to get more specific error messages. + {BridgeType, _Name} -> + Schema = emqx_bridge_v2_schema:action_api_schema("put", BridgeType), + emqx_dashboard_swagger:filter_check_request(Request, refine_api_schema(Schema, ReqMeta)) + catch + throw:#{reason := Reason} -> + ?NOT_FOUND(<<"Invalid bridge ID, ", Reason/binary>>) + end; +check_api_schema(Request, ReqMeta = #{path := "/sources/:id", method := put}) -> + SourceId = emqx_utils_maps:deep_get([bindings, id], Request), + try emqx_bridge_resource:parse_bridge_id(SourceId, #{atom_name => false}) of + %% NOTE + %% Source type is known, refine the API schema to get more specific error messages. + {BridgeType, _Name} -> + Schema = emqx_bridge_v2_schema:source_api_schema("put", BridgeType), + emqx_dashboard_swagger:filter_check_request(Request, refine_api_schema(Schema, ReqMeta)) + catch + throw:#{reason := Reason} -> + ?NOT_FOUND(<<"Invalid source ID, ", Reason/binary>>) + end; +check_api_schema(Request, ReqMeta) -> + emqx_dashboard_swagger:filter_check_request(Request, ReqMeta). + +refine_api_schema(Schema, ReqMeta = #{path := Path, method := Method}) -> + Spec = maps:get(Method, schema(Path)), + SpecRefined = Spec#{'requestBody' => Schema}, + ReqMeta#{apispec => SpecRefined}. + %%------------------------------------------------------------------------------ %% Thin Handlers %%------------------------------------------------------------------------------ diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index caec1f53c..6dbad456b 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -31,6 +31,7 @@ actions_get_response/0, actions_put_request/0, actions_post_request/0, + action_api_schema/2, actions_examples/1, action_values/4 ]). @@ -39,6 +40,7 @@ sources_get_response/0, sources_put_request/0, sources_post_request/0, + source_api_schema/2, sources_examples/1, source_values/4 ]). @@ -100,6 +102,15 @@ actions_api_schema(Method) -> APISchemas = ?MODULE:registered_actions_api_schemas(Method), hoconsc:union(bridge_api_union(APISchemas)). +action_api_schema(Method, BridgeV2Type) -> + APISchemas = ?MODULE:registered_actions_api_schemas(Method), + case lists:keyfind(atom_to_binary(BridgeV2Type), 1, APISchemas) of + {_, SchemaRef} -> + hoconsc:mk(SchemaRef); + false -> + unknown_bridge_schema(BridgeV2Type) + end. + registered_actions_api_schemas(Method) -> RegisteredSchemas = emqx_action_info:registered_schema_modules_actions(), [ @@ -159,6 +170,15 @@ sources_api_schema(Method) -> APISchemas = ?MODULE:registered_sources_api_schemas(Method), hoconsc:union(bridge_api_union(APISchemas)). +source_api_schema(Method, SourceType) -> + APISchemas = ?MODULE:registered_sources_api_schemas(Method), + case lists:keyfind(atom_to_binary(SourceType), 1, APISchemas) of + {_, SchemaRef} -> + hoconsc:mk(SchemaRef); + false -> + unknown_source_schema(SourceType) + end. + registered_sources_api_schemas(Method) -> RegisteredSchemas = emqx_action_info:registered_schema_modules_sources(), [ @@ -231,6 +251,25 @@ bridge_api_union(Refs) -> end end. +unknown_bridge_schema(BridgeV2Type) -> + erroneous_value_schema(BridgeV2Type, <<"unknown bridge type">>). + +unknown_source_schema(SourceType) -> + erroneous_value_schema(SourceType, <<"unknown source type">>). + +%% @doc Construct a schema that always emits validation error. +%% We need to silence dialyzer because inner anonymous function always throws. +-dialyzer({nowarn_function, [erroneous_value_schema/2]}). +erroneous_value_schema(Value, Reason) -> + hoconsc:mk(typerefl:any(), #{ + validator => fun(_) -> + throw(#{ + value => Value, + reason => Reason + }) + end + }). + -spec method_values(action | source, http_method(), atom()) -> schema_example_map(). method_values(Kind, post, Type) -> KindBin = atom_to_binary(Kind), diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl index 09cf12329..b7c17bbaa 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl @@ -156,12 +156,15 @@ t_create_via_http(Config) -> t_on_get_status(Config) -> emqx_bridge_v2_testlib:t_on_get_status(Config, #{}). -t_invalid_config(Config) -> +t_create_invalid_config(Config) -> ?assertMatch( {error, {_Status, _, #{ <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := #{<<"kind">> := <<"validation_error">>} + <<"message">> := #{ + <<"kind">> := <<"validation_error">>, + <<"reason">> := <<"Inconsistent 'min_part_size'", _/bytes>> + } }}}, emqx_bridge_v2_testlib:create_bridge_api( Config, @@ -174,6 +177,28 @@ t_invalid_config(Config) -> ) ). +t_update_invalid_config(Config) -> + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + ?assertMatch( + {error, + {_Status, _, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := #{ + <<"kind">> := <<"validation_error">>, + <<"reason">> := <<"Inconsistent 'min_part_size'", _/bytes>> + } + }}}, + emqx_bridge_v2_testlib:update_bridge_api( + Config, + _Overrides = #{ + <<"parameters">> => #{ + <<"min_part_size">> => <<"5GB">>, + <<"max_part_size">> => <<"100MB">> + } + } + ) + ). + t_aggreg_upload(Config) -> Bucket = ?config(s3_bucket, Config), BridgeName = ?config(bridge_name, Config), diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src index 0f977e81e..4b3135f5f 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_sqlserver, [ {description, "EMQX Enterprise SQL Server Bridge"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, odbc]}, {env, [ diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl index 726d2656a..840715ac3 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl @@ -478,7 +478,7 @@ worker_do_insert( {error, {unrecoverable_error, {invalid_request, Reason}}} end. --spec execute(pid(), sql()) -> +-spec execute(connection_reference(), sql()) -> updated_tuple() | selected_tuple() | [updated_tuple()] @@ -487,7 +487,7 @@ worker_do_insert( execute(Conn, SQL) -> odbc:sql_query(Conn, str(SQL)). --spec execute(pid(), sql(), time_out()) -> +-spec execute(connection_reference(), sql(), time_out()) -> updated_tuple() | selected_tuple() | [updated_tuple()] diff --git a/apps/emqx_conf/include/emqx_conf.hrl b/apps/emqx_conf/include/emqx_conf.hrl index d79f59bc7..db66e6f29 100644 --- a/apps/emqx_conf/include/emqx_conf.hrl +++ b/apps/emqx_conf/include/emqx_conf.hrl @@ -73,11 +73,21 @@ (?CE_AUTHN_PROVIDER_SCHEMA_MODS ++ ?EE_AUTHN_PROVIDER_SCHEMA_MODS) ). +-define(OTHER_INJECTING_CONFIGS, []). + -else. -define(AUTHZ_SOURCE_SCHEMA_MODS, ?CE_AUTHZ_SOURCE_SCHEMA_MODS). -define(AUTHN_PROVIDER_SCHEMA_MODS, ?CE_AUTHN_PROVIDER_SCHEMA_MODS). --endif. +-define(OTHER_INJECTING_CONFIGS, []). + +-endif. + +-define(INJECTING_CONFIGS, [ + {emqx_authn_schema, ?AUTHN_PROVIDER_SCHEMA_MODS}, + {emqx_authz_schema, ?AUTHZ_SOURCE_SCHEMA_MODS} + | ?OTHER_INJECTING_CONFIGS +]). -endif. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index d5b902ba4..d24c73683 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -70,10 +70,6 @@ emqx_otel_schema, emqx_mgmt_api_key_schema ]). --define(INJECTING_CONFIGS, [ - {emqx_authn_schema, ?AUTHN_PROVIDER_SCHEMA_MODS}, - {emqx_authz_schema, ?AUTHZ_SOURCE_SCHEMA_MODS} -]). %% 1 million default ports counter -define(DEFAULT_MAX_PORTS, 1024 * 1024). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 9112578dc..ae31f4871 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -92,7 +92,14 @@ -define(DEFAULT_ROW, 100). -type request() :: #{bindings => map(), query_string => map(), body => map()}. --type request_meta() :: #{module => module(), path => string(), method => atom()}. +-type request_meta() :: #{ + module := module(), + path := string(), + method := atom(), + %% API Operation specification override. + %% Takes precedence over the API specification defined in the module. + apispec => map() +}. %% More exact types are defined in minirest.hrl, but we don't want to include it %% because it defines a lot of types and they may clash with the types declared locally. @@ -360,8 +367,8 @@ filter_check_request_and_translate_body(Request, RequestMeta) -> filter_check_request(Request, RequestMeta) -> translate_req(Request, RequestMeta, fun check_only/3). -translate_req(Request, #{module := Module, path := Path, method := Method}, CheckFun) -> - #{Method := Spec} = apply(Module, schema, [Path]), +translate_req(Request, ReqMeta = #{module := Module}, CheckFun) -> + Spec = find_req_apispec(ReqMeta), try Params = maps:get(parameters, Spec, []), Body = maps:get('requestBody', Spec, []), @@ -378,6 +385,12 @@ translate_req(Request, #{module := Module, path := Path, method := Method}, Chec {400, 'BAD_REQUEST', Msg} end. +find_req_apispec(#{apispec := Spec}) -> + Spec; +find_req_apispec(#{module := Module, path := Path, method := Method}) -> + #{Method := Spec} = apply(Module, schema, [Path]), + Spec. + check_and_translate(Schema, Map, Opts) -> hocon_tconf:check_plain(Schema, Map, Opts). diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index a1432f367..f01f5c7b9 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.2.0"}, + {vsn, "5.3.0"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [ diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index ecef5b720..754209257 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -24,6 +24,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_utils/include/emqx_utils_api.hrl"). +-include_lib("emqx/include/emqx_durable_session_metadata.hrl"). -include("emqx_mgmt.hrl"). @@ -1739,7 +1740,7 @@ format_channel_info(undefined, {ClientId, PSInfo0 = #{}}, _Opts) -> format_persistent_session_info( _ClientId, #{ - metadata := #{offline_info := #{chan_info := ChanInfo, stats := Stats} = OfflineInfo} = + metadata := #{?offline_info := #{chan_info := ChanInfo, stats := Stats} = OfflineInfo} = Metadata } = PSInfo diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_2_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_2_SUITE.erl new file mode 100644 index 000000000..93ffa3eb8 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_2_SUITE.erl @@ -0,0 +1,154 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_api_configs_2_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + _ = application:load(emqx), + CertDir = filename:join([code:lib_dir(emqx), "etc", "certs"]), + Cert = fun(Name) -> filename:join(CertDir, Name) end, + %% keep it the same as default conf in emqx_dashboard.conf + Conf = + [ + "dashboard.listeners.http { enable = true, bind = 18083 }", + "dashboard.listeners.https {\n", + " bind = 0 # disabled by default\n", + " ssl_options {\n", + " certfile = \"" ++ Cert("cert.pem") ++ "\"\n", + " keyfile = \"" ++ Cert("key.pem") ++ "\"\n", + " }\n" + "}\n" + ], + Apps = emqx_cth_suite:start( + [ + emqx_conf, + emqx_management, + emqx_mgmt_api_test_util:emqx_dashboard(Conf) + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{suite_apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(suite_apps, Config)). + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, Config) -> + Config. + +t_dashboard(_Config) -> + {ok, Dashboard = #{<<"listeners">> := Listeners}} = get_config("dashboard"), + Https1 = #{enable => true, bind => 18084}, + %% Ensure HTTPS listener can be enabled with just changing bind to a non-zero number + %% i.e. the default certs should work + ?assertMatch( + {ok, _}, + update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}}) + ), + + Https2 = #{ + <<"bind">> => 18084, + <<"ssl_options">> => + #{ + <<"keyfile">> => "etc/certs/badkey.pem", + <<"cacertfile">> => "etc/certs/badcacert.pem", + <<"certfile">> => "etc/certs/badcert.pem" + } + }, + Dashboard2 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https2}}, + ?assertMatch( + {error, {"HTTP/1.1", 400, _}}, + update_config("dashboard", Dashboard2) + ), + + FilePath = fun(Name) -> + iolist_to_binary( + emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", Name])) + ) + end, + KeyFile = FilePath("key.pem"), + CertFile = FilePath("cert.pem"), + CacertFile = FilePath("cacert.pem"), + Https3 = #{ + <<"bind">> => 18084, + <<"ssl_options">> => #{ + <<"keyfile">> => KeyFile, + <<"cacertfile">> => CacertFile, + <<"certfile">> => CertFile + } + }, + Dashboard3 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https3}}, + ?assertMatch({ok, _}, update_config("dashboard", Dashboard3)), + + Dashboard4 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => #{<<"bind">> => 0}}}, + ?assertMatch({ok, _}, update_config("dashboard", Dashboard4)), + {ok, Dashboard41} = get_config("dashboard"), + ?assertMatch( + #{ + <<"bind">> := 0, + <<"ssl_options">> := + #{ + <<"keyfile">> := KeyFile, + <<"cacertfile">> := CacertFile, + <<"certfile">> := CertFile + } + }, + read_conf([<<"dashboard">>, <<"listeners">>, <<"https">>]), + Dashboard41 + ), + + ?assertMatch({ok, _}, update_config("dashboard", Dashboard)), + {ok, Dashboard1} = get_config("dashboard"), + ?assertEqual(Dashboard, Dashboard1), + timer:sleep(1500), + ok. + +%% Helpers + +get_config(Name) -> + Path = emqx_mgmt_api_test_util:api_path(["configs", Name]), + case emqx_mgmt_api_test_util:request_api(get, Path) of + {ok, Res} -> + {ok, emqx_utils_json:decode(Res, [return_maps])}; + Error -> + Error + end. + +update_config(Name, Change) -> + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + UpdatePath = emqx_mgmt_api_test_util:api_path(["configs", Name]), + case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of + {ok, Update} -> {ok, emqx_utils_json:decode(Update, [return_maps])}; + Error -> Error + end. + +read_conf(RootKeys) when is_list(RootKeys) -> + case emqx_config:read_override_conf(#{override_to => cluster}) of + undefined -> undefined; + Conf -> emqx_utils_maps:deep_get(RootKeys, Conf, undefined) + end; +read_conf(RootKey) -> + read_conf([RootKey]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 678a00cad..496192e39 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -225,71 +225,6 @@ update_global_zone(Change) -> % ?assertEqual(undefined, emqx_config:get_raw([zones, new_zone], undefined)), % ok. -t_dashboard(_Config) -> - {ok, Dashboard = #{<<"listeners">> := Listeners}} = get_config("dashboard"), - Https1 = #{enable => true, bind => 18084}, - ?assertMatch( - {ok, _}, - update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}}) - ), - - Https2 = #{ - <<"bind">> => 18084, - <<"ssl_options">> => - #{ - <<"keyfile">> => "etc/certs/badkey.pem", - <<"cacertfile">> => "etc/certs/badcacert.pem", - <<"certfile">> => "etc/certs/badcert.pem" - } - }, - Dashboard2 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https2}}, - ?assertMatch( - {error, {"HTTP/1.1", 400, _}}, - update_config("dashboard", Dashboard2) - ), - - FilePath = fun(Name) -> - iolist_to_binary( - emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", Name])) - ) - end, - KeyFile = FilePath("key.pem"), - CertFile = FilePath("cert.pem"), - CacertFile = FilePath("cacert.pem"), - Https3 = #{ - <<"bind">> => 18084, - <<"ssl_options">> => #{ - <<"keyfile">> => KeyFile, - <<"cacertfile">> => CacertFile, - <<"certfile">> => CertFile - } - }, - Dashboard3 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https3}}, - ?assertMatch({ok, _}, update_config("dashboard", Dashboard3)), - - Dashboard4 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => #{<<"bind">> => 0}}}, - ?assertMatch({ok, _}, update_config("dashboard", Dashboard4)), - {ok, Dashboard41} = get_config("dashboard"), - ?assertMatch( - #{ - <<"bind">> := 0, - <<"ssl_options">> := - #{ - <<"keyfile">> := KeyFile, - <<"cacertfile">> := CacertFile, - <<"certfile">> := CertFile - } - }, - read_conf([<<"dashboard">>, <<"listeners">>, <<"https">>]), - Dashboard41 - ), - - ?assertMatch({ok, _}, update_config("dashboard", Dashboard)), - {ok, Dashboard1} = get_config("dashboard"), - ?assertEqual(Dashboard, Dashboard1), - timer:sleep(1500), - ok. - %% v1 version json t_configs_node({'init', Config}) -> Node = node(), diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 9a269a509..22df66011 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -375,6 +375,7 @@ t_clean(Config) -> [{qos, 0}, {retain, true}], Config ), + ct:sleep(100), {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/#">>, [{qos, 0}, {rh, 0}]), ?assertEqual(3, length(receive_messages(3))), diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl index 8e14af449..179c72de3 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl @@ -284,6 +284,22 @@ t_rule_test_smoke(_Config) -> <<"sql">> => <<"SELECT\n *\nFROM\n \"t/#\"">> } }, + #{ + expected => #{code => 412}, + input => + #{ + <<"context">> => + #{ + <<"clientid">> => <<"c_emqx">>, + <<"event_type">> => <<"client_check_authn_complete">>, + <<"reason_code">> => <<"sucess">>, + <<"is_superuser">> => true, + <<"is_anonymous">> => false, + <<"username">> => <<"u_emqx">> + }, + <<"sql">> => <<"SELECT\n *\nFROM\n \"t/#\"">> + } + }, #{ expected => #{code => 412}, input =>