diff --git a/README-CN.md b/README-CN.md index c6efab682..974a0a126 100644 --- a/README-CN.md +++ b/README-CN.md @@ -1,3 +1,5 @@ +简体中文 | [English](./README.md) | [Русский](./README-RU.md) + # EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) diff --git a/README-RU.md b/README-RU.md index 45bf08102..e4fe92696 100644 --- a/README-RU.md +++ b/README-RU.md @@ -1,3 +1,5 @@ +Русский | [简体中文](./README-CN.md) | [English](./README.md) + # Брокер EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) diff --git a/README.md b/README.md index 98f78c297..9fb2ae61a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +English | [简体中文](./README-CN.md) | [Русский](./README-RU.md) + # EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) diff --git a/apps/emqx/include/emqx_metrics.hrl b/apps/emqx/include/emqx_metrics.hrl index ddb537e6c..6fcbfd3f1 100644 --- a/apps/emqx/include/emqx_metrics.hrl +++ b/apps/emqx/include/emqx_metrics.hrl @@ -119,12 +119,12 @@ %% All Messages received {counter, 'messages.received', << "Number of messages received from the client, equal to the sum of " - "messages.qos0.received\fmessages.qos1.received and messages.qos2.received" + "messages.qos0.received, messages.qos1.received and messages.qos2.received" >>}, %% All Messages sent {counter, 'messages.sent', << "Number of messages sent to the client, equal to the sum of " - "messages.qos0.sent\fmessages.qos1.sent and messages.qos2.sent" + "messages.qos0.sent, messages.qos1.sent and messages.qos2.sent" >>}, %% QoS0 Messages received {counter, 'messages.qos0.received', <<"Number of QoS 0 messages received from clients">>}, diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index 195189635..90c4e4fec 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -100,7 +100,7 @@ update_config(Path, ConfigRequest) -> override_to => cluster }). --spec parse_http_resp_body(binary(), binary()) -> allow | deny | ignore | error. +-spec parse_http_resp_body(binary(), binary()) -> allow | deny | ignore | error | {error, term()}. parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> try result(maps:from_list(cow_qs:parse_qs(Body))) @@ -112,7 +112,9 @@ parse_http_resp_body(<<"application/json", _/binary>>, Body) -> result(emqx_utils_json:decode(Body, [return_maps])) catch _:_ -> error - end. + end; +parse_http_resp_body(ContentType = <<_/binary>>, _Body) -> + {error, <<"unsupported content-type: ", ContentType/binary>>}. result(#{<<"result">> := <<"allow">>}) -> allow; result(#{<<"result">> := <<"deny">>}) -> deny; diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index 7877b24fc..b1c2ef0ab 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -106,6 +106,9 @@ authorize( body => Body }), nomatch; + {error, Reason} -> + ?tp(error, bad_authz_http_response, #{reason => Reason}), + nomatch; Result -> {matched, Result} end; diff --git a/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl index 06e46e726..d165d288d 100644 --- a/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl @@ -463,6 +463,72 @@ t_placeholder_and_body(_Config) -> emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ). +%% Checks that we don't crash when receiving an unsupported content-type back. +t_bad_response_content_type(_Config) -> + ok = setup_handler_and_config( + fun(Req0, State) -> + ?assertEqual( + <<"/authz/users/">>, + cowboy_req:path(Req0) + ), + + {ok, _PostVars, Req1} = cowboy_req:read_urlencoded_body(Req0), + + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"text/csv">>}, + "hi", + Req1 + ), + {ok, Req, State} + end, + #{ + <<"method">> => <<"post">>, + <<"body">> => #{ + <<"username">> => <<"${username}">>, + <<"clientid">> => <<"${clientid}">>, + <<"peerhost">> => <<"${peerhost}">>, + <<"proto_name">> => <<"${proto_name}">>, + <<"mountpoint">> => <<"${mountpoint}">>, + <<"topic">> => <<"${topic}">>, + <<"action">> => <<"${action}">>, + <<"access">> => <<"${access}">>, + <<"CN">> => ?PH_CERT_CN_NAME, + <<"CS">> => ?PH_CERT_SUBJECT + }, + <<"headers">> => #{ + <<"accept">> => <<"text/plain">>, + <<"content-type">> => <<"application/json">> + } + } + ), + + ClientInfo = #{ + clientid => <<"client id">>, + username => <<"user name">>, + peerhost => {127, 0, 0, 1}, + protocol => <<"MQTT">>, + mountpoint => <<"MOUNTPOINT">>, + zone => default, + listener => {tcp, default}, + cn => ?PH_CERT_CN_NAME, + dn => ?PH_CERT_SUBJECT + }, + + ?check_trace( + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) + ), + fun(Trace) -> + ?assertMatch( + [#{reason := <<"unsupported content-type", _/binary>>}], + ?of_kind(bad_authz_http_response, Trace) + ), + ok + end + ). + t_no_value_for_placeholder(_Config) -> ok = setup_handler_and_config( fun(Req0, State) -> diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index 99caba625..89b7f6e17 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -32,6 +32,7 @@ %% Swagger specs from hocon schema -export([ api_spec/0, + check_api_schema/2, paths/0, schema/1, namespace/0 @@ -96,7 +97,7 @@ namespace() -> "actions_and_sources". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun check_api_schema/2}). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun ?MODULE:check_api_schema/2}). paths() -> [ diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index f7154d879..a36253c34 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mongodb, [ {description, "EMQX Enterprise MongoDB Bridge"}, - {vsn, "0.3.0"}, + {vsn, "0.3.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl index 593bf6ff8..5e3f12035 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl @@ -100,8 +100,10 @@ fields(mongodb_action) -> ); fields(action_parameters) -> [ - {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}, - {payload_template, mk(binary(), #{required => false, desc => ?DESC("payload_template")})} + {collection, + mk(emqx_schema:template(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}, + {payload_template, + mk(emqx_schema:template(), #{required => false, desc => ?DESC("payload_template")})} ]; fields(connector_resource_opts) -> emqx_connector_schema:resource_opts_fields(); diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index f02600336..a7ec0c31e 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mysql, [ {description, "EMQX Enterprise MySQL Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl index 24b11b930..e468e7407 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl @@ -146,7 +146,7 @@ fields(action_parameters) -> [ {sql, mk( - binary(), + emqx_schema:template(), #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} )} ]; diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 070f86bab..9ccb88c7a 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -144,17 +144,27 @@ reset(Node, KeyPath, Opts) -> %% @doc Called from build script. %% TODO: move to a external escript after all refactoring is done dump_schema(Dir, SchemaModule) -> - %% TODO: Load all apps instead of only emqx_dashboard + %% Load all apps in ERL_LIBS %% as this will help schemas that searches for apps with %% relevant schema definitions - _ = application:load(emqx_dashboard), + lists:foreach( + fun(LibPath) -> + Lib = list_to_atom(lists:last(filename:split(LibPath))), + load(SchemaModule, Lib) + end, + string:lexemes(os:getenv("ERL_LIBS"), ":;") + ), ok = emqx_dashboard_desc_cache:init(), lists:foreach( fun(Lang) -> ok = gen_schema_json(Dir, SchemaModule, Lang) end, ["en", "zh"] - ). + ), + emqx_dashboard:save_dispatch_eterm(SchemaModule). + +load(emqx_enterprise_schema, emqx_telemetry) -> ignore; +load(_, Lib) -> ok = application:load(Lib). %% for scripts/spellcheck. gen_schema_json(Dir, SchemaModule, Lang) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 5a6a4d8f9..83e28597e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -28,11 +28,15 @@ %% Authorization -export([authorize/1]). +-export([save_dispatch_eterm/1]). + -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/http_api.hrl"). -include_lib("emqx/include/emqx_release.hrl"). +-dialyzer({[no_opaque, no_match, no_return], [init_cache_dispatch/2, start_listeners/1]}). -define(EMQX_MIDDLE, emqx_dashboard_middleware). +-define(DISPATCH_FILE, "dispatch.eterm"). %%-------------------------------------------------------------------- %% Start/Stop Listeners @@ -46,6 +50,42 @@ stop_listeners() -> start_listeners(Listeners) -> {ok, _} = application:ensure_all_started(minirest), + SwaggerSupport = emqx:get_config([dashboard, swagger_support], true), + InitDispatch = dispatch(), + {OkListeners, ErrListeners} = + lists:foldl( + fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) -> + ok = init_cache_dispatch(Name, InitDispatch), + Options = #{ + dispatch => InitDispatch, + swagger_support => SwaggerSupport, + protocol => Protocol, + protocol_options => ProtoOpts + }, + Minirest = minirest_option(Options), + case minirest:start(Name, RanchOptions, Minirest) of + {ok, _} -> + ?ULOG("Listener ~ts on ~ts started.~n", [ + Name, emqx_listeners:format_bind(Bind) + ]), + {[Name | OkAcc], ErrAcc}; + {error, _Reason} -> + %% Don't record the reason because minirest already does(too much logs noise). + {OkAcc, [Name | ErrAcc]} + end + end, + {[], []}, + listeners(ensure_ssl_cert(Listeners)) + ), + case ErrListeners of + [] -> + optvar:set(emqx_dashboard_listeners_ready, OkListeners), + ok; + _ -> + {error, ErrListeners} + end. + +minirest_option(Options) -> Authorization = {?MODULE, authorize}, GlobalSpec = #{ openapi => "3.0.0", @@ -68,42 +108,33 @@ start_listeners(Listeners) -> } } }, - BaseMinirest = #{ - base_path => emqx_dashboard_swagger:base_path(), - modules => minirest_api:find_api_modules(apps()), - authorization => Authorization, - log => audit_log_fun(), - security => [#{'basicAuth' => []}, #{'bearerAuth' => []}], - swagger_global_spec => GlobalSpec, - dispatch => dispatch(), - middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler], - swagger_support => emqx:get_config([dashboard, swagger_support], true) - }, - {OkListeners, ErrListeners} = - lists:foldl( - fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) -> - Minirest = BaseMinirest#{protocol => Protocol, protocol_options => ProtoOpts}, - case minirest:start(Name, RanchOptions, Minirest) of - {ok, _} -> - ?ULOG("Listener ~ts on ~ts started.~n", [ - Name, emqx_listeners:format_bind(Bind) - ]), - {[Name | OkAcc], ErrAcc}; - {error, _Reason} -> - %% Don't record the reason because minirest already does(too much logs noise). - {OkAcc, [Name | ErrAcc]} - end - end, - {[], []}, - listeners(ensure_ssl_cert(Listeners)) - ), - case ErrListeners of - [] -> - optvar:set(emqx_dashboard_listeners_ready, OkListeners), - ok; - _ -> - {error, ErrListeners} - end. + Base = + #{ + base_path => emqx_dashboard_swagger:base_path(), + modules => minirest_api:find_api_modules(apps()), + authorization => Authorization, + log => audit_log_fun(), + security => [#{'basicAuth' => []}, #{'bearerAuth' => []}], + swagger_global_spec => GlobalSpec, + dispatch => static_dispatch(), + middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler], + swagger_support => true + }, + maps:merge(Base, Options). + +%% save dispatch to priv dir. +save_dispatch_eterm(SchemaMod) -> + Dir = code:priv_dir(emqx_dashboard), + emqx_config:put([dashboard], #{i18n_lang => en, swagger_support => false}), + os:putenv("SCHEMA_MOD", atom_to_list(SchemaMod)), + DispatchFile = filename:join([Dir, ?DISPATCH_FILE]), + io:format(user, "===< Generating: ~s~n", [DispatchFile]), + #{dispatch := Dispatch} = generate_dispatch(), + IoData = io_lib:format("~p.~n", [Dispatch]), + ok = file:write_file(DispatchFile, IoData), + {ok, [SaveDispatch]} = file:consult(DispatchFile), + SaveDispatch =/= Dispatch andalso erlang:error("bad dashboard dispatch.eterm file generated"), + ok. stop_listeners(Listeners) -> optvar:unset(emqx_dashboard_listeners_ready), @@ -127,6 +158,34 @@ wait_for_listeners() -> %%-------------------------------------------------------------------- %% internal +%%-------------------------------------------------------------------- + +init_cache_dispatch(Name, Dispatch0) -> + Dispatch1 = [{_, _, Rules}] = trails:single_host_compile(Dispatch0), + FileName = filename:join(code:priv_dir(emqx_dashboard), ?DISPATCH_FILE), + Dispatch2 = + case file:consult(FileName) of + {ok, [[{Host, Path, CacheRules}]]} -> + Trails = trails:trails([{cowboy_swagger_handler, #{server => 'http:dashboard'}}]), + [{_, _, SwaggerRules}] = trails:single_host_compile(Trails), + [{Host, Path, CacheRules ++ SwaggerRules ++ Rules}]; + {error, _} -> + Dispatch1 + end, + persistent_term:put(Name, Dispatch2). + +generate_dispatch() -> + Options = #{ + dispatch => [], + swagger_support => false, + protocol => http, + protocol_options => proto_opts(#{}) + }, + Minirest = minirest_option(Options), + minirest:generate_dispatch(Minirest). + +dispatch() -> + static_dispatch() ++ dynamic_dispatch(). apps() -> [ @@ -287,9 +346,6 @@ ensure_ssl_cert(Listeners = #{https := Https0 = #{ssl_options := SslOpts}}) -> ensure_ssl_cert(Listeners) -> Listeners. -dispatch() -> - static_dispatch() ++ dynamic_dispatch(). - static_dispatch() -> StaticFiles = ["/editor.worker.js", "/json.worker.js", "/version"], [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard_app.erl b/apps/emqx_dashboard/src/emqx_dashboard_app.erl index 76704ca59..dabfb171d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_app.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_app.erl @@ -25,6 +25,8 @@ -include("emqx_dashboard.hrl"). +-dialyzer({nowarn_function, [start/2]}). + start(_StartType, _StartArgs) -> Tables = lists:append([ emqx_dashboard_admin:create_tables(), diff --git a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl index 61c5570de..d7c8b389d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl @@ -21,28 +21,14 @@ -export([execute/2]). execute(Req, Env) -> - case check_dispatch_ready(Env) of - true -> add_cors_flag(Req, Env); - false -> {stop, cowboy_req:reply(503, #{<<"retry-after">> => <<"15">>}, Req)} - end. + add_cors_flag(Req, Env). add_cors_flag(Req, Env) -> CORS = emqx_conf:get([dashboard, cors], false), - Origin = cowboy_req:header(<<"origin">>, Req, undefined), - case CORS andalso Origin =/= undefined of + case CORS andalso cowboy_req:header(<<"origin">>, Req, undefined) =/= undefined of false -> {ok, Req, Env}; true -> Req2 = cowboy_req:set_resp_header(<<"Access-Control-Allow-Origin">>, <<"*">>, Req), {ok, Req2, Env} end. - -check_dispatch_ready(Env) -> - case maps:is_key(options, Env) of - false -> - true; - true -> - %% dashboard should always ready, if not, is_ready/1 will block until ready. - %% if not ready, dashboard will return 503. - emqx_dashboard_listener:is_ready(timer:seconds(20)) - end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 87c71c2c4..e8cea1db5 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -348,14 +348,7 @@ compose_filters(undefined, Filter2) -> compose_filters(Filter1, undefined) -> Filter1; compose_filters(Filter1, Filter2) -> - fun(Request, RequestMeta) -> - case Filter1(Request, RequestMeta) of - {ok, Request1} -> - Filter2(Request1, RequestMeta); - Response -> - Response - end - end. + [Filter1, Filter2]. %%------------------------------------------------------------------------------ %% Private functions diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 356bd51bf..d17f3a6d0 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -69,6 +69,9 @@ init_per_suite(Config) -> ], #{work_dir => emqx_cth_suite:work_dir(Config)} ), + _ = emqx_conf_schema:roots(), + ok = emqx_dashboard_desc_cache:init(), + emqx_dashboard:save_dispatch_eterm(emqx_conf:schema_module()), emqx_common_test_http:create_default_app(), [{suite_apps, SuiteApps} | Config]. @@ -87,6 +90,79 @@ t_overview(_) -> || Overview <- ?OVERVIEWS ]. +t_dashboard_restart(Config) -> + emqx_config:put([dashboard], #{ + i18n_lang => en, + swagger_support => true, + listeners => + #{ + http => + #{ + inet6 => false, + bind => 18083, + ipv6_v6only => false, + send_timeout => 10000, + num_acceptors => 8, + max_connections => 512, + backlog => 1024, + proxy_header => false + } + } + }), + application:stop(emqx_dashboard), + application:start(emqx_dashboard), + Name = 'http:dashboard', + t_overview(Config), + [{'_', [], Rules}] = Dispatch = persistent_term:get(Name), + %% complete dispatch has more than 150 rules. + ?assertNotMatch([{[], [], cowboy_static, _} | _], Rules), + ?assert(erlang:length(Rules) > 150), + CheckRules = fun(Tag) -> + [{'_', [], NewRules}] = persistent_term:get(Name, Tag), + ?assertEqual(length(Rules), length(NewRules), Tag), + ?assertEqual(lists:sort(Rules), lists:sort(NewRules), Tag) + end, + ?check_trace( + ?wait_async_action( + begin + ok = application:stop(emqx_dashboard), + ?assertEqual(Dispatch, persistent_term:get(Name)), + ok = application:start(emqx_dashboard), + %% After we restart the dashboard, the dispatch rules should be the same. + CheckRules(step_1) + end, + #{?snk_kind := regenerate_minirest_dispatch}, + 30_000 + ), + fun(Trace) -> + ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)), + %% The dispatch is updated after being regenerated. + CheckRules(step_2) + end + ), + t_overview(Config), + ?check_trace( + ?wait_async_action( + begin + %% erase to mock the initial dashboard startup. + persistent_term:erase(Name), + ok = application:stop(emqx_dashboard), + ok = application:start(emqx_dashboard), + ct:sleep(800), + %% regenerate the dispatch rules again + CheckRules(step_3) + end, + #{?snk_kind := regenerate_minirest_dispatch}, + 30_000 + ), + fun(Trace) -> + ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)), + CheckRules(step_4) + end + ), + t_overview(Config), + ok. + t_admins_add_delete(_) -> mnesia:clear_table(?ADMIN), Desc = <<"simple description">>, @@ -196,28 +272,41 @@ t_disable_swagger_json(_Config) -> {ok, {{"HTTP/1.1", 200, "OK"}, __, _}}, httpc:request(get, {Url, []}, [], [{body_format, binary}]) ), - DashboardCfg = emqx:get_raw_config([dashboard]), - DashboardCfg2 = DashboardCfg#{<<"swagger_support">> => false}, - emqx:update_config([dashboard], DashboardCfg2), - ?retry( - _Sleep = 1000, - _Attempts = 5, - ?assertMatch( - {ok, {{"HTTP/1.1", 404, "Not Found"}, _, _}}, - httpc:request(get, {Url, []}, [], [{body_format, binary}]) - ) - ), - DashboardCfg3 = DashboardCfg#{<<"swagger_support">> => true}, - emqx:update_config([dashboard], DashboardCfg3), - ?retry( - _Sleep0 = 1000, - _Attempts0 = 5, - ?assertMatch( - {ok, {{"HTTP/1.1", 200, "OK"}, __, _}}, - httpc:request(get, {Url, []}, [], [{body_format, binary}]) - ) + ?check_trace( + ?wait_async_action( + begin + DashboardCfg2 = DashboardCfg#{<<"swagger_support">> => false}, + emqx:update_config([dashboard], DashboardCfg2) + end, + #{?snk_kind := regenerate_minirest_dispatch}, + 30_000 + ), + fun(Trace) -> + ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)), + ?assertMatch( + {ok, {{"HTTP/1.1", 404, "Not Found"}, _, _}}, + httpc:request(get, {Url, []}, [], [{body_format, binary}]) + ) + end + ), + ?check_trace( + ?wait_async_action( + begin + DashboardCfg3 = DashboardCfg#{<<"swagger_support">> => true}, + emqx:update_config([dashboard], DashboardCfg3) + end, + #{?snk_kind := regenerate_minirest_dispatch}, + 30_000 + ), + fun(Trace) -> + ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)), + ?assertMatch( + {ok, {{"HTTP/1.1", 200, "OK"}, __, _}}, + httpc:request(get, {Url, []}, [], [{body_format, binary}]) + ) + end ), ok. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl index ce5d89930..9db73d201 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl @@ -20,6 +20,7 @@ -export([ api_spec/0, + check_api_schema/2, paths/0, schema/1, namespace/0 @@ -40,11 +41,12 @@ namespace() -> "dashboard_sso". api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{ translate_body => false, - check_schema => fun(Params, Meta) -> - emqx_dashboard_swagger:validate_content_type(Params, Meta, <<"application/xml">>) - end + check_schema => fun ?MODULE:check_api_schema/2 }). +check_api_schema(Params, Meta) -> + emqx_dashboard_swagger:validate_content_type(Params, Meta, <<"application/xml">>). + paths() -> [ "/sso/saml/acs", diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 754209257..547324925 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -815,7 +815,8 @@ fields(mqueue_message) -> fields(requested_client_fields) -> %% NOTE: some Client fields actually returned in response are missing in schema: %% enable_authn, is_persistent, listener, peerport - ClientFields = [element(1, F) || F <- fields(client)], + ClientFields0 = [element(1, F) || F <- fields(client)], + ClientFields = [client_attrs | ClientFields0], [ {fields, hoconsc:mk( diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index d51085eea..2c71e9822 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -1032,6 +1032,7 @@ t_query_multiple_clients_urlencode(_) -> t_query_clients_with_fields(_) -> process_flag(trap_exit, true), TCBin = atom_to_binary(?FUNCTION_NAME), + APIPort = 18083, ClientId = <>, Username = <>, {ok, C} = emqtt:start_link(#{clientid => ClientId, username => Username}), @@ -1040,6 +1041,13 @@ t_query_clients_with_fields(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), ?assertEqual([#{<<"clientid">> => ClientId}], get_clients_all_fields(Auth, "fields=clientid")), + ?assertMatch( + {ok, + {{_, 200, _}, _, #{ + <<"data">> := [#{<<"client_attrs">> := #{}}] + }}}, + list_request(APIPort, "fields=client_attrs") + ), ?assertEqual( [#{<<"clientid">> => ClientId, <<"username">> => Username}], get_clients_all_fields(Auth, "fields=clientid,username") @@ -1072,6 +1080,7 @@ get_clients(Auth, Qs, ExpectError, ClientIdOnly) -> Resp = emqx_mgmt_api_test_util:request_api(get, ClientsPath, Qs, Auth), case ExpectError of false -> + ct:pal("get clients response:\n ~p", [Resp]), {ok, Body} = Resp, #{<<"data">> := Clients} = emqx_utils_json:decode(Body), case ClientIdOnly of diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 9de7b0173..af8ac4603 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -155,6 +155,7 @@ replace/4, regex_match/2, regex_replace/3, + regex_extract/2, ascii/1, find/2, find/3, @@ -805,6 +806,8 @@ regex_match(Str, RE) -> emqx_variform_bif:regex_match(Str, RE). regex_replace(SrcStr, RE, RepStr) -> emqx_variform_bif:regex_replace(SrcStr, RE, RepStr). +regex_extract(SrcStr, RE) -> emqx_variform_bif:regex_extract(SrcStr, RE). + ascii(Char) -> emqx_variform_bif:ascii(Char). find(S, P) -> emqx_variform_bif:find(S, P). diff --git a/changes/ce/feat-13242.en.md b/changes/ce/feat-13242.en.md new file mode 100644 index 000000000..e216b7908 --- /dev/null +++ b/changes/ce/feat-13242.en.md @@ -0,0 +1 @@ +Significantly increased the startup speed of EMQX dashboard listener. diff --git a/changes/ce/fix-13216.en.md b/changes/ce/fix-13216.en.md index 2fe85c6b2..8a82de518 100644 --- a/changes/ce/fix-13216.en.md +++ b/changes/ce/fix-13216.en.md @@ -1,10 +1,10 @@ Respcet `clientid_prefix` config for MQTT bridges. -As of version 5.4.1, EMQX limits MQTT Client ID lengths to 23 bytes. -Previously, the system included the `clientid_prefix` in the hash calculation of the original, excessively long Client ID, thereby impacting the resulting shortened ID. +As of version 5.4.1, EMQX limits MQTT client ID lengths to 23 bytes. +Previously, the system included the `clientid_prefix` in the hash calculation of the original unique, but long client ID, thereby impacting the resulting shortened ID. Change Details: -- Without Prefix: Behavior remains unchanged; EMQX will hash the entire Client ID into a 23-byte space (when longer than 23 bytes). +- Without Prefix: Behavior remains unchanged; EMQX will hash the long (> 23 bytes) client ID into a 23-byte space. - With Prefix: - - Prefix no more than 19 bytes: The prefix is preserved, and the remaining suffix is hashed into a 4-byte space. - - Prefix is 20 or more bytes: EMQX no longer attempts to shorten the Client ID, respecting the configured prefix in its entirety. + - Prefix no more than 19 bytes: The prefix is preserved, and the client ID is hashed into a 4-byte space capping the length within 23 bytes. + - Prefix is 20 or more bytes: EMQX no longer attempts to shorten the client ID, respecting the configured prefix in its entirety. diff --git a/changes/ce/fix-13238.en.md b/changes/ce/fix-13238.en.md new file mode 100644 index 000000000..b9f039c33 --- /dev/null +++ b/changes/ce/fix-13238.en.md @@ -0,0 +1 @@ +Improved the logged error messages when an HTTP authorization request with an unsupported content-type header is returned. diff --git a/mix.exs b/mix.exs index ef32c6ac9..1dcbd8147 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.19.4", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.4.1", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.4.3", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.8", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, diff --git a/rebar.config b/rebar.config index 4190ef7f0..e1d8d23f3 100644 --- a/rebar.config +++ b/rebar.config @@ -86,7 +86,7 @@ {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.4"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}}, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}}, - {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.4.1"}}}, + {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.4.3"}}}, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.7"}}}, {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.8"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},