From 09f91159c919d2214d1b57658f1de96b4b0bb00d Mon Sep 17 00:00:00 2001 From: zmstone Date: Tue, 11 Jun 2024 21:14:15 +0200 Subject: [PATCH 01/11] fix(emqx_rule_funcs): expose regex_extract function to rule engine previoulsy only available in variform expressions, now made available for rule-engine --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 9de7b0173..af8ac4603 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -155,6 +155,7 @@ replace/4, regex_match/2, regex_replace/3, + regex_extract/2, ascii/1, find/2, find/3, @@ -805,6 +806,8 @@ regex_match(Str, RE) -> emqx_variform_bif:regex_match(Str, RE). regex_replace(SrcStr, RE, RepStr) -> emqx_variform_bif:regex_replace(SrcStr, RE, RepStr). +regex_extract(SrcStr, RE) -> emqx_variform_bif:regex_extract(SrcStr, RE). + ascii(Char) -> emqx_variform_bif:ascii(Char). find(S, P) -> emqx_variform_bif:find(S, P). From 0874768c1dc777eb2ba0785cbf1915ffb2ba0e7c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 11 Jun 2024 17:14:33 -0300 Subject: [PATCH 02/11] fix(client mgmt api): allow projecting `client_attrs` from client fields Fixes https://emqx.atlassian.net/browse/EMQX-12511 --- apps/emqx_management/src/emqx_mgmt_api_clients.erl | 3 ++- .../emqx_management/test/emqx_mgmt_api_clients_SUITE.erl | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 754209257..547324925 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -815,7 +815,8 @@ fields(mqueue_message) -> fields(requested_client_fields) -> %% NOTE: some Client fields actually returned in response are missing in schema: %% enable_authn, is_persistent, listener, peerport - ClientFields = [element(1, F) || F <- fields(client)], + ClientFields0 = [element(1, F) || F <- fields(client)], + ClientFields = [client_attrs | ClientFields0], [ {fields, hoconsc:mk( diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index d51085eea..2c71e9822 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -1032,6 +1032,7 @@ t_query_multiple_clients_urlencode(_) -> t_query_clients_with_fields(_) -> process_flag(trap_exit, true), TCBin = atom_to_binary(?FUNCTION_NAME), + APIPort = 18083, ClientId = <>, Username = <>, {ok, C} = emqtt:start_link(#{clientid => ClientId, username => Username}), @@ -1040,6 +1041,13 @@ t_query_clients_with_fields(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), ?assertEqual([#{<<"clientid">> => ClientId}], get_clients_all_fields(Auth, "fields=clientid")), + ?assertMatch( + {ok, + {{_, 200, _}, _, #{ + <<"data">> := [#{<<"client_attrs">> := #{}}] + }}}, + list_request(APIPort, "fields=client_attrs") + ), ?assertEqual( [#{<<"clientid">> => ClientId, <<"username">> => Username}], get_clients_all_fields(Auth, "fields=clientid,username") @@ -1072,6 +1080,7 @@ get_clients(Auth, Qs, ExpectError, ClientIdOnly) -> Resp = emqx_mgmt_api_test_util:request_api(get, ClientsPath, Qs, Auth), case ExpectError of false -> + ct:pal("get clients response:\n ~p", [Resp]), {ok, Body} = Resp, #{<<"data">> := Clients} = emqx_utils_json:decode(Body), case ClientIdOnly of From 46ab3be1f4ed9afd455d5491be2fd196226ad035 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 11 Jun 2024 17:25:53 -0300 Subject: [PATCH 03/11] chore: change types of mysql and mongodb fields to `template()` Fixes https://emqx.atlassian.net/browse/EMQX-12395 --- apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src | 2 +- apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl | 6 ++++-- apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src | 2 +- apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index f7154d879..a36253c34 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mongodb, [ {description, "EMQX Enterprise MongoDB Bridge"}, - {vsn, "0.3.0"}, + {vsn, "0.3.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl index 593bf6ff8..5e3f12035 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl @@ -100,8 +100,10 @@ fields(mongodb_action) -> ); fields(action_parameters) -> [ - {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}, - {payload_template, mk(binary(), #{required => false, desc => ?DESC("payload_template")})} + {collection, + mk(emqx_schema:template(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}, + {payload_template, + mk(emqx_schema:template(), #{required => false, desc => ?DESC("payload_template")})} ]; fields(connector_resource_opts) -> emqx_connector_schema:resource_opts_fields(); diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index f02600336..a7ec0c31e 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mysql, [ {description, "EMQX Enterprise MySQL Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl index 24b11b930..e468e7407 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl @@ -146,7 +146,7 @@ fields(action_parameters) -> [ {sql, mk( - binary(), + emqx_schema:template(), #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} )} ]; From 2c264d9a4bef858d4c12398408f837e3312f37e4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 12 Jun 2024 09:47:21 -0300 Subject: [PATCH 04/11] fix(http authz): handle unknown content types in responses Fixes https://emqx.atlassian.net/browse/EMQX-12530 --- .../src/emqx_authz/emqx_authz_utils.erl | 6 +- apps/emqx_auth_http/src/emqx_authz_http.erl | 3 + .../test/emqx_authz_http_SUITE.erl | 66 +++++++++++++++++++ changes/ce/fix-13238.en.md | 1 + 4 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 changes/ce/fix-13238.en.md diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index 26b8d000f..5c7b0965c 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -188,7 +188,7 @@ render_sql_params(ParamList, Values) -> ), Row. --spec parse_http_resp_body(binary(), binary()) -> allow | deny | ignore | error. +-spec parse_http_resp_body(binary(), binary()) -> allow | deny | ignore | error | {error, term()}. parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> try result(maps:from_list(cow_qs:parse_qs(Body))) @@ -200,7 +200,9 @@ parse_http_resp_body(<<"application/json", _/binary>>, Body) -> result(emqx_utils_json:decode(Body, [return_maps])) catch _:_ -> error - end. + end; +parse_http_resp_body(ContentType = <<_/binary>>, _Body) -> + {error, <<"unsupported content-type: ", ContentType/binary>>}. result(#{<<"result">> := <<"allow">>}) -> allow; result(#{<<"result">> := <<"deny">>}) -> deny; diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index 49296f690..e4d4326ff 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -105,6 +105,9 @@ authorize( body => Body }), nomatch; + {error, Reason} -> + ?tp(error, bad_authz_http_response, #{reason => Reason}), + nomatch; Result -> {matched, Result} end; diff --git a/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl index 93990791d..3fb2c0572 100644 --- a/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl @@ -463,6 +463,72 @@ t_placeholder_and_body(_Config) -> emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ). +%% Checks that we don't crash when receiving an unsupported content-type back. +t_bad_response_content_type(_Config) -> + ok = setup_handler_and_config( + fun(Req0, State) -> + ?assertEqual( + <<"/authz/users/">>, + cowboy_req:path(Req0) + ), + + {ok, _PostVars, Req1} = cowboy_req:read_urlencoded_body(Req0), + + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"text/csv">>}, + "hi", + Req1 + ), + {ok, Req, State} + end, + #{ + <<"method">> => <<"post">>, + <<"body">> => #{ + <<"username">> => <<"${username}">>, + <<"clientid">> => <<"${clientid}">>, + <<"peerhost">> => <<"${peerhost}">>, + <<"proto_name">> => <<"${proto_name}">>, + <<"mountpoint">> => <<"${mountpoint}">>, + <<"topic">> => <<"${topic}">>, + <<"action">> => <<"${action}">>, + <<"access">> => <<"${access}">>, + <<"CN">> => ?PH_CERT_CN_NAME, + <<"CS">> => ?PH_CERT_SUBJECT + }, + <<"headers">> => #{ + <<"accept">> => <<"text/plain">>, + <<"content-type">> => <<"application/json">> + } + } + ), + + ClientInfo = #{ + clientid => <<"client id">>, + username => <<"user name">>, + peerhost => {127, 0, 0, 1}, + protocol => <<"MQTT">>, + mountpoint => <<"MOUNTPOINT">>, + zone => default, + listener => {tcp, default}, + cn => ?PH_CERT_CN_NAME, + dn => ?PH_CERT_SUBJECT + }, + + ?check_trace( + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) + ), + fun(Trace) -> + ?assertMatch( + [#{reason := <<"unsupported content-type", _/binary>>}], + ?of_kind(bad_authz_http_response, Trace) + ), + ok + end + ). + t_no_value_for_placeholder(_Config) -> ok = setup_handler_and_config( fun(Req0, State) -> diff --git a/changes/ce/fix-13238.en.md b/changes/ce/fix-13238.en.md new file mode 100644 index 000000000..b9f039c33 --- /dev/null +++ b/changes/ce/fix-13238.en.md @@ -0,0 +1 @@ +Improved the logged error messages when an HTTP authorization request with an unsupported content-type header is returned. From 6823c79ae068646a74c6329ddef6e1470ba6be31 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 14 Jun 2024 16:04:11 +0800 Subject: [PATCH 05/11] chore: fix typo --- apps/emqx/include/emqx_metrics.hrl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/include/emqx_metrics.hrl b/apps/emqx/include/emqx_metrics.hrl index ddb537e6c..6fcbfd3f1 100644 --- a/apps/emqx/include/emqx_metrics.hrl +++ b/apps/emqx/include/emqx_metrics.hrl @@ -119,12 +119,12 @@ %% All Messages received {counter, 'messages.received', << "Number of messages received from the client, equal to the sum of " - "messages.qos0.received\fmessages.qos1.received and messages.qos2.received" + "messages.qos0.received, messages.qos1.received and messages.qos2.received" >>}, %% All Messages sent {counter, 'messages.sent', << "Number of messages sent to the client, equal to the sum of " - "messages.qos0.sent\fmessages.qos1.sent and messages.qos2.sent" + "messages.qos0.sent, messages.qos1.sent and messages.qos2.sent" >>}, %% QoS0 Messages received {counter, 'messages.qos0.received', <<"Number of QoS 0 messages received from clients">>}, From a6e3a091185eda88fcc96219d608a85eca9408f7 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 13 Jun 2024 16:25:30 +0800 Subject: [PATCH 06/11] feat: make the dashboard restart quicker --- README-CN.md | 2 + README-RU.md | 2 + README.md | 2 + apps/emqx/include/logger.hrl | 10 +- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 3 +- apps/emqx_conf/src/emqx_conf.erl | 16 ++- apps/emqx_dashboard/src/emqx_dashboard.erl | 133 +++++++++++++----- .../src/emqx_dashboard_middleware.erl | 18 +-- .../src/emqx_dashboard_swagger.erl | 9 +- .../test/emqx_dashboard_SUITE.erl | 108 +++++++++++--- .../src/emqx_dashboard_sso.app.src | 2 +- .../src/emqx_dashboard_sso_saml_api.erl | 4 + changes/ce/feat-13242.en.md | 1 + mix.exs | 2 +- rebar.config | 2 +- 15 files changed, 222 insertions(+), 92 deletions(-) create mode 100644 changes/ce/feat-13242.en.md diff --git a/README-CN.md b/README-CN.md index c6efab682..974a0a126 100644 --- a/README-CN.md +++ b/README-CN.md @@ -1,3 +1,5 @@ +简体中文 | [English](./README.md) | [Русский](./README-RU.md) + # EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) diff --git a/README-RU.md b/README-RU.md index 45bf08102..e4fe92696 100644 --- a/README-RU.md +++ b/README-RU.md @@ -1,3 +1,5 @@ +Русский | [简体中文](./README-CN.md) | [English](./README.md) + # Брокер EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) diff --git a/README.md b/README.md index 98f78c297..9fb2ae61a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +English | [简体中文](./README-CN.md) | [Русский](./README-RU.md) + # EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases) diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 31fe0e36a..d8ff3fe60 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -119,7 +119,13 @@ end). -endif. %% print to 'user' group leader --define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). --define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)). +-define(ULOG(Fmt, Args), + io:format(user, "~ts " ++ Fmt, [emqx_utils_calendar:now_to_rfc3339(millisecond) | Args]) +). +-define(ELOG(Fmt, Args), + io:format(standard_error, "~ts " ++ Fmt, [ + emqx_utils_calendar:now_to_rfc3339(millisecond) | Args + ]) +). -endif. diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index 99caba625..89b7f6e17 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -32,6 +32,7 @@ %% Swagger specs from hocon schema -export([ api_spec/0, + check_api_schema/2, paths/0, schema/1, namespace/0 @@ -96,7 +97,7 @@ namespace() -> "actions_and_sources". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun check_api_schema/2}). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun ?MODULE:check_api_schema/2}). paths() -> [ diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 23dda6b02..09883dc63 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -144,17 +144,27 @@ reset(Node, KeyPath, Opts) -> %% @doc Called from build script. %% TODO: move to a external escript after all refactoring is done dump_schema(Dir, SchemaModule) -> - %% TODO: Load all apps instead of only emqx_dashboard + %% Load all apps in ERL_LIBS %% as this will help schemas that searches for apps with %% relevant schema definitions - _ = application:load(emqx_dashboard), + lists:foreach( + fun(LibPath) -> + Lib = list_to_atom(lists:last(filename:split(LibPath))), + load(SchemaModule, Lib) + end, + string:lexemes(os:getenv("ERL_LIBS"), ":;") + ), ok = emqx_dashboard_desc_cache:init(), lists:foreach( fun(Lang) -> ok = gen_schema_json(Dir, SchemaModule, Lang) end, ["en", "zh"] - ). + ), + emqx_dashboard:save_dispatch_eterm(SchemaModule). + +load(emqx_enterprise_schema, emqx_telemetry) -> ignore; +load(_, Lib) -> ok = application:load(Lib). %% for scripts/spellcheck. gen_schema_json(Dir, SchemaModule, Lang) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 5a6a4d8f9..a685bfa07 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -28,11 +28,14 @@ %% Authorization -export([authorize/1]). +-export([save_dispatch_eterm/1]). + -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/http_api.hrl"). -include_lib("emqx/include/emqx_release.hrl"). -define(EMQX_MIDDLE, emqx_dashboard_middleware). +-define(DISPATCH_FILE, "dispatch.eterm"). %%-------------------------------------------------------------------- %% Start/Stop Listeners @@ -46,6 +49,42 @@ stop_listeners() -> start_listeners(Listeners) -> {ok, _} = application:ensure_all_started(minirest), + SwaggerSupport = emqx:get_config([dashboard, swagger_support], true), + InitDispatch = dispatch(), + {OkListeners, ErrListeners} = + lists:foldl( + fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) -> + init_cache_dispatch(Name, InitDispatch), + Options = #{ + dispatch => InitDispatch, + swagger_support => SwaggerSupport, + protocol => Protocol, + protocol_options => ProtoOpts + }, + Minirest = minirest_option(Options), + case minirest:start(Name, RanchOptions, Minirest) of + {ok, _} -> + ?ULOG("Listener ~ts on ~ts started.~n", [ + Name, emqx_listeners:format_bind(Bind) + ]), + {[Name | OkAcc], ErrAcc}; + {error, _Reason} -> + %% Don't record the reason because minirest already does(too much logs noise). + {OkAcc, [Name | ErrAcc]} + end + end, + {[], []}, + listeners(ensure_ssl_cert(Listeners)) + ), + case ErrListeners of + [] -> + optvar:set(emqx_dashboard_listeners_ready, OkListeners), + ok; + _ -> + {error, ErrListeners} + end. + +minirest_option(Options) -> Authorization = {?MODULE, authorize}, GlobalSpec = #{ openapi => "3.0.0", @@ -68,42 +107,33 @@ start_listeners(Listeners) -> } } }, - BaseMinirest = #{ - base_path => emqx_dashboard_swagger:base_path(), - modules => minirest_api:find_api_modules(apps()), - authorization => Authorization, - log => audit_log_fun(), - security => [#{'basicAuth' => []}, #{'bearerAuth' => []}], - swagger_global_spec => GlobalSpec, - dispatch => dispatch(), - middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler], - swagger_support => emqx:get_config([dashboard, swagger_support], true) - }, - {OkListeners, ErrListeners} = - lists:foldl( - fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) -> - Minirest = BaseMinirest#{protocol => Protocol, protocol_options => ProtoOpts}, - case minirest:start(Name, RanchOptions, Minirest) of - {ok, _} -> - ?ULOG("Listener ~ts on ~ts started.~n", [ - Name, emqx_listeners:format_bind(Bind) - ]), - {[Name | OkAcc], ErrAcc}; - {error, _Reason} -> - %% Don't record the reason because minirest already does(too much logs noise). - {OkAcc, [Name | ErrAcc]} - end - end, - {[], []}, - listeners(ensure_ssl_cert(Listeners)) - ), - case ErrListeners of - [] -> - optvar:set(emqx_dashboard_listeners_ready, OkListeners), - ok; - _ -> - {error, ErrListeners} - end. + Base = + #{ + base_path => emqx_dashboard_swagger:base_path(), + modules => minirest_api:find_api_modules(apps()), + authorization => Authorization, + log => audit_log_fun(), + security => [#{'basicAuth' => []}, #{'bearerAuth' => []}], + swagger_global_spec => GlobalSpec, + dispatch => static_dispatch(), + middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler], + swagger_support => true + }, + maps:merge(Base, Options). + +%% save dispatch to priv dir. +save_dispatch_eterm(SchemaMod) -> + Dir = code:priv_dir(emqx_dashboard), + emqx_config:put([dashboard], #{i18n_lang => en, swagger_support => false}), + os:putenv("SCHEMA_MOD", atom_to_list(SchemaMod)), + DispatchFile = filename:join([Dir, ?DISPATCH_FILE]), + io:format(user, "===< Generating: ~s~n", [DispatchFile]), + #{dispatch := Dispatch} = generate_dispatch(), + IoData = io_lib:format("~p.~n", [Dispatch]), + ok = file:write_file(DispatchFile, IoData), + {ok, [SaveDispatch]} = file:consult(DispatchFile), + SaveDispatch =/= Dispatch andalso erlang:error("bad dashboard dispatch.eterm file generated"), + ok. stop_listeners(Listeners) -> optvar:unset(emqx_dashboard_listeners_ready), @@ -127,6 +157,34 @@ wait_for_listeners() -> %%-------------------------------------------------------------------- %% internal +%%-------------------------------------------------------------------- + +init_cache_dispatch(Name, Dispatch0) -> + Dispatch1 = [{_, _, Rules}] = trails:single_host_compile(Dispatch0), + FileName = filename:join(code:priv_dir(emqx_dashboard), ?DISPATCH_FILE), + Dispatch2 = + case file:consult(FileName) of + {ok, [[{Host, Path, CacheRules}]]} -> + Trails = trails:trails([{cowboy_swagger_handler, #{server => 'http:dashboard'}}]), + [{_, _, SwaggerRules}] = trails:single_host_compile(Trails), + [{Host, Path, CacheRules ++ SwaggerRules ++ Rules}]; + {error, _} -> + Dispatch1 + end, + persistent_term:put(Name, Dispatch2). + +generate_dispatch() -> + Options = #{ + dispatch => [], + swagger_support => false, + protocol => http, + protocol_options => proto_opts(#{}) + }, + Minirest = minirest_option(Options), + minirest:generate_dispatch(Minirest). + +dispatch() -> + static_dispatch() ++ dynamic_dispatch(). apps() -> [ @@ -287,9 +345,6 @@ ensure_ssl_cert(Listeners = #{https := Https0 = #{ssl_options := SslOpts}}) -> ensure_ssl_cert(Listeners) -> Listeners. -dispatch() -> - static_dispatch() ++ dynamic_dispatch(). - static_dispatch() -> StaticFiles = ["/editor.worker.js", "/json.worker.js", "/version"], [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl index 61c5570de..d7c8b389d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl @@ -21,28 +21,14 @@ -export([execute/2]). execute(Req, Env) -> - case check_dispatch_ready(Env) of - true -> add_cors_flag(Req, Env); - false -> {stop, cowboy_req:reply(503, #{<<"retry-after">> => <<"15">>}, Req)} - end. + add_cors_flag(Req, Env). add_cors_flag(Req, Env) -> CORS = emqx_conf:get([dashboard, cors], false), - Origin = cowboy_req:header(<<"origin">>, Req, undefined), - case CORS andalso Origin =/= undefined of + case CORS andalso cowboy_req:header(<<"origin">>, Req, undefined) =/= undefined of false -> {ok, Req, Env}; true -> Req2 = cowboy_req:set_resp_header(<<"Access-Control-Allow-Origin">>, <<"*">>, Req), {ok, Req2, Env} end. - -check_dispatch_ready(Env) -> - case maps:is_key(options, Env) of - false -> - true; - true -> - %% dashboard should always ready, if not, is_ready/1 will block until ready. - %% if not ready, dashboard will return 503. - emqx_dashboard_listener:is_ready(timer:seconds(20)) - end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index a6005254a..e8673d6bf 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -323,14 +323,7 @@ compose_filters(undefined, Filter2) -> compose_filters(Filter1, undefined) -> Filter1; compose_filters(Filter1, Filter2) -> - fun(Request, RequestMeta) -> - case Filter1(Request, RequestMeta) of - {ok, Request1} -> - Filter2(Request1, RequestMeta); - Response -> - Response - end - end. + [Filter1, Filter2]. %%------------------------------------------------------------------------------ %% Private functions diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 356bd51bf..4302e297f 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -61,6 +61,7 @@ init_per_suite(Config) -> Apps = emqx_machine_boot:reboot_apps(), ct:pal("load apps:~p~n", [Apps]), lists:foreach(fun(App) -> application:load(App) end, Apps), + emqx_dashboard:save_dispatch_eterm(emqx_conf:schema_module()), SuiteApps = emqx_cth_suite:start( [ emqx_conf, @@ -87,6 +88,60 @@ t_overview(_) -> || Overview <- ?OVERVIEWS ]. +t_dashboard_restart(Config) -> + Name = 'http:dashboard', + t_overview(Config), + [{'_', [], Rules}] = Dispatch = persistent_term:get(Name), + %% complete dispatch has more than 150 rules. + ?assertNotMatch([{[], [], cowboy_static, _} | _], Rules), + ?assert(erlang:length(Rules) > 150), + CheckRules = fun(Tag) -> + io:format("zhongwen:~p~n", [Tag]), + [{'_', [], NewRules}] = persistent_term:get(Name), + ?assertEqual(length(NewRules), length(Rules), Tag), + ?assertEqual(lists:sort(NewRules), lists:sort(Rules), Tag) + end, + ?check_trace( + ?wait_async_action( + begin + ok = application:stop(emqx_dashboard), + ?assertEqual(Dispatch, persistent_term:get(Name)), + ok = application:start(emqx_dashboard), + %% After we restart the dashboard, the dispatch rules should be the same. + CheckRules(step_1) + end, + #{?snk_kind := regenerate_minirest_dispatch}, + 30_000 + ), + fun(Trace) -> + ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)), + %% The dispatch is updated after being regenerated. + CheckRules(step_2) + end + ), + t_overview(Config), + ?check_trace( + ?wait_async_action( + begin + ok = application:stop(emqx_dashboard), + %% erase to mock the initial dashboard startup. + persistent_term:erase(Name), + ok = application:start(emqx_dashboard), + ct:sleep(800), + %% regenerate the dispatch rules again + CheckRules(step_3) + end, + #{?snk_kind := regenerate_minirest_dispatch}, + 30_000 + ), + fun(Trace) -> + ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)), + CheckRules(step_4) + end + ), + t_overview(Config), + ok. + t_admins_add_delete(_) -> mnesia:clear_table(?ADMIN), Desc = <<"simple description">>, @@ -196,28 +251,41 @@ t_disable_swagger_json(_Config) -> {ok, {{"HTTP/1.1", 200, "OK"}, __, _}}, httpc:request(get, {Url, []}, [], [{body_format, binary}]) ), - DashboardCfg = emqx:get_raw_config([dashboard]), - DashboardCfg2 = DashboardCfg#{<<"swagger_support">> => false}, - emqx:update_config([dashboard], DashboardCfg2), - ?retry( - _Sleep = 1000, - _Attempts = 5, - ?assertMatch( - {ok, {{"HTTP/1.1", 404, "Not Found"}, _, _}}, - httpc:request(get, {Url, []}, [], [{body_format, binary}]) - ) - ), - DashboardCfg3 = DashboardCfg#{<<"swagger_support">> => true}, - emqx:update_config([dashboard], DashboardCfg3), - ?retry( - _Sleep0 = 1000, - _Attempts0 = 5, - ?assertMatch( - {ok, {{"HTTP/1.1", 200, "OK"}, __, _}}, - httpc:request(get, {Url, []}, [], [{body_format, binary}]) - ) + ?check_trace( + ?wait_async_action( + begin + DashboardCfg2 = DashboardCfg#{<<"swagger_support">> => false}, + emqx:update_config([dashboard], DashboardCfg2) + end, + #{?snk_kind := regenerate_minirest_dispatch}, + 30_000 + ), + fun(Trace) -> + ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)), + ?assertMatch( + {ok, {{"HTTP/1.1", 404, "Not Found"}, _, _}}, + httpc:request(get, {Url, []}, [], [{body_format, binary}]) + ) + end + ), + ?check_trace( + ?wait_async_action( + begin + DashboardCfg3 = DashboardCfg#{<<"swagger_support">> => true}, + emqx:update_config([dashboard], DashboardCfg3) + end, + #{?snk_kind := regenerate_minirest_dispatch}, + 30_000 + ), + fun(Trace) -> + ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)), + ?assertMatch( + {ok, {{"HTTP/1.1", 200, "OK"}, __, _}}, + httpc:request(get, {Url, []}, [], [{body_format, binary}]) + ) + end ), ok. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src index 7f5e8f04a..0ed5d8025 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_sso, [ {description, "EMQX Dashboard Single Sign-On"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, [emqx_dashboard_sso_sup]}, {applications, [ kernel, diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl index ec9396828..36850ff56 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl @@ -20,6 +20,7 @@ -export([ api_spec/0, + validate_xml_content_type/2, paths/0, schema/1, namespace/0 @@ -40,6 +41,9 @@ namespace() -> "dashboard_sso". api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false, translate_body => false}). +validate_xml_content_type(Params, Meta) -> + emqx_dashboard_swagger:validate_content_type(Params, Meta, <<"application/xml">>). + paths() -> [ "/sso/saml/acs", diff --git a/changes/ce/feat-13242.en.md b/changes/ce/feat-13242.en.md new file mode 100644 index 000000000..0fa875020 --- /dev/null +++ b/changes/ce/feat-13242.en.md @@ -0,0 +1 @@ +Significantly increased the startup speed of the EMQX management dashboard. diff --git a/mix.exs b/mix.exs index 868c57804..157039274 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.19.3", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.4.0", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.4.3", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.8", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, diff --git a/rebar.config b/rebar.config index ba7cc63dc..346014c17 100644 --- a/rebar.config +++ b/rebar.config @@ -86,7 +86,7 @@ {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.3"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}}, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}}, - {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.4.0"}}}, + {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.4.3"}}}, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.7"}}}, {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.8"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, From 945ea785ae3f7a0683d06dca9163e051f6394c72 Mon Sep 17 00:00:00 2001 From: zmstone Date: Fri, 14 Jun 2024 13:12:08 +0200 Subject: [PATCH 07/11] docs: refine change log --- changes/ce/fix-13216.en.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/changes/ce/fix-13216.en.md b/changes/ce/fix-13216.en.md index 2fe85c6b2..8a82de518 100644 --- a/changes/ce/fix-13216.en.md +++ b/changes/ce/fix-13216.en.md @@ -1,10 +1,10 @@ Respcet `clientid_prefix` config for MQTT bridges. -As of version 5.4.1, EMQX limits MQTT Client ID lengths to 23 bytes. -Previously, the system included the `clientid_prefix` in the hash calculation of the original, excessively long Client ID, thereby impacting the resulting shortened ID. +As of version 5.4.1, EMQX limits MQTT client ID lengths to 23 bytes. +Previously, the system included the `clientid_prefix` in the hash calculation of the original unique, but long client ID, thereby impacting the resulting shortened ID. Change Details: -- Without Prefix: Behavior remains unchanged; EMQX will hash the entire Client ID into a 23-byte space (when longer than 23 bytes). +- Without Prefix: Behavior remains unchanged; EMQX will hash the long (> 23 bytes) client ID into a 23-byte space. - With Prefix: - - Prefix no more than 19 bytes: The prefix is preserved, and the remaining suffix is hashed into a 4-byte space. - - Prefix is 20 or more bytes: EMQX no longer attempts to shorten the Client ID, respecting the configured prefix in its entirety. + - Prefix no more than 19 bytes: The prefix is preserved, and the client ID is hashed into a 4-byte space capping the length within 23 bytes. + - Prefix is 20 or more bytes: EMQX no longer attempts to shorten the client ID, respecting the configured prefix in its entirety. From f713f13b2c4d8da15e5fde639c6d70a1de331c0f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Jun 2024 21:52:58 +0800 Subject: [PATCH 08/11] test: generate dispatch.eterm in dashboard test --- apps/emqx_dashboard/src/emqx_dashboard.erl | 3 +- .../emqx_dashboard/src/emqx_dashboard_app.erl | 2 ++ .../test/emqx_dashboard_SUITE.erl | 33 +++++++++++++++---- .../src/emqx_dashboard_sso_saml_api.erl | 4 --- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index a685bfa07..83e28597e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -33,6 +33,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/http_api.hrl"). -include_lib("emqx/include/emqx_release.hrl"). +-dialyzer({[no_opaque, no_match, no_return], [init_cache_dispatch/2, start_listeners/1]}). -define(EMQX_MIDDLE, emqx_dashboard_middleware). -define(DISPATCH_FILE, "dispatch.eterm"). @@ -54,7 +55,7 @@ start_listeners(Listeners) -> {OkListeners, ErrListeners} = lists:foldl( fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) -> - init_cache_dispatch(Name, InitDispatch), + ok = init_cache_dispatch(Name, InitDispatch), Options = #{ dispatch => InitDispatch, swagger_support => SwaggerSupport, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_app.erl b/apps/emqx_dashboard/src/emqx_dashboard_app.erl index 76704ca59..dabfb171d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_app.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_app.erl @@ -25,6 +25,8 @@ -include("emqx_dashboard.hrl"). +-dialyzer({nowarn_function, [start/2]}). + start(_StartType, _StartArgs) -> Tables = lists:append([ emqx_dashboard_admin:create_tables(), diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 4302e297f..d17f3a6d0 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -61,7 +61,6 @@ init_per_suite(Config) -> Apps = emqx_machine_boot:reboot_apps(), ct:pal("load apps:~p~n", [Apps]), lists:foreach(fun(App) -> application:load(App) end, Apps), - emqx_dashboard:save_dispatch_eterm(emqx_conf:schema_module()), SuiteApps = emqx_cth_suite:start( [ emqx_conf, @@ -70,6 +69,9 @@ init_per_suite(Config) -> ], #{work_dir => emqx_cth_suite:work_dir(Config)} ), + _ = emqx_conf_schema:roots(), + ok = emqx_dashboard_desc_cache:init(), + emqx_dashboard:save_dispatch_eterm(emqx_conf:schema_module()), emqx_common_test_http:create_default_app(), [{suite_apps, SuiteApps} | Config]. @@ -89,6 +91,26 @@ t_overview(_) -> ]. t_dashboard_restart(Config) -> + emqx_config:put([dashboard], #{ + i18n_lang => en, + swagger_support => true, + listeners => + #{ + http => + #{ + inet6 => false, + bind => 18083, + ipv6_v6only => false, + send_timeout => 10000, + num_acceptors => 8, + max_connections => 512, + backlog => 1024, + proxy_header => false + } + } + }), + application:stop(emqx_dashboard), + application:start(emqx_dashboard), Name = 'http:dashboard', t_overview(Config), [{'_', [], Rules}] = Dispatch = persistent_term:get(Name), @@ -96,10 +118,9 @@ t_dashboard_restart(Config) -> ?assertNotMatch([{[], [], cowboy_static, _} | _], Rules), ?assert(erlang:length(Rules) > 150), CheckRules = fun(Tag) -> - io:format("zhongwen:~p~n", [Tag]), - [{'_', [], NewRules}] = persistent_term:get(Name), - ?assertEqual(length(NewRules), length(Rules), Tag), - ?assertEqual(lists:sort(NewRules), lists:sort(Rules), Tag) + [{'_', [], NewRules}] = persistent_term:get(Name, Tag), + ?assertEqual(length(Rules), length(NewRules), Tag), + ?assertEqual(lists:sort(Rules), lists:sort(NewRules), Tag) end, ?check_trace( ?wait_async_action( @@ -123,9 +144,9 @@ t_dashboard_restart(Config) -> ?check_trace( ?wait_async_action( begin - ok = application:stop(emqx_dashboard), %% erase to mock the initial dashboard startup. persistent_term:erase(Name), + ok = application:stop(emqx_dashboard), ok = application:start(emqx_dashboard), ct:sleep(800), %% regenerate the dispatch rules again diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl index 36850ff56..ec9396828 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl @@ -20,7 +20,6 @@ -export([ api_spec/0, - validate_xml_content_type/2, paths/0, schema/1, namespace/0 @@ -41,9 +40,6 @@ namespace() -> "dashboard_sso". api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false, translate_body => false}). -validate_xml_content_type(Params, Meta) -> - emqx_dashboard_swagger:validate_content_type(Params, Meta, <<"application/xml">>). - paths() -> [ "/sso/saml/acs", From d95f17fe777de2ed3242d4849849464e066d790e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Jun 2024 22:04:20 +0800 Subject: [PATCH 09/11] chore: revert ULOG/ELOG --- apps/emqx/include/logger.hrl | 10 ++-------- apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src | 2 +- changes/ce/feat-13242.en.md | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index d8ff3fe60..31fe0e36a 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -119,13 +119,7 @@ end). -endif. %% print to 'user' group leader --define(ULOG(Fmt, Args), - io:format(user, "~ts " ++ Fmt, [emqx_utils_calendar:now_to_rfc3339(millisecond) | Args]) -). --define(ELOG(Fmt, Args), - io:format(standard_error, "~ts " ++ Fmt, [ - emqx_utils_calendar:now_to_rfc3339(millisecond) | Args - ]) -). +-define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). +-define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)). -endif. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src index 0ed5d8025..7f5e8f04a 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_sso, [ {description, "EMQX Dashboard Single Sign-On"}, - {vsn, "0.1.5"}, + {vsn, "0.1.4"}, {registered, [emqx_dashboard_sso_sup]}, {applications, [ kernel, diff --git a/changes/ce/feat-13242.en.md b/changes/ce/feat-13242.en.md index 0fa875020..e216b7908 100644 --- a/changes/ce/feat-13242.en.md +++ b/changes/ce/feat-13242.en.md @@ -1 +1 @@ -Significantly increased the startup speed of the EMQX management dashboard. +Significantly increased the startup speed of EMQX dashboard listener. From f7ac829f2858a73502e6780ece36ee95f55abf81 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 2 May 2024 19:50:15 +0300 Subject: [PATCH 10/11] fix(auth,http): improve URI handling --- apps/emqx_auth/src/emqx_auth.app.src | 3 +- apps/emqx_auth/src/emqx_auth_utils.erl | 78 ++++ .../src/emqx_authn/emqx_authn_utils.erl | 20 - apps/emqx_auth_http/src/emqx_authn_http.erl | 21 +- apps/emqx_auth_http/src/emqx_authz_http.erl | 33 +- .../test/emqx_authn_http_SUITE.erl | 6 +- .../test/emqx_authz_http_SUITE.erl | 44 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 40 +- .../src/emqx_bridge_dynamo.app.src | 2 +- .../src/emqx_bridge_dynamo_connector.erl | 4 +- .../emqx_bridge_dynamo_connector_client.erl | 4 +- .../emqx_bridge_es/src/emqx_bridge_es.app.src | 2 +- .../src/emqx_bridge_es_connector.erl | 6 +- .../src/emqx_bridge_http.app.src | 2 +- .../src/emqx_bridge_http_action_info.erl | 15 +- .../src/emqx_bridge_http_connector.erl | 20 +- .../test/emqx_bridge_http_SUITE.erl | 50 +++ .../test/emqx_bridge_http_connector_tests.erl | 5 +- .../src/emqx_bridge_iotdb.app.src | 2 +- .../src/emqx_bridge_iotdb_connector.erl | 10 +- .../src/emqx_connector_resource.erl | 40 +- apps/emqx_utils/rebar.config | 3 +- apps/emqx_utils/src/emqx_utils.app.src | 3 +- apps/emqx_utils/src/emqx_utils_uri.erl | 392 ++++++++++++++++++ changes/ce/fix-13273.en.md | 4 + 25 files changed, 656 insertions(+), 153 deletions(-) create mode 100644 apps/emqx_auth/src/emqx_auth_utils.erl create mode 100644 apps/emqx_utils/src/emqx_utils_uri.erl create mode 100644 changes/ce/fix-13273.en.md diff --git a/apps/emqx_auth/src/emqx_auth.app.src b/apps/emqx_auth/src/emqx_auth.app.src index db4b69ded..6db2d6213 100644 --- a/apps/emqx_auth/src/emqx_auth.app.src +++ b/apps/emqx_auth/src/emqx_auth.app.src @@ -7,7 +7,8 @@ {applications, [ kernel, stdlib, - emqx + emqx, + emqx_utils ]}, {mod, {emqx_auth_app, []}}, {env, []}, diff --git a/apps/emqx_auth/src/emqx_auth_utils.erl b/apps/emqx_auth/src/emqx_auth_utils.erl new file mode 100644 index 000000000..0abb336af --- /dev/null +++ b/apps/emqx_auth/src/emqx_auth_utils.erl @@ -0,0 +1,78 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_auth_utils). + +%% TODO +%% Move more identical authn and authz helpers here + +-export([parse_url/1]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec parse_url(binary()) -> + {_Base :: emqx_utils_uri:request_base(), _Path :: binary(), _Query :: binary()}. +parse_url(Url) -> + Parsed = emqx_utils_uri:parse(Url), + case Parsed of + #{scheme := undefined} -> + throw({invalid_url, {no_scheme, Url}}); + #{authority := undefined} -> + throw({invalid_url, {no_host, Url}}); + #{authority := #{userinfo := Userinfo}} when Userinfo =/= undefined -> + throw({invalid_url, {userinfo_not_supported, Url}}); + #{fragment := Fragment} when Fragment =/= undefined -> + throw({invalid_url, {fragments_not_supported, Url}}); + _ -> + case emqx_utils_uri:request_base(Parsed) of + {ok, Base} -> + {Base, emqx_utils_uri:path(Parsed), + emqx_maybe:define(emqx_utils_uri:query(Parsed), <<>>)}; + {error, Reason} -> + throw({invalid_url, {invalid_base, Reason, Url}}) + end + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +templates_test_() -> + [ + ?_assertEqual( + { + #{port => 80, scheme => http, host => "example.com"}, + <<"">>, + <<"client=${clientid}">> + }, + parse_url(<<"http://example.com?client=${clientid}">>) + ), + ?_assertEqual( + { + #{port => 80, scheme => http, host => "example.com"}, + <<"/path">>, + <<"client=${clientid}">> + }, + parse_url(<<"http://example.com/path?client=${clientid}">>) + ), + ?_assertEqual( + {#{port => 80, scheme => http, host => "example.com"}, <<"/path">>, <<>>}, + parse_url(<<"http://example.com/path">>) + ) + ]. + +-endif. diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index 8d4d245da..a08ac260c 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -39,7 +39,6 @@ make_resource_id/1, without_password/1, to_bool/1, - parse_url/1, convert_headers/1, convert_headers_no_content_type/1, default_headers/0, @@ -287,25 +286,6 @@ to_bool(MaybeBinInt) when is_binary(MaybeBinInt) -> to_bool(_) -> false. -parse_url(Url) -> - case string:split(Url, "//", leading) of - [Scheme, UrlRem] -> - case string:split(UrlRem, "/", leading) of - [HostPort, Remaining] -> - BaseUrl = iolist_to_binary([Scheme, "//", HostPort]), - case string:split(Remaining, "?", leading) of - [Path, QueryString] -> - {BaseUrl, <<"/", Path/binary>>, QueryString}; - [Path] -> - {BaseUrl, <<"/", Path/binary>>, <<>>} - end; - [HostPort] -> - {iolist_to_binary([Scheme, "//", HostPort]), <<>>, <<>>} - end; - [Url] -> - throw({invalid_url, Url}) - end. - convert_headers(Headers) -> transform_header_name(Headers). diff --git a/apps/emqx_auth_http/src/emqx_authn_http.erl b/apps/emqx_auth_http/src/emqx_authn_http.erl index 96eb1215a..18995cb9d 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http.erl @@ -134,21 +134,25 @@ parse_config( request_timeout := RequestTimeout } = Config ) -> - {BaseUrl0, Path, Query} = emqx_authn_utils:parse_url(RawUrl), - {ok, BaseUrl} = emqx_http_lib:uri_parse(BaseUrl0), + {RequestBase, Path, Query} = emqx_auth_utils:parse_url(RawUrl), State = #{ method => Method, path => Path, headers => ensure_header_name_type(Headers), base_path_template => emqx_authn_utils:parse_str(Path), base_query_template => emqx_authn_utils:parse_deep( - cow_qs:parse_qs(to_bin(Query)) + cow_qs:parse_qs(Query) ), body_template => emqx_authn_utils:parse_deep(maps:get(body, Config, #{})), request_timeout => RequestTimeout, url => RawUrl }, - {ok, Config#{base_url => BaseUrl, pool_type => random}, State}. + {ok, + Config#{ + request_base => RequestBase, + pool_type => random + }, + State}. generate_request(Credential, #{ method := Method, @@ -246,14 +250,14 @@ request_for_log(Credential, #{url := Url, method := Method} = State) -> {PathQuery, Headers} -> #{ method => Method, - base_url => Url, + url => Url, path_query => PathQuery, headers => Headers }; {PathQuery, Headers, Body} -> #{ method => Method, - base_url => Url, + url => Url, path_query => PathQuery, headers => Headers, body => Body @@ -274,11 +278,6 @@ to_list(B) when is_binary(B) -> to_list(L) when is_list(L) -> L. -to_bin(B) when is_binary(B) -> - B; -to_bin(L) when is_list(L) -> - list_to_binary(L). - ensure_header_name_type(Headers) -> Fun = fun (Key, _Val, Acc) when is_binary(Key) -> diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index e4d4326ff..6b0152b7d 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -29,8 +29,7 @@ update/1, destroy/1, authorize/4, - merge_defaults/1, - parse_url/1 + merge_defaults/1 ]). -ifdef(TEST). @@ -160,15 +159,14 @@ parse_config( request_timeout := ReqTimeout } = Conf ) -> - {BaseUrl0, Path, Query} = parse_url(RawUrl), - {ok, BaseUrl} = emqx_http_lib:uri_parse(BaseUrl0), + {RequestBase, Path, Query} = emqx_auth_utils:parse_url(RawUrl), Conf#{ method => Method, - base_url => BaseUrl, + request_base => RequestBase, headers => Headers, base_path_template => emqx_authz_utils:parse_str(Path, allowed_vars()), base_query_template => emqx_authz_utils:parse_deep( - cow_qs:parse_qs(to_bin(Query)), + cow_qs:parse_qs(Query), allowed_vars() ), body_template => emqx_authz_utils:parse_deep( @@ -180,25 +178,6 @@ parse_config( pool_type => random }. -parse_url(Url) -> - case string:split(Url, "//", leading) of - [Scheme, UrlRem] -> - case string:split(UrlRem, "/", leading) of - [HostPort, Remaining] -> - BaseUrl = iolist_to_binary([Scheme, "//", HostPort]), - case string:split(Remaining, "?", leading) of - [Path, QueryString] -> - {BaseUrl, <<"/", Path/binary>>, QueryString}; - [Path] -> - {BaseUrl, <<"/", Path/binary>>, <<>>} - end; - [HostPort] -> - {iolist_to_binary([Scheme, "//", HostPort]), <<>>, <<>>} - end; - [Url] -> - throw({invalid_url, Url}) - end. - generate_request( Action, Topic, @@ -272,10 +251,6 @@ to_list(B) when is_binary(B) -> to_list(L) when is_list(L) -> L. -to_bin(B) when is_binary(B) -> B; -to_bin(L) when is_list(L) -> list_to_binary(L); -to_bin(X) -> X. - allowed_vars() -> allowed_vars(emqx_authz:feature_available(rich_actions)). diff --git a/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl index 864aa6e0e..dc1443b19 100644 --- a/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl @@ -112,7 +112,11 @@ t_create_invalid(_Config) -> InvalidConfigs = [ AuthConfig#{<<"headers">> => []}, - AuthConfig#{<<"method">> => <<"delete">>} + AuthConfig#{<<"method">> => <<"delete">>}, + AuthConfig#{<<"url">> => <<"localhost">>}, + AuthConfig#{<<"url">> => <<"http://foo.com/xxx#fragment">>}, + AuthConfig#{<<"url">> => <<"http://${foo}.com/xxx">>}, + AuthConfig#{<<"url">> => <<"//foo.com/xxx">>} ], lists:foreach( diff --git a/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl index 3fb2c0572..70822e802 100644 --- a/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl @@ -639,6 +639,8 @@ t_create_replace(_Config) -> listener => {tcp, default} }, + ValidConfig = raw_http_authz_config(), + %% Create with valid URL ok = setup_handler_and_config( fun(Req0, State) -> @@ -656,13 +658,10 @@ t_create_replace(_Config) -> ), %% Changing to valid config - OkConfig = maps:merge( - raw_http_authz_config(), - #{ - <<"url">> => - <<"http://127.0.0.1:33333/authz/users/?topic=${topic}&action=${action}">> - } - ), + OkConfig = ValidConfig#{ + <<"url">> => + <<"http://127.0.0.1:33333/authz/users/?topic=${topic}&action=${action}">> + }, ?assertMatch( {ok, _}, @@ -672,6 +671,37 @@ t_create_replace(_Config) -> ?assertEqual( allow, emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) + ), + + ?assertMatch( + {error, _}, + emqx_authz:update({?CMD_REPLACE, http}, ValidConfig#{ + <<"url">> => <<"localhost">> + }) + ), + + ?assertMatch( + {error, _}, + emqx_authz:update({?CMD_REPLACE, http}, ValidConfig#{ + <<"url">> => <<"//foo.bar/x/y?q=z">> + }) + ), + + ?assertMatch( + {error, _}, + emqx_authz:update({?CMD_REPLACE, http}, ValidConfig#{ + <<"url">> => + <<"http://127.0.0.1:33333/authz/users/?topic=${topic}&action=${action}#fragment">> + }) + ). + +t_uri_normalization(_Config) -> + ok = emqx_authz_test_lib:setup_config( + raw_http_authz_config(), + #{ + <<"url">> => + <<"http://127.0.0.1:33333?topic=${topic}&action=${action}">> + } ). %%------------------------------------------------------------------------------ diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index f334a8607..d2408ca73 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -336,21 +336,13 @@ parse_confs( } = Conf ) -> Url1 = bin(Url), - {BaseUrl, Path} = parse_url(Url1), - BaseUrl1 = - case emqx_http_lib:uri_parse(BaseUrl) of - {ok, BUrl} -> - BUrl; - {error, Reason} -> - Reason1 = emqx_utils:readable_error_msg(Reason), - invalid_data(<<"Invalid URL: ", Url1/binary, ", details: ", Reason1/binary>>) - end, + {RequestBase, Path} = parse_url(Url1), RequestTTL = emqx_utils_maps:deep_get( [resource_opts, request_ttl], Conf ), Conf#{ - base_url => BaseUrl1, + request_base => RequestBase, request => #{ path => Path, @@ -422,16 +414,24 @@ connector_config(BridgeType, Config) -> end. parse_url(Url) -> - case string:split(Url, "//", leading) of - [Scheme, UrlRem] -> - case string:split(UrlRem, "/", leading) of - [HostPort, Path] -> - {iolist_to_binary([Scheme, "//", HostPort]), Path}; - [HostPort] -> - {iolist_to_binary([Scheme, "//", HostPort]), <<>>} - end; - [Url] -> - invalid_data(<<"Missing scheme in URL: ", Url/binary>>) + Parsed = emqx_utils_uri:parse(Url), + case Parsed of + #{scheme := undefined} -> + invalid_data(<<"Missing scheme in URL: ", Url/binary>>); + #{authority := undefined} -> + invalid_data(<<"Missing host in URL: ", Url/binary>>); + #{authority := #{userinfo := Userinfo}} when Userinfo =/= undefined -> + invalid_data(<<"Userinfo is not supported in URL: ", Url/binary>>); + #{fragment := Fragment} when Fragment =/= undefined -> + invalid_data(<<"Fragments are not supported in URL: ", Url/binary>>); + _ -> + case emqx_utils_uri:request_base(Parsed) of + {ok, Base} -> + {Base, emqx_maybe:define(emqx_utils_uri:path(Parsed), <<>>)}; + {error, Reason0} -> + Reason1 = emqx_utils:readable_error_msg(Reason0), + invalid_data(<<"Invalid URL: ", Url/binary, ", details: ", Reason1/binary>>) + end end. bin(Bin) when is_binary(Bin) -> Bin; diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src index ca7d46ec5..0ffd143dc 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_dynamo, [ {description, "EMQX Enterprise Dynamo Bridge"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index 4e974a8a9..82f5fb18d 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -86,7 +86,7 @@ on_start( config => redact(Config) }), - {Schema, Server, DefaultPort} = get_host_info(to_str(Url)), + {Scheme, Server, DefaultPort} = get_host_info(to_str(Url)), #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, #{ default_port => DefaultPort }), @@ -97,7 +97,7 @@ on_start( port => Port, aws_access_key_id => to_str(AccessKeyID), aws_secret_access_key => SecretAccessKey, - schema => Schema + scheme => Scheme }}, {pool_size, PoolSize} ], diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl index cc37c8dd0..0613ca3bb 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl @@ -63,11 +63,11 @@ init(#{ aws_secret_access_key := Secret, host := Host, port := Port, - schema := Schema + scheme := Scheme }) -> %% TODO: teach `erlcloud` to to accept 0-arity closures as passwords. SecretAccessKey = to_str(emqx_secret:unwrap(Secret)), - erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema), + erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Scheme), {ok, #{}}. handle_call(is_connected, _From, State) -> diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src index 9ed9d572d..7e5f6203b 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_es, [ {description, "EMQX Enterprise Elastic Search Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {modules, [ emqx_bridge_es, emqx_bridge_es_connector diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl index d94ce8e15..20de92e6e 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -44,11 +44,10 @@ -type config() :: #{ - base_url := #{ + request_base := #{ scheme := http | https, host := iolist(), - port := inet:port_number(), - path := _ + port := inet:port_number() }, connect_timeout := pos_integer(), pool_type := random | hash, @@ -59,7 +58,6 @@ -type state() :: #{ - base_path := _, connect_timeout := pos_integer(), pool_type := random | hash, channels := map(), diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src index bb89461ec..57fceae74 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src +++ b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_http, [ {description, "EMQX HTTP Bridge and Connector Application"}, - {vsn, "0.3.0"}, + {vsn, "0.3.1"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, ehttpc]}, {env, [ diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl index f84112700..6bb65babc 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl @@ -98,15 +98,6 @@ validate_webhook_url(undefined) -> required_field => <<"url">> }); validate_webhook_url(Url) -> - {BaseUrl, _Path} = emqx_connector_resource:parse_url(Url), - case emqx_http_lib:uri_parse(BaseUrl) of - {ok, _} -> - ok; - {error, Reason} -> - throw(#{ - kind => validation_error, - reason => invalid_url, - url => Url, - error => emqx_utils:readable_error_msg(Reason) - }) - end. + %% parse_url throws if the URL is invalid + {_RequestBase, _Path} = emqx_connector_resource:parse_url(Url), + ok. diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index f639596b6..785424c67 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -53,7 +53,7 @@ %% for other http-like connectors. -export([redact_request/1]). --export([validate_method/1, join_paths/2, formalize_request/3, transform_result/1]). +-export([validate_method/1, join_paths/2, formalize_request/2, transform_result/1]). -define(DEFAULT_PIPELINE_SIZE, 100). -define(DEFAULT_REQUEST_TIMEOUT_MS, 30_000). @@ -189,11 +189,10 @@ callback_mode() -> async_if_possible. on_start( InstId, #{ - base_url := #{ + request_base := #{ scheme := Scheme, host := Host, - port := Port, - path := BasePath + port := Port }, connect_timeout := ConnectTimeout, pool_type := PoolType, @@ -227,13 +226,13 @@ on_start( {transport_opts, NTransportOpts}, {enable_pipelining, maps:get(enable_pipelining, Config, ?DEFAULT_PIPELINE_SIZE)} ], + State = #{ pool_name => InstId, pool_type => PoolType, host => Host, port => Port, connect_timeout => ConnectTimeout, - base_path => BasePath, scheme => Scheme, request => preprocess_request(maps:get(request, Config, undefined)) }, @@ -362,7 +361,7 @@ on_query(InstId, {Method, Request, Timeout}, State) -> on_query( InstId, {ActionId, KeyOrNum, Method, Request, Timeout, Retry}, - #{base_path := BasePath, host := Host, scheme := Scheme, port := Port} = State + #{host := Host, scheme := Scheme, port := Port} = State ) -> ?TRACE( "QUERY", @@ -375,7 +374,7 @@ on_query( state => redact(State) } ), - NRequest = formalize_request(Method, BasePath, Request), + NRequest = formalize_request(Method, Request), trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout), Worker = resolve_pool_worker(State, KeyOrNum), Result0 = ehttpc:request( @@ -472,7 +471,7 @@ on_query_async( InstId, {ActionId, KeyOrNum, Method, Request, Timeout}, ReplyFunAndArgs, - #{base_path := BasePath, host := Host, port := Port, scheme := Scheme} = State + #{host := Host, port := Port, scheme := Scheme} = State ) -> Worker = resolve_pool_worker(State, KeyOrNum), ?TRACE( @@ -485,7 +484,7 @@ on_query_async( state => redact(State) } ), - NRequest = formalize_request(Method, BasePath, Request), + NRequest = formalize_request(Method, Request), trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout), MaxAttempts = maps:get(max_attempts, State, 3), Context = #{ @@ -824,6 +823,9 @@ make_method(M) when M == <<"PUT">>; M == <<"put">> -> put; make_method(M) when M == <<"GET">>; M == <<"get">> -> get; make_method(M) when M == <<"DELETE">>; M == <<"delete">> -> delete. +formalize_request(Method, Request) -> + formalize_request(Method, "/", Request). + formalize_request(Method, BasePath, {Path, Headers, _Body}) when Method =:= get; Method =:= delete -> diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl index 7f9418bd3..e1d0e9724 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl @@ -85,6 +85,16 @@ init_per_testcase(t_path_not_found, Config) -> ), ok = emqx_bridge_http_connector_test_server:set_handler(not_found_http_handler()), [{http_server, #{port => HTTPPort, path => HTTPPath}} | Config]; +init_per_testcase(t_empty_path, Config) -> + HTTPPath = <<"/">>, + ServerSSLOpts = false, + {ok, {HTTPPort, _Pid}} = emqx_bridge_http_connector_test_server:start_link( + _Port = random, HTTPPath, ServerSSLOpts + ), + ok = emqx_bridge_http_connector_test_server:set_handler( + emqx_bridge_http_test_lib:success_http_handler() + ), + [{http_server, #{port => HTTPPort, path => HTTPPath}} | Config]; init_per_testcase(t_too_many_requests, Config) -> HTTPPath = <<"/path">>, ServerSSLOpts = false, @@ -122,6 +132,7 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(TestCase, _Config) when TestCase =:= t_path_not_found; + TestCase =:= t_empty_path; TestCase =:= t_too_many_requests; TestCase =:= t_service_unavailable; TestCase =:= t_rule_action_expired; @@ -588,6 +599,45 @@ t_path_not_found(Config) -> ), ok. +t_empty_path(Config) -> + ?check_trace( + begin + #{port := Port, path := _Path} = ?config(http_server, Config), + MQTTTopic = <<"t/webhook">>, + BridgeConfig = emqx_bridge_http_test_lib:bridge_async_config(#{ + type => ?BRIDGE_TYPE, + name => ?BRIDGE_NAME, + local_topic => MQTTTopic, + port => Port, + path => <<"">> + }), + {ok, _} = emqx_bridge:create(?BRIDGE_TYPE, ?BRIDGE_NAME, BridgeConfig), + Msg = emqx_message:make(MQTTTopic, <<"{}">>), + emqx:publish(Msg), + wait_http_request(), + ?retry( + _Interval = 500, + _NAttempts = 20, + ?assertMatch( + #{ + counters := #{ + matched := 1, + failed := 0, + success := 1 + } + }, + get_metrics(?BRIDGE_NAME) + ) + ), + ok + end, + fun(Trace) -> + ?assertEqual([], ?of_kind(http_will_retry_async, Trace)), + ok + end + ), + ok. + t_too_many_requests(Config) -> check_send_is_retried(Config). diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl index 94cc4c01a..becf221a4 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl @@ -32,11 +32,10 @@ wrap_auth_headers_test_() -> end, fun meck:unload/1, fun(_) -> Config = #{ - base_url => #{ + request_base => #{ scheme => http, host => "localhost", - port => 18083, - path => "/status" + port => 18083 }, connect_timeout => 1000, pool_type => random, diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index 08d8c9ccc..b3c5767db 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_iotdb, [ {description, "EMQX Enterprise Apache IoTDB Bridge"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {modules, [ emqx_bridge_iotdb, emqx_bridge_iotdb_connector diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl index 65fbda936..78866ef79 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl @@ -44,11 +44,10 @@ -type config() :: #{ - base_url := #{ + request_base := #{ scheme := http | https, host := iolist(), - port := inet:port_number(), - path := _ + port := inet:port_number() }, connect_timeout := pos_integer(), pool_type := random | hash, @@ -60,7 +59,6 @@ -type state() :: #{ - base_path := _, connect_timeout := pos_integer(), pool_type := random | hash, channels := map(), @@ -245,10 +243,10 @@ on_stop(InstanceId, State) -> -spec on_get_status(manager_id(), state()) -> connected | connecting | {disconnected, state(), term()}. -on_get_status(InstanceId, #{base_path := BasePath} = State) -> +on_get_status(InstanceId, State) -> Func = fun(Worker, Timeout) -> Request = {?IOTDB_PING_PATH, [], undefined}, - NRequest = emqx_bridge_http_connector:formalize_request(get, BasePath, Request), + NRequest = emqx_bridge_http_connector:formalize_request(get, Request), Result0 = ehttpc:request(Worker, get, NRequest, Timeout), case emqx_bridge_http_connector:transform_result(Result0) of {ok, 200, _, Body} -> diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index f0848b599..c71afca60 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -281,17 +281,9 @@ parse_confs( } = Conf ) -> Url1 = bin(Url), - {BaseUrl, Path} = parse_url(Url1), - BaseUrl1 = - case emqx_http_lib:uri_parse(BaseUrl) of - {ok, BUrl} -> - BUrl; - {error, Reason} -> - Reason1 = emqx_utils:readable_error_msg(Reason), - invalid_data(<<"Invalid URL: ", Url1/binary, ", details: ", Reason1/binary>>) - end, + {RequestBase, Path} = parse_url(Url1), Conf#{ - base_url => BaseUrl1, + request_base => RequestBase, request => #{ path => Path, @@ -324,16 +316,24 @@ connector_config(ConnectorType, Name, Config) -> end. parse_url(Url) -> - case string:split(Url, "//", leading) of - [Scheme, UrlRem] -> - case string:split(UrlRem, "/", leading) of - [HostPort, Path] -> - {iolist_to_binary([Scheme, "//", HostPort]), Path}; - [HostPort] -> - {iolist_to_binary([Scheme, "//", HostPort]), <<>>} - end; - [Url] -> - invalid_data(<<"Missing scheme in URL: ", Url/binary>>) + Parsed = emqx_utils_uri:parse(Url), + case Parsed of + #{scheme := undefined} -> + invalid_data(<<"Missing scheme in URL: ", Url/binary>>); + #{authority := undefined} -> + invalid_data(<<"Missing host in URL: ", Url/binary>>); + #{authority := #{userinfo := Userinfo}} when Userinfo =/= undefined -> + invalid_data(<<"Userinfo is not supported in URL: ", Url/binary>>); + #{fragment := Fragment} when Fragment =/= undefined -> + invalid_data(<<"Fragments are not supported in URL: ", Url/binary>>); + _ -> + case emqx_utils_uri:request_base(Parsed) of + {ok, Base} -> + {Base, emqx_maybe:define(emqx_utils_uri:path(Parsed), <<>>)}; + {error, Reason0} -> + Reason1 = emqx_utils:readable_error_msg(Reason0), + invalid_data(<<"Invalid URL: ", Url/binary, ", details: ", Reason1/binary>>) + end end. -spec invalid_data(binary()) -> no_return(). diff --git a/apps/emqx_utils/rebar.config b/apps/emqx_utils/rebar.config index 7aa4d34d0..3852ac87e 100644 --- a/apps/emqx_utils/rebar.config +++ b/apps/emqx_utils/rebar.config @@ -5,7 +5,8 @@ ]}. {deps, [ - {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.6"}}} + {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.6"}}}, + {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}} ]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index bac23cefb..959a3a37a 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -15,7 +15,8 @@ {applications, [ kernel, stdlib, - jiffy + jiffy, + emqx_http_lib ]}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_utils/src/emqx_utils_uri.erl b/apps/emqx_utils/src/emqx_utils_uri.erl new file mode 100644 index 000000000..d36cd9050 --- /dev/null +++ b/apps/emqx_utils/src/emqx_utils_uri.erl @@ -0,0 +1,392 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% This module provides a loose parser for URIs. +%% The standard library's `uri_string' module is strict and does not allow +%% to parse invalid URIs, like templates: `http://example.com/${username}'. + +-module(emqx_utils_uri). + +-export([parse/1, format/1]). + +-export([ + scheme/1, + userinfo/1, + host/1, + port/1, + path/1, + query/1, + fragment/1, + base_url/1, + request_base/1 +]). + +-type scheme() :: binary(). +-type userinfo() :: binary(). +-type host() :: binary(). +-type port_number() :: inet:port_number(). +-type path() :: binary(). +-type query() :: binary(). +-type fragment() :: binary(). +-type request_base() :: #{ + scheme := http | https, + host := iolist(), + port := inet:port_number() +}. + +-type authority() :: #{ + userinfo := emqx_maybe:t(userinfo()), + host := host(), + %% Types: + %% ipv6: `\[[a-z\d:\.]*\]` — bracketed "ivp6-like" address + %% regular: `example.com` — arbitrary host not containg `:` which is forbidden in hostnames other than ipv6 + %% loose: non ipv6-like host containing `:`, probably invalid for a strictly valid URI + host_type := ipv6 | regular | loose, + port := emqx_maybe:t(port_number()) +}. + +-type uri() :: #{ + scheme := emqx_maybe:t(scheme()), + authority := emqx_maybe:t(authority()), + path := path(), + query := emqx_maybe:t(query()), + fragment := emqx_maybe:t(fragment()) +}. + +-export_type([ + scheme/0, + userinfo/0, + host/0, + port_number/0, + path/0, + query/0, + fragment/0, + authority/0, + uri/0, + request_base/0 +]). + +%%-------------------------------------------------------------------- +%% API +%%------------------------------------------------------------------- + +-spec parse(binary()) -> uri(). +parse(URIString) -> + {match, [SchemeMatch, AuthorityMatch, PathMatch, QueryMatch, FragmentMatch]} = re:run( + URIString, uri_regexp(), [{capture, [scheme, authority, path, query, fragment], binary}] + ), + Scheme = parse_scheme(SchemeMatch), + Authority = parse_authority(AuthorityMatch), + Path = PathMatch, + Query = parse_query(QueryMatch), + Fragment = parse_fragment(FragmentMatch), + + #{ + scheme => Scheme, + authority => Authority, + path => Path, + query => Query, + fragment => Fragment + }. + +-spec base_url(uri()) -> iodata(). +base_url(#{scheme := Scheme, authority := Authority}) -> + [format_scheme(Scheme), format_authority(Authority)]. + +-spec format(uri()) -> iodata(). +format(#{path := Path, query := Query, fragment := Fragment} = URI) -> + [ + base_url(URI), + Path, + format_query(Query), + format_fragment(Fragment) + ]. + +-spec scheme(uri()) -> emqx_maybe:t(scheme()). +scheme(#{scheme := Scheme}) -> Scheme. + +-spec userinfo(uri()) -> emqx_maybe:t(userinfo()). +userinfo(#{authority := undefined}) -> undefined; +userinfo(#{authority := #{userinfo := UserInfo}}) -> UserInfo. + +-spec host(uri()) -> emqx_maybe:t(host()). +host(#{authority := undefined}) -> undefined; +host(#{authority := #{host := Host}}) -> Host. + +-spec port(uri()) -> emqx_maybe:t(port_number()). +port(#{authority := undefined}) -> undefined; +port(#{authority := #{port := Port}}) -> Port. + +-spec path(uri()) -> path(). +path(#{path := Path}) -> Path. + +-spec query(uri()) -> emqx_maybe:t(query()). +query(#{query := Query}) -> Query. + +-spec fragment(uri()) -> emqx_maybe:t(fragment()). +fragment(#{fragment := Fragment}) -> Fragment. + +-spec request_base(uri()) -> {ok, request_base()} | {error, term()}. +request_base(URI) when is_map(URI) -> + case emqx_http_lib:uri_parse(iolist_to_binary(base_url(URI))) of + {error, Reason} -> {error, Reason}; + {ok, URIMap} -> {ok, maps:with([scheme, host, port], URIMap)} + end; +request_base(URIString) when is_list(URIString) orelse is_binary(URIString) -> + request_base(parse(URIString)). + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +parse_scheme(<<>>) -> undefined; +parse_scheme(Scheme) -> Scheme. + +parse_query(<<>>) -> undefined; +parse_query(<<$?, Query/binary>>) -> Query. + +parse_fragment(<<>>) -> undefined; +parse_fragment(<<$#, Fragment/binary>>) -> Fragment. + +-define(AUTHORITY_REGEX, + ("^(?.*@)?" + "(?:(?:\\[(?[a-z\\d\\.:]*)\\])|(?[^:]*?)|(?.*?))" + "(?:\\d+)?$") +). + +authority_regexp() -> + {ok, RE} = re:compile(?AUTHORITY_REGEX, [caseless]), + RE. + +parse_authority(<<>>) -> + undefined; +parse_authority(<<$/, $/, Authority/binary>>) -> + %% Authority regexp always matches + {match, [UserInfoMatch, HostIPv6, HostRegular, HostLoose, PortMatch]} = re:run( + Authority, authority_regexp(), [ + {capture, [userinfo, host_ipv6, host_regular, host_loose, port], binary} + ] + ), + UserInfo = parse_userinfo(UserInfoMatch), + {HostType, Host} = parse_host(HostIPv6, HostRegular, HostLoose), + Port = parse_port(PortMatch), + #{ + userinfo => UserInfo, + host => Host, + host_type => HostType, + port => Port + }. + +parse_userinfo(<<>>) -> undefined; +parse_userinfo(UserInfoMatch) -> binary:part(UserInfoMatch, 0, byte_size(UserInfoMatch) - 1). + +parse_host(<<>>, <<>>, Host) -> {loose, Host}; +parse_host(<<>>, Host, <<>>) -> {regular, Host}; +parse_host(Host, <<>>, <<>>) -> {ipv6, Host}. + +parse_port(<<>>) -> undefined; +parse_port(<<$:, Port/binary>>) -> binary_to_integer(Port). + +%% https://datatracker.ietf.org/doc/html/rfc3986#appendix-B +%% +%% > The following line is the regular expression for breaking-down a +%% > well-formed URI reference into its components. +%% +%% > ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? +%% +%% We skip capturing some unused parts of the regex. + +-define(URI_REGEX, + ("^(?:(?[^:/?#]+):)?(?//[^/?#]*)?" + "(?[^?#]*)(?\\?[^#]*)?(?#.*)?") +). + +uri_regexp() -> + {ok, RE} = re:compile(?URI_REGEX, [caseless]), + RE. + +format_scheme(undefined) -> <<>>; +format_scheme(Scheme) -> [Scheme, $:]. + +format_authority(undefined) -> + <<>>; +format_authority(#{userinfo := UserInfo, host := Host, host_type := HostType, port := Port}) -> + [$/, $/, format_userinfo(UserInfo), format_host(HostType, Host), format_port(Port)]. + +format_userinfo(undefined) -> <<>>; +format_userinfo(UserInfo) -> [UserInfo, $@]. + +format_host(ipv6, Host) -> [$[, Host, $]]; +format_host(_, Host) -> Host. + +format_port(undefined) -> <<>>; +format_port(Port) -> [$:, integer_to_binary(Port)]. + +format_query(undefined) -> <<>>; +format_query(Query) -> [$?, Query]. + +format_fragment(undefined) -> <<>>; +format_fragment(Fragment) -> [$#, Fragment]. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +-define(URLS, [ + "https://www.example.com/page", + "http://subdomain.example.com/path/to/page", + "https://www.example.com:8080/path/to/page", + "https://user:password@example.com/path/to/page", + "https://www.example.com/path%20with%20${spaces}", + "http://192.0.2.1/path/to/page", + "http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]/${path}/to/page", + "http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]/to/page", + "http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:4444/to/page", + "ftp://ftp.example.com/${path}/to/file", + "ftps://ftp.example.com/path/to/file", + "mailto:user@example.com", + "tel:+1234567890", + "sms:+1234567890?body=Hello%20World", + "git://github.com/user/repo.git", + "a:b:c", + "svn://svn.example.com/project/trunk", + "https://www.${example}.com/path/to/page?query_param=value", + "https://www.example.com/path/to/page?query_param1=value1&query_param2=value2", + "https://www.example.com?query_param1=value1&query_param2=value2", + "https://www.example.com/path/to/page#section1", + "https://www.example.com/path/to/page?query_param=value#section1", + "https://www.example.com/path/to/page?query_param1=value1&query_param2=${value2}#section1", + "https://www.example.com?query_param1=value1&query_param2=value2#section1", + "file:///path/to/file.txt", + "localhost", + "localhost:8080", + "localhost:8080/path/to/page", + "localhost:8080/path/to/page?query_param=value", + "localhost:8080/path/to/page?query_param1=value1&query_param2=value2", + "/abc/${def}", + "/abc/def?query_param=value", + "?query_param=value", + "#section1" +]). + +parse_format_test_() -> + [ + {URI, ?_assertEqual(list_to_binary(URI), iolist_to_binary(format(parse(URI))))} + || URI <- ?URLS + ]. + +base_url_test_() -> + [ + {URI, ?_assert(is_prefix(iolist_to_binary(base_url(parse(URI))), list_to_binary(URI)))} + || URI <- ?URLS + ]. + +scheme_test_() -> + [ + if_parseable_by_uri_string(URI, fun(Expected, Parsed) -> + ?assertEqual(maybe_get_bin(scheme, Expected), scheme(Parsed)) + end) + || URI <- ?URLS + ]. + +host_test_() -> + [ + if_parseable_by_uri_string(URI, fun(Expected, Parsed) -> + ?assertEqual(maybe_get_bin(host, Expected), host(Parsed)) + end) + || URI <- ?URLS + ]. + +path_test_() -> + [ + if_parseable_by_uri_string(URI, fun(Expected, Parsed) -> + ?assertEqual(maybe_get_bin(path, Expected), path(Parsed)) + end) + || URI <- ?URLS + ]. + +query_test_() -> + [ + if_parseable_by_uri_string(URI, fun(Expected, Parsed) -> + ?assertEqual(maybe_get_bin(query, Expected), query(Parsed)) + end) + || URI <- ?URLS + ]. + +fragment_test_() -> + [ + if_parseable_by_uri_string(URI, fun(Expected, Parsed) -> + ?assertEqual(maybe_get_bin(fragment, Expected), fragment(Parsed)) + end) + || URI <- ?URLS + ]. + +templates_test_() -> + [ + {"template in path", + ?_assertEqual( + <<"/${client_attrs.group}">>, + path(parse("https://www.example.com/${client_attrs.group}")) + )}, + {"template in query, no path", + ?_assertEqual( + <<"group=${client_attrs.group}">>, + query(parse("https://www.example.com?group=${client_attrs.group}")) + )}, + {"template in query, path", + ?_assertEqual( + <<"group=${client_attrs.group}">>, + query(parse("https://www.example.com/path/?group=${client_attrs.group}")) + )} + ]. + +request_target_test_() -> + [ + ?_assertEqual( + {ok, #{port => 443, scheme => https, host => "www.example.com"}}, + request_base(parse("https://www.example.com/path/to/page?query_param=value#fr")) + ), + ?_assertEqual( + {error, empty_host_not_allowed}, + request_base(parse("localhost?query_param=value#fr")) + ), + ?_assertEqual( + {error, {unsupported_scheme, <<"ftp">>}}, + request_base(parse("ftp://localhost")) + ) + ]. + +is_prefix(Prefix, Binary) -> + case Binary of + <> -> true; + _ -> false + end. + +if_parseable_by_uri_string(URI, Fun) -> + case uri_string:parse(URI) of + {error, _, _} -> + {"skipped", fun() -> true end}; + ExpectedMap -> + ParsedMap = parse(URI), + {URI, fun() -> Fun(ExpectedMap, ParsedMap) end} + end. + +maybe_get_bin(Key, Map) -> + maybe_bin(maps:get(Key, Map, undefined)). + +maybe_bin(String) when is_list(String) -> list_to_binary(String); +maybe_bin(undefined) -> undefined. + +-endif. diff --git a/changes/ce/fix-13273.en.md b/changes/ce/fix-13273.en.md new file mode 100644 index 000000000..c917cebb1 --- /dev/null +++ b/changes/ce/fix-13273.en.md @@ -0,0 +1,4 @@ +Fixed and improved handling of URIs in several configurations. +Previously, +* In authentication or authorization configurations, valid pathless URIs (`https://example.com?q=x`) were not accepted as valid. +* In bridge connectors, some kinds of URIs that couldn't be correctly handled were nevertheless accepted. E.g., URIs with user info or fragment parts. From 19c9f0d76feeb17127425b336d00b1c8dbf0f8f9 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 6 May 2024 11:46:18 +0300 Subject: [PATCH 11/11] chore(auth,http): cache REs for parsing URIs --- apps/emqx_utils/src/emqx_utils_uri.erl | 65 ++++++++++++++++---------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/apps/emqx_utils/src/emqx_utils_uri.erl b/apps/emqx_utils/src/emqx_utils_uri.erl index d36cd9050..e0566b677 100644 --- a/apps/emqx_utils/src/emqx_utils_uri.erl +++ b/apps/emqx_utils/src/emqx_utils_uri.erl @@ -79,7 +79,44 @@ request_base/0 ]). -%%-------------------------------------------------------------------- +-on_load(init/0). + +%% https://datatracker.ietf.org/doc/html/rfc3986#appendix-B +%% +%% > The following line is the regular expression for breaking-down a +%% > well-formed URI reference into its components. +%% +%% > ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? +%% +%% We skip capturing some unused parts of the regex. + +-define(URI_REGEX, + ("^(?:(?[^:/?#]+):)?(?//[^/?#]*)?" + "(?[^?#]*)(?\\?[^#]*)?(?#.*)?") +). + +-define(URI_REGEX_PT_KEY, {?MODULE, uri_re}). + +-define(AUTHORITY_REGEX, + ("^(?.*@)?" + "(?:(?:\\[(?[a-z\\d\\.:]*)\\])|(?[^:]*?)|(?.*?))" + "(?:\\d+)?$") +). + +-define(AUTHORITY_REGEX_PT_KEY, {?MODULE, authority_re}). + +%%------------------------------------------------------------------- +%% Internal API +%%------------------------------------------------------------------- + +init() -> + {ok, UriRE} = re:compile(?URI_REGEX), + persistent_term:put(?URI_REGEX_PT_KEY, UriRE), + + {ok, AuthorityRE} = re:compile(?AUTHORITY_REGEX, [caseless]), + persistent_term:put(?AUTHORITY_REGEX_PT_KEY, AuthorityRE). + +%%------------------------------------------------------------------- %% API %%------------------------------------------------------------------- @@ -161,15 +198,8 @@ parse_query(<<$?, Query/binary>>) -> Query. parse_fragment(<<>>) -> undefined; parse_fragment(<<$#, Fragment/binary>>) -> Fragment. --define(AUTHORITY_REGEX, - ("^(?.*@)?" - "(?:(?:\\[(?[a-z\\d\\.:]*)\\])|(?[^:]*?)|(?.*?))" - "(?:\\d+)?$") -). - authority_regexp() -> - {ok, RE} = re:compile(?AUTHORITY_REGEX, [caseless]), - RE. + persistent_term:get(?AUTHORITY_REGEX_PT_KEY). parse_authority(<<>>) -> undefined; @@ -200,23 +230,8 @@ parse_host(Host, <<>>, <<>>) -> {ipv6, Host}. parse_port(<<>>) -> undefined; parse_port(<<$:, Port/binary>>) -> binary_to_integer(Port). -%% https://datatracker.ietf.org/doc/html/rfc3986#appendix-B -%% -%% > The following line is the regular expression for breaking-down a -%% > well-formed URI reference into its components. -%% -%% > ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? -%% -%% We skip capturing some unused parts of the regex. - --define(URI_REGEX, - ("^(?:(?[^:/?#]+):)?(?//[^/?#]*)?" - "(?[^?#]*)(?\\?[^#]*)?(?#.*)?") -). - uri_regexp() -> - {ok, RE} = re:compile(?URI_REGEX, [caseless]), - RE. + persistent_term:get(?URI_REGEX_PT_KEY). format_scheme(undefined) -> <<>>; format_scheme(Scheme) -> [Scheme, $:].