From b9f258b73708ecf6c5937d3e2ac570fab1a93475 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 9 Jan 2023 11:39:42 -0300 Subject: [PATCH 01/31] feat(mongodb): add `payload_template` field for bridge (e5.0) https://emqx.atlassian.net/browse/EMQX-8705 Adds a `payload_template` fields that allows users to customize the payload to publish to MongoDB. --- .../i18n/emqx_ee_bridge_mongodb.conf | 11 +++ lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 6 +- .../src/emqx_ee_bridge_mongodb.erl | 3 +- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 24 +++++- .../src/emqx_ee_connector_mongodb.erl | 78 +++++++++++++++++++ 5 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf index f8009f0a4..4880148f9 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf @@ -86,4 +86,15 @@ emqx_ee_bridge_mongodb { zh: "桥接名称" } } + + payload_template { + desc { + en: "The template for formatting the outgoing messages. If undefined, will send all the available context in JSON format." + zh: "用于格式化外发信息的模板。 如果未定义，将以JSON格式发送所有可用的上下文。" + } + label: { + en: "Payload template" + zh: "有效载荷模板" + } + } } diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index d0099db1c..f47829870 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -57,9 +57,9 @@ resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, u resource_type(kafka) -> emqx_bridge_impl_kafka; resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; resource_type(gcp_pubsub) -> emqx_ee_connector_gcp_pubsub; -resource_type(mongodb_rs) -> emqx_connector_mongo; -resource_type(mongodb_sharded) -> emqx_connector_mongo; -resource_type(mongodb_single) -> emqx_connector_mongo; +resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; +resource_type(mongodb_sharded) -> emqx_ee_connector_mongodb; +resource_type(mongodb_single) -> emqx_ee_connector_mongodb; resource_type(mysql) -> emqx_connector_mysql; resource_type(influxdb_api_v1) -> emqx_ee_connector_influxdb; resource_type(influxdb_api_v2) -> emqx_ee_connector_influxdb; diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl index 516c75f65..bb4082681 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl @@ -37,7 +37,8 @@ roots() -> fields("config") -> [ {enable, mk(boolean(), #{desc => ?DESC("enable"), default => true})}, - {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})} + {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}, + {payload_template, mk(binary(), #{required => false, desc => ?DESC("payload_template")})} ]; fields(mongodb_rs) -> emqx_connector_mongo:fields(rs) ++ fields("config"); diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index fb8f1fcc3..7e44347f3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -25,7 +25,8 @@ all() -> group_tests() -> [ t_setup_via_config_and_publish, - t_setup_via_http_api_and_publish + t_setup_via_http_api_and_publish, + t_payload_template ]. groups() -> @@ -196,9 +197,14 @@ parse_and_check(ConfigString, Type, Name) -> Config. create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> Type = mongo_type_bin(?config(mongo_type, Config)), Name = ?config(mongo_name, Config), - MongoConfig = ?config(mongo_config, Config), + MongoConfig0 = ?config(mongo_config, Config), + MongoConfig = emqx_map_lib:deep_merge(MongoConfig0, Overrides), + ct:pal("creating ~p bridge with config:\n ~p", [Type, MongoConfig]), emqx_bridge:create(Type, Name, MongoConfig). delete_bridge(Config) -> @@ -219,7 +225,8 @@ clear_db(Config) -> Name = ?config(mongo_name, Config), #{<<"collection">> := Collection} = ?config(mongo_config, Config), ResourceID = emqx_bridge_resource:resource_id(Type, Name), - {ok, _, #{state := #{poolname := PoolName}}} = emqx_resource:get_instance(ResourceID), + {ok, _, #{state := #{connector_state := #{poolname := PoolName}}}} = + emqx_resource:get_instance(ResourceID), Selector = #{}, {true, _} = ecpool:pick_and_do( PoolName, {mongo_api, delete, [Collection, Selector]}, no_handover @@ -275,3 +282,14 @@ t_setup_via_http_api_and_publish(Config) -> find_all(Config) ), ok. + +t_payload_template(Config) -> + {ok, _} = create_bridge(Config, #{<<"payload_template">> => <<"{\"foo\": \"${clientid}\"}">>}), + Val = erlang:unique_integer(), + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + ok = send_message(Config, #{key => Val, clientid => ClientId}), + ?assertMatch( + {ok, [#{<<"foo">> := ClientId}]}, + find_all(Config) + ), + ok. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl new file mode 100644 index 000000000..b1327fef6 --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl @@ -0,0 +1,78 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_mongodb). + +-behaviour(emqx_resource). + +-include_lib("emqx_connector/include/emqx_connector_tables.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% `emqx_resource' API +-export([ + callback_mode/0, + is_buffer_supported/0, + on_start/2, + on_stop/2, + on_query/3, + on_get_status/2 +]). + +%%======================================================================================== +%% `emqx_resource' API +%%======================================================================================== + +callback_mode() -> emqx_connector_mongo:callback_mode(). + +is_buffer_supported() -> false. + +on_start(InstanceId, Config) -> + case emqx_connector_mongo:on_start(InstanceId, Config) of + {ok, ConnectorState} -> + PayloadTemplate0 = maps:get(payload_template, Config, undefined), + PayloadTemplate = preprocess_template(PayloadTemplate0), + State = #{ + payload_template => PayloadTemplate, + connector_state => ConnectorState + }, + {ok, State}; + Error -> + Error + end. + +on_stop(InstanceId, _State = #{connector_state := ConnectorState}) -> + emqx_connector_mongo:on_stop(InstanceId, ConnectorState). + +on_query(InstanceId, {send_message, Message0}, State) -> + #{ + payload_template := PayloadTemplate, + connector_state := ConnectorState + } = State, + Message = render_message(PayloadTemplate, Message0), + emqx_connector_mongo:on_query(InstanceId, {send_message, Message}, ConnectorState); +on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) -> + emqx_connector_mongo:on_query(InstanceId, Request, ConnectorState). + +on_get_status(InstanceId, _State = #{connector_state := ConnectorState}) -> + emqx_connector_mongo:on_get_status(InstanceId, ConnectorState). + +%%======================================================================================== +%% Helper fns +%%======================================================================================== + +preprocess_template(undefined = _PayloadTemplate) -> + undefined; +preprocess_template(PayloadTemplate) -> + emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate). + +render_message(undefined = _PayloadTemplate, Message) -> + Message; +render_message(PayloadTemplate, Message) -> + %% Note: mongo expects a map as a document, so the rendered result + %% must be JSON-serializable + Rendered = emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Message), + emqx_json:decode(Rendered, [return_maps]). From 0bd5bec982f4ea75592df7e65f1f391604bf1c77 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 9 Jan 2023 18:14:11 +0100 Subject: [PATCH 02/31] chore: upgrade mongodb-erlang to v3.0.19 This upgrades mongodb-erlang from v3.0.18 to v3.0.19. This fixes a bug that occurred when one issued a command specified as a map*. This fix should not affect present EMQX as this functionality is currently not used by EMQX. However, I think it is good to do the upgrade anyway in case we will use the fixed functionality in the future. There is no need to update the changelog. * https://github.com/emqx/mongodb-erlang/pull/36 --- apps/emqx_connector/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index b84d87e38..ed0bc827d 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -12,7 +12,7 @@ {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7-emqx.2"}}}, %% NOTE: mind poolboy version when updating mongodb-erlang version - {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.18"}}}, + {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.19"}}}, %% NOTE: mind poolboy version when updating eredis_cluster version {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.5"}}}, %% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git From f3cc722172ff2af873a25e57cb425d92f4baa059 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 10 Jan 2023 09:06:12 -0300 Subject: [PATCH 03/31] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf index 4880148f9..81ebc1e31 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf @@ -89,8 +89,8 @@ emqx_ee_bridge_mongodb { payload_template { desc { - en: "The template for formatting the outgoing messages. If undefined, will send all the available context in JSON format." - zh: "用于格式化外发信息的模板。 如果未定义，将以JSON格式发送所有可用的上下文。" + en: "The template for formatting the outgoing messages. If undefined, rule engine will use JSON format to serialize all visible inputs, such as clientid, topic, payload etc." + zh: "用于格式化写入 MongoDB 的消息模板。 如果未定义，规则引擎会使用 JSON 格式序列化所有的可见输入，例如 clientid, topic, payload 等。" } label: { en: "Payload template" From 50b7ac6a22799f16511d21c2e3a385dedb1abb67 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 3 Jan 2023 12:55:07 +0800 Subject: [PATCH 04/31] feat(prom): support headers for pushing --- .../i18n/emqx_prometheus_schema_i18n.conf | 10 +++++++++ apps/emqx_prometheus/src/emqx_prometheus.erl | 21 ++++++++++++++----- .../src/emqx_prometheus_api.erl | 1 + .../src/emqx_prometheus_schema.erl | 9 ++++++++ .../test/emqx_prometheus_SUITE.erl | 20 ++++++++++++++++++ 5 files changed, 56 insertions(+), 5 deletions(-) diff --git a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf b/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf index 7f251ff4b..391637570 100644 --- a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf +++ b/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf @@ -24,6 +24,16 @@ emqx_prometheus_schema { zh: """数据推送间隔""" } } + + headers { + desc { + en: """A list of HTTP Headers when pushing to Push Gateway.
+For example, { Authorization = "some-authz-tokens"}""" + zh: """推送到 Push Gateway 的 HTTP Headers 列表。
+例如， { Authorization = "some-authz-tokens"}""" + } + } + enable { desc { en: """Turn Prometheus data pushing on or off""" diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 5424c4e24..623ae43f0 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -98,8 +98,12 @@ handle_cast(_Msg, State) -> {noreply, State}. handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) -> - #{interval := Interval, push_gateway_server := Server} = opts(), - PushRes = push_to_push_gateway(Server), + #{ + interval := Interval, + headers := Headers, + push_gateway_server := Server + } = opts(), + PushRes = push_to_push_gateway(Server, Headers), NewTimer = ensure_timer(Interval), NewState = maps:update_with(PushRes, fun(C) -> C + 1 end, 1, State#{timer => NewTimer}), %% Data is too big, hibernate for saving memory and stop system monitor warning. @@ -107,12 +111,19 @@ handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) -> handle_info(_Msg, State) -> {noreply, State}. -push_to_push_gateway(Uri) -> +push_to_push_gateway(Uri, Headers0) when is_map(Headers0) -> [Name, Ip] = string:tokens(atom_to_list(node()), "@"), Url = lists:concat([Uri, "/metrics/job/", Name, "/instance/", Name, "~", Ip]), Data = prometheus_text_format:format(), - case httpc:request(post, {Url, [], "text/plain", Data}, ?HTTP_OPTIONS, []) of - {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, _Body}} -> + Headers = maps:fold( + fun(K, V, Acc) -> + [{atom_to_list(K), binary_to_list(V)} | Acc] + end, + [], + Headers0 + ), + case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of + {ok, {{"HTTP/1.1", 200, _}, _Headers, _Body}} -> ok; Error -> ?SLOG(error, #{ diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index 7466a1fd1..1acdf99a7 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -121,6 +121,7 @@ prometheus_config_example() -> enable => true, interval => "15s", push_gateway_server => <<"http://127.0.0.1:9091">>, + headers => #{'header-name' => 'header-value'}, vm_dist_collector => enabled, mnesia_collector => enabled, vm_statistics_collector => enabled, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 688c9be58..40b9ed6a1 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -52,6 +52,15 @@ fields("prometheus") -> desc => ?DESC(interval) } )}, + {headers, + ?HOCON( + map(), + #{ + default => #{}, + required => false, + desc => ?DESC(headers) + } + )}, {enable, ?HOCON( boolean(), diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index b9df1103b..effdfc914 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -27,6 +27,7 @@ "prometheus {\n" " push_gateway_server = \"http://127.0.0.1:9091\"\n" " interval = \"1s\"\n" + " headers = { Authorization = \"some-authz-tokens\"}\n" " enable = true\n" " vm_dist_collector = enabled\n" " mnesia_collector = enabled\n" @@ -85,6 +86,25 @@ t_collector_no_crash_test(_) -> prometheus_text_format:format(), ok. +t_assert_push(_) -> + meck:new(httpc, [passthrough]), + Self = self(), + AssertPush = fun(Method, Req = {Url, Headers, ContentType, _Data}, HttpOpts, Opts) -> + ?assertEqual(post, Method), + ?assertMatch("http://127.0.0.1:9091/metrics/job/" ++ _, Url), + ?assertEqual([{"Authorization", "some-authz-tokens"}], Headers), + ?assertEqual("text/plain", ContentType), + Self ! pass, + meck:passthrough([Method, Req, HttpOpts, Opts]) + end, + meck:expect(httpc, request, AssertPush), + ?assertMatch(ok, emqx_prometheus_sup:start_child(emqx_prometheus)), + receive + pass -> ok + after 2000 -> + ct:fail(assert_push_request_failed) + end. + t_only_for_coverage(_) -> ?assertEqual("5.0.0", emqx_prometheus_proto_v1:introduced_in()), ok. From b1292d6d4f610e6a455c1665fc157a1381a5768c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 3 Jan 2023 18:27:41 +0800 Subject: [PATCH 05/31] feat(prom): support configuring job_name for pushing mode --- .../i18n/emqx_prometheus_schema_i18n.conf | 19 +++++++++++++++++++ apps/emqx_prometheus/src/emqx_prometheus.erl | 15 ++++++++++++--- .../src/emqx_prometheus_api.erl | 1 + .../src/emqx_prometheus_schema.erl | 10 ++++++++++ .../test/emqx_prometheus_SUITE.erl | 3 ++- 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf b/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf index 391637570..f25e35219 100644 --- a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf +++ b/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf @@ -34,6 +34,25 @@ For example, { Authorization = "some-authz-tokens"}""" } } + job_name { + desc { + en: """Job Name that is pushed to the Push Gateway. Available variables:
+- ${name}: Name of EMQX node.
+- ${host}: Host name of EMQX node.
+For example, when the EMQX node name is emqx@127.0.0.1 then the name variable takes value emqx and the host variable takes value 127.0.0.1.
+ +Default value is: ${name}/instance/${name}~${host} +""" + zh: """推送到 Push Gateway 的 Job 名称。可用变量为：
+- ${name}: EMQX 节点的名称。 +- ${host}: EMQX 节点主机名。 + +例如，当 EMQX 节点名为 emqx@127.0.0.1 则 name 变量的值为 emqx，host 变量的值为 127.0.0.1。
+ +默认值为: ${name}/instance/${name}~${host}""" + } + } + enable { desc { en: """Turn Prometheus data pushing on or off""" diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 623ae43f0..91c1f22fc 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -101,9 +101,10 @@ handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) -> #{ interval := Interval, headers := Headers, + job_name := JobName, push_gateway_server := Server } = opts(), - PushRes = push_to_push_gateway(Server, Headers), + PushRes = push_to_push_gateway(Server, Headers, JobName), NewTimer = ensure_timer(Interval), NewState = maps:update_with(PushRes, fun(C) -> C + 1 end, 1, State#{timer => NewTimer}), %% Data is too big, hibernate for saving memory and stop system monitor warning. @@ -111,9 +112,17 @@ handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) -> handle_info(_Msg, State) -> {noreply, State}. -push_to_push_gateway(Uri, Headers0) when is_map(Headers0) -> +push_to_push_gateway(Uri, Headers0, JobName) when is_map(Headers0) -> [Name, Ip] = string:tokens(atom_to_list(node()), "@"), - Url = lists:concat([Uri, "/metrics/job/", Name, "/instance/", Name, "~", Ip]), + JobName1 = emqx_placeholder:preproc_tmpl(JobName), + JobName2 = binary_to_list( + emqx_placeholder:proc_tmpl( + JobName1, + #{<<"name">> => Name, <<"host">> => Ip} + ) + ), + + Url = lists:concat([Uri, "/metrics/job/", JobName2]), Data = prometheus_text_format:format(), Headers = maps:fold( fun(K, V, Acc) -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index 1acdf99a7..945c6eba9 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -122,6 +122,7 @@ prometheus_config_example() -> interval => "15s", push_gateway_server => <<"http://127.0.0.1:9091">>, headers => #{'header-name' => 'header-value'}, + job_name => <<"${name}/instance/${name}~${host}">>, vm_dist_collector => enabled, mnesia_collector => enabled, vm_statistics_collector => enabled, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 40b9ed6a1..da65e02c4 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -61,6 +61,16 @@ fields("prometheus") -> desc => ?DESC(headers) } )}, + {job_name, + ?HOCON( + binary(), + #{ + default => <<"${name}/instance/${name}~${host}">>, + required => true, + desc => ?DESC(job_name) + } + )}, + {enable, ?HOCON( boolean(), diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index effdfc914..77d9902a2 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -28,6 +28,7 @@ " push_gateway_server = \"http://127.0.0.1:9091\"\n" " interval = \"1s\"\n" " headers = { Authorization = \"some-authz-tokens\"}\n" + " job_name = \"${name}~${host}\"\n" " enable = true\n" " vm_dist_collector = enabled\n" " mnesia_collector = enabled\n" @@ -91,7 +92,7 @@ t_assert_push(_) -> Self = self(), AssertPush = fun(Method, Req = {Url, Headers, ContentType, _Data}, HttpOpts, Opts) -> ?assertEqual(post, Method), - ?assertMatch("http://127.0.0.1:9091/metrics/job/" ++ _, Url), + ?assertMatch("http://127.0.0.1:9091/metrics/job/test~127.0.0.1", Url), ?assertEqual([{"Authorization", "some-authz-tokens"}], Headers), ?assertEqual("text/plain", ContentType), Self ! pass, From 371b42eef295ebcb08a9c9fd88e9430ecc187fbb Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 11 Jan 2023 17:46:00 +0800 Subject: [PATCH 06/31] chore: update changes --- changes/v5.0.15/feat-9722.en.md | 3 +++ changes/v5.0.15/feat-9722.zh.md | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 changes/v5.0.15/feat-9722.en.md create mode 100644 changes/v5.0.15/feat-9722.zh.md diff --git a/changes/v5.0.15/feat-9722.en.md b/changes/v5.0.15/feat-9722.en.md new file mode 100644 index 000000000..b86f37b83 --- /dev/null +++ b/changes/v5.0.15/feat-9722.en.md @@ -0,0 +1,3 @@ +Add the following configuration options for Pushing metrics to Prometheus Push Gateway: +- `headers`: Allows custom HTTP request headers. +- `job_name`: allows to customize the name of the Job pushed to Push Gateway. diff --git a/changes/v5.0.15/feat-9722.zh.md b/changes/v5.0.15/feat-9722.zh.md new file mode 100644 index 000000000..a806cb1de --- /dev/null +++ b/changes/v5.0.15/feat-9722.zh.md @@ -0,0 +1,3 @@ +为 Prometheus 推送到 Push Gateway 新增以下配置项： +- `headers`：允许自定义 HTTP 请求头。 +- `job_name`：允许自定义推送到 Push Gateway 的 Job 名称。 From 9f6df2775960ac096a00d573962a17427938a537 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 11 Jan 2023 17:46:57 +0800 Subject: [PATCH 07/31] chore: bump app.src vsn --- apps/emqx_prometheus/src/emqx_prometheus.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index d95c89c3b..31f8cbfaf 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.3"}, + {vsn, "5.0.4"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx]}, From 48e1ba48322c818c67786de1951bb0a721b202ee Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 6 Dec 2022 16:47:55 -0300 Subject: [PATCH 08/31] feat(docs): add tags to schemas This'll allow us to split the generated `schema.json` file into subsections for better documentation navigation. --- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx_authn/src/emqx_authn_schema.erl | 4 ++++ .../enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl | 4 ++++ apps/emqx_authn/src/simple_authn/emqx_authn_http.erl | 4 ++++ apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl | 4 ++++ apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl | 4 ++++ apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl | 4 ++++ apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl | 4 ++++ apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl | 4 ++++ apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl | 4 ++++ apps/emqx_authz/src/emqx_authz_schema.erl | 4 ++++ apps/emqx_bridge/src/schema/emqx_bridge_schema.erl | 5 ++++- apps/emqx_conf/src/emqx_conf_schema.erl | 7 ++++++- apps/emqx_gateway/src/emqx_gateway_schema.erl | 5 ++++- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 4 ++++ lib-ee/emqx_license/src/emqx_license_schema.erl | 5 ++++- 16 files changed, 63 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 4cd78b575..f113089dd 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2342,7 +2342,7 @@ authentication(Which) -> undefined -> hoconsc:array(typerefl:map()); Module -> Module:root_type() end, - %% It is a lazy type because when handing runtime update requests + %% It is a lazy type because when handling runtime update requests %% the config is not checked by emqx_schema, but by the injected schema Type = hoconsc:lazy(Type0), #{ diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 88d8955c5..f40e759f0 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -22,6 +22,7 @@ -export([ common_fields/0, roots/0, + tags/0, fields/1, authenticator_type/0, authenticator_type_without_scram/0, @@ -32,6 +33,9 @@ roots() -> []. +tags() -> + [<<"Authentication">>]. + common_fields() -> [{enable, fun enable/1}]. diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index ba13bd069..ac39e2cda 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -25,6 +25,7 @@ -export([ namespace/0, + tags/0, roots/0, fields/1, desc/1 @@ -105,6 +106,9 @@ mnesia(boot) -> namespace() -> "authn-scram-builtin_db". +tags() -> + [<<"Authentication">>]. + roots() -> [?CONF_NS]. fields(?CONF_NS) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 0a9aaa825..faa06b71a 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -26,6 +26,7 @@ -export([ namespace/0, + tags/0, roots/0, fields/1, desc/1, @@ -51,6 +52,9 @@ namespace() -> "authn-http". +tags() -> + [<<"Authentication">>]. + roots() -> [ {?CONF_NS, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 5709a1fe7..1c44b4d1f 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -25,6 +25,7 @@ -export([ namespace/0, + tags/0, roots/0, fields/1, desc/1 @@ -44,6 +45,9 @@ namespace() -> "authn-jwt". +tags() -> + [<<"Authentication">>]. + roots() -> [ {?CONF_NS, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index e915744e1..7c51644b7 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -26,6 +26,7 @@ -export([ namespace/0, + tags/0, roots/0, fields/1, desc/1 @@ -107,6 +108,9 @@ mnesia(boot) -> namespace() -> "authn-builtin_db". +tags() -> + [<<"Authentication">>]. + roots() -> [?CONF_NS]. fields(?CONF_NS) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 3fac0ed7d..3f140a8eb 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -25,6 +25,7 @@ -export([ namespace/0, + tags/0, roots/0, fields/1, desc/1 @@ -44,6 +45,9 @@ namespace() -> "authn-mongodb". +tags() -> + [<<"Authentication">>]. + roots() -> [ {?CONF_NS, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 68913443f..ffce42bb3 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -27,6 +27,7 @@ -export([ namespace/0, + tags/0, roots/0, fields/1, desc/1 @@ -46,6 +47,9 @@ namespace() -> "authn-mysql". +tags() -> + [<<"Authentication">>]. + roots() -> [?CONF_NS]. fields(?CONF_NS) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 1cadf9c56..2d7974301 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -26,6 +26,7 @@ -export([ namespace/0, + tags/0, roots/0, fields/1, desc/1 @@ -50,6 +51,9 @@ namespace() -> "authn-postgresql". +tags() -> + [<<"Authentication">>]. + roots() -> [?CONF_NS]. fields(?CONF_NS) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 0c8fedfb5..12b7422b5 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -25,6 +25,7 @@ -export([ namespace/0, + tags/0, roots/0, fields/1, desc/1 @@ -44,6 +45,9 @@ namespace() -> "authn-redis". +tags() -> + [<<"Authentication">>]. + roots() -> [ {?CONF_NS, diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index a684ae6ba..e2da9b41e 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -33,6 +33,7 @@ -export([ namespace/0, roots/0, + tags/0, fields/1, validations/0, desc/1 @@ -65,6 +66,9 @@ type_names() -> namespace() -> authz. +tags() -> + [<<"Authorization">>]. + %% @doc authorization schema is not exported %% but directly used by emqx_schema roots() -> []. diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 1ad5d7aba..845c1ef90 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -20,7 +20,7 @@ -import(hoconsc, [mk/2, ref/2]). --export([roots/0, fields/1, desc/1, namespace/0]). +-export([roots/0, fields/1, desc/1, namespace/0, tags/0]). -export([ get_response/0, @@ -104,6 +104,9 @@ metrics_status_fields() -> namespace() -> "bridge". +tags() -> + [<<"Bridge">>]. + roots() -> [bridges]. fields(bridges) -> diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index a7b388964..71d2ab3fd 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -38,7 +38,9 @@ cipher/0 ]). --export([namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1]). +-export([ + namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1, tags/0 +]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). %% Static apps which merge their configs into the merged emqx.conf @@ -67,6 +69,9 @@ %% root config should not have a namespace namespace() -> undefined. +tags() -> + [<<"EMQX">>]. + roots() -> PtKey = ?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, case persistent_term:get(PtKey, undefined) of diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index e89280f14..804e1f862 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -49,12 +49,15 @@ ]). -elvis([{elvis_style, dont_repeat_yourself, disable}]). --export([namespace/0, roots/0, fields/1, desc/1]). +-export([namespace/0, roots/0, fields/1, desc/1, tags/0]). -export([proxy_protocol_opts/0]). namespace() -> gateway. +tags() -> + [<<"Gateway">>]. + roots() -> [gateway]. fields(gateway) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index cbe7dae82..d6913cbc6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -23,6 +23,7 @@ -export([ namespace/0, + tags/0, roots/0, fields/1, desc/1, @@ -33,6 +34,9 @@ namespace() -> rule_engine. +tags() -> + [<<"Rule Engine">>]. + roots() -> ["rule_engine"]. fields("rule_engine") -> diff --git a/lib-ee/emqx_license/src/emqx_license_schema.erl b/lib-ee/emqx_license/src/emqx_license_schema.erl index bde4ed076..00c253ec8 100644 --- a/lib-ee/emqx_license/src/emqx_license_schema.erl +++ b/lib-ee/emqx_license/src/emqx_license_schema.erl @@ -13,7 +13,7 @@ -behaviour(hocon_schema). --export([roots/0, fields/1, validations/0, desc/1]). +-export([roots/0, fields/1, validations/0, desc/1, tags/0]). -export([ default_license/0, @@ -31,6 +31,9 @@ roots() -> )} ]. +tags() -> + [<<"License">>]. + fields(key_license) -> [ {key, #{ From a9d0143d5fe0dd94629b9fc1b16e112c2dd7ed6d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 7 Dec 2022 09:15:42 -0300 Subject: [PATCH 09/31] chore(license): change `key` type to `binary()` So that the `raw_default` field in the schema JSON renders properly. --- bin/nodetool | 11 +++++++++-- lib-ee/emqx_license/src/emqx_license_schema.erl | 12 +++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/bin/nodetool b/bin/nodetool index b4f0a0183..9a5d5e069 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -24,12 +24,19 @@ main(Args) -> ["hocon" | Rest] -> %% forward the call to hocon_cli hocon_cli:main(Rest); - ["check_license_key", Key] -> - check_license(#{key => list_to_binary(Key)}); + ["check_license_key", Key0] -> + Key = cleanup_key(Key0), + check_license(#{key => Key}); _ -> do(Args) end. +%% the key is a string (list) representation of a binary, so we need +%% to remove the leading and trailing angle brackets. +cleanup_key(Str0) -> + Str1 = iolist_to_binary(string:replace(Str0, "<<", "", leading)), + iolist_to_binary(string:replace(Str1, ">>", "", trailing)). + do(Args) -> ok = do_with_halt(Args, "mnesia_dir", fun create_mnesia_dir/2), ok = do_with_halt(Args, "chkconfig", fun("-config", X) -> chkconfig(X) end), diff --git a/lib-ee/emqx_license/src/emqx_license_schema.erl b/lib-ee/emqx_license/src/emqx_license_schema.erl index 00c253ec8..9d16f697c 100644 --- a/lib-ee/emqx_license/src/emqx_license_schema.erl +++ b/lib-ee/emqx_license/src/emqx_license_schema.erl @@ -37,7 +37,7 @@ tags() -> fields(key_license) -> [ {key, #{ - type => string(), + type => binary(), default => default_license(), %% so it's not logged sensitive => true, @@ -85,7 +85,9 @@ check_license_watermark(Conf) -> %% NOTE: when updating a new key, the schema doc in emqx_license_schema_i18n.conf %% should be updated accordingly default_license() -> - "MjIwMTExCjAKMTAKRXZhbHVhdGlvbgpjb250YWN0QGVtcXguaW8KZ" - "GVmYXVsdAoyMDIzMDEwOQoxODI1CjEwMAo=.MEUCIG62t8W15g05f" - "1cKx3tA3YgJoR0dmyHOPCdbUxBGxgKKAiEAhHKh8dUwhU+OxNEaOn" - "8mgRDtiT3R8RZooqy6dEsOmDI=". + << + "MjIwMTExCjAKMTAKRXZhbHVhdGlvbgpjb250YWN0QGVtcXguaW8KZ" + "GVmYXVsdAoyMDIzMDEwOQoxODI1CjEwMAo=.MEUCIG62t8W15g05f" + "1cKx3tA3YgJoR0dmyHOPCdbUxBGxgKKAiEAhHKh8dUwhU+OxNEaOn" + "8mgRDtiT3R8RZooqy6dEsOmDI=" + >>. From e6a8be1ff612c770bf39f13109c78f93b6450fd7 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 16 Dec 2022 10:02:40 -0300 Subject: [PATCH 10/31] docs: fix closing tag --- apps/emqx/i18n/emqx_schema_i18n.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 750c0c2cd..045b06da0 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -2050,7 +2050,7 @@ base_listener_enable_authn { Set true (default) to enable client authentication on this listener, the authentication process goes through the configured authentication chain. When set to false to allow any clients with or without authentication information such as username or password to log in. -When set to quick_deny_anonymous, it behaves like when set to true but clients will be +When set to quick_deny_anonymous, it behaves like when set to true but clients will be denied immediately without going through any authenticators if username is not provided. This is useful to fence off anonymous clients early. """ From d5a462486eb2936eba10b55085b654fcb92ff331 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 16 Dec 2022 14:27:00 -0300 Subject: [PATCH 11/31] feat: tag another emqx schema as `emqx` --- apps/emqx/src/emqx_schema.erl | 5 ++++- apps/emqx_authn/src/emqx_authn.app.src | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index f113089dd..ed7e0a016 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -111,7 +111,7 @@ comma_separated_atoms/0 ]). --export([namespace/0, roots/0, roots/1, fields/1, desc/1]). +-export([namespace/0, roots/0, roots/1, fields/1, desc/1, tags/0]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1]). -export([sc/2, map/2]). @@ -120,6 +120,9 @@ namespace() -> broker. +tags() -> + [<<"EMQX">>]. + roots() -> %% TODO change config importance to a field metadata roots(high) ++ roots(medium) ++ roots(low). diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index ea21e5bdc..7f01d94c0 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.11"}, + {vsn, "0.1.12"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, From 254b92a3c17789cba73a2ed72e87c048b9c9d7a3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 11 Jan 2023 10:31:23 -0300 Subject: [PATCH 12/31] docs: add comma to please spellcheck --- apps/emqx/i18n/emqx_schema_i18n.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 045b06da0..0665cfb09 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -2050,7 +2050,7 @@ base_listener_enable_authn { Set true (default) to enable client authentication on this listener, the authentication process goes through the configured authentication chain. When set to false to allow any clients with or without authentication information such as username or password to log in. -When set to quick_deny_anonymous, it behaves like when set to true but clients will be +When set to quick_deny_anonymous, it behaves like when set to true, but clients will be denied immediately without going through any authenticators if username is not provided. This is useful to fence off anonymous clients early. """ From bf259e360ab3d82a1c7ed6ced014b2c31f2fa1e9 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 11 Jan 2023 21:30:20 +0800 Subject: [PATCH 13/31] chore: update the change log for #9725 --- changes/v5.0.15/feat-9725.en.md | 11 +++++++++++ changes/v5.0.15/feat-9725.zh.md | 8 ++++++++ 2 files changed, 19 insertions(+) create mode 100644 changes/v5.0.15/feat-9725.en.md create mode 100644 changes/v5.0.15/feat-9725.zh.md diff --git a/changes/v5.0.15/feat-9725.en.md b/changes/v5.0.15/feat-9725.en.md new file mode 100644 index 000000000..832aa6bf9 --- /dev/null +++ b/changes/v5.0.15/feat-9725.en.md @@ -0,0 +1,11 @@ +Remove the config `auto_reconnect` from the emqx_authz, emqx_authn and data-bridge componets. +This is because we have another config with similar functions: `resource_opts.auto_restart_interval`。 + +The functions of these two config are difficult to distinguish, which will lead to confusion. +After this change, `auto_reconnect` will not be configurable (always be true), and the underlying +drivers that support this config will automatically reconnect the abnormally disconnected +connection every `2s`. + +And the config `resource_opts.auto_restart_interval` is still available for user. +It is the time interval that emqx restarts the resource when the connection cannot be +established for some reason. diff --git a/changes/v5.0.15/feat-9725.zh.md b/changes/v5.0.15/feat-9725.zh.md new file mode 100644 index 000000000..e7a2412d4 --- /dev/null +++ b/changes/v5.0.15/feat-9725.zh.md @@ -0,0 +1,8 @@ +从认证、鉴权和数据桥接功能中，删除 `auto_reconnect` 配置项，因为我们还有另一个功能类似的配置项： +`resource_opts.auto_restart_interval`。 + +这两个配置项的功能难以区分，会导致困惑。此修改之后，`auto_reconnect` 将不可配置(永远为 true)， +支持此配置的底层驱动将以 `2s` 为周期自动重连异常断开的连接。 + +而 `resource_opts.auto_restart_interval` 配置项仍然开放给用户配置，它是资源因为某些原因 +无法建立连接的时候，emqx 重新启动该资源的时间间隔。 From 3e9c4f444f03f3ebc31ce67db556025fecbeef92 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 11 Jan 2023 19:24:51 +0800 Subject: [PATCH 14/31] refactor: remove the auto_reconnect field --- apps/emqx_authz/README.md | 3 --- apps/emqx_connector/README.md | 2 +- .../i18n/emqx_connector_schema_lib.conf | 10 ++++----- .../emqx_connector/include/emqx_connector.hrl | 2 ++ .../src/emqx_connector_ldap.erl | 8 ++----- .../src/emqx_connector_mqtt.erl | 11 +++++----- .../src/emqx_connector_mysql.erl | 20 ++++++------------ .../src/emqx_connector_pgsql.erl | 20 ++++++------------ .../src/emqx_connector_redis.erl | 21 +++++++------------ .../src/emqx_connector_schema_lib.erl | 1 + .../src/emqx_ee_bridge_mysql.erl | 1 - .../src/emqx_ee_bridge_pgsql.erl | 1 - .../src/emqx_ee_bridge_redis.erl | 1 - mix.exs | 2 +- rebar.config | 2 +- 15 files changed, 38 insertions(+), 67 deletions(-) diff --git a/apps/emqx_authz/README.md b/apps/emqx_authz/README.md index 8c05f21be..af543e478 100644 --- a/apps/emqx_authz/README.md +++ b/apps/emqx_authz/README.md @@ -15,7 +15,6 @@ authz:{ pool_size: 1 username: root password: public - auto_reconnect: true ssl: { enable: true cacertfile: "etc/certs/cacert.pem" @@ -33,7 +32,6 @@ authz:{ pool_size: 1 username: root password: public - auto_reconnect: true ssl: {enable: false} } sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or username = '$all' or clientid = ${clientid}" @@ -45,7 +43,6 @@ authz:{ database: 0 pool_size: 1 password: public - auto_reconnect: true ssl: {enable: false} } cmd: "HGETALL mqtt_authz:${username}" diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md index 7ef3a8c4a..6baba29de 100644 --- a/apps/emqx_connector/README.md +++ b/apps/emqx_connector/README.md @@ -14,7 +14,7 @@ An MySQL connector can be used as following: ``` (emqx@127.0.0.1)5> emqx_resource:list_instances_verbose(). [#{config => - #{auto_reconnect => true,cacertfile => [],certfile => [], + #{cacertfile => [],certfile => [], database => "mqtt",keyfile => [],password => "public", pool_size => 1, server => {{127,0,0,1},3306}, diff --git a/apps/emqx_connector/i18n/emqx_connector_schema_lib.conf b/apps/emqx_connector/i18n/emqx_connector_schema_lib.conf index f5caf29c4..8f25e0352 100644 --- a/apps/emqx_connector/i18n/emqx_connector_schema_lib.conf +++ b/apps/emqx_connector/i18n/emqx_connector_schema_lib.conf @@ -68,13 +68,13 @@ emqx_connector_schema_lib { auto_reconnect { desc { - en: "Enable automatic reconnect to the database." - zh: "自动重连数据库。" + en: "Deprecated. Enable automatic reconnect to the database." + zh: "已弃用。自动重连数据库。" } label: { - en: "Auto Reconnect Database" - zh: "自动重连数据库" - } + en: "Deprecated. Auto Reconnect Database" + zh: "已弃用。自动重连数据库" + } } } diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_connector/include/emqx_connector.hrl index 96b6ba4d6..82c946cfc 100644 --- a/apps/emqx_connector/include/emqx_connector.hrl +++ b/apps/emqx_connector/include/emqx_connector.hrl @@ -24,6 +24,8 @@ -define(REDIS_DEFAULT_PORT, 6379). -define(PGSQL_DEFAULT_PORT, 5432). +-define(AUTO_RECONNECT_INTERVAL, 2). + -define(SERVERS_DESC, "A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].`
" "For each Node should be: " diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 1cb65034d..1930d9e68 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -59,7 +59,6 @@ on_start( bind_password := BindPassword, timeout := Timeout, pool_size := PoolSize, - auto_reconnect := AutoReconn, ssl := SSL } = Config ) -> @@ -86,11 +85,11 @@ on_start( {bind_password, BindPassword}, {timeout, Timeout}, {pool_size, PoolSize}, - {auto_reconnect, reconn_interval(AutoReconn)} + {auto_reconnect, ?AUTO_RECONNECT_INTERVAL} ], PoolName = emqx_plugin_libs_pool:pool_name(InstId), case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ SslOpts) of - ok -> {ok, #{poolname => PoolName, auto_reconnect => AutoReconn}}; + ok -> {ok, #{poolname => PoolName}}; {error, Reason} -> {error, Reason} end. @@ -129,9 +128,6 @@ on_query(InstId, {search, Base, Filter, Attributes}, #{poolname := PoolName} = S on_get_status(_InstId, _State) -> connected. -reconn_interval(true) -> 15; -reconn_interval(false) -> false. - search(Conn, Base, Filter, Attributes) -> eldap2:search(Conn, [ {base, Base}, diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 522f15ccf..c28aa2514 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -15,6 +15,8 @@ %%-------------------------------------------------------------------- -module(emqx_connector_mqtt). +-include("emqx_connector.hrl"). + -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -198,12 +200,10 @@ on_query_async( ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => InstanceId}), emqx_connector_mqtt_worker:send_to_remote_async(InstanceId, Msg, {ReplyFun, Args}). -on_get_status(_InstId, #{name := InstanceId, bridge_conf := Conf}) -> - AutoReconn = maps:get(auto_reconnect, Conf, true), +on_get_status(_InstId, #{name := InstanceId}) -> case emqx_connector_mqtt_worker:status(InstanceId) of connected -> connected; - _ when AutoReconn == true -> connecting; - _ when AutoReconn == false -> disconnected + _ -> connecting end. ensure_mqtt_worker_started(InstanceId, BridgeConf) -> @@ -236,7 +236,6 @@ make_forward_confs(FrowardConf) -> basic_config( #{ server := Server, - reconnect_interval := ReconnIntv, proto_ver := ProtoVer, bridge_mode := BridgeMode, clean_start := CleanStart, @@ -252,7 +251,7 @@ basic_config( %% 30s connect_timeout => 30, auto_reconnect => true, - reconnect_interval => ReconnIntv, + reconnect_interval => ?AUTO_RECONNECT_INTERVAL, proto_ver => ProtoVer, %% Opening bridge_mode will form a non-standard mqtt connection message. %% A load balancing server (such as haproxy) is often set up before the emqx broker server. diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 6c0ff7210..d119a01e9 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -52,7 +52,6 @@ -type state() :: #{ poolname := atom(), - auto_reconnect := boolean(), prepare_statement := prepares(), params_tokens := params_tokens(), batch_inserts := sqls(), @@ -84,8 +83,6 @@ on_start( server := Server, database := DB, username := User, - password := Password, - auto_reconnect := AutoReconn, pool_size := PoolSize, ssl := SSL } = Config @@ -107,14 +104,14 @@ on_start( {host, Host}, {port, Port}, {user, User}, - {password, Password}, + {password, maps:get(password, Config, <<>>)}, {database, DB}, - {auto_reconnect, reconn_interval(AutoReconn)}, + {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, {pool_size, PoolSize} ], PoolName = emqx_plugin_libs_pool:pool_name(InstId), Prepares = parse_prepare_sql(Config), - State = maps:merge(#{poolname => PoolName, auto_reconnect => AutoReconn}, Prepares), + State = maps:merge(#{poolname => PoolName}, Prepares), case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of ok -> {ok, init_prepare(State)}; @@ -194,7 +191,7 @@ mysql_function(prepared_query) -> mysql_function(_) -> mysql_function(prepared_query). -on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) -> +on_get_status(_InstId, #{poolname := Pool} = State) -> case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of true -> case do_check_prepares(State) of @@ -205,10 +202,10 @@ on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State {connected, NState}; {error, _Reason} -> %% do not log error, it is logged in prepare_sql_to_conn - conn_status(AutoReconn) + connecting end; false -> - conn_status(AutoReconn) + connecting end. do_get_status(Conn) -> @@ -227,11 +224,6 @@ do_check_prepares(State = #{poolname := PoolName, prepare_statement := {error, P end. %% =================================================================== -conn_status(_AutoReconn = true) -> connecting; -conn_status(_AutoReconn = false) -> disconnected. - -reconn_interval(true) -> 15; -reconn_interval(false) -> false. connect(Options) -> mysql:start_link(Options). diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 4b565a614..25a22455d 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -56,7 +56,6 @@ -type state() :: #{ poolname := atom(), - auto_reconnect := boolean(), prepare_sql := prepares(), params_tokens := params_tokens(), prepare_statement := epgsql:statement() @@ -87,8 +86,6 @@ on_start( server := Server, database := DB, username := User, - password := Password, - auto_reconnect := AutoReconn, pool_size := PoolSize, ssl := SSL } = Config @@ -113,14 +110,14 @@ on_start( {host, Host}, {port, Port}, {username, User}, - {password, emqx_secret:wrap(Password)}, + {password, emqx_secret:wrap(maps:get(password, Config, ""))}, {database, DB}, - {auto_reconnect, reconn_interval(AutoReconn)}, + {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, {pool_size, PoolSize} ], PoolName = emqx_plugin_libs_pool:pool_name(InstId), Prepares = parse_prepare_sql(Config), - InitState = #{poolname => PoolName, auto_reconnect => AutoReconn, prepare_statement => #{}}, + InitState = #{poolname => PoolName, prepare_statement => #{}}, State = maps:merge(InitState, Prepares), case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of ok -> @@ -247,7 +244,7 @@ on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> end, Result. -on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State) -> +on_get_status(_InstId, #{poolname := Pool} = State) -> case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of true -> case do_check_prepares(State) of @@ -258,10 +255,10 @@ on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn} = State {connected, NState}; false -> %% do not log error, it is logged in prepare_sql_to_conn - conn_status(AutoReconn) + connecting end; false -> - conn_status(AutoReconn) + connecting end. do_get_status(Conn) -> @@ -280,11 +277,6 @@ do_check_prepares(State = #{poolname := PoolName, prepare_sql := {error, Prepare end. %% =================================================================== -conn_status(_AutoReconn = true) -> connecting; -conn_status(_AutoReconn = false) -> disconnected. - -reconn_interval(true) -> 15; -reconn_interval(false) -> false. connect(Opts) -> Host = proplists:get_value(host, Opts), diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 726af2d9b..6143dbf0c 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -117,7 +117,6 @@ on_start( #{ redis_type := Type, pool_size := PoolSize, - auto_reconnect := AutoReconn, ssl := SSL } = Config ) -> @@ -142,7 +141,7 @@ on_start( [ {pool_size, PoolSize}, {password, maps:get(password, Config, "")}, - {auto_reconnect, reconn_interval(AutoReconn)} + {auto_reconnect, ?AUTO_RECONNECT_INTERVAL} ] ++ Database ++ Servers, Options = case maps:get(enable, SSL) of @@ -155,7 +154,7 @@ on_start( [{ssl, false}] end ++ [{sentinel, maps:get(sentinel, Config, undefined)}], PoolName = emqx_plugin_libs_pool:pool_name(InstId), - State = #{poolname => PoolName, type => Type, auto_reconnect => AutoReconn}, + State = #{poolname => PoolName, type => Type}, case Type of cluster -> case eredis_cluster:start_pool(PoolName, Opts ++ [{options, Options}]) of @@ -229,18 +228,18 @@ eredis_cluster_workers_exist_and_are_connected(Workers) -> Workers ). -on_get_status(_InstId, #{type := cluster, poolname := PoolName, auto_reconnect := AutoReconn}) -> +on_get_status(_InstId, #{type := cluster, poolname := PoolName}) -> case eredis_cluster:pool_exists(PoolName) of true -> Workers = extract_eredis_cluster_workers(PoolName), Health = eredis_cluster_workers_exist_and_are_connected(Workers), - status_result(Health, AutoReconn); + status_result(Health); false -> disconnected end; -on_get_status(_InstId, #{poolname := Pool, auto_reconnect := AutoReconn}) -> +on_get_status(_InstId, #{poolname := Pool}) -> Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1), - status_result(Health, AutoReconn). + status_result(Health). do_get_status(Conn) -> case eredis:q(Conn, ["PING"]) of @@ -248,12 +247,8 @@ do_get_status(Conn) -> _ -> false end. -status_result(_Status = true, _AutoReconn) -> connected; -status_result(_Status = false, _AutoReconn = true) -> connecting; -status_result(_Status = false, _AutoReconn = false) -> disconnected. - -reconn_interval(true) -> 15; -reconn_interval(false) -> false. +status_result(_Status = true) -> connected; +status_result(_Status = false) -> connecting. do_cmd(PoolName, cluster, {cmd, Command}) -> eredis_cluster:q(PoolName, Command); diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 2364aeeaa..5d8f6941c 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -106,4 +106,5 @@ password(_) -> undefined. auto_reconnect(type) -> boolean(); auto_reconnect(desc) -> ?DESC("auto_reconnect"); auto_reconnect(default) -> true; +auto_reconnect(deprecated) -> {since, "v5.0.15"}; auto_reconnect(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index bf5d2e140..114459149 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -51,7 +51,6 @@ values(post) -> pool_size => 8, username => <<"root">>, password => <<"">>, - auto_reconnect => true, sql => ?DEFAULT_SQL, local_topic => <<"local/topic/#">>, resource_opts => #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl index dc8697e37..be9fc9dc8 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl @@ -53,7 +53,6 @@ values(post, Type) -> pool_size => 8, username => <<"root">>, password => <<"public">>, - auto_reconnect => true, sql => ?DEFAULT_SQL, local_topic => <<"local/topic/#">>, resource_opts => #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl index a0c6ba834..5c273e050 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl @@ -79,7 +79,6 @@ values(common, RedisType, SpecificOpts) -> local_topic => <<"local/topic/#">>, pool_size => 8, password => <<"secret">>, - auto_reconnect => true, command_template => [<<"LPUSH">>, <<"MSGS">>, <<"${payload}">>], resource_opts => #{ batch_size => 1, diff --git a/mix.exs b/mix.exs index 45bc820e0..832f6e7ae 100644 --- a/mix.exs +++ b/mix.exs @@ -69,7 +69,7 @@ defmodule EMQXUmbrella.MixProject do {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, {:hocon, github: "emqx/hocon", tag: "0.35.0", override: true}, - {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, + {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, # in conflict by ehttpc and emqtt diff --git a/rebar.config b/rebar.config index a61bfe16a..f05624477 100644 --- a/rebar.config +++ b/rebar.config @@ -69,7 +69,7 @@ , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.35.0"}}} - , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} + , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} , {telemetry, "1.1.0"} From 1194d07a07598d602a4597c61623231051ceb470 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 12 Jan 2023 12:48:26 +0800 Subject: [PATCH 15/31] refactor: update emqx_prometheus's headers from map() to list() --- .../src/emqx_dashboard_swagger.erl | 2 ++ apps/emqx_prometheus/src/emqx_prometheus.erl | 14 ++++---------- .../src/emqx_prometheus_schema.erl | 17 +++++++++++++++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 102b95f4e..1b5b6ca9c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -708,6 +708,8 @@ typename_to_spec("qos()", _Mod) -> #{type => integer, minimum => 0, maximum => 2, example => 0}; typename_to_spec("{binary(), binary()}", _Mod) -> #{type => object, example => #{}}; +typename_to_spec("{string(), string()}", _Mod) -> + #{type => object, example => #{}}; typename_to_spec("comma_separated_list()", _Mod) -> #{type => string, example => <<"item1,item2">>}; typename_to_spec("comma_separated_binary()", _Mod) -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 91c1f22fc..a66f275f8 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -112,7 +112,7 @@ handle_info({timeout, Timer, ?TIMER_MSG}, State = #{timer := Timer}) -> handle_info(_Msg, State) -> {noreply, State}. -push_to_push_gateway(Uri, Headers0, JobName) when is_map(Headers0) -> +push_to_push_gateway(Uri, Headers, JobName) when is_list(Headers) -> [Name, Ip] = string:tokens(atom_to_list(node()), "@"), JobName1 = emqx_placeholder:preproc_tmpl(JobName), JobName2 = binary_to_list( @@ -124,21 +124,15 @@ push_to_push_gateway(Uri, Headers0, JobName) when is_map(Headers0) -> Url = lists:concat([Uri, "/metrics/job/", JobName2]), Data = prometheus_text_format:format(), - Headers = maps:fold( - fun(K, V, Acc) -> - [{atom_to_list(K), binary_to_list(V)} | Acc] - end, - [], - Headers0 - ), case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of - {ok, {{"HTTP/1.1", 200, _}, _Headers, _Body}} -> + {ok, {{"HTTP/1.1", 200, _}, _RespHeaders, _RespBody}} -> ok; Error -> ?SLOG(error, #{ msg => "post_to_push_gateway_failed", error => Error, - url => Url + url => Url, + headers => Headers }), failed end. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index da65e02c4..c13d198a2 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -25,7 +25,8 @@ roots/0, fields/1, desc/1, - translation/1 + translation/1, + convert_headers/1 ]). namespace() -> "prometheus". @@ -54,10 +55,11 @@ fields("prometheus") -> )}, {headers, ?HOCON( - map(), + list({string(), string()}), #{ default => #{}, required => false, + converter => fun ?MODULE:convert_headers/1, desc => ?DESC(headers) } )}, @@ -145,6 +147,17 @@ fields("prometheus") -> desc("prometheus") -> ?DESC(prometheus); desc(_) -> undefined. +convert_headers(Headers) when is_map(Headers) -> + maps:fold( + fun(K, V, Acc) -> + [{binary_to_list(K), binary_to_list(V)} | Acc] + end, + [], + Headers + ); +convert_headers(Headers) when is_list(Headers) -> + Headers. + %% for CI test, CI don't load the whole emqx_conf_schema. translation(Name) -> emqx_conf_schema:translation(Name). From e07aa2086bdd9e0c428a5da3436024ff55eb0ca7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 11 Jan 2023 16:53:47 +0300 Subject: [PATCH 16/31] fix(api): augment paged search responses with `hasnext` flag This flag indicates whether there are more results available on the next pages. It is needed in cases when the total number of search results is not known in advance. Also, in such cases there's no `count` field in responses anymore because responding with `0` was confusing for clients: it's not possible to differentiate between "there are no results" and "we don't know how much". Co-authored-by: Thales Macedo Garitezi --- .../src/emqx_dashboard_swagger.erl | 16 +- apps/emqx_management/src/emqx_mgmt_api.erl | 187 ++++++++++-------- .../test/emqx_mgmt_api_SUITE.erl | 7 +- .../test/emqx_mgmt_api_subscription_SUITE.erl | 79 ++++++-- 4 files changed, 180 insertions(+), 109 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 102b95f4e..42fdcd46d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -139,14 +139,20 @@ fields(limit) -> [{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}]; fields(count) -> Desc = << - "Total number of records counted.
" - "Note: this field is 0 when the queryed table is empty, " - "or if the query can not be optimized and requires a full table scan." + "Total number of records matching the query.
" + "Note: this field is present only if the query can be optimized and does " + "not require a full table scan." + >>, + Meta = #{desc => Desc, required => false}, + [{count, hoconsc:mk(non_neg_integer(), Meta)}]; +fields(hasnext) -> + Desc = << + "Flag indicating whether there are more results available on next pages." >>, Meta = #{desc => Desc, required => true}, - [{count, hoconsc:mk(non_neg_integer(), Meta)}]; + [{hasnext, hoconsc:mk(boolean(), Meta)}]; fields(meta) -> - fields(page) ++ fields(limit) ++ fields(count). + fields(page) ++ fields(limit) ++ fields(count) ++ fields(hasnext). -spec schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema_map(). schema_with_example(Type, Example) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 893007ebf..614949f16 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -20,7 +20,6 @@ -elvis([{elvis_style, dont_repeat_yourself, #{min_complexity => 100}}]). --define(FRESH_SELECT, fresh_select). -define(LONG_QUERY_TIMEOUT, 50000). -export([ @@ -174,13 +173,12 @@ do_node_query( case do_query(Node, QueryState) of {error, {badrpc, R}} -> {error, Node, {badrpc, R}}; - {Rows, NQueryState = #{continuation := ?FRESH_SELECT}} -> - {_, NResultAcc} = accumulate_query_rows(Node, Rows, NQueryState, ResultAcc), - NResultAcc; - {Rows, NQueryState} -> + {Rows, NQueryState = #{complete := Complete}} -> case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of {enough, NResultAcc} -> - NResultAcc; + finalize_query(NResultAcc, NQueryState); + {_, NResultAcc} when Complete -> + finalize_query(NResultAcc, NQueryState); {more, NResultAcc} -> do_node_query(Node, NQueryState, NResultAcc) end @@ -212,8 +210,6 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) -> end. %% @private -do_cluster_query([], _QueryState, ResultAcc) -> - ResultAcc; do_cluster_query( [Node | Tail] = Nodes, QueryState, @@ -222,31 +218,29 @@ do_cluster_query( case do_query(Node, QueryState) of {error, {badrpc, R}} -> {error, Node, {badrpc, R}}; - {Rows, NQueryState} -> + {Rows, NQueryState = #{complete := Complete}} -> case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of {enough, NResultAcc} -> - maybe_collect_total_from_tail_nodes(Tail, NQueryState, NResultAcc); + FQueryState = maybe_collect_total_from_tail_nodes(Tail, NQueryState), + FComplete = Complete andalso Tail =:= [], + finalize_query(NResultAcc, mark_complete(FQueryState, FComplete)); + {more, NResultAcc} when not Complete -> + do_cluster_query(Nodes, NQueryState, NResultAcc); + {more, NResultAcc} when Tail =/= [] -> + do_cluster_query(Tail, reset_query_state(NQueryState), NResultAcc); {more, NResultAcc} -> - NextNodes = - case NQueryState of - #{continuation := ?FRESH_SELECT} -> Tail; - _ -> Nodes - end, - do_cluster_query(NextNodes, NQueryState, NResultAcc) + finalize_query(NResultAcc, NQueryState) end end. -maybe_collect_total_from_tail_nodes([], _QueryState, ResultAcc) -> - ResultAcc; -maybe_collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc) -> - case counting_total_fun(QueryState) of - false -> - ResultAcc; - _Fun -> - collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc) - end. +maybe_collect_total_from_tail_nodes([], QueryState) -> + QueryState; +maybe_collect_total_from_tail_nodes(Nodes, QueryState = #{total := _}) -> + collect_total_from_tail_nodes(Nodes, QueryState); +maybe_collect_total_from_tail_nodes(_Nodes, QueryState) -> + QueryState. -collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc}) -> +collect_total_from_tail_nodes(Nodes, QueryState = #{total := TotalAcc}) -> %% XXX: badfun risk? if the FuzzyFun is an anonumous func in local node case rpc:multicall(Nodes, ?MODULE, apply_total_query, [QueryState], ?LONG_QUERY_TIMEOUT) of {_, [Node | _]} -> @@ -257,7 +251,8 @@ collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc [{Node, {badrpc, Reason}} | _] -> {error, Node, {badrpc, Reason}}; [] -> - ResultAcc#{total => ResL ++ TotalAcc} + NTotalAcc = maps:merge(TotalAcc, maps:from_list(ResL)), + QueryState#{total := NTotalAcc} end end. @@ -266,13 +261,14 @@ collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc %%-------------------------------------------------------------------- %% QueryState :: -%% #{continuation := ets:continuation(), +%% #{continuation => ets:continuation(), %% page := pos_integer(), %% limit := pos_integer(), -%% total := [{node(), non_neg_integer()}], +%% total => #{node() => non_neg_integer()}, %% table := atom(), -%% qs := {Qs, Fuzzy} %% parsed query params -%% msfun := query_to_match_spec_fun() +%% qs := {Qs, Fuzzy}, %% parsed query params +%% msfun := query_to_match_spec_fun(), +%% complete := boolean() %% } init_query_state(Tab, QString, MsFun, _Meta = #{page := Page, limit := Limit}) -> #{match_spec := Ms, fuzzy_fun := FuzzyFun} = erlang:apply(MsFun, [Tab, QString]), @@ -285,17 +281,31 @@ init_query_state(Tab, QString, MsFun, _Meta = #{page := Page, limit := Limit}) - true = is_list(Args), {type, external} = erlang:fun_info(NamedFun, type) end, - #{ + QueryState = #{ page => Page, limit => Limit, table => Tab, qs => QString, msfun => MsFun, - mactch_spec => Ms, + match_spec => Ms, fuzzy_fun => FuzzyFun, - total => [], - continuation => ?FRESH_SELECT - }. + complete => false + }, + case counting_total_fun(QueryState) of + false -> + QueryState; + Fun when is_function(Fun) -> + QueryState#{total => #{}} + end. + +reset_query_state(QueryState) -> + maps:remove(continuation, mark_complete(QueryState, false)). + +mark_complete(QueryState) -> + mark_complete(QueryState, true). + +mark_complete(QueryState, Complete) -> + QueryState#{complete => Complete}. %% @private This function is exempt from BPAPI do_query(Node, QueryState) when Node =:= node() -> @@ -318,47 +328,50 @@ do_select( Node, QueryState0 = #{ table := Tab, - mactch_spec := Ms, - fuzzy_fun := FuzzyFun, - continuation := Continuation, - limit := Limit + match_spec := Ms, + limit := Limit, + complete := false } ) -> QueryState = maybe_apply_total_query(Node, QueryState0), Result = - case Continuation of - ?FRESH_SELECT -> + case maps:get(continuation, QueryState, undefined) of + undefined -> ets:select(Tab, Ms, Limit); - _ -> + Continuation -> %% XXX: Repair is necessary because we pass Continuation back %% and forth through the nodes in the `do_cluster_query` ets:select(ets:repair_continuation(Continuation, Ms)) end, case Result of - '$end_of_table' -> - {[], QueryState#{continuation => ?FRESH_SELECT}}; + {Rows, '$end_of_table'} -> + NRows = maybe_apply_fuzzy_filter(Rows, QueryState), + {NRows, mark_complete(QueryState)}; {Rows, NContinuation} -> - NRows = - case FuzzyFun of - undefined -> - Rows; - {FilterFun, Args0} when is_function(FilterFun), is_list(Args0) -> - lists:filter( - fun(E) -> erlang:apply(FilterFun, [E | Args0]) end, - Rows - ) - end, - {NRows, QueryState#{continuation => NContinuation}} + NRows = maybe_apply_fuzzy_filter(Rows, QueryState), + {NRows, QueryState#{continuation => NContinuation}}; + '$end_of_table' -> + {[], mark_complete(QueryState)} end. -maybe_apply_total_query(Node, QueryState = #{total := TotalAcc}) -> - case proplists:get_value(Node, TotalAcc, undefined) of - undefined -> - Total = apply_total_query(QueryState), - QueryState#{total := [{Node, Total} | TotalAcc]}; - _ -> - QueryState - end. +maybe_apply_fuzzy_filter(Rows, #{fuzzy_fun := undefined}) -> + Rows; +maybe_apply_fuzzy_filter(Rows, #{fuzzy_fun := {FilterFun, Args}}) -> + lists:filter( + fun(E) -> erlang:apply(FilterFun, [E | Args]) end, + Rows + ). + +maybe_apply_total_query(Node, QueryState = #{total := Acc}) -> + case Acc of + #{Node := _} -> + QueryState; + #{} -> + NodeTotal = apply_total_query(QueryState), + QueryState#{total := Acc#{Node => NodeTotal}} + end; +maybe_apply_total_query(_Node, QueryState = #{}) -> + QueryState. apply_total_query(QueryState = #{table := Tab}) -> case counting_total_fun(QueryState) of @@ -371,7 +384,7 @@ apply_total_query(QueryState = #{table := Tab}) -> counting_total_fun(_QueryState = #{qs := {[], []}}) -> fun(Tab) -> ets:info(Tab, size) end; -counting_total_fun(_QueryState = #{mactch_spec := Ms, fuzzy_fun := undefined}) -> +counting_total_fun(_QueryState = #{match_spec := Ms, fuzzy_fun := undefined}) -> %% XXX: Calculating the total number of data that match a certain %% condition under a large table is very expensive because the %% entire ETS table needs to be scanned. @@ -390,15 +403,16 @@ counting_total_fun(_QueryState = #{fuzzy_fun := FuzzyFun}) when FuzzyFun =/= und %% ResultAcc :: #{count := integer(), %% cursor := integer(), %% rows := [{node(), Rows :: list()}], -%% total := [{node() => integer()}] +%% partial := boolean(), +%% hasnext => boolean() %% } init_query_result() -> - #{cursor => 0, count => 0, rows => [], total => []}. + #{cursor => 0, count => 0, rows => [], partial => false}. accumulate_query_rows( Node, Rows, - _QueryState = #{page := Page, limit := Limit, total := TotalAcc}, + _QueryState = #{page := Page, limit := Limit}, ResultAcc = #{cursor := Cursor, count := Count, rows := RowsAcc} ) -> PageStart = (Page - 1) * Limit + 1, @@ -406,12 +420,11 @@ accumulate_query_rows( Len = length(Rows), case Cursor + Len of NCursor when NCursor < PageStart -> - {more, ResultAcc#{cursor => NCursor, total => TotalAcc}}; + {more, ResultAcc#{cursor => NCursor}}; NCursor when NCursor < PageEnd -> {more, ResultAcc#{ cursor => NCursor, count => Count + length(Rows), - total => TotalAcc, rows => [{Node, Rows} | RowsAcc] }}; NCursor when NCursor >= PageEnd -> @@ -419,11 +432,21 @@ accumulate_query_rows( {enough, ResultAcc#{ cursor => NCursor, count => Count + length(SubRows), - total => TotalAcc, - rows => [{Node, SubRows} | RowsAcc] + rows => [{Node, SubRows} | RowsAcc], + partial => (Limit - Count) < Len }} end. +finalize_query(Result = #{partial := Partial}, QueryState = #{complete := Complete}) -> + HasNext = Partial orelse not Complete, + maybe_accumulate_totals(Result#{hasnext => HasNext}, QueryState). + +maybe_accumulate_totals(Result, #{total := TotalAcc}) -> + QueryTotal = maps:fold(fun(_Node, T, N) -> N + T end, 0, TotalAcc), + Result#{total => QueryTotal}; +maybe_accumulate_totals(Result, _QueryState) -> + Result. + %%-------------------------------------------------------------------- %% Internal Functions %%-------------------------------------------------------------------- @@ -520,16 +543,22 @@ is_fuzzy_key(<<"match_", _/binary>>) -> is_fuzzy_key(_) -> false. -format_query_result(_FmtFun, _Meta, Error = {error, _Node, _Reason}) -> +format_query_result(_FmtFun, _MetaIn, Error = {error, _Node, _Reason}) -> Error; format_query_result( - FmtFun, Meta, _ResultAcc = #{total := TotalAcc, rows := RowsAcc} + FmtFun, MetaIn, ResultAcc = #{hasnext := HasNext, rows := RowsAcc} ) -> - Total = lists:foldr(fun({_Node, T}, N) -> N + T end, 0, TotalAcc), + Meta = + case ResultAcc of + #{total := QueryTotal} -> + %% The `count` is used in HTTP API to indicate the total number of + %% queries that can be read + MetaIn#{hasnext => HasNext, count => QueryTotal}; + #{} -> + MetaIn#{hasnext => HasNext} + end, #{ - %% The `count` is used in HTTP API to indicate the total number of - %% queries that can be read - meta => Meta#{count => Total}, + meta => Meta, data => lists:flatten( lists:foldl( fun({Node, Rows}, Acc) -> @@ -552,7 +581,7 @@ parse_pager_params(Params) -> Limit = b2i(limit(Params)), case Page > 0 andalso Limit > 0 of true -> - #{page => Page, limit => Limit, count => 0}; + #{page => Page, limit => Limit}; false -> false end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index a14305d8b..a8bbfa6d9 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -88,10 +88,9 @@ t_cluster_query(_Config) -> %% fuzzy searching can't return total {200, ClientsNode2} = query_clients(Node2, #{<<"like_username">> => <<"corenode2">>}), - ?assertMatch( - #{count := 0}, - maps:get(meta, ClientsNode2) - ), + MetaNode2 = maps:get(meta, ClientsNode2), + ?assertNotMatch(#{count := _}, MetaNode2), + ?assertMatch(#{hasnext := false}, MetaNode2), ?assertMatch(10, length(maps:get(data, ClientsNode2))), _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1), diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index 965ed0997..2ab213e30 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -44,9 +44,8 @@ init_per_suite(Config) -> end_per_suite(_) -> emqx_mgmt_api_test_util:end_suite(). -t_subscription_api(_) -> - {ok, Client} = emqtt:start_link(#{username => ?USERNAME, clientid => ?CLIENTID, proto_ver => v5}), - {ok, _} = emqtt:connect(Client), +t_subscription_api(Config) -> + Client = proplists:get_value(client, Config), {ok, _, _} = emqtt:subscribe( Client, [ {?TOPIC1, [{rh, ?TOPIC1RH}, {rap, ?TOPIC1RAP}, {nl, ?TOPIC1NL}, {qos, ?TOPIC1QOS}]} @@ -84,40 +83,78 @@ t_subscription_api(_) -> ?assertEqual(maps:get(<<"topic">>, Subscriptions2), ?TOPIC2), ?assertEqual(maps:get(<<"clientid">>, Subscriptions2), ?CLIENTID), - QS = uri_string:compose_query([ + QS = [ {"clientid", ?CLIENTID}, {"topic", ?TOPIC2_TOPIC_ONLY}, {"node", atom_to_list(node())}, {"qos", "0"}, {"share_group", "test_group"}, {"match_topic", "t/#"} - ]), + ], Headers = emqx_mgmt_api_test_util:auth_header_(), - {ok, ResponseTopic2} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers), - DataTopic2 = emqx_json:decode(ResponseTopic2, [return_maps]), - Meta2 = maps:get(<<"meta">>, DataTopic2), + DataTopic2 = #{<<"meta">> := Meta2} = request_json(get, QS, Headers), ?assertEqual(1, maps:get(<<"page">>, Meta2)), ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta2)), ?assertEqual(1, maps:get(<<"count">>, Meta2)), SubscriptionsList2 = maps:get(<<"data">>, DataTopic2), - ?assertEqual(length(SubscriptionsList2), 1), + ?assertEqual(length(SubscriptionsList2), 1). - MatchQs = uri_string:compose_query([ +t_subscription_fuzzy_search(Config) -> + Client = proplists:get_value(client, Config), + Topics = [ + <<"t/foo">>, + <<"t/foo/bar">>, + <<"t/foo/baz">>, + <<"topic/foo/bar">>, + <<"topic/foo/baz">> + ], + _ = [{ok, _, _} = emqtt:subscribe(Client, T) || T <- Topics], + + Headers = emqx_mgmt_api_test_util:auth_header_(), + MatchQs = [ {"clientid", ?CLIENTID}, {"node", atom_to_list(node())}, - {"qos", "0"}, {"match_topic", "t/#"} - ]), + ], - {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(get, Path, MatchQs, Headers), - MatchData = emqx_json:decode(MatchRes, [return_maps]), - MatchMeta = maps:get(<<"meta">>, MatchData), - ?assertEqual(1, maps:get(<<"page">>, MatchMeta)), - ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta)), - %% count equals 0 in fuzzy searching - ?assertEqual(0, maps:get(<<"count">>, MatchMeta)), - MatchSubs = maps:get(<<"data">>, MatchData), - ?assertEqual(1, length(MatchSubs)), + MatchData1 = #{<<"meta">> := MatchMeta1} = request_json(get, MatchQs, Headers), + ?assertEqual(1, maps:get(<<"page">>, MatchMeta1)), + ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta1)), + %% count is undefined in fuzzy searching + ?assertNot(maps:is_key(<<"count">>, MatchMeta1)), + ?assertMatch(3, length(maps:get(<<"data">>, MatchData1))), + ?assertEqual(false, maps:get(<<"hasnext">>, MatchMeta1)), + LimitMatchQuery = [ + {"clientid", ?CLIENTID}, + {"match_topic", "+/+/+"}, + {"limit", "3"} + ], + + MatchData2 = #{<<"meta">> := MatchMeta2} = request_json(get, LimitMatchQuery, Headers), + ?assertEqual(#{<<"page">> => 1, <<"limit">> => 3, <<"hasnext">> => true}, MatchMeta2), + ?assertEqual(3, length(maps:get(<<"data">>, MatchData2))), + + MatchData2P2 = + #{<<"meta">> := MatchMeta2P2} = + request_json(get, [{"page", "2"} | LimitMatchQuery], Headers), + ?assertEqual(#{<<"page">> => 2, <<"limit">> => 3, <<"hasnext">> => false}, MatchMeta2P2), + ?assertEqual(1, length(maps:get(<<"data">>, MatchData2P2))). + +request_json(Method, Query, Headers) when is_list(Query) -> + Qs = uri_string:compose_query(Query), + {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers), + emqx_json:decode(MatchRes, [return_maps]). + +path() -> + emqx_mgmt_api_test_util:api_path(["subscriptions"]). + +init_per_testcase(_TC, Config) -> + {ok, Client} = emqtt:start_link(#{username => ?USERNAME, clientid => ?CLIENTID, proto_ver => v5}), + {ok, _} = emqtt:connect(Client), + [{client, Client} | Config]. + +end_per_testcase(_TC, Config) -> + Client = proplists:get_value(client, Config), emqtt:disconnect(Client). From abf6f143e5e4b66a4fe42b286d2d7dabceaeca99 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 11 Jan 2023 16:59:16 +0300 Subject: [PATCH 17/31] fix(paging): return subset of rows if page start is in the middle This change should cover this example scenario where select ended returning 4 rows. Row 3 Row 4 Row 5 <- page start Row 6 Here, only rows 5 and 6 should end up in the response. Before this change some rows could be duplicated across adjacent search pages. --- apps/emqx_management/src/emqx_mgmt_api.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 614949f16..5f4d8f78a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -422,10 +422,11 @@ accumulate_query_rows( NCursor when NCursor < PageStart -> {more, ResultAcc#{cursor => NCursor}}; NCursor when NCursor < PageEnd -> + SubRows = lists:nthtail(max(0, PageStart - Cursor - 1), Rows), {more, ResultAcc#{ cursor => NCursor, - count => Count + length(Rows), - rows => [{Node, Rows} | RowsAcc] + count => Count + length(SubRows), + rows => [{Node, SubRows} | RowsAcc] }}; NCursor when NCursor >= PageEnd -> SubRows = lists:sublist(Rows, Limit - Count), From c89b2276872d0d9f9b86bb80e45f8e2c51eb4e79 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 11 Jan 2023 17:38:45 +0300 Subject: [PATCH 18/31] fix(test): adapt affected testcases --- apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl | 6 +++--- .../test/emqx_enhanced_authn_scram_mnesia_SUITE.erl | 6 +++--- apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl | 7 ++++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index 8191fe2e9..cd97a15d9 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -197,7 +197,7 @@ t_list_users(_) -> #{is_superuser := false, user_id := _}, #{is_superuser := false, user_id := _} ], - meta := #{page := 1, limit := 2, count := 3} + meta := #{page := 1, limit := 2, count := 3, hasnext := true} } = emqx_authn_mnesia:list_users( #{<<"page">> => 1, <<"limit">> => 2}, State @@ -205,7 +205,7 @@ t_list_users(_) -> #{ data := [#{is_superuser := false, user_id := _}], - meta := #{page := 2, limit := 2, count := 3} + meta := #{page := 2, limit := 2, count := 3, hasnext := false} } = emqx_authn_mnesia:list_users( #{<<"page">> => 2, <<"limit">> => 2}, State @@ -213,7 +213,7 @@ t_list_users(_) -> #{ data := [#{is_superuser := false, user_id := <<"u3">>}], - meta := #{page := 1, limit := 20, count := 0} + meta := #{page := 1, limit := 20, hasnext := false} } = emqx_authn_mnesia:list_users( #{ <<"page">> => 1, diff --git a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl index e1a2586cd..b143903b5 100644 --- a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl @@ -300,14 +300,14 @@ t_list_users(_) -> #{ data := [?USER_MAP, ?USER_MAP], - meta := #{page := 1, limit := 2, count := 3} + meta := #{page := 1, limit := 2, count := 3, hasnext := true} } = emqx_enhanced_authn_scram_mnesia:list_users( #{<<"page">> => 1, <<"limit">> => 2}, State ), #{ data := [?USER_MAP], - meta := #{page := 2, limit := 2, count := 3} + meta := #{page := 2, limit := 2, count := 3, hasnext := false} } = emqx_enhanced_authn_scram_mnesia:list_users( #{<<"page">> => 2, <<"limit">> => 2}, State @@ -319,7 +319,7 @@ t_list_users(_) -> is_superuser := _ } ], - meta := #{page := 1, limit := 3, count := 0} + meta := #{page := 1, limit := 3, hasnext := false} } = emqx_enhanced_authn_scram_mnesia:list_users( #{ <<"page">> => 1, diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index 62c747433..1819cf66f 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -92,7 +92,8 @@ t_api(_) -> <<"meta">> := #{ <<"count">> := 1, <<"limit">> := 100, - <<"page">> := 1 + <<"page">> := 1, + <<"hasnext">> := false } } = jsx:decode(Request1), ?assertEqual(3, length(Rules1)), @@ -111,9 +112,9 @@ t_api(_) -> #{ <<"data">> := [], <<"meta">> := #{ - <<"count">> := 0, <<"limit">> := 20, - <<"page">> := 1 + <<"page">> := 1, + <<"hasnext">> := false } } = jsx:decode(Request1_1), From 50c9321b9ed969a618691d809db590b28e37619b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 11 Jan 2023 22:47:28 +0300 Subject: [PATCH 19/31] fix(api): anticipate empty set of nodes handling cluster query --- apps/emqx_management/src/emqx_mgmt_api.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 5f4d8f78a..9a1d0ce4e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -210,6 +210,8 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) -> end. %% @private +do_cluster_query([], QueryState, ResultAcc) -> + finalize_query(ResultAcc, mark_complete(QueryState)); do_cluster_query( [Node | Tail] = Nodes, QueryState, From 82e80e56104e83c533fbccaae2649d11866a9900 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 12 Jan 2023 09:01:11 +0100 Subject: [PATCH 20/31] fix: remove password from info logs Information log messages for several connectors contained sensitive password information. This commit fixes this by hiding the sensitive information from the log messages. Fixes: https://emqx.atlassian.net/browse/EMQX-8718 --- apps/emqx_connector/src/emqx_connector_http.erl | 2 +- apps/emqx_connector/src/emqx_connector_ldap.erl | 2 +- apps/emqx_connector/src/emqx_connector_mongo.erl | 2 +- apps/emqx_connector/src/emqx_connector_mqtt.erl | 2 +- apps/emqx_connector/src/emqx_connector_mysql.erl | 2 +- apps/emqx_connector/src/emqx_connector_pgsql.erl | 2 +- apps/emqx_connector/src/emqx_connector_redis.erl | 2 +- changes/v5.0.15/fix-8718-en.md | 1 + changes/v5.0.15/fix-8718-zh.md | 1 + 9 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 changes/v5.0.15/fix-8718-en.md create mode 100644 changes/v5.0.15/fix-8718-zh.md diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index a04850746..817cd44f6 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -209,7 +209,7 @@ on_start( ?SLOG(info, #{ msg => "starting_http_connector", connector => InstId, - config => Config + config => emqx_misc:redact(Config) }), {Transport, TransportOpts} = case Scheme of diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 1930d9e68..82d622e09 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -65,7 +65,7 @@ on_start( ?SLOG(info, #{ msg => "starting_ldap_connector", connector => InstId, - config => Config + config => emqx_misc:redact(Config) }), Servers = emqx_schema:parse_servers(Servers0, ?LDAP_HOST_OPTIONS), SslOpts = diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 0bcc39208..1b0bcf94d 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -155,7 +155,7 @@ on_start( rs -> "starting_mongodb_replica_set_connector"; sharded -> "starting_mongodb_sharded_connector" end, - ?SLOG(info, #{msg => Msg, connector => InstId, config => Config}), + ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_misc:redact(Config)}), NConfig = #{hosts := Hosts} = maybe_resolve_srv_and_txt_records(Config), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index c28aa2514..585122539 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -149,7 +149,7 @@ on_start(InstId, Conf) -> ?SLOG(info, #{ msg => "starting_mqtt_connector", connector => InstanceId, - config => Conf + config => emqx_misc:redact(Conf) }), BasicConf = basic_config(Conf), BridgeConf = BasicConf#{ diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index d119a01e9..ea6a84240 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -91,7 +91,7 @@ on_start( ?SLOG(info, #{ msg => "starting_mysql_connector", connector => InstId, - config => Config + config => emqx_misc:redact(Config) }), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 25a22455d..9965ff3b4 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -94,7 +94,7 @@ on_start( ?SLOG(info, #{ msg => "starting_postgresql_connector", connector => InstId, - config => Config + config => emqx_misc:redact(Config) }), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 6143dbf0c..7fdf9d28d 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -123,7 +123,7 @@ on_start( ?SLOG(info, #{ msg => "starting_redis_connector", connector => InstId, - config => Config + config => emqx_misc:redact(Config) }), ConfKey = case Type of diff --git a/changes/v5.0.15/fix-8718-en.md b/changes/v5.0.15/fix-8718-en.md new file mode 100644 index 000000000..6085adecd --- /dev/null +++ b/changes/v5.0.15/fix-8718-en.md @@ -0,0 +1 @@ +Password information has been removed from information log messages for http, ldap, mongo, mqtt, mysql, pgsql and redis. diff --git a/changes/v5.0.15/fix-8718-zh.md b/changes/v5.0.15/fix-8718-zh.md new file mode 100644 index 000000000..d8aa81fd1 --- /dev/null +++ b/changes/v5.0.15/fix-8718-zh.md @@ -0,0 +1 @@ +密码信息已从http、ldap、mongo、mqtt、mysql、pgsql和redis的信息日志消息中删除。 From 49e1f5176527069dc79805da569d0919406b0c10 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 29 Dec 2022 20:30:32 +0100 Subject: [PATCH 21/31] ci: simplify build_and_push_docker_images workflow --- .github/actions/docker-meta/action.yaml | 81 ----- .../build_and_push_docker_images.yaml | 314 ++++-------------- deploy/docker/Dockerfile | 3 +- deploy/docker/Dockerfile.alpine | 3 +- scripts/docker-create-push-manifests.sh | 27 -- 5 files changed, 76 insertions(+), 352 deletions(-) delete mode 100644 .github/actions/docker-meta/action.yaml delete mode 100755 scripts/docker-create-push-manifests.sh diff --git a/.github/actions/docker-meta/action.yaml b/.github/actions/docker-meta/action.yaml deleted file mode 100644 index 13ab21da6..000000000 --- a/.github/actions/docker-meta/action.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: 'Docker meta' -inputs: - profile: - required: true - type: string - registry: - required: true - type: string - arch: - required: true - type: string - otp: - required: true - type: string - elixir: - required: false - type: string - default: '' - builder_base: - required: true - type: string - owner: - required: true - type: string - docker_tags: - required: true - type: string - -outputs: - emqx_name: - description: "EMQX name" - value: ${{ steps.pre-meta.outputs.emqx_name }} - version: - description: "docker image version" - value: ${{ steps.meta.outputs.version }} - tags: - description: "docker image tags" - value: ${{ steps.meta.outputs.tags }} - labels: - description: "docker image labels" - value: ${{ steps.meta.outputs.labels }} - -runs: - using: composite - steps: - - name: prepare for docker/metadata-action - id: pre-meta - shell: bash - run: | - emqx_name=${{ inputs.profile }} - img_suffix=${{ inputs.arch }} - img_labels="org.opencontainers.image.otp.version=${{ inputs.otp }}" - if [ -n "${{ inputs.elixir }}" ]; then - emqx_name="emqx-elixir" - img_suffix="elixir-${{ inputs.arch }}" - img_labels="org.opencontainers.image.elixir.version=${{ inputs.elixir }}\n${img_labels}" - fi - if [ "${{ inputs.profile }}" = "emqx" ]; then - img_labels="org.opencontainers.image.edition=Opensource\n${img_labels}" - fi - if [ "${{ inputs.profile }}" = "emqx-enterprise" ]; then - img_labels="org.opencontainers.image.edition=Enterprise\n${img_labels}" - fi - if [[ "${{ inputs.builder_base }}" =~ "alpine" ]]; then - img_suffix="${img_suffix}-alpine" - fi - echo "emqx_name=${emqx_name}" >> $GITHUB_OUTPUT - echo "img_suffix=${img_suffix}" >> $GITHUB_OUTPUT - echo "img_labels=${img_labels}" >> $GITHUB_OUTPUT - echo "img_name=${{ inputs.registry }}/${{ inputs.owner }}/${{ inputs.profile }}" >> $GITHUB_OUTPUT - - uses: docker/metadata-action@v4 - id: meta - with: - images: - ${{ steps.pre-meta.outputs.img_name }} - flavor: | - suffix=-${{ steps.pre-meta.outputs.img_suffix }} - tags: | - type=raw,value=${{ inputs.docker_tags }} - labels: - ${{ steps.pre-meta.outputs.img_labels }} diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 76238c75f..71515f699 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -9,15 +9,17 @@ on: tags: - v* - e* - release: - types: - - published + - docker-latest-* workflow_dispatch: inputs: branch_or_tag: required: false profile: required: false + default: 'emqx' + is_latest: + required: false + default: false jobs: prepare: @@ -26,10 +28,11 @@ jobs: container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: - BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} - IS_DOCKER_LATEST: ${{ steps.get_profile.outputs.IS_DOCKER_LATEST }} + PROFILE: ${{ steps.get_profile.outputs.PROFILE }} + EDITION: ${{ steps.get_profile.outputs.EDITION }} + IS_LATEST: ${{ steps.get_profile.outputs.IS_LATEST }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} - DOCKER_TAG_VERSION: ${{ steps.get_profile.outputs.DOCKER_TAG_VERSION }} + VERSION: ${{ steps.get_profile.outputs.VERSION }} steps: - uses: actions/checkout@v3 @@ -45,14 +48,14 @@ jobs: tag=${{ github.ref }} # tag docker-latest-ce or docker-latest-ee if git describe --tags --exact --match 'docker-latest-*' 2>/dev/null; then - echo 'docker_latest=true due to docker-latest-* tag' - docker_latest=true - elif [ "${{ github.event_name }}" = "release" ]; then - echo 'docker_latest=true due to release' - docker_latest=true + echo 'is_latest=true due to docker-latest-* tag' + is_latest=true + elif [ "${{ inputs.is_latest }}" = "true" ]; then + echo 'is_latest=true due to manual input from workflow_dispatch' + is_latest=true else - echo 'docker_latest=false' - docker_latest=false + echo 'is_latest=false' + is_latest=false fi if git describe --tags --match "[v|e]*" --exact; then echo "This is an exact git tag, will publish images" @@ -64,18 +67,20 @@ jobs: case $tag in refs/tags/v*) PROFILE='emqx' + EDITION='Opensource' ;; refs/tags/e*) PROFILE=emqx-enterprise + EDITION='Enterprise' ;; *) PROFILE=${{ github.event.inputs.profile }} case "$PROFILE" in emqx) - true + EDITION='Opensource' ;; emqx-enterprise) - true + EDITION='Enterprise' ;; *) echo "ERROR: Failed to resolve build profile" @@ -85,14 +90,18 @@ jobs: ;; esac VSN="$(./pkg-vsn.sh "$PROFILE")" - echo "Building $PROFILE image with tag $VSN (latest=$docker_latest)" - echo "IS_DOCKER_LATEST=$docker_latest" >> $GITHUB_OUTPUT + echo "Building emqx/$PROFILE:$VSN image (latest=$is_latest)" + echo "Push = $is_exact" + echo "IS_LATEST=$is_latest" >> $GITHUB_OUTPUT echo "IS_EXACT_TAG=$is_exact" >> $GITHUB_OUTPUT - echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT - echo "DOCKER_TAG_VERSION=$VSN" >> $GITHUB_OUTPUT + echo "PROFILE=$PROFILE" >> $GITHUB_OUTPUT + echo "EDITION=$EDITION" >> $GITHUB_OUTPUT + echo "VERSION=$VSN" >> $GITHUB_OUTPUT - name: get_all_deps + env: + PROFILE: ${{ steps.get_profile.outputs.PROFILE }} run: | - make -C source deps-all + PROFILE=$PROFILE make -C source deps-$PROFILE zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v3 with: @@ -100,17 +109,17 @@ jobs: path: source.zip docker: - runs-on: ${{ matrix.arch[1] }} + runs-on: ubuntu-20.04 needs: prepare strategy: fail-fast: false matrix: - arch: - - [amd64, ubuntu-20.04] - - [arm64, aws-arm64] profile: - - ${{ needs.prepare.outputs.BUILD_PROFILE }} + - "${{ needs.prepare.outputs.PROFILE }}" + flavor: + - '' + - '-elixir' registry: - 'docker.io' - 'public.ecr.aws' @@ -128,9 +137,10 @@ jobs: exclude: # TODO: publish enterprise to ecr too? - registry: 'public.ecr.aws' profile: emqx-enterprise + - flavor: '-elixir' + os: [alpine3.15.1, "alpine:3.15.1", "deploy/docker/Dockerfile.alpine"] + steps: - - uses: AutoModality/action-clean@v1 - if: matrix.arch[1] == 'aws-arm64' - uses: actions/download-artifact@v3 with: name: source @@ -138,16 +148,17 @@ jobs: - name: unzip source code run: unzip -q source.zip + - uses: docker/setup-qemu-action@v2 - uses: docker/setup-buildx-action@v2 - - name: Login for docker. + - name: Login to hub.docker.com uses: docker/login-action@v2 if: matrix.registry == 'docker.io' with: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: Login for AWS ECR + - name: Login to AWS ECR uses: docker/login-action@v2 if: matrix.registry == 'public.ecr.aws' with: @@ -156,229 +167,48 @@ jobs: password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ecr: true - - uses: ./source/.github/actions/docker-meta + - name: prepare for docker/metadata-action + id: pre-meta + shell: bash + run: | + extra_labels= + img_suffix= + flavor="${{ matrix.flavor }}" + if [ "${{ matrix.flavor }}" = '-elixir' ]; then + img_suffix="-elixir" + extra_labels="org.opencontainers.image.elixir.version=${{ matrix.elixir }}" + fi + if [[ "${{ matrix.os[0] }}" =~ "alpine" ]]; then + img_suffix="${img_suffix}-alpine" + fi + + echo "img_suffix=$img_suffix" >> $GITHUB_OUTPUT + echo "extra_labels=$extra_labels" >> $GITHUB_OUTPUT + + - uses: docker/metadata-action@v4 id: meta with: - profile: ${{ matrix.profile }} - registry: ${{ matrix.registry }} - arch: ${{ matrix.arch[0] }} - otp: ${{ matrix.otp }} - builder_base: ${{ matrix.os[0] }} - owner: ${{ github.repository_owner }} - docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }} + images: | + ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} + flavor: | + suffix=${{ steps.pre-meta.outputs.img_suffix }} + tags: | + type=raw,value=${{ needs.prepare.outputs.VERSION }} + type=raw,value=latest,enable=${{ needs.prepare.outputs.IS_LATEST }} + labels: | + org.opencontainers.image.otp.version=${{ matrix.otp }} + org.opencontainers.image.edition=${{ needs.prepare.outputs.EDITION }} + ${{ steps.pre-meta.outputs.extra_labels }} - uses: docker/build-push-action@v3 with: push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }} pull: true no-cache: true - platforms: linux/${{ matrix.arch[0] }} + platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} - RUN_FROM=${{ matrix.os[1] }} - EMQX_NAME=${{ steps.meta.outputs.emqx_name }} + EMQX_NAME=${{ matrix.profile }}${{ matrix.flavor }} file: source/${{ matrix.os[2] }} context: source - - - name: Docker Hub Description - if: matrix.registry == 'docker.io' - uses: peter-evans/dockerhub-description@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - repository: "emqx/${{ needs.prepare.outputs.BUILD_PROFILE }}" - readme-filepath: ./source/deploy/docker/README.md - short-description: "The most scalable open-source MQTT broker for IoT, IIoT, connected vehicles, and more." - - docker-elixir: - runs-on: ${{ matrix.arch[1] }} - needs: prepare - # do not build elixir images for ee for now - if: needs.prepare.outputs.BUILD_PROFILE == 'emqx' - - strategy: - fail-fast: false - matrix: - arch: - - [amd64, ubuntu-20.04] - - [arm64, aws-arm64] - profile: - - ${{ needs.prepare.outputs.BUILD_PROFILE }} - registry: - - 'docker.io' - os: - - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - builder: - - 5.0-26 # update to latest - otp: - - 25.1.2-2 # update to latest - elixir: - - 1.13.4 # update to latest - - steps: - - uses: AutoModality/action-clean@v1 - if: matrix.arch[1] == 'aws-arm64' - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: unzip -q source.zip - - - uses: docker/setup-buildx-action@v2 - - - name: Login for docker. - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - uses: ./source/.github/actions/docker-meta - id: meta - with: - profile: ${{ matrix.profile }} - registry: ${{ matrix.registry }} - arch: ${{ matrix.arch[0] }} - otp: ${{ matrix.otp }} - elixir: ${{ matrix.elixir }} - builder_base: ${{ matrix.os[0] }} - owner: ${{ github.repository_owner }} - docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }} - - - uses: docker/build-push-action@v3 - with: - push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }} - pull: true - no-cache: true - platforms: linux/${{ matrix.arch[0] }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} - RUN_FROM=${{ matrix.os[1] }} - EMQX_NAME=${{ steps.meta.outputs.emqx_name }} - file: source/${{ matrix.os[2] }} - context: source - - docker-push-multi-arch-manifest: - # note, we only run on amd64 - if: needs.prepare.outputs.IS_EXACT_TAG - needs: - - prepare - - docker - runs-on: ${{ matrix.arch[1] }} - strategy: - fail-fast: false - matrix: - arch: - - [amd64, ubuntu-20.04] - profile: - - ${{ needs.prepare.outputs.BUILD_PROFILE }} - os: - - [alpine3.15.1, "alpine:3.15.1", "deploy/docker/Dockerfile.alpine"] - - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] - # NOTE: only support latest otp version, not a matrix - otp: - - 24.3.4.2-1 # switch to 25 once ready to release 5.1 - registry: - - 'docker.io' - - 'public.ecr.aws' - exclude: - - registry: 'public.ecr.aws' - profile: emqx-enterprise - - steps: - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - - name: unzip source code - run: unzip -q source.zip - - - uses: docker/login-action@v2 - if: matrix.registry == 'docker.io' - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - uses: docker/login-action@v2 - if: matrix.registry == 'public.ecr.aws' - with: - registry: public.ecr.aws - username: ${{ secrets.AWS_ACCESS_KEY_ID }} - password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - ecr: true - - - uses: ./source/.github/actions/docker-meta - id: meta - with: - profile: ${{ matrix.profile }} - registry: ${{ matrix.registry }} - arch: ${{ matrix.arch[0] }} - otp: ${{ matrix.otp }} - builder_base: ${{ matrix.os[0] }} - owner: ${{ github.repository_owner }} - docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }} - - - name: update manifest for multiarch image - working-directory: source - run: | - is_latest="${{ needs.prepare.outputs.IS_DOCKER_LATEST }}" - scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" "$is_latest" - - docker-elixir-push-multi-arch-manifest: - # note, we only run on amd64 - # do not build enterprise elixir images for now - if: needs.prepare.outputs.IS_EXACT_TAG == 'true' && needs.prepare.outputs.BUILD_PROFILE == 'emqx' - needs: - - prepare - - docker-elixir - runs-on: ${{ matrix.arch[1] }} - strategy: - fail-fast: false - matrix: - arch: - - [amd64, ubuntu-20.04] - profile: - - ${{ needs.prepare.outputs.BUILD_PROFILE }} - # NOTE: for docker, only support latest otp version, not a matrix - otp: - - 25.1.2-2 # update to latest - elixir: - - 1.13.4 # update to latest - registry: - - 'docker.io' - - steps: - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - - name: unzip source code - run: unzip -q source.zip - - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - uses: ./source/.github/actions/docker-meta - id: meta - with: - profile: ${{ matrix.profile }} - registry: ${{ matrix.registry }} - arch: ${{ matrix.arch[0] }} - otp: ${{ matrix.otp }} - elixir: ${{ matrix.elixir }} - builder_base: ${{ matrix.os[0] }} - owner: ${{ github.repository_owner }} - docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }} - - - name: update manifest for multiarch image - working-directory: source - run: | - scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" false diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 4a00c68fb..03533eec4 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,6 +1,7 @@ ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11 ARG RUN_FROM=debian:11-slim -FROM ${BUILD_FROM} AS builder +ARG BUILDPLATFORM=linux/amd64 +FROM --platform=$BUILDPLATFORM ${BUILD_FROM} AS builder COPY . /emqx diff --git a/deploy/docker/Dockerfile.alpine b/deploy/docker/Dockerfile.alpine index 0f72be9ab..ebce2f539 100644 --- a/deploy/docker/Dockerfile.alpine +++ b/deploy/docker/Dockerfile.alpine @@ -1,6 +1,7 @@ ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-alpine3.15.1 ARG RUN_FROM=alpine:3.15.1 -FROM ${BUILD_FROM} AS builder +ARG BUILDPLATFORM=linux/amd64 +FROM --platform=$BUILDPLATFORM ${BUILD_FROM} AS builder RUN apk add --no-cache \ autoconf \ diff --git a/scripts/docker-create-push-manifests.sh b/scripts/docker-create-push-manifests.sh deleted file mode 100755 index db9c01bfb..000000000 --- a/scripts/docker-create-push-manifests.sh +++ /dev/null @@ -1,27 +0,0 @@ -##!/usr/bin/env bash -set -exuo pipefail - -img_amd64=$1 -push_latest=${2:-false} - -img_arm64=$(echo ${img_amd64} | sed 's/-amd64$/-arm64/g') -img_name=${img_amd64%-amd64} -docker pull "$img_amd64" -docker pull --platform linux/arm64 "$img_arm64" -img_amd64_digest=$(docker inspect --format='{{index .RepoDigests 0}}' "$img_amd64") -img_arm64_digest=$(docker inspect --format='{{index .RepoDigests 0}}' "$img_arm64") -echo "sha256 of amd64 is $img_amd64_digest" -echo "sha256 of arm64 is $img_arm64_digest" -docker manifest create "${img_name}" \ - --amend "$img_amd64_digest" \ - --amend "$img_arm64_digest" -docker manifest push "${img_name}" - -# PUSH latest if it is a release build -if [ "$push_latest" = "true" ]; then - img_latest=$(echo "$img_arm64" | cut -d: -f 1):latest - docker manifest create "${img_latest}" \ - --amend "$img_amd64_digest" \ - --amend "$img_arm64_digest" - docker manifest push "${img_latest}" -fi From 4d3b6405f60095eaf851c091aa4995a43651db3d Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 12 Jan 2023 18:13:14 +0800 Subject: [PATCH 22/31] chore: bump dsahboard tag to `e1.0.1-beta.12` --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 180028899..faa866753 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v1.1.5 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.9 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.12 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) From 1f57e7b5383033041d003e5ecb41b7bf87fa52d5 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 28 Dec 2022 11:11:26 +0100 Subject: [PATCH 23/31] fix: enable authorization cache by default --- apps/emqx_authz/etc/emqx_authz.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index e7fd73498..3bdc180c5 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -1,6 +1,7 @@ authorization { deny_action = ignore no_match = allow + cache = { enable = true } sources = [ { type = file From 32f75197f2982381aa87f637144671e35d52d146 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 29 Dec 2022 10:16:40 +0100 Subject: [PATCH 24/31] docs: add changelog --- changes/v5.0.14-en.md | 8 ++++++++ changes/v5.0.14-zh.md | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 changes/v5.0.14-en.md create mode 100644 changes/v5.0.14-zh.md diff --git a/changes/v5.0.14-en.md b/changes/v5.0.14-en.md new file mode 100644 index 000000000..060e86652 --- /dev/null +++ b/changes/v5.0.14-en.md @@ -0,0 +1,8 @@ +# v5.0.14 + +## Enhancements + +- Enable authorization cache by default [#9626](https://github.com/emqx/emqx/pull/9626). + +## Bug fixes + diff --git a/changes/v5.0.14-zh.md b/changes/v5.0.14-zh.md new file mode 100644 index 000000000..62c40e638 --- /dev/null +++ b/changes/v5.0.14-zh.md @@ -0,0 +1,8 @@ +# v5.0.14 + +## 增强 + +- 默认启用授权缓存 [#9626](https://github.com/emqx/emqx/pull/9626)。 + +## 修复 + From f90c41f769174fa1ac312ed73a2342c7565f9b18 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 12 Jan 2023 13:47:50 +0100 Subject: [PATCH 25/31] fix: set default value in schema --- apps/emqx/src/emqx_hocon.erl | 6 ++- apps/emqx/src/emqx_schema.erl | 54 ++++++++++--------- .../src/emqx_authz_api_settings.erl | 6 ++- .../emqx_authz/src/emqx_authz_api_sources.erl | 2 +- apps/emqx_authz/src/emqx_authz_schema.erl | 40 +++++++------- apps/emqx_conf/src/emqx_conf_schema.erl | 4 +- changes/v5.0.14-en.md | 8 --- changes/v5.0.14-zh.md | 8 --- changes/v5.0.15/fix-9626.en.md | 2 + changes/v5.0.15/fix-9626.zh.md | 3 ++ 10 files changed, 68 insertions(+), 65 deletions(-) delete mode 100644 changes/v5.0.14-en.md delete mode 100644 changes/v5.0.14-zh.md create mode 100644 changes/v5.0.15/fix-9626.en.md create mode 100644 changes/v5.0.15/fix-9626.zh.md diff --git a/apps/emqx/src/emqx_hocon.erl b/apps/emqx/src/emqx_hocon.erl index 7e9dbca77..4391a9a0b 100644 --- a/apps/emqx/src/emqx_hocon.erl +++ b/apps/emqx/src/emqx_hocon.erl @@ -21,7 +21,8 @@ format_path/1, check/2, format_error/1, - format_error/2 + format_error/2, + make_schema/1 ]). %% @doc Format hocon config field path to dot-separated string in iolist format. @@ -79,6 +80,9 @@ format_error({_Schema, [#{kind := K} = First | Rest] = All}, Opts) when format_error(_Other, _) -> false. +make_schema(Fields) -> + #{roots => Fields, fields => #{}}. + %% Ensure iolist() iol(B) when is_binary(B) -> B; iol(A) when is_atom(A) -> atom_to_binary(A, utf8); diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ed7e0a016..043b57b99 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -114,6 +114,7 @@ -export([namespace/0, roots/0, roots/1, fields/1, desc/1, tags/0]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1]). +-export([authz_fields/0]). -export([sc/2, map/2]). -elvis([{elvis_style, god_modules, disable}]). @@ -326,31 +327,7 @@ fields("stats") -> )} ]; fields("authorization") -> - [ - {"no_match", - sc( - hoconsc:enum([allow, deny]), - #{ - default => allow, - required => true, - desc => ?DESC(fields_authorization_no_match) - } - )}, - {"deny_action", - sc( - hoconsc:enum([ignore, disconnect]), - #{ - default => ignore, - required => true, - desc => ?DESC(fields_authorization_deny_action) - } - )}, - {"cache", - sc( - ref(?MODULE, "cache"), - #{} - )} - ]; + authz_fields(); fields("cache") -> [ {"enable", @@ -2091,6 +2068,33 @@ do_default_ciphers(_) -> %% otherwise resolve default ciphers list at runtime []. +authz_fields() -> + [ + {"no_match", + sc( + hoconsc:enum([allow, deny]), + #{ + default => allow, + required => true, + desc => ?DESC(fields_authorization_no_match) + } + )}, + {"deny_action", + sc( + hoconsc:enum([ignore, disconnect]), + #{ + default => ignore, + required => true, + desc => ?DESC(fields_authorization_deny_action) + } + )}, + {"cache", + sc( + ref(?MODULE, "cache"), + #{} + )} + ]. + %% @private return a list of keys in a parent field -spec keys(string(), hocon:config()) -> [string()]. keys(Parent, Conf) -> diff --git a/apps/emqx_authz/src/emqx_authz_api_settings.erl b/apps/emqx_authz/src/emqx_authz_api_settings.erl index 72a2db35c..db915a795 100644 --- a/apps/emqx_authz/src/emqx_authz_api_settings.erl +++ b/apps/emqx_authz/src/emqx_authz_api_settings.erl @@ -64,7 +64,7 @@ schema("/authorization/settings") -> }. ref_authz_schema() -> - proplists:delete(sources, emqx_conf_schema:fields("authorization")). + emqx_schema:authz_fields(). settings(get, _Params) -> {200, authorization_settings()}; @@ -83,4 +83,6 @@ settings(put, #{ {200, authorization_settings()}. authorization_settings() -> - maps:remove(<<"sources">>, emqx:get_raw_config([authorization], #{})). + C = maps:remove(<<"sources">>, emqx:get_raw_config([authorization], #{})), + Schema = emqx_hocon:make_schema(emqx_schema:authz_fields()), + hocon_tconf:make_serializable(Schema, C, #{}). diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index f5570f1f1..c692154b1 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -449,7 +449,7 @@ is_ok(ResL) -> get_raw_sources() -> RawSources = emqx:get_raw_config([authorization, sources], []), - Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}}, + Schema = emqx_hocon:make_schema(emqx_authz_schema:authz_fields()), Conf = #{<<"sources">> => RawSources}, #{<<"sources">> := Sources} = hocon_tconf:make_serializable(Schema, Conf, #{}), merge_default_headers(Sources). diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index e2da9b41e..5527c26d6 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -36,7 +36,8 @@ tags/0, fields/1, validations/0, - desc/1 + desc/1, + authz_fields/0 ]). -export([ @@ -74,23 +75,7 @@ tags() -> roots() -> []. fields("authorization") -> - Types = [?R_REF(Type) || Type <- type_names()], - UnionMemberSelector = - fun - (all_union_members) -> Types; - %% must return list - ({value, Value}) -> [select_union_member(Value)] - end, - [ - {sources, - ?HOCON( - ?ARRAY(?UNION(UnionMemberSelector)), - #{ - default => [], - desc => ?DESC(sources) - } - )} - ]; + authz_fields(); fields(file) -> authz_common_fields(file) ++ [{path, ?HOCON(string(), #{required => true, desc => ?DESC(path)})}]; @@ -492,3 +477,22 @@ select_union_member_loop(TypeValue, [Type | Types]) -> false -> select_union_member_loop(TypeValue, Types) end. + +authz_fields() -> + Types = [?R_REF(Type) || Type <- type_names()], + UnionMemberSelector = + fun + (all_union_members) -> Types; + %% must return list + ({value, Value}) -> [select_union_member(Value)] + end, + [ + {sources, + ?HOCON( + ?ARRAY(?UNION(UnionMemberSelector)), + #{ + default => [], + desc => ?DESC(sources) + } + )} + ]. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 71d2ab3fd..90af47aca 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -947,8 +947,8 @@ fields("log_burst_limit") -> )} ]; fields("authorization") -> - emqx_schema:fields("authorization") ++ - emqx_authz_schema:fields("authorization"). + emqx_schema:authz_fields() ++ + emqx_authz_schema:authz_fields(). desc("cluster") -> ?DESC("desc_cluster"); diff --git a/changes/v5.0.14-en.md b/changes/v5.0.14-en.md deleted file mode 100644 index 060e86652..000000000 --- a/changes/v5.0.14-en.md +++ /dev/null @@ -1,8 +0,0 @@ -# v5.0.14 - -## Enhancements - -- Enable authorization cache by default [#9626](https://github.com/emqx/emqx/pull/9626). - -## Bug fixes - diff --git a/changes/v5.0.14-zh.md b/changes/v5.0.14-zh.md deleted file mode 100644 index 62c40e638..000000000 --- a/changes/v5.0.14-zh.md +++ /dev/null @@ -1,8 +0,0 @@ -# v5.0.14 - -## 增强 - -- 默认启用授权缓存 [#9626](https://github.com/emqx/emqx/pull/9626)。 - -## 修复 - diff --git a/changes/v5.0.15/fix-9626.en.md b/changes/v5.0.15/fix-9626.en.md new file mode 100644 index 000000000..cc1c86d3e --- /dev/null +++ b/changes/v5.0.15/fix-9626.en.md @@ -0,0 +1,2 @@ +Return authorization settings with default values. +The authorization cache is enabled by default, but due to the missing default value in `GET` response of `/authorization/settings`, it seemed to be disabled from the dashboard. diff --git a/changes/v5.0.15/fix-9626.zh.md b/changes/v5.0.15/fix-9626.zh.md new file mode 100644 index 000000000..bc2391f48 --- /dev/null +++ b/changes/v5.0.15/fix-9626.zh.md @@ -0,0 +1,3 @@ +为授权设置 API 返回默认值。 +授权缓存默认为开启，但是在此修复前，因为默认值在 `/authorization/settings` 这个 API 的返回值中缺失， +使得在仪表盘配置页面中看起来是关闭了。 From b5d3e9d8b89a1a7f2071cb4629581e57448f3949 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 12 Jan 2023 14:14:21 +0100 Subject: [PATCH 26/31] fix: remove time unit from duration fields description --- apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index 52756f70d..0b6cbd0a2 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -36,8 +36,8 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise health_check_interval { desc { - en: """Health check interval, in milliseconds.""" - zh: """健康检查间隔，单位毫秒。""" + en: """Health check interval.""" + zh: """健康检查间隔。""" } label { en: """Health Check Interval""" @@ -69,8 +69,8 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise auto_restart_interval { desc { - en: """The auto restart interval after the resource is disconnected, in milliseconds.""" - zh: """资源断开以后，自动重连的时间间隔，单位毫秒。""" + en: """The auto restart interval after the resource is disconnected.""" + zh: """资源断开以后，自动重连的时间间隔。""" } label { en: """Auto Restart Interval""" From c5f557e315cad4c0ccded1051458e2a1a6c72dbe Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 11 Jan 2023 13:35:42 +0800 Subject: [PATCH 27/31] fix: disable basic auth for HTTP API --- .github/workflows/run_fvt_tests.yaml | 17 ++++++- .github/workflows/run_jmeter_tests.yaml | 10 ++-- apps/emqx/include/http_api.hrl | 10 ++-- apps/emqx/test/emqx_common_test_http.erl | 19 ++++++-- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 15 ++---- .../test/emqx_authz_api_cache_SUITE.erl | 12 ++--- .../test/emqx_authz_api_mnesia_SUITE.erl | 8 ++-- .../test/emqx_authz_api_settings_SUITE.erl | 6 +-- .../test/emqx_authz_api_sources_SUITE.erl | 8 ++-- .../test/emqx_auto_subscribe_SUITE.erl | 13 ++--- .../test/emqx_bridge_api_SUITE.erl | 17 ++----- apps/emqx_dashboard/src/emqx_dashboard.erl | 48 +++++++++---------- .../emqx_dashboard/src/emqx_dashboard_api.erl | 6 +-- .../test/emqx_dashboard_SUITE.erl | 4 +- .../test/emqx_dashboard_bad_api_SUITE.erl | 29 ++--------- .../test/emqx_dashboard_monitor_SUITE.erl | 6 +-- .../test/emqx_exhook_api_SUITE.erl | 8 +--- .../test/emqx_gateway_test_utils.erl | 5 +- apps/emqx_management/src/emqx_mgmt_auth.erl | 20 +++++--- .../test/emqx_mgmt_api_api_keys_SUITE.erl | 15 +++--- .../test/emqx_mgmt_api_test_util.erl | 41 +++++++++++++--- .../test/emqx_mgmt_api_trace_SUITE.erl | 9 +--- .../test/emqx_delayed_api_SUITE.erl | 14 ++---- .../test/emqx_rewrite_api_SUITE.erl | 16 ++----- .../test/emqx_telemetry_api_SUITE.erl | 10 ++-- .../test/emqx_topic_metrics_api_SUITE.erl | 17 ++----- .../test/emqx_slow_subs_api_SUITE.erl | 8 +--- 27 files changed, 183 insertions(+), 208 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 9e44034fb..a54bb68dd 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -201,12 +201,25 @@ jobs: echo "waiting emqx started"; sleep 10; done + - name: Get Token + timeout-minutes: 1 + run: | + kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null & + + while + [ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ] + do + echo "waiting emqx" + sleep 1 + done + + echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV + - name: Check cluster timeout-minutes: 10 run: | - kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null & while - [ "$(curl --silent --basic -u admin:public -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ]; + [ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ]; do echo "waiting ${{ matrix.profile }} cluster scale" sleep 1 diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index 6eaf4aa75..4ba246d8c 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -92,7 +92,7 @@ jobs: - uses: actions/checkout@v3 with: repository: emqx/emqx-fvt - ref: broker-autotest-v2 + ref: broker-autotest-v4 path: scripts - uses: actions/setup-java@v3 with: @@ -191,7 +191,7 @@ jobs: - uses: actions/checkout@v3 with: repository: emqx/emqx-fvt - ref: broker-autotest-v2 + ref: broker-autotest-v4 path: scripts - uses: actions/setup-java@v3 with: @@ -297,7 +297,7 @@ jobs: - uses: actions/checkout@v3 with: repository: emqx/emqx-fvt - ref: broker-autotest-v2 + ref: broker-autotest-v4 path: scripts - uses: actions/setup-java@v3 with: @@ -396,7 +396,7 @@ jobs: - uses: actions/checkout@v3 with: repository: emqx/emqx-fvt - ref: broker-autotest-v2 + ref: broker-autotest-v4 path: scripts - name: run jwks_server timeout-minutes: 10 @@ -496,7 +496,7 @@ jobs: - uses: actions/checkout@v3 with: repository: emqx/emqx-fvt - ref: broker-autotest-v2 + ref: broker-autotest-v4 path: scripts - uses: actions/setup-java@v3 with: diff --git a/apps/emqx/include/http_api.hrl b/apps/emqx/include/http_api.hrl index 858ce96ce..08dd08362 100644 --- a/apps/emqx/include/http_api.hrl +++ b/apps/emqx/include/http_api.hrl @@ -15,10 +15,8 @@ %%-------------------------------------------------------------------- %% HTTP API Auth --define(WRONG_USERNAME_OR_PWD, 'WRONG_USERNAME_OR_PWD'). --define(WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET, - 'WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET' -). +-define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD'). +-define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET'). %% Bad Request -define(BAD_REQUEST, 'BAD_REQUEST'). @@ -57,8 +55,8 @@ %% All codes -define(ERROR_CODES, [ - {'WRONG_USERNAME_OR_PWD', <<"Wrong username or pwd">>}, - {'WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET', <<"Wrong username & pwd or key & secret">>}, + {?BAD_USERNAME_OR_PWD, <<"Bad username or password">>}, + {?BAD_API_KEY_OR_SECRET, <<"Bad API key or secret">>}, {'BAD_REQUEST', <<"Request parameters are not legal">>}, {'NOT_MATCH', <<"Conditions are not matched">>}, {'ALREADY_EXISTS', <<"Resource already existed">>}, diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl index 87a35a1e2..575bed5c3 100644 --- a/apps/emqx/test/emqx_common_test_http.erl +++ b/apps/emqx/test/emqx_common_test_http.erl @@ -29,6 +29,9 @@ auth_header/2 ]). +-define(DEFAULT_APP_ID, <<"default_appid">>). +-define(DEFAULT_APP_SECRET, <<"default_app_secret">>). + request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). @@ -74,12 +77,18 @@ auth_header(User, Pass) -> {"Authorization", "Basic " ++ Encoded}. default_auth_header() -> - AppId = <<"myappid">>, - AppSecret = emqx_mgmt_auth:get_appsecret(AppId), - auth_header(erlang:binary_to_list(AppId), erlang:binary_to_list(AppSecret)). + {ok, #{api_key := APIKey}} = emqx_mgmt_auth:read(?DEFAULT_APP_ID), + auth_header( + erlang:binary_to_list(APIKey), erlang:binary_to_list(?DEFAULT_APP_SECRET) + ). create_default_app() -> - emqx_mgmt_auth:add_app(<<"myappid">>, <<"test">>). + Now = erlang:system_time(second), + ExpiredAt = Now + timer:minutes(10), + emqx_mgmt_auth:create( + ?DEFAULT_APP_ID, ?DEFAULT_APP_SECRET, true, ExpiredAt, <<"default app key for test">> + ), + ok. delete_default_app() -> - emqx_mgmt_auth:del_app(<<"myappid">>). + emqx_mgmt_auth:delete(?DEFAULT_APP_ID). diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 1a867b0be..11e2c6773 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -18,7 +18,8 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_dashboard_api_test_helpers, [request/3, uri/1, multipart_formdata_request/3]). +-import(emqx_dashboard_api_test_helpers, [multipart_formdata_request/3]). +-import(emqx_mgmt_api_test_util, [request/3, uri/1]). -include("emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -65,9 +66,8 @@ end_per_testcase(_, Config) -> init_per_suite(Config) -> emqx_config:erase(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY), _ = application:load(emqx_conf), - ok = emqx_common_test_helpers:start_apps( - [emqx_authn, emqx_dashboard], - fun set_special_configs/1 + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_authn] ), ?AUTHN:delete_chain(?GLOBAL), @@ -76,12 +76,7 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authn]), - ok. - -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(); -set_special_configs(_App) -> + emqx_mgmt_api_test_util:end_suite([emqx_authn]), ok. %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index 24b8fe25e..45e6d7287 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_dashboard_api_test_helpers, [request/2, uri/1]). +-import(emqx_mgmt_api_test_util, [request/2, uri/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -32,8 +32,8 @@ groups() -> []. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authz, emqx_dashboard, emqx_management], + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_conf, emqx_authz], fun set_special_configs/1 ), Config. @@ -47,7 +47,7 @@ end_per_suite(_Config) -> <<"sources">> => [] } ), - emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf, emqx_management]), + emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]), ok. set_special_configs(emqx_dashboard) -> @@ -67,12 +67,12 @@ t_clean_cahce(_) -> ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0), {ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(2, length(jsx:decode(Result3))), + ?assertEqual(2, length(emqx_json:decode(Result3))), request(delete, uri(["authorization", "cache"])), {ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(0, length(jsx:decode(Result4))), + ?assertEqual(0, length(emqx_json:decode(Result4))), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index 1819cf66f..62bce770e 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --import(emqx_dashboard_api_test_helpers, [request/3, uri/1]). +-import(emqx_mgmt_api_test_util, [request/3, uri/1]). all() -> emqx_common_test_helpers:all(?MODULE). @@ -31,8 +31,8 @@ groups() -> []. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authz, emqx_dashboard], + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_conf, emqx_authz], fun set_special_configs/1 ), Config. @@ -46,7 +46,7 @@ end_per_suite(_Config) -> <<"sources">> => [] } ), - emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]), + emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]), ok. set_special_configs(emqx_dashboard) -> diff --git a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl index 275b04e40..41eba109e 100644 --- a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_dashboard_api_test_helpers, [request/3, uri/1]). +-import(emqx_mgmt_api_test_util, [request/3, uri/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -30,7 +30,7 @@ groups() -> []. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps( + ok = emqx_mgmt_api_test_util:init_suite( [emqx_conf, emqx_authz, emqx_dashboard], fun set_special_configs/1 ), @@ -46,7 +46,7 @@ end_per_suite(_Config) -> } ), ok = stop_apps([emqx_resource]), - emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]), + emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]), ok. set_special_configs(emqx_dashboard) -> diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 34638d0aa..76b025716 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_dashboard_api_test_helpers, [request/3, uri/1]). +-import(emqx_mgmt_api_test_util, [request/3, uri/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -115,8 +115,8 @@ init_per_suite(Config) -> end ), - ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authz, emqx_dashboard], + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_conf, emqx_authz], fun set_special_configs/1 ), ok = start_apps([emqx_resource]), @@ -134,7 +134,7 @@ end_per_suite(_Config) -> %% resource and connector should be stop first, %% or authz_[mysql|pgsql|redis..]_SUITE would be failed ok = stop_apps([emqx_resource]), - emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authz, emqx_conf]), + emqx_mgmt_api_test_util:end_suite([emqx_authz, emqx_conf]), meck:unload(emqx_resource), ok. diff --git a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl index 959f8ec1b..900f39ebb 100644 --- a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl +++ b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl @@ -93,9 +93,8 @@ init_per_suite(Config) -> " }" >> ), - emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_dashboard, ?APP], - fun set_special_configs/1 + emqx_mgmt_api_test_util:init_suite( + [emqx_conf, ?APP] ), Config. @@ -111,12 +110,6 @@ end_per_testcase(t_get_basic_usage_info, _Config) -> end_per_testcase(_TestCase, _Config) -> ok. -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(), - ok; -set_special_configs(_) -> - ok. - topic_config(T) -> #{ topic => T, @@ -132,7 +125,7 @@ end_per_suite(_) -> application:unload(?APP), meck:unload(emqx_resource), meck:unload(emqx_schema), - emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_conf, ?APP]). + emqx_mgmt_api_test_util:end_suite([emqx_conf, ?APP]). t_auto_subscribe(_) -> emqx_auto_subscribe:update([#{<<"topic">> => Topic} || Topic <- ?TOPICS]), diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 557eced13..4d16f1692 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_dashboard_api_test_helpers, [request/4, uri/1]). +-import(emqx_mgmt_api_test_util, [request/3, uri/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -60,9 +60,8 @@ init_per_suite(Config) -> %% some testcases (may from other app) already get emqx_connector started _ = application:stop(emqx_resource), _ = application:stop(emqx_connector), - ok = emqx_common_test_helpers:start_apps( - [emqx_rule_engine, emqx_bridge, emqx_dashboard], - fun set_special_configs/1 + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_rule_engine, emqx_bridge] ), ok = emqx_common_test_helpers:load_config( emqx_rule_engine_schema, @@ -72,12 +71,7 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_rule_engine, emqx_bridge, emqx_dashboard]), - ok. - -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(<<"bridge_admin">>); -set_special_configs(_) -> + emqx_mgmt_api_test_util:end_suite([emqx_rule_engine, emqx_bridge]), ok. init_per_testcase(_, Config) -> @@ -605,9 +599,6 @@ t_with_redact_update(_Config) -> ?assertEqual(Password, Value), ok. -request(Method, Url, Body) -> - request(<<"bridge_admin">>, Method, Url, Body). - operation_path(node, Oper, BridgeID) -> uri(["nodes", node(), "bridges", BridgeID, "operation", Oper]); operation_path(cluster, Oper, BridgeID) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index f15467658..36c7660cc 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -65,8 +65,12 @@ start_listeners(Listeners) -> components => #{ schemas => #{}, 'securitySchemes' => #{ - 'basicAuth' => #{type => http, scheme => basic}, - 'bearerAuth' => #{type => http, scheme => bearer} + 'basicAuth' => #{ + type => http, + scheme => basic, + description => + <<"Authorize with [API Keys](https://www.emqx.io/docs/en/v5.0/admin/api.html#api-keys)">> + } } } }, @@ -215,28 +219,7 @@ listener_name(Protocol) -> authorize(Req) -> case cowboy_req:parse_header(<<"authorization">>, Req) of {basic, Username, Password} -> - case emqx_dashboard_admin:check(Username, Password) of - ok -> - ok; - {error, <<"username_not_found">>} -> - Path = cowboy_req:path(Req), - case emqx_mgmt_auth:authorize(Path, Username, Password) of - ok -> - ok; - {error, <<"not_allowed">>} -> - return_unauthorized( - ?WRONG_USERNAME_OR_PWD, - <<"Check username/password">> - ); - {error, _} -> - return_unauthorized( - ?WRONG_USERNAME_OR_PWD_OR_API_KEY_OR_API_SECRET, - <<"Check username/password or api_key/api_secret">> - ) - end; - {error, _} -> - return_unauthorized(?WRONG_USERNAME_OR_PWD, <<"Check username/password">>) - end; + api_key_authorize(Req, Username, Password); {bearer, Token} -> case emqx_dashboard_admin:verify_token(Token) of ok -> @@ -269,3 +252,20 @@ i18n_file() -> listeners() -> emqx_conf:get([dashboard, listeners], []). + +api_key_authorize(Req, Key, Secret) -> + Path = cowboy_req:path(Req), + case emqx_mgmt_auth:authorize(Path, Key, Secret) of + ok -> + ok; + {error, <<"not_allowed">>} -> + return_unauthorized( + ?BAD_API_KEY_OR_SECRET, + <<"Not allowed, Check api_key/api_secret">> + ); + {error, _} -> + return_unauthorized( + ?BAD_API_KEY_OR_SECRET, + <<"Check api_key/api_secret">> + ) + end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index 9facac59c..a4322c696 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -47,7 +47,7 @@ -define(EMPTY(V), (V == undefined orelse V == <<>>)). --define(WRONG_USERNAME_OR_PWD, 'WRONG_USERNAME_OR_PWD'). +-define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD'). -define(WRONG_TOKEN_OR_USERNAME, 'WRONG_TOKEN_OR_USERNAME'). -define(USER_NOT_FOUND, 'USER_NOT_FOUND'). -define(ERROR_PWD_NOT_MATCH, 'ERROR_PWD_NOT_MATCH'). @@ -164,7 +164,7 @@ schema("/users/:username/change_pwd") -> }. response_schema(401) -> - emqx_dashboard_swagger:error_codes([?WRONG_USERNAME_OR_PWD], ?DESC(login_failed401)); + emqx_dashboard_swagger:error_codes([?BAD_USERNAME_OR_PWD], ?DESC(login_failed401)); response_schema(404) -> emqx_dashboard_swagger:error_codes([?USER_NOT_FOUND], ?DESC(users_api404)). @@ -223,7 +223,7 @@ login(post, #{body := Params}) -> }}; {error, R} -> ?SLOG(info, #{msg => "Dashboard login failed", username => Username, reason => R}), - {401, ?WRONG_USERNAME_OR_PWD, <<"Auth failed">>} + {401, ?BAD_USERNAME_OR_PWD, <<"Auth failed">>} end. logout(_, #{ diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 934d6055d..23d1b40c1 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -114,9 +114,9 @@ t_admin_delete_self_failed(_) -> ?assertEqual(1, length(Admins)), Header = auth_header_(<<"username1">>, <<"password">>), {error, {_, 400, _}} = request_dashboard(delete, api_path(["users", "username1"]), Header), - Token = erlang:iolist_to_binary(["Basic ", base64:encode("username1:password")]), + Token = ["Basic ", base64:encode("username1:password")], Header2 = {"Authorization", Token}, - {error, {_, 400, _}} = request_dashboard(delete, api_path(["users", "username1"]), Header2), + {error, {_, 401, _}} = request_dashboard(delete, api_path(["users", "username1"]), Header2), mnesia:clear_table(?ADMIN). t_rest_api(_Config) -> diff --git a/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl index b7fbf889e..a9b448662 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl @@ -25,43 +25,24 @@ -define(SERVER, "http://127.0.0.1:18083/api/v5"). +-import(emqx_mgmt_api_test_util, [request/2]). + all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> mria:start(), - application:load(emqx_dashboard), - emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1), + emqx_mgmt_api_test_util:init_suite([emqx_conf]), Config. -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(), - ok; -set_special_configs(_) -> - ok. - end_per_suite(Config) -> end_suite(), Config. end_suite() -> - application:unload(emqx_management), - emqx_common_test_helpers:stop_apps([emqx_dashboard]). + emqx_mgmt_api_test_util:end_suite([emqx_conf]). t_bad_api_path(_) -> Url = ?SERVER ++ "/for/test/some/path/not/exist", - {error, {"HTTP/1.1", 404, "Not Found"}} = request(Url), + {ok, 404, _} = request(get, Url), ok. - -request(Url) -> - Request = {Url, []}, - case httpc:request(get, Request, [], []) of - {error, Reason} -> - {error, Reason}; - {ok, {{"HTTP/1.1", Code, _}, _, Return}} when - Code >= 200 andalso Code =< 299 - -> - {ok, emqx_json:decode(Return, [return_maps])}; - {ok, {Reason, _, _}} -> - {error, Reason} - end. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index 6f4a0e0fd..7d4980320 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -19,6 +19,8 @@ -compile(nowarn_export_all). -compile(export_all). +-import(emqx_dashboard_SUITE, [auth_header_/0]). + -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("emqx/include/emqx.hrl"). @@ -153,10 +155,6 @@ do_request_api(Method, Request) -> {error, Reason} end. -auth_header_() -> - Basic = binary_to_list(base64:encode(<<"admin:public">>)), - {"Authorization", "Basic " ++ Basic}. - restart_monitor() -> OldMonitor = erlang:whereis(emqx_dashboard_monitor), erlang:exit(OldMonitor, kill), diff --git a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl index 7be940a53..8a4fb7a44 100644 --- a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl @@ -347,13 +347,7 @@ do_request_api(Method, Request) -> end. auth_header_() -> - AppId = <<"admin">>, - AppSecret = <<"public">>, - auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). - -auth_header_(User, Pass) -> - Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), - {"Authorization", "Basic " ++ Encoded}. + emqx_mgmt_api_test_util:auth_header_(). api_path(Parts) -> ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index a6791a36b..deb602bc7 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -106,8 +106,6 @@ assert_fields_exist(Ks, Map) -> %% http -define(http_api_host, "http://127.0.0.1:18083/api/v5"). --define(default_user, "admin"). --define(default_pass, "public"). request(delete = Mth, Path) -> do_request(Mth, req(Path, [])); @@ -176,5 +174,4 @@ url(Path, Qs) -> lists:concat([?http_api_host, Path, "?", binary_to_list(cow_qs:qs(Qs))]). auth(Headers) -> - Token = base64:encode(?default_user ++ ":" ++ ?default_pass), - [{"Authorization", "Basic " ++ binary_to_list(Token)}] ++ Headers. + [emqx_mgmt_api_test_util:auth_header_() | Headers]. diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 0bf849d3c..6f2a27414 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -40,6 +40,10 @@ do_force_create_app/3 ]). +-ifdef(TEST). +-export([create/5]). +-endif. + -define(APP, emqx_app). -record(?APP, { @@ -68,8 +72,12 @@ init_bootstrap_file() -> init_bootstrap_file(File). create(Name, Enable, ExpiredAt, Desc) -> + ApiSecret = generate_api_secret(), + create(Name, ApiSecret, Enable, ExpiredAt, Desc). + +create(Name, ApiSecret, Enable, ExpiredAt, Desc) -> case mnesia:table_info(?APP, size) < 100 of - true -> create_app(Name, Enable, ExpiredAt, Desc); + true -> create_app(Name, ApiSecret, Enable, ExpiredAt, Desc); false -> {error, "Maximum ApiKey"} end. @@ -157,8 +165,7 @@ to_map(#?APP{name = N, api_key = K, enable = E, expired_at = ET, created_at = CT is_expired(undefined) -> false; is_expired(ExpiredTime) -> ExpiredTime < erlang:system_time(second). -create_app(Name, Enable, ExpiredAt, Desc) -> - ApiSecret = generate_api_secret(), +create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) -> App = #?APP{ name = Name, @@ -170,9 +177,10 @@ create_app(Name, Enable, ExpiredAt, Desc) -> api_key = list_to_binary(emqx_misc:gen_id(16)) }, case create_app(App) of - {error, api_key_already_existed} -> create_app(Name, Enable, ExpiredAt, Desc); - {ok, Res} -> {ok, Res#{api_secret => ApiSecret}}; - Error -> Error + {ok, Res} -> + {ok, Res#{api_secret => ApiSecret}}; + Error -> + Error end. create_app(App = #?APP{api_key = ApiKey, name = Name}) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl index 079351538..241a73dc4 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl @@ -225,21 +225,23 @@ t_create_unexpired_app(_Config) -> ok. list_app() -> + AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key"]), - case emqx_mgmt_api_test_util:request_api(get, Path) of + case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; Error -> Error end. read_app(Name) -> + AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key", Name]), - case emqx_mgmt_api_test_util:request_api(get, Path) of + case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; Error -> Error end. create_app(Name) -> - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key"]), ExpiredAt = to_rfc3339(erlang:system_time(second) + 1000), App = #{ @@ -254,7 +256,7 @@ create_app(Name) -> end. create_unexpired_app(Name, Params) -> - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key"]), App = maps:merge(#{name => Name, desc => <<"Note"/utf8>>, enable => true}, Params), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of @@ -263,11 +265,12 @@ create_unexpired_app(Name, Params) -> end. delete_app(Name) -> + AuthHeader = emqx_dashboard_SUITE:auth_header_(), DeletePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]), - emqx_mgmt_api_test_util:request_api(delete, DeletePath). + emqx_mgmt_api_test_util:request_api(delete, DeletePath, AuthHeader). update_app(Name, Change) -> - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + AuthHeader = emqx_dashboard_SUITE:auth_header_(), UpdatePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]), case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of {ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])}; diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 5bb0ba818..82d55bb6a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -24,14 +24,19 @@ init_suite() -> init_suite([]). init_suite(Apps) -> + init_suite(Apps, fun set_special_configs/1). + +init_suite(Apps, SetConfigs) -> mria:start(), application:load(emqx_management), - emqx_common_test_helpers:start_apps(Apps ++ [emqx_dashboard], fun set_special_configs/1). + emqx_common_test_helpers:start_apps(Apps ++ [emqx_dashboard], SetConfigs), + emqx_common_test_http:create_default_app(). end_suite() -> end_suite([]). end_suite(Apps) -> + emqx_common_test_http:delete_default_app(), application:unload(emqx_management), emqx_common_test_helpers:stop_apps(Apps ++ [emqx_dashboard]), emqx_config:delete_override_conf_files(), @@ -43,8 +48,23 @@ set_special_configs(emqx_dashboard) -> set_special_configs(_App) -> ok. +%% there is no difference between the 'request' and 'request_api' +%% the 'request' is only to be compatible with the 'emqx_dashboard_api_test_helpers:request' +request(Method, Url) -> + request(Method, Url, []). + +request(Method, Url, Body) -> + request_api_with_body(Method, Url, Body). + +uri(Parts) -> + emqx_dashboard_api_test_helpers:uri(Parts). + +%% compatible_mode will return as same as 'emqx_dashboard_api_test_helpers:request' +request_api_with_body(Method, Url, Body) -> + request_api(Method, Url, [], auth_header_(), Body, #{compatible_mode => true}). + request_api(Method, Url) -> - request_api(Method, Url, [], [], [], #{}). + request_api(Method, Url, auth_header_()). request_api(Method, Url, AuthOrHeaders) -> request_api(Method, Url, [], AuthOrHeaders, [], #{}). @@ -90,10 +110,20 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when do_request_api(Method, Request, Opts) -> ReturnAll = maps:get(return_all, Opts, false), + CompatibleMode = maps:get(compatible_mode, Opts, false), + ReqOpts = + case CompatibleMode of + true -> + [{body_format, binary}]; + _ -> + [] + end, ct:pal("Method: ~p, Request: ~p", [Method, Request]), - case httpc:request(Method, Request, [], []) of + case httpc:request(Method, Request, [], ReqOpts) of {error, socket_closed_remotely} -> {error, socket_closed_remotely}; + {ok, {{_, Code, _}, _Headers, Body}} when CompatibleMode -> + {ok, Code, Body}; {ok, {{"HTTP/1.1", Code, _} = Reason, Headers, Body}} when Code >= 200 andalso Code =< 299 andalso ReturnAll -> @@ -109,10 +139,7 @@ do_request_api(Method, Request, Opts) -> end. auth_header_() -> - Username = <<"admin">>, - Password = <<"public">>, - {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), - {"Authorization", "Bearer " ++ binary_to_list(Token)}. + emqx_common_test_http:default_auth_header(). build_http_header(X) when is_list(X) -> X; diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 8e8c5b06f..0ba05b280 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -30,6 +30,8 @@ -define(API_VERSION, "v5"). -define(BASE_PATH, "api"). +-import(emqx_dashboard_SUITE, [auth_header_/0]). + %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -330,13 +332,6 @@ t_stream_log(_Config) -> to_rfc3339(Second) -> list_to_binary(calendar:system_time_to_rfc3339(Second)). -auth_header_() -> - auth_header_("admin", "public"). - -auth_header_(User, Pass) -> - Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), - {"Authorization", "Basic " ++ Encoded}. - request_api(Method, Url, Auth) -> do_request_api(Method, {Url, [Auth]}). request_api(Method, Url, Auth, Body) -> diff --git a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl index 96cdf7840..ed3cd9292 100644 --- a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl @@ -26,7 +26,7 @@ <<"max_delayed_messages">> => <<"0">> }). --import(emqx_dashboard_api_test_helpers, [request/2, request/3, uri/1]). +-import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1]). all() -> emqx_common_test_helpers:all(?MODULE). @@ -36,27 +36,21 @@ init_per_suite(Config) -> raw_with_default => true }), - ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_modules, emqx_dashboard], - fun set_special_configs/1 + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_conf, emqx_modules] ), emqx_delayed:load(), Config. end_per_suite(Config) -> ok = emqx_delayed:unload(), - emqx_common_test_helpers:stop_apps([emqx_conf, emqx_dashboard, emqx_modules]), + emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_modules]), Config. init_per_testcase(_, Config) -> {ok, _} = emqx_cluster_rpc:start_link(), Config. -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(); -set_special_configs(_App) -> - ok. - %%------------------------------------------------------------------------------ %% Test Cases %%------------------------------------------------------------------------------ diff --git a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl index 90e90d788..ddb136f1e 100644 --- a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_dashboard_api_test_helpers, [request/3, uri/1]). +-import(emqx_mgmt_api_test_util, [request/3, uri/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -37,20 +37,14 @@ init_per_suite(Config) -> raw_with_default => true }), - ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_modules, emqx_dashboard], - fun set_special_configs/1 + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_conf, emqx_modules] ), Config. end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_conf, emqx_dashboard, emqx_modules]), - ok. - -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(); -set_special_configs(_App) -> + emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_modules]), ok. %%------------------------------------------------------------------------------ @@ -81,7 +75,7 @@ t_mqtt_topic_rewrite(_) -> ?assertEqual( Rules, - jsx:decode(Result) + emqx_json:decode(Result, [return_maps]) ). t_mqtt_topic_rewrite_limit(_) -> diff --git a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl index 288a155d9..16f942bc0 100644 --- a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_dashboard_api_test_helpers, [request/2, request/3, uri/1]). +-import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -33,8 +33,8 @@ init_per_suite(Config) -> raw_with_default => true }), - ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authn, emqx_authz, emqx_modules, emqx_dashboard], + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_conf, emqx_authn, emqx_authz, emqx_modules], fun set_special_configs/1 ), @@ -49,8 +49,8 @@ end_per_suite(_Config) -> <<"sources">> => [] } ), - emqx_common_test_helpers:stop_apps([ - emqx_dashboard, emqx_conf, emqx_authn, emqx_authz, emqx_modules + emqx_mgmt_api_test_util:end_suite([ + emqx_conf, emqx_authn, emqx_authz, emqx_modules ]), ok. diff --git a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl index 8c23d042c..ea85d1fe9 100644 --- a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_dashboard_api_test_helpers, [request/3, uri/1]). +-import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -44,9 +44,8 @@ init_per_suite(Config) -> raw_with_default => true }), - ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_modules, emqx_dashboard], - fun set_special_configs/1 + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_conf, emqx_modules] ), %% When many tests run in an obscure order, it may occur that @@ -59,15 +58,10 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_conf, emqx_dashboard, emqx_modules]), + emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_modules]), application:stop(gen_rpc), ok. -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(); -set_special_configs(_App) -> - ok. - %%------------------------------------------------------------------------------ %% Tests %%------------------------------------------------------------------------------ @@ -315,6 +309,3 @@ t_badrpc(_) -> %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ - -request(Method, Url) -> - request(Method, Url, []). diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl index 5b5ed063f..6b0721e3d 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl @@ -203,13 +203,7 @@ do_request_api(Method, Request) -> end. auth_header_() -> - AppId = <<"admin">>, - AppSecret = <<"public">>, - auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). - -auth_header_(User, Pass) -> - Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), - {"Authorization", "Basic " ++ Encoded}. + emqx_mgmt_api_test_util:auth_header_(). api_path(Parts) -> ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). From 45c3b372dd4adaa12137bbbd16b9a777b87760dc Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 12 Jan 2023 11:56:09 +0800 Subject: [PATCH 28/31] chore: update changes --- changes/v5.0.15/feat-9586.en.md | 1 + changes/v5.0.15/feat-9586.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/v5.0.15/feat-9586.en.md create mode 100644 changes/v5.0.15/feat-9586.zh.md diff --git a/changes/v5.0.15/feat-9586.en.md b/changes/v5.0.15/feat-9586.en.md new file mode 100644 index 000000000..777fb81df --- /dev/null +++ b/changes/v5.0.15/feat-9586.en.md @@ -0,0 +1 @@ +Basic auth is no longer allowed for API calls, must use API key instead. diff --git a/changes/v5.0.15/feat-9586.zh.md b/changes/v5.0.15/feat-9586.zh.md new file mode 100644 index 000000000..102266a46 --- /dev/null +++ b/changes/v5.0.15/feat-9586.zh.md @@ -0,0 +1 @@ +API 调用不再支持基于 `username:password` 的 `baisc` 认证, 现在 API 必须通过 API Key 才能进行调用。 From 1711823487859d1cf572a1e42febb14efb57f8b9 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 12 Jan 2023 14:37:07 +0800 Subject: [PATCH 29/31] refactor: remove default value of timestamp field of influxdb --- .../src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_plugin_libs_rule.erl | 2 + .../src/emqx_ee_bridge_influxdb.erl | 36 ++--- .../test/emqx_ee_bridge_influxdb_SUITE.erl | 1 - lib-ee/emqx_ee_connector/rebar.config | 2 +- .../src/emqx_ee_connector_influxdb.erl | 135 ++++++++++-------- mix.exs | 2 +- 7 files changed, 92 insertions(+), 88 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index bcdcfe420..3120b8503 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugin_libs, [ {description, "EMQX Plugin utility libs"}, - {vsn, "4.3.4"}, + {vsn, "4.3.5"}, {modules, []}, {applications, [kernel, stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index 57bdd16e5..969374309 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -63,6 +63,8 @@ can_topic_match_oneof/2 ]). +-export_type([tmpl_token/0]). + -compile({no_auto_import, [float/1]}). -define(EX_PLACE_HOLDER, "(\\$\\{[a-zA-Z0-9\\._]+\\})"). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 62c8b6ab7..b42c27832 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -196,33 +196,25 @@ to_influx_lines(RawLines) -> converter_influx_line(Line, AccIn) -> case string:tokens(str(Line), " ") of [MeasurementAndTags, Fields, Timestamp] -> - {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags), - [ - #{ - measurement => Measurement, - tags => kv_pairs(Tags), - fields => kv_pairs(string:tokens(Fields, ",")), - timestamp => Timestamp - } - | AccIn - ]; + append_influx_item(MeasurementAndTags, Fields, Timestamp, AccIn); [MeasurementAndTags, Fields] -> - {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags), - %% TODO: fix here both here and influxdb driver. - %% Default value should evaluated by InfluxDB. - [ - #{ - measurement => Measurement, - tags => kv_pairs(Tags), - fields => kv_pairs(string:tokens(Fields, ",")), - timestamp => "${timestamp}" - } - | AccIn - ]; + append_influx_item(MeasurementAndTags, Fields, undefined, AccIn); _ -> throw("Bad InfluxDB Line Protocol schema") end. +append_influx_item(MeasurementAndTags, Fields, Timestamp, Acc) -> + {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags), + [ + #{ + measurement => Measurement, + tags => kv_pairs(Tags), + fields => kv_pairs(string:tokens(Fields, ",")), + timestamp => Timestamp + } + | Acc + ]. + split_measurement_and_tags(Subject) -> case string:tokens(Subject, ",") of [] -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl index 6331611d0..bb87a9f37 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl @@ -525,7 +525,6 @@ t_start_ok(Config) -> SentData = #{ <<"clientid">> => ClientId, <<"topic">> => atom_to_binary(?FUNCTION_NAME), - <<"timestamp">> => erlang:system_time(nanosecond), <<"payload">> => Payload }, ?check_trace( diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index ab4c88396..262641d44 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.5"}}}, + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.6"}}}, {emqx, {path, "../../apps/emqx"}} ]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 7974bf028..db99c4475 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -142,7 +142,11 @@ fields(common) -> [ {server, server()}, {precision, - mk(enum([ns, us, ms, s, m, h]), #{ + %% The influxdb only supports these 4 precision: + %% See "https://github.com/influxdata/influxdb/blob/ + %% 6b607288439a991261307518913eb6d4e280e0a7/models/points.go#L487" for + %% more information. + mk(enum([ns, us, ms, s]), #{ required => false, default => ms, desc => ?DESC("precision") })} ]; @@ -210,9 +214,7 @@ start_client(InstId, Config) -> do_start_client( InstId, ClientConfig, - Config = #{ - write_syntax := Lines - } + Config = #{write_syntax := Lines} ) -> case influxdb:start_client(ClientConfig) of {ok, Client} -> @@ -220,7 +222,9 @@ do_start_client( true -> State = #{ client => Client, - write_syntax => to_config(Lines) + write_syntax => to_config( + Lines, proplists:get_value(precision, ClientConfig) + ) }, ?SLOG(info, #{ msg => "starting influxdb connector success", @@ -348,30 +352,33 @@ do_async_query(InstId, Client, Points, ReplyFunAndArgs) -> %% ------------------------------------------------------------------------------------------------- %% Tags & Fields Config Trans -to_config(Lines) -> - to_config(Lines, []). +to_config(Lines, Precision) -> + to_config(Lines, [], Precision). -to_config([], Acc) -> +to_config([], Acc, _Precision) -> lists:reverse(Acc); -to_config( - [ - #{ - measurement := Measurement, - timestamp := Timestamp, - tags := Tags, - fields := Fields - } - | Rest - ], - Acc -) -> - Res = #{ - measurement => emqx_plugin_libs_rule:preproc_tmpl(Measurement), - timestamp => emqx_plugin_libs_rule:preproc_tmpl(Timestamp), - tags => to_kv_config(Tags), - fields => to_kv_config(Fields) +to_config([Item0 | Rest], Acc, Precision) -> + Ts = maps:get(timestamp, Item0, undefined), + Item = #{ + measurement => emqx_plugin_libs_rule:preproc_tmpl(maps:get(measurement, Item0)), + timestamp => preproc_tmpl_timestamp(Ts, Precision), + tags => to_kv_config(maps:get(tags, Item0)), + fields => to_kv_config(maps:get(fields, Item0)) }, - to_config(Rest, [Res | Acc]). + to_config(Rest, [Item | Acc], Precision). + +preproc_tmpl_timestamp(undefined, <<"ns">>) -> + erlang:system_time(nanosecond); +preproc_tmpl_timestamp(undefined, <<"us">>) -> + erlang:system_time(microsecond); +preproc_tmpl_timestamp(undefined, <<"ms">>) -> + erlang:system_time(millisecond); +preproc_tmpl_timestamp(undefined, <<"s">>) -> + erlang:system_time(second); +preproc_tmpl_timestamp(Ts, _) when is_integer(Ts) -> + Ts; +preproc_tmpl_timestamp(Ts, _) when is_binary(Ts); is_list(Ts) -> + emqx_plugin_libs_rule:preproc_tmpl(Ts). to_kv_config(KVfields) -> maps:fold(fun to_maps_config/3, #{}, proplists:to_map(KVfields)). @@ -414,7 +421,7 @@ parse_batch_data(InstId, BatchData, SyntaxLines) -> fields := [{binary(), binary()}], measurement := binary(), tags := [{binary(), binary()}], - timestamp := binary() + timestamp := emqx_plugin_libs_rule:tmpl_token() | integer() } ]) -> {ok, [map()]} | {error, term()}. data_to_points(Data, SyntaxLines) -> @@ -430,46 +437,50 @@ lines_to_points(_, [], Points, ErrorPoints) -> %% ignore trans succeeded points {error, ErrorPoints} end; -lines_to_points( - Data, - [ - #{ - measurement := Measurement, - timestamp := Timestamp, - tags := Tags, - fields := Fields - } - | Rest - ], - ResultPointsAcc, - ErrorPointsAcc -) -> +lines_to_points(Data, [#{timestamp := Ts} = Item | Rest], ResultPointsAcc, ErrorPointsAcc) when + is_list(Ts) +-> TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, - case emqx_plugin_libs_rule:proc_tmpl(Timestamp, Data, TransOptions) of - [TimestampInt] when is_integer(TimestampInt) -> - {_, EncodedTags} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags), - {_, EncodedFields} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields), - Point = #{ - measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Data), - timestamp => TimestampInt, - tags => EncodedTags, - fields => EncodedFields - }, - case map_size(EncodedFields) =:= 0 of - true -> - %% influxdb client doesn't like empty field maps... - lines_to_points(Data, Rest, ResultPointsAcc, [ - {error, no_fields} | ErrorPointsAcc - ]); - false -> - lines_to_points(Data, Rest, [Point | ResultPointsAcc], ErrorPointsAcc) - end; - BadTimestamp -> + case emqx_plugin_libs_rule:proc_tmpl(Ts, Data, TransOptions) of + [TsInt] when is_integer(TsInt) -> + Item1 = Item#{timestamp => TsInt}, + continue_lines_to_points(Data, Item1, Rest, ResultPointsAcc, ErrorPointsAcc); + BadTs -> lines_to_points(Data, Rest, ResultPointsAcc, [ - {error, {bad_timestamp, BadTimestamp}} | ErrorPointsAcc + {error, {bad_timestamp, BadTs}} | ErrorPointsAcc ]) + end; +lines_to_points(Data, [#{timestamp := Ts} = Item | Rest], ResultPointsAcc, ErrorPointsAcc) when + is_integer(Ts) +-> + continue_lines_to_points(Data, Item, Rest, ResultPointsAcc, ErrorPointsAcc). + +continue_lines_to_points(Data, Item, Rest, ResultPointsAcc, ErrorPointsAcc) -> + case line_to_point(Data, Item) of + #{fields := Fields} when map_size(Fields) =:= 0 -> + %% influxdb client doesn't like empty field maps... + ErrorPointsAcc1 = [{error, no_fields} | ErrorPointsAcc], + lines_to_points(Data, Rest, ResultPointsAcc, ErrorPointsAcc1); + Point -> + lines_to_points(Data, Rest, [Point | ResultPointsAcc], ErrorPointsAcc) end. +line_to_point( + Data, + #{ + measurement := Measurement, + tags := Tags, + fields := Fields + } = Item +) -> + {_, EncodedTags} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags), + {_, EncodedFields} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields), + Item#{ + measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Data), + tags => EncodedTags, + fields => EncodedFields + }. + maps_config_to_data(K, V, {Data, Res}) -> KTransOptions = #{return => rawlist, var_trans => fun key_filter/1}, VTransOptions = #{return => rawlist, var_trans => fun data_filter/1}, diff --git a/mix.exs b/mix.exs index 832f6e7ae..27d9b43ce 100644 --- a/mix.exs +++ b/mix.exs @@ -131,7 +131,7 @@ defmodule EMQXUmbrella.MixProject do defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do [ {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}, - {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.4", override: true}, + {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.6", override: true}, {:wolff, github: "kafka4beam/wolff", tag: "1.7.4"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.2", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"}, From 9adc6017545a4586c053517a096a53ce5d738dd7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 12 Jan 2023 18:29:46 +0300 Subject: [PATCH 30/31] chore: add changelog for the fuzzy search API fix Co-authored-by: Zaiming (Stone) Shi --- changes/v5.0.15/fix-9726-en.md | 1 + changes/v5.0.15/fix-9726-zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/v5.0.15/fix-9726-en.md create mode 100644 changes/v5.0.15/fix-9726-zh.md diff --git a/changes/v5.0.15/fix-9726-en.md b/changes/v5.0.15/fix-9726-en.md new file mode 100644 index 000000000..9aa522690 --- /dev/null +++ b/changes/v5.0.15/fix-9726-en.md @@ -0,0 +1 @@ +Client fuzzy search API results were missing information which could tell if more results are available in the next pages, this is now fixed by providing `hasnext` flag in the response. diff --git a/changes/v5.0.15/fix-9726-zh.md b/changes/v5.0.15/fix-9726-zh.md new file mode 100644 index 000000000..3554d2db7 --- /dev/null +++ b/changes/v5.0.15/fix-9726-zh.md @@ -0,0 +1 @@ +在此修复前，客户端模糊搜索 API 缺少一些可以用于判断是否可以继续翻页的信息，现在通过在响应中提供 `hasnext` 标志来解决这个问题。 From 90e837783fc1a8d2c1d237cf57d3f43ca995e31d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 12 Jan 2023 19:25:40 +0300 Subject: [PATCH 31/31] fix(paging): rename `partial` to `overflow` for clarity --- apps/emqx_management/src/emqx_mgmt_api.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 9a1d0ce4e..ba232daf3 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -405,11 +405,11 @@ counting_total_fun(_QueryState = #{fuzzy_fun := FuzzyFun}) when FuzzyFun =/= und %% ResultAcc :: #{count := integer(), %% cursor := integer(), %% rows := [{node(), Rows :: list()}], -%% partial := boolean(), +%% overflow := boolean(), %% hasnext => boolean() %% } init_query_result() -> - #{cursor => 0, count => 0, rows => [], partial => false}. + #{cursor => 0, count => 0, rows => [], overflow => false}. accumulate_query_rows( Node, @@ -436,12 +436,13 @@ accumulate_query_rows( cursor => NCursor, count => Count + length(SubRows), rows => [{Node, SubRows} | RowsAcc], - partial => (Limit - Count) < Len + % there are more rows than can fit in the page + overflow => (Limit - Count) < Len }} end. -finalize_query(Result = #{partial := Partial}, QueryState = #{complete := Complete}) -> - HasNext = Partial orelse not Complete, +finalize_query(Result = #{overflow := Overflow}, QueryState = #{complete := Complete}) -> + HasNext = Overflow orelse not Complete, maybe_accumulate_totals(Result#{hasnext => HasNext}, QueryState). maybe_accumulate_totals(Result, #{total := TotalAcc}) ->