From 974380a3d4cb560c615e6632c59f50ad7aca4151 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 27 Apr 2022 09:21:16 +0200 Subject: [PATCH 1/7] feat(dashboard): add listener enable/disable config toggle --- .../i18n/emqx_dashboard_i18n.conf | 10 +++ apps/emqx_dashboard/src/emqx_dashboard.erl | 9 ++- .../src/emqx_dashboard_schema.erl | 61 +++++++++++++++---- 3 files changed, 65 insertions(+), 15 deletions(-) diff --git a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf index 07bd75f9b..9badd75e0 100644 --- a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf +++ b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf @@ -132,6 +132,16 @@ Note: `sample_interval` should be a divisor of 60.""" zh: "HTTPS" } } + listener_enable { + desc { + en: "Ignore or enable this listener" + zh: "忽略或启用该监听器配置" + } + label { + en: "Enable" + zh: "启用" + } + } bind { desc { en: "Port without IP(18083) or port with specified IP(127.0.0.1:18083)." diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index c13e20b00..af96bb15a 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -153,10 +153,13 @@ apps() -> ]. listeners(Listeners) -> - lists:map( + lists:filtermap( fun({Protocol, Conf}) -> - {Conf1, Bind} = ip_port(Conf), - {listener_name(Protocol, Conf1), Protocol, Bind, ranch_opts(Conf1)} + maps:get(enable, Conf) andalso + begin + {Conf1, Bind} = ip_port(Conf), + {true, {listener_name(Protocol, Conf1), Protocol, Bind, ranch_opts(Conf1)}} + end end, maps:to_list(Listeners) ). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 27a7aa571..0f4221c84 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -76,6 +76,44 @@ fields("listeners") -> )} ]; fields("http") -> + [ + {"enable", + sc( + boolean(), + #{ + default => true, + desc => ?DESC(listener_enable) + } + )} + | common_listener_fields() + ]; +fields("https") -> + [ + {"enable", + sc( + boolean(), + #{ + default => false, + desc => ?DESC(listener_enable) + } + )} + | common_listener_fields() ++ + exclude_fields( + ["enable", "fail_if_no_peer_cert"], + emqx_schema:server_ssl_opts_schema(#{}, true) + ) + ]. + +exclude_fields([], Fields) -> + Fields; +exclude_fields([FieldName | Rest], Fields) -> + %% assert field exists + case lists:keytake(FieldName, 1, Fields) of + {value, _, New} -> exclude_fields(Rest, New); + false -> error({FieldName, Fields}) + end. + +common_listener_fields() -> [ {"bind", fun bind/1}, {"num_acceptors", @@ -126,19 +164,18 @@ fields("http") -> desc => ?DESC(ipv6_v6only) } )} - ]; -fields("https") -> - fields("http") ++ - proplists:delete( - "fail_if_no_peer_cert", - emqx_schema:server_ssl_opts_schema(#{}, true) - ). + ]. -desc("dashboard") -> ?DESC(desc_dashboard); -desc("listeners") -> ?DESC(desc_listeners); -desc("http") -> ?DESC(desc_http); -desc("https") -> ?DESC(desc_https); -desc(_) -> undefined. +desc("dashboard") -> + ?DESC(desc_dashboard); +desc("listeners") -> + ?DESC(desc_listeners); +desc("http") -> + ?DESC(desc_http); +desc("https") -> + ?DESC(desc_https); +desc(_) -> + undefined. bind(type) -> hoconsc:union([non_neg_integer(), emqx_schema:ip_port()]); bind(default) -> 18083; From 204f04be655c4589a2513600199a4fa0f25caf8d Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 27 Apr 2022 19:51:00 +0800 Subject: [PATCH 2/7] fix: handshake_timeout is ranch option not socket options --- apps/emqx_dashboard/src/emqx_dashboard.erl | 27 ++++++---------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index af96bb15a..36daed7db 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -175,34 +175,21 @@ init_i18n() -> Lang = emqx_conf:get([dashboard, i18n_lang], en), init_i18n(File, Lang). -ranch_opts(RanchOptions) -> +ranch_opts(Options) -> Keys = [ - {ack_timeout, handshake_timeout}, + handshake_timeout, connection_type, max_connections, num_acceptors, shutdown, socket ], - {S, R} = lists:foldl(fun key_take/2, {RanchOptions, #{}}, Keys), - R#{socket_opts => maps:fold(fun key_only/3, [], S)}. + RanchOpts = maps:with(Keys, Options), + SocketOpts = maps:fold(fun filter_false/3, [], maps:without([enable | Keys], Options)), + RanchOpts#{socket_opts => SocketOpts}. -key_take(Key, {All, R}) -> - {K, KX} = - case Key of - {K1, K2} -> {K1, K2}; - _ -> {Key, Key} - end, - case maps:get(K, All, undefined) of - undefined -> - {All, R}; - V -> - {maps:remove(K, All), R#{KX => V}} - end. - -key_only(K, true, S) -> [K | S]; -key_only(_K, false, S) -> S; -key_only(K, V, S) -> [{K, V} | S]. +filter_false(_K, false, S) -> S; +filter_false(K, V, S) -> [{K, V} | S]. listener_name(Protocol, #{port := Port, ip := IP}) -> Name = From b06747d96116ef8eae658f1d3c56c42800997b9b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 27 Apr 2022 20:19:19 +0800 Subject: [PATCH 3/7] chore: bump typeref to 0.9.1 to fix flatten error --- mix.exs | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index b999480ab..b87579a10 100644 --- a/mix.exs +++ b/mix.exs @@ -48,7 +48,7 @@ defmodule EMQXUmbrella.MixProject do [ {:lc, github: "emqx/lc", tag: "0.2.1"}, {:redbug, "2.0.7"}, - {:typerefl, github: "ieQu1/typerefl", tag: "0.9.0", override: true}, + {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, diff --git a/rebar.config b/rebar.config index 95e3f705c..54114c608 100644 --- a/rebar.config +++ b/rebar.config @@ -47,7 +47,7 @@ [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.2.1"}}} , {redbug, "2.0.7"} , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps - , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.0"}}} + , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.12"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} From 5c08c4ff4e64deaf1fe7c9363ae4dc3759bb4faa Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 27 Apr 2022 22:24:32 +0800 Subject: [PATCH 4/7] chore: more detailed about dashboard inet6 option --- .../i18n/emqx_dashboard_i18n.conf | 6 +++--- apps/emqx_dashboard/src/emqx_dashboard.erl | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf index 9badd75e0..d6587c203 100644 --- a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf +++ b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf @@ -74,8 +74,8 @@ Note: `sample_interval` should be a divisor of 60.""" } inet6 { desc { - en: "Enable IPv6 support." - zh: "启用IPv6" + en: "Enable IPv6 support, default is false, which means IPv4 only." + zh: "启用IPv6， 如果机器不支持IPv6，请关闭此选项，否则会导致仪表盘无法使用。" } label { en: "IPv6" @@ -85,7 +85,7 @@ Note: `sample_interval` should be a divisor of 60.""" ipv6_v6only { desc { en: "Disable IPv4-to-IPv6 mapping for the listener." - zh: "禁用IPv4-to-IPv6映射" + zh: "当开启 inet6 功能的同时禁用 IPv4-to-IPv6 映射。该配置仅在 inet6 功能开启时有效。" } label { en: "IPv6 only" diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 36daed7db..b76858d4b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -185,8 +185,21 @@ ranch_opts(Options) -> socket ], RanchOpts = maps:with(Keys, Options), - SocketOpts = maps:fold(fun filter_false/3, [], maps:without([enable | Keys], Options)), - RanchOpts#{socket_opts => SocketOpts}. + SocketOpts = maps:fold( + fun filter_false/3, + [], + maps:without([enable, inet6, ipv6_v6only | Keys], Options) + ), + InetOpts = + case Options of + #{inet6 := true, ipv6_v6only := true} -> + [inet6, {ipv6_v6only, true}]; + #{inet6 := true, ipv6_v6only := false} -> + [inet6]; + _ -> + [inet] + end, + RanchOpts#{socket_opts => InetOpts ++ SocketOpts}. filter_false(_K, false, S) -> S; filter_false(K, V, S) -> [{K, V} | S]. From 2eab3f1cdbb5347f6e93ee6c440db63dcb14c693 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 27 Apr 2022 23:17:02 +0800 Subject: [PATCH 5/7] fix: dashboard https's password can't update --- .../src/emqx_dashboard_config.erl | 44 ++++++++++++----- .../src/emqx_dashboard_schema.erl | 49 ++++++++++--------- 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_config.erl b/apps/emqx_dashboard/src/emqx_dashboard_config.erl index aea8304da..1061661f5 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_config.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_config.erl @@ -63,22 +63,42 @@ remove_handler() -> ok. pre_config_update(_Path, UpdateConf0, RawConf) -> - UpdateConf = - case UpdateConf0 of - #{<<"default_password">> := <<"******">>} -> - maps:remove(<<"default_password">>, UpdateConf0); - _ -> - UpdateConf0 - end, + UpdateConf = remove_sensitive_data(UpdateConf0), NewConf = emqx_map_lib:deep_merge(RawConf, UpdateConf), {ok, NewConf}. +-define(SENSITIVE_PASSWORD, <<"******">>). + +remove_sensitive_data(Conf0) -> + Conf1 = + case Conf0 of + #{<<"default_password">> := ?SENSITIVE_PASSWORD} -> + maps:remove(<<"default_password">>, Conf0); + _ -> + Conf0 + end, + case Conf1 of + #{<<"listeners">> := #{<<"https">> := #{<<"password">> := ?SENSITIVE_PASSWORD}}} -> + emqx_map_lib:deep_remove([<<"listeners">>, <<"https">>, <<"password">>], Conf1); + _ -> + Conf1 + end. + post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) -> - #{listeners := NewListeners} = NewConf, - #{listeners := OldListeners} = OldConf, + #{listeners := #{http := NewHttp, https := NewHttps}} = NewConf, + #{listeners := #{http := OldHttp, https := OldHttps}} = OldConf, _ = - case NewListeners =:= OldListeners of - true -> ok; - false -> erlang:send_after(500, ?MODULE, {update_listeners, OldListeners, NewListeners}) + case diff_listeners(OldHttp, NewHttp, OldHttps, NewHttps) of + identical -> ok; + {Stop, Start} -> erlang:send_after(500, ?MODULE, {update_listeners, Stop, Start}) end, ok. + +diff_listeners(Http, Http, Https, Https) -> + identical; +diff_listeners(OldHttp, NewHttp, Https, Https) -> + {#{http => OldHttp}, #{http => NewHttp}}; +diff_listeners(Http, Http, OldHttps, NewHttps) -> + {#{https => OldHttps}, #{https => NewHttps}}; +diff_listeners(OldHttp, NewHttp, OldHttps, NewHttps) -> + {#{http => OldHttp, https => OldHttps}, #{http => NewHttp, https => NewHttps}}. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 0f4221c84..4bfc97797 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -77,26 +77,14 @@ fields("listeners") -> ]; fields("http") -> [ - {"enable", - sc( - boolean(), - #{ - default => true, - desc => ?DESC(listener_enable) - } - )} + enable(true), + bind(18803) | common_listener_fields() ]; fields("https") -> [ - {"enable", - sc( - boolean(), - #{ - default => false, - desc => ?DESC(listener_enable) - } - )} + enable(false), + bind(18804) | common_listener_fields() ++ exclude_fields( ["enable", "fail_if_no_peer_cert"], @@ -115,7 +103,6 @@ exclude_fields([FieldName | Rest], Fields) -> common_listener_fields() -> [ - {"bind", fun bind/1}, {"num_acceptors", sc( integer(), @@ -166,6 +153,28 @@ common_listener_fields() -> )} ]. +enable(Bool) -> + {"enable", + sc( + boolean(), + #{ + default => Bool, + required => true, + desc => ?DESC(listener_enable) + } + )}. + +bind(Port) -> + {"bind", + sc( + hoconsc:union([non_neg_integer(), emqx_schema:ip_port()]), + #{ + default => Port, + required => true, + desc => ?DESC(bind) + } + )}. + desc("dashboard") -> ?DESC(desc_dashboard); desc("listeners") -> @@ -177,12 +186,6 @@ desc("https") -> desc(_) -> undefined. -bind(type) -> hoconsc:union([non_neg_integer(), emqx_schema:ip_port()]); -bind(default) -> 18083; -bind(required) -> true; -bind(desc) -> ?DESC(bind); -bind(_) -> undefined. - default_username(type) -> binary(); default_username(default) -> "admin"; default_username(required) -> true; From 3e8bedda76ba3eb1031a91cf4b3fadcb35e8a82b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 28 Apr 2022 09:41:00 +0800 Subject: [PATCH 6/7] fix: dashboard ct add enable options --- apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index 3501494cb..84eb46cc8 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -37,6 +37,7 @@ set_default_config(DefaultUsername) -> Config = #{ listeners => #{ http => #{ + enable => true, port => 18083 } }, From c5b42ea0d993cba91c99b7d822bb2baaa512c648 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 28 Apr 2022 14:06:57 +0800 Subject: [PATCH 7/7] chore: reformat emqx_prometheus_SUITE --- apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index 03e8d6d78..16590b114 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -22,14 +22,14 @@ -compile(export_all). -define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard). --define(CONF_DEFAULT, - <<"\n" +-define(CONF_DEFAULT, << + "\n" "prometheus {\n" " push_gateway_server = \"http://127.0.0.1:9091\"\n" " interval = \"1s\"\n" " enable = true\n" - "}\n">> -). + "}\n" +>>). %%-------------------------------------------------------------------- %% Setups