From 813ccd7df1357ee7234ee82be2580465b3d1a4e9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 21 Mar 2023 16:14:12 +0100 Subject: [PATCH 01/11] chore: hide node.perf configs these configs are rarely needed by regular users and if they are to be changed, it should be done by someone who really understands what they mean --- apps/emqx/src/emqx_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 19fc3aae4..bf509b4e5 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1492,7 +1492,7 @@ fields("broker") -> {"perf", sc( ref("broker_perf"), - #{} + #{hidden => true} )}, {"shared_subscription_group", sc( From c302405408ba4233a97f63a42da7515de36e4b5c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Mar 2023 20:16:07 +0100 Subject: [PATCH 02/11] fix: fail fast if not able to load config --- apps/emqx_conf/src/emqx_conf_app.erl | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 34224c3f2..09478e304 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -28,7 +28,18 @@ -define(DEFAULT_INIT_TXN_ID, -1). start(_StartType, _StartArgs) -> - init_conf(), + try + ok = init_conf() + catch + C:E:St -> + ?SLOG(critical, #{ + msg => failed_to_init_config, + exception => C, + reason => E, + stacktrace => St + }), + init:stop() + end, ok = emqx_config_logger:refresh_config(), emqx_conf_sup:start_link(). @@ -85,9 +96,9 @@ init_load() -> init_conf() -> {ok, TnxId} = copy_override_conf_from_core_node(), - emqx_app:set_init_tnx_id(TnxId), - init_load(), - emqx_app:set_init_config_load_done(). + _ = emqx_app:set_init_tnx_id(TnxId), + ok = init_load(), + ok = emqx_app:set_init_config_load_done(). cluster_nodes() -> maps:get(running_nodes, ekka_cluster:info()) -- [node()]. From fb29f8035bb3107866e71da8f0ef60f1df643d1b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Mar 2023 22:06:00 +0100 Subject: [PATCH 03/11] feat: make possible to dump full schema (including hidden fields) --- apps/emqx_conf/src/emqx_conf.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 33214946d..a840c3f1b 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -156,7 +156,11 @@ dump_schema(Dir, SchemaModule, I18nFile) -> gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> SchemaJsonFile = filename:join([Dir, "schema-" ++ Lang ++ ".json"]), io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]), - Opts = #{desc_file => I18nFile, lang => Lang}, + %% EMQX_SCHEMA_FULL_DUMP is quite a hidden API + %% it is used to dump the full schema for EMQX developers and supporters + IncludeHidden = (os:getenv("EMQX_SCHEMA_FULL_DUMP") =:= "1"), + IncludeHidden andalso io:format(user, "===< Including hidden fields~n", []), + Opts = #{desc_file => I18nFile, lang => Lang, include_hidden_fields => IncludeHidden}, JsonMap = hocon_schema_json:gen(SchemaModule, Opts), IoData = jsx:encode(JsonMap, [space, {indent, 4}]), ok = file:write_file(SchemaJsonFile, IoData). From 486352eb6f978a565fea4c9a6d4a966f858f9a6c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Mar 2023 19:32:00 +0100 Subject: [PATCH 04/11] fix(swagger): do not generate scheam for hidden args --- .../src/emqx_dashboard_swagger.erl | 80 ++++++++++++------- .../test/emqx_swagger_requestBody_SUITE.erl | 2 +- .../test/emqx_swagger_response_SUITE.erl | 2 +- 3 files changed, 52 insertions(+), 32 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 2290dbedb..18196425f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -830,36 +830,8 @@ to_bin(X) -> X. parse_object(PropList = [_ | _], Module, Options) when is_list(PropList) -> - {Props, Required, Refs} = - lists:foldl( - fun({Name, Hocon}, {Acc, RequiredAcc, RefsAcc}) -> - NameBin = to_bin(Name), - case hoconsc:is_schema(Hocon) of - true -> - HoconType = hocon_schema:field_schema(Hocon, type), - Init0 = init_prop([default | ?DEFAULT_FIELDS], #{}, Hocon), - SchemaToSpec = schema_converter(Options), - Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin), - {Prop, Refs1} = SchemaToSpec(HoconType, Module), - NewRequiredAcc = - case is_required(Hocon) of - true -> [NameBin | RequiredAcc]; - false -> RequiredAcc - end, - { - [{NameBin, maps:merge(Prop, Init)} | Acc], - NewRequiredAcc, - Refs1 ++ RefsAcc - }; - false -> - {SubObject, SubRefs} = parse_object(Hocon, Module, Options), - {[{NameBin, SubObject} | Acc], RequiredAcc, SubRefs ++ RefsAcc} - end - end, - {[], [], []}, - PropList - ), - Object = #{<<"type">> => object, <<"properties">> => lists:reverse(Props)}, + {Props, Required, Refs} = parse_object_loop(PropList, Module, Options), + Object = #{<<"type">> => object, <<"properties">> => Props}, case Required of [] -> {Object, Refs}; _ -> {maps:put(required, Required, Object), Refs} @@ -874,6 +846,54 @@ parse_object(Other, Module, Options) -> }} ). +parse_object_loop(PropList0, Module, Options) -> + PropList = lists:filter( + fun({_, Hocon}) -> + case hoconsc:is_schema(Hocon) andalso is_hidden(Hocon) of + true -> false; + false -> true + end + end, + PropList0 + ), + parse_object_loop(PropList, Module, Options, _Props = [], _Required = [], _Refs = []). + +parse_object_loop([], _Modlue, _Options, Props, Required, Refs) -> + {lists:reverse(Props), lists:usort(Required), Refs}; +parse_object_loop([{Name, Hocon} | Rest], Module, Options, Props, Required, Refs) -> + NameBin = to_bin(Name), + case hoconsc:is_schema(Hocon) of + true -> + HoconType = hocon_schema:field_schema(Hocon, type), + Init0 = init_prop([default | ?DEFAULT_FIELDS], #{}, Hocon), + SchemaToSpec = schema_converter(Options), + Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin), + {Prop, Refs1} = SchemaToSpec(HoconType, Module), + NewRequiredAcc = + case is_required(Hocon) of + true -> [NameBin | Required]; + false -> Required + end, + parse_object_loop( + Rest, + Module, + Options, + [{NameBin, maps:merge(Prop, Init)} | Props], + NewRequiredAcc, + Refs1 ++ Refs + ); + false -> + %% TODO: there is only a handful of such + %% refactor the schema to unify the two cases + {SubObject, SubRefs} = parse_object(Hocon, Module, Options), + parse_object_loop( + Rest, Module, Options, [{NameBin, SubObject} | Props], Required, SubRefs ++ Refs + ) + end. + +is_hidden(Hocon) -> + hocon_schema:is_hidden(Hocon). + is_required(Hocon) -> hocon_schema:field_schema(Hocon, required) =:= true. diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 717a7d4ca..3150ed097 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -61,7 +61,7 @@ t_object(_Config) -> #{ <<"schema">> => #{ - required => [<<"timeout">>, <<"per_page">>], + required => [<<"per_page">>, <<"timeout">>], <<"properties">> => [ {<<"per_page">>, #{ description => <<"good per page desc">>, diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 314154633..4d1501dae 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -59,7 +59,7 @@ t_object(_config) -> <<"application/json">> => #{ <<"schema">> => #{ - required => [<<"timeout">>, <<"per_page">>], + required => [<<"per_page">>, <<"timeout">>], <<"properties">> => [ {<<"per_page">>, #{ description => <<"good per page desc">>, From 5a19b6f73e986250960d135a18a628e36455575d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 27 Mar 2023 16:09:15 +0200 Subject: [PATCH 05/11] chore: pin hocon 0.38.0 (with config importance level annotation) --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 3ec5b6c30..9079322eb 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.2"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/mix.exs b/mix.exs index 3c3ce0766..ffa2ce056 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.37.2", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.38.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 04470030a..b641077ea 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.2"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From d07987288afe562932891a3b9ced1a13c68638e8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 27 Mar 2023 16:10:36 +0200 Subject: [PATCH 06/11] chore: add some example annotations for config importance level --- apps/emqx/src/emqx_schema.erl | 8 ++--- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf.erl | 13 +++++--- apps/emqx_conf/src/emqx_conf_schema.erl | 33 ++++++++++++++++--- .../src/emqx_dashboard_swagger.erl | 4 +-- .../src/emqx_prometheus.app.src | 2 +- .../src/emqx_prometheus_schema.erl | 12 +++---- .../src/schema/emqx_resource_schema.erl | 2 +- .../src/emqx_ee_bridge_kafka.erl | 2 +- 9 files changed, 54 insertions(+), 24 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index bf509b4e5..94e73cecb 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -230,7 +230,7 @@ roots(low) -> {"crl_cache", sc( ref("crl_cache"), - #{hidden => true} + #{importance => ?IMPORTANCE_HIDDEN} )} ]. @@ -1492,7 +1492,7 @@ fields("broker") -> {"perf", sc( ref("broker_perf"), - #{hidden => true} + #{importance => ?IMPORTANCE_HIDDEN} )}, {"shared_subscription_group", sc( @@ -2997,7 +2997,7 @@ quic_feature_toggle(Desc) -> typerefl:alias("boolean", typerefl:union([true, false, 0, 1])), #{ desc => Desc, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, required => false, converter => fun (true) -> 1; @@ -3012,7 +3012,7 @@ quic_lowlevel_settings_uint(Low, High, Desc) -> range(Low, High), #{ required => false, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => Desc } ). diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index fbbffba1f..37707431a 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.14"}, + {vsn, "0.1.15"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index a840c3f1b..d03cf9c27 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -158,9 +158,13 @@ gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]), %% EMQX_SCHEMA_FULL_DUMP is quite a hidden API %% it is used to dump the full schema for EMQX developers and supporters - IncludeHidden = (os:getenv("EMQX_SCHEMA_FULL_DUMP") =:= "1"), - IncludeHidden andalso io:format(user, "===< Including hidden fields~n", []), - Opts = #{desc_file => I18nFile, lang => Lang, include_hidden_fields => IncludeHidden}, + IncludeImportance = + case os:getenv("EMQX_SCHEMA_FULL_DUMP") =:= "1" of + true -> ?IMPORTANCE_HIDDEN; + false -> ?IMPORTANCE_LOW + end, + io:format(user, "===< Including fields from importance level: ~p~n", [IncludeImportance]), + Opts = #{desc_file => I18nFile, lang => Lang, include_importance_up_from => IncludeImportance}, JsonMap = hocon_schema_json:gen(SchemaModule, Opts), IoData = jsx:encode(JsonMap, [space, {indent, 4}]), ok = file:write_file(SchemaJsonFile, IoData). @@ -224,7 +228,8 @@ gen_example(File, SchemaModule, I18nFile, Lang) -> title => <<"EMQX Configuration Example">>, body => <<"">>, desc_file => I18nFile, - lang => Lang + lang => Lang, + include_importance_up_from => ?IMPORTANCE_MEDIUM }, Example = hocon_schema_example:gen(SchemaModule, Opts), file:write_file(File, Example). diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 4862be5fe..58bcf9700 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -397,6 +397,7 @@ fields("node") -> #{ default => <<"emqx@127.0.0.1">>, 'readOnly' => true, + importance => ?IMPORTANCE_HIGH, desc => ?DESC(node_name) } )}, @@ -409,6 +410,7 @@ fields("node") -> 'readOnly' => true, sensitive => true, desc => ?DESC(node_cookie), + importance => ?IMPORTANCE_HIGH, converter => fun emqx_schema:password_converter/2 } )}, @@ -419,6 +421,7 @@ fields("node") -> mapping => "vm_args.+P", desc => ?DESC(process_limit), default => 2097152, + importance => ?IMPORTANCE_MEDIUM, 'readOnly' => true } )}, @@ -429,6 +432,7 @@ fields("node") -> mapping => "vm_args.+Q", desc => ?DESC(max_ports), default => 1048576, + importance => ?IMPORTANCE_HIGH, 'readOnly' => true } )}, @@ -439,6 +443,7 @@ fields("node") -> mapping => "vm_args.+zdbbl", desc => ?DESC(dist_buffer_size), default => 8192, + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -449,6 +454,7 @@ fields("node") -> mapping => "vm_args.+e", desc => ?DESC(max_ets_tables), default => 262144, + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -459,6 +465,10 @@ fields("node") -> required => true, 'readOnly' => true, mapping => "emqx.data_dir", + %% for now, it's tricky to use a different data_dir + %% otherwise data paths in cluster config may differ + %% TODO: change configurable data file paths to relative + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(node_data_dir) } )}, @@ -467,7 +477,7 @@ fields("node") -> hoconsc:array(string()), #{ mapping => "emqx.config_files", - hidden => true, + importance => ?IMPORTANCE_HIDDEN, required => false, 'readOnly' => true } @@ -479,6 +489,7 @@ fields("node") -> mapping => "emqx_machine.global_gc_interval", default => <<"15m">>, desc => ?DESC(node_global_gc_interval), + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -489,6 +500,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP", desc => ?DESC(node_crash_dump_file), default => crash_dump_file_default(), + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -499,6 +511,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP_SECONDS", default => <<"30s">>, desc => ?DESC(node_crash_dump_seconds), + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -509,6 +522,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP_BYTES", default => <<"100MB">>, desc => ?DESC(node_crash_dump_bytes), + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -519,6 +533,7 @@ fields("node") -> mapping => "vm_args.-kernel net_ticktime", default => <<"2m">>, 'readOnly' => true, + importance => ?IMPORTANCE_LOW, desc => ?DESC(node_dist_net_ticktime) } )}, @@ -529,6 +544,7 @@ fields("node") -> mapping => "emqx_machine.backtrace_depth", default => 23, 'readOnly' => true, + importance => ?IMPORTANCE_LOW, desc => ?DESC(node_backtrace_depth) } )}, @@ -539,6 +555,7 @@ fields("node") -> mapping => "emqx_machine.applications", default => [], 'readOnly' => true, + importance => ?IMPORTANCE_LOW, desc => ?DESC(node_applications) } )}, @@ -548,13 +565,17 @@ fields("node") -> #{ desc => ?DESC(node_etc_dir), 'readOnly' => true, + importance => ?IMPORTANCE_LOW, deprecated => {since, "5.0.8"} } )}, {"cluster_call", sc( ?R_REF("cluster_call"), - #{'readOnly' => true} + #{ + 'readOnly' => true, + importance => ?IMPORTANCE_LOW + } )}, {"db_backend", sc( @@ -563,6 +584,7 @@ fields("node") -> mapping => "mria.db_backend", default => rlog, 'readOnly' => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(db_backend) } )}, @@ -573,6 +595,7 @@ fields("node") -> mapping => "mria.node_role", default => core, 'readOnly' => true, + importance => ?IMPORTANCE_HIGH, desc => ?DESC(db_role) } )}, @@ -583,6 +606,7 @@ fields("node") -> mapping => "mria.rlog_rpc_module", default => gen_rpc, 'readOnly' => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(db_rpc_module) } )}, @@ -593,6 +617,7 @@ fields("node") -> mapping => "mria.tlog_push_mode", default => async, 'readOnly' => true, + importance => ?IMPORTANCE_LOW, desc => ?DESC(db_tlog_push_mode) } )}, @@ -601,7 +626,7 @@ fields("node") -> hoconsc:enum([gen_rpc, distr]), #{ mapping => "mria.shard_transport", - hidden => true, + importance => ?IMPORTANCE_HIDDEN, default => gen_rpc, desc => ?DESC(db_default_shard_transport) } @@ -611,7 +636,7 @@ fields("node") -> map(shard, hoconsc:enum([gen_rpc, distr])), #{ desc => ?DESC(db_shard_transports), - hidden => true, + importance => ?IMPORTANCE_HIDDEN, mapping => "emqx_machine.custom_shard_transports", default => #{} } diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 18196425f..e2872c0d7 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -891,9 +891,9 @@ parse_object_loop([{Name, Hocon} | Rest], Module, Options, Props, Required, Refs ) end. +%% return true if the field has 'importance' set to 'hidden' is_hidden(Hocon) -> - hocon_schema:is_hidden(Hocon). - + hocon_schema:is_hidden(Hocon, #{include_importance_up_from => ?IMPORTANCE_LOW}). is_required(Hocon) -> hocon_schema:field_schema(Hocon, required) =:= true. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 07ae38d75..1e7e59f7a 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.7"}, + {vsn, "5.0.8"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 6ced0bf42..f8005f06b 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -90,7 +90,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(vm_dist_collector) } )}, @@ -100,7 +100,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(mnesia_collector) } )}, @@ -110,7 +110,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(vm_statistics_collector) } )}, @@ -120,7 +120,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(vm_system_info_collector) } )}, @@ -130,7 +130,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(vm_memory_collector) } )}, @@ -140,7 +140,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(vm_msacc_collector) } )} diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index b9ed176fe..04a0ffd72 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -83,7 +83,7 @@ worker_pool_size(required) -> false; worker_pool_size(_) -> undefined. resume_interval(type) -> emqx_schema:duration_ms(); -resume_interval(hidden) -> true; +resume_interval(importance) -> hidden; resume_interval(desc) -> ?DESC("resume_interval"); resume_interval(required) -> false; resume_interval(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index 3db8dd5f1..d9401b7fd 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -364,7 +364,7 @@ fields(consumer_kafka_opts) -> })}, {max_rejoin_attempts, mk(non_neg_integer(), #{ - hidden => true, + importance => ?IMPORTANCE_HIDDEN, default => 5, desc => ?DESC(consumer_max_rejoin_attempts) })}, From 3a88e7739da941cf2d5bf2271e5eee5580b83ff9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Mar 2023 21:22:38 +0100 Subject: [PATCH 07/11] test: add swagger check to smoke tests --- scripts/test/emqx-smoke-test.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/test/emqx-smoke-test.sh b/scripts/test/emqx-smoke-test.sh index 361137bc0..82de296ba 100755 --- a/scripts/test/emqx-smoke-test.sh +++ b/scripts/test/emqx-smoke-test.sh @@ -8,6 +8,7 @@ IP=$1 PORT=$2 URL="http://$IP:$PORT/status" +## Check if EMQX is responding ATTEMPTS=10 while ! curl "$URL" >/dev/null 2>&1; do if [ $ATTEMPTS -eq 0 ]; then @@ -17,3 +18,23 @@ while ! curl "$URL" >/dev/null 2>&1; do sleep 5 ATTEMPTS=$((ATTEMPTS-1)) done + +## Check if the API docs are available +API_DOCS_URL="http://$IP:$PORT/api-docs/index.html" +API_DOCS_STATUS="$(curl -s -o /dev/null -w "%{http_code}" "$API_DOCS_URL")" +if [ "$API_DOCS_STATUS" != "200" ]; then + echo "emqx is not responding on $API_DOCS_URL" + exit 1 +fi + +## Check if the swagger.json contains hidden fields +## fail if it does +SWAGGER_JSON_URL="http://$IP:$PORT/api-docs/swagger.json" +## assert swagger.json is valid json +JSON="$(curl -s "$SWAGGER_JSON_URL")" +echo "$JSON" | jq . >/dev/null +## assert swagger.json does not contain trie_compaction (which is a hidden field) +if echo "$JSON" | grep -q trie_compaction; then + echo "swagger.json contains hidden fields" + exit 1 +fi From d35c5e0516c74c553fcb0228b1f7cfce3e96181e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Mar 2023 21:35:10 +0100 Subject: [PATCH 08/11] docs: add changelogs --- changes/ce/fix-10211.en.md | 3 +++ changes/ce/fix-10211.zh.md | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 changes/ce/fix-10211.en.md create mode 100644 changes/ce/fix-10211.zh.md diff --git a/changes/ce/fix-10211.en.md b/changes/ce/fix-10211.en.md new file mode 100644 index 000000000..9474f2027 --- /dev/null +++ b/changes/ce/fix-10211.en.md @@ -0,0 +1,3 @@ +Hide `broker.broker_perf` config and API documents. +The two configs `route_lock_type` and `trie_compaction` are rarely used and requires a full cluster restart to take effect. They are not suitable for being exposed to users. +Detailed changes can be found here: https://gist.github.com/zmstone/01ad5754b9beaeaf3f5b86d14d49a0b7/revisions diff --git a/changes/ce/fix-10211.zh.md b/changes/ce/fix-10211.zh.md new file mode 100644 index 000000000..e8db64f86 --- /dev/null +++ b/changes/ce/fix-10211.zh.md @@ -0,0 +1,3 @@ +隐藏 `broker.broker_perf` 配置项，不再在 配置和 API 的文档中展示。 +`route_lock_type` 和 `trie_compaction` 这两个配置项很少使用，且需要全集群重启才能生效，不适合暴露给用户。 +详细对比： https://gist.github.com/zmstone/01ad5754b9beaeaf3f5b86d14d49a0b7/revisions From 3f9ed0830810f66be86f5bd1440d6afd5bceecec Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 14:18:50 +0200 Subject: [PATCH 09/11] fix(emqx_schema): ocsp field use 'importance' to replace 'hidden' --- apps/emqx/src/emqx_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 94e73cecb..3177294ee 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2299,7 +2299,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> #{ required => false, %% TODO: remove after e5.0.2 - hidden => true, + importance => ?IMPORTANCE_HIDDEN, validator => fun ocsp_inner_validator/1 } )}, From f5e9d3eab282b98b5605fd295267d87c6ea0add5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 14:28:49 +0200 Subject: [PATCH 10/11] test: use valid config for jq function timeout --- .../test/emqx_rule_funcs_SUITE.erl | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 5d78f5e4a..209332fe7 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -28,7 +28,7 @@ init_per_suite(Config) -> application:load(emqx_conf), - ConfigConf = <<"rule_engine {jq_function_default_timeout {}}">>, + ConfigConf = <<"rule_engine {jq_function_default_timeout=10s}">>, ok = emqx_common_test_helpers:load_config(emqx_rule_engine_schema, ConfigConf), Config. @@ -691,20 +691,10 @@ t_jq(_) -> ConfigRootKey, jq_function_default_timeout ]), - case DefaultTimeOut =< 15000 of - true -> - got_timeout = - try - apply_func(jq, [TOProgram, <<"-2">>]) - catch - throw:{jq_exception, {timeout, _}} -> - %% Got timeout as expected - got_timeout - end; - false -> - %% Skip test as we don't want it to take to long time to run - ok - end. + ?assertThrow( + {jq_exception, {timeout, _}}, + apply_func(jq, [TOProgram, <<"-2">>]) + ). ascii_string() -> list(range(0, 127)). From 0aab24d9173cf77b417c6ef99bdd69099278349e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 16:03:55 +0200 Subject: [PATCH 11/11] ci: do not check hidden fields if running older version --- .github/workflows/build_slim_packages.yaml | 3 +++ scripts/test/emqx-smoke-test.sh | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 7f7f35e16..d6e4fc961 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -194,12 +194,15 @@ jobs: run: | CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) + export EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS='yes' ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT docker stop $CID - name: test two nodes cluster with proto_dist=inet_tls in docker run: | ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) + # versions before 5.0.22 have hidden fields included in the API spec + export EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS='no' ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT # cleanup ./scripts/test/start-two-nodes-in-docker.sh -c diff --git a/scripts/test/emqx-smoke-test.sh b/scripts/test/emqx-smoke-test.sh index 82de296ba..ce8116b39 100755 --- a/scripts/test/emqx-smoke-test.sh +++ b/scripts/test/emqx-smoke-test.sh @@ -33,8 +33,11 @@ SWAGGER_JSON_URL="http://$IP:$PORT/api-docs/swagger.json" ## assert swagger.json is valid json JSON="$(curl -s "$SWAGGER_JSON_URL")" echo "$JSON" | jq . >/dev/null -## assert swagger.json does not contain trie_compaction (which is a hidden field) -if echo "$JSON" | grep -q trie_compaction; then - echo "swagger.json contains hidden fields" - exit 1 + +if [ "${EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS:-yes}" = 'yes' ]; then + ## assert swagger.json does not contain trie_compaction (which is a hidden field) + if echo "$JSON" | grep -q trie_compaction; then + echo "swagger.json contains hidden fields" + exit 1 + fi fi