From 47f826f99a5d645ac51e8c8e22803938206797a4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 8 Jun 2023 18:32:33 +0200 Subject: [PATCH 1/5] chore: fix typo --- apps/emqx/src/emqx_schema.erl | 4 ++-- .../test/emqx_authz_redis_SUITE.erl | 2 +- .../src/emqx_bridge_influxdb.erl | 18 +++++++------- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- .../src/emqx_ee_bridge_redis.erl | 24 +++++++++---------- rel/emqx_conf.template.en.md | 4 ++-- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index dd04fad78..8c520d14f 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3018,7 +3018,7 @@ non_empty_string(_) -> {error, invalid_string}. %% hosts can be successfully parsed. %% 3. parsing: Done at runtime in each module which uses this config servers_sc(Meta0, ParseOpts) -> - %% if this filed has a default value + %% if this field has a default value %% then it is not NOT required %% NOTE: maps:is_key is not the solution because #{default => undefined} is legit HasDefault = (maps:get(default, Meta0, undefined) =/= undefined), @@ -3079,7 +3079,7 @@ servers_validator(Opts, Required) -> %% we should remove field from config if it's empty throw("cannot_be_empty"); "undefined" when Required -> - %% when the filed is not set in config file + %% when the field is not set in config file %% NOTE: assuming nobody is going to name their server "undefined" throw("cannot_be_empty"); "undefined" -> diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index d68ea342e..35aa0449f 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -181,7 +181,7 @@ t_create_with_config_values_wont_work(_Config) -> InvalidConfigs ). -%% creating without a require filed should return error +%% creating without a require field should return error t_create_invalid_config(_Config) -> AuthzConfig = raw_redis_authz_config(), C = maps:without([<<"server">>], AuthzConfig), diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl index c2a04e93d..dd340d15e 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl @@ -95,33 +95,33 @@ namespace() -> "bridge_influxdb". roots() -> []. fields("post_api_v1") -> - method_fileds(post, influxdb_api_v1); + method_fields(post, influxdb_api_v1); fields("post_api_v2") -> - method_fileds(post, influxdb_api_v2); + method_fields(post, influxdb_api_v2); fields("put_api_v1") -> - method_fileds(put, influxdb_api_v1); + method_fields(put, influxdb_api_v1); fields("put_api_v2") -> - method_fileds(put, influxdb_api_v2); + method_fields(put, influxdb_api_v2); fields("get_api_v1") -> - method_fileds(get, influxdb_api_v1); + method_fields(get, influxdb_api_v1); fields("get_api_v2") -> - method_fileds(get, influxdb_api_v2); + method_fieds(get, influxdb_api_v2); fields(Type) when Type == influxdb_api_v1 orelse Type == influxdb_api_v2 -> influxdb_bridge_common_fields() ++ connector_fields(Type). -method_fileds(post, ConnectorType) -> +method_fields(post, ConnectorType) -> influxdb_bridge_common_fields() ++ connector_fields(ConnectorType) ++ type_name_fields(ConnectorType); -method_fileds(get, ConnectorType) -> +method_fields(get, ConnectorType) -> influxdb_bridge_common_fields() ++ connector_fields(ConnectorType) ++ type_name_fields(ConnectorType) ++ emqx_bridge_schema:status_fields(); -method_fileds(put, ConnectorType) -> +method_fields(put, ConnectorType) -> influxdb_bridge_common_fields() ++ connector_fields(ConnectorType). diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index bf500de26..9725c2da9 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1323,7 +1323,7 @@ roots(Module) -> lists:map(fun({_BinName, Root}) -> Root end, hocon_schema:roots(Module)). %% Like authentication schema, authorization schema is incomplete in emqx_schema -%% module, this function replaces the root filed "authorization" with a new schema +%% module, this function replaces the root field "authorization" with a new schema emqx_schema_high_prio_roots() -> Roots = emqx_schema:roots(high), Authz = diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl index a728ecb7e..ff15aa00f 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl @@ -101,23 +101,23 @@ namespace() -> "bridge_redis". roots() -> []. fields("post_single") -> - method_fileds(post, redis_single); + method_fields(post, redis_single); fields("post_sentinel") -> - method_fileds(post, redis_sentinel); + method_fields(post, redis_sentinel); fields("post_cluster") -> - method_fileds(post, redis_cluster); + method_fields(post, redis_cluster); fields("put_single") -> - method_fileds(put, redis_single); + method_fields(put, redis_single); fields("put_sentinel") -> - method_fileds(put, redis_sentinel); + method_fields(put, redis_sentinel); fields("put_cluster") -> - method_fileds(put, redis_cluster); + method_fields(put, redis_cluster); fields("get_single") -> - method_fileds(get, redis_single); + method_fields(get, redis_single); fields("get_sentinel") -> - method_fileds(get, redis_sentinel); + method_fields(get, redis_sentinel); fields("get_cluster") -> - method_fileds(get, redis_cluster); + method_fields(get, redis_cluster); fields(Type) when Type == redis_single orelse Type == redis_sentinel orelse Type == redis_cluster -> @@ -126,16 +126,16 @@ fields(Type) when fields("creation_opts_" ++ Type) -> resource_creation_fields(Type). -method_fileds(post, ConnectorType) -> +method_fields(post, ConnectorType) -> redis_bridge_common_fields(ConnectorType) ++ connector_fields(ConnectorType) ++ type_name_fields(ConnectorType); -method_fileds(get, ConnectorType) -> +method_fields(get, ConnectorType) -> redis_bridge_common_fields(ConnectorType) ++ connector_fields(ConnectorType) ++ type_name_fields(ConnectorType) ++ emqx_bridge_schema:status_fields(); -method_fileds(put, ConnectorType) -> +method_fields(put, ConnectorType) -> redis_bridge_common_fields(ConnectorType) ++ connector_fields(ConnectorType). diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index c1259869c..2dcb83896 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -84,7 +84,7 @@ There are 4 complex data types in EMQX's HOCON config: 1. Array: `[ElementType]` ::: tip Tip -If map filed name is a positive integer number, it is interpreted as an alternative representation of an `Array`. +If map field name is a positive integer number, it is interpreted as an alternative representation of an `Array`. For example: ``` myarray.1 = 74 @@ -120,7 +120,7 @@ If we consider the whole EMQX config as a tree, to reference a primitive value, we can use a dot-separated names form string for the path from the tree-root (always a Struct) down to the primitive values at tree-leaves. -Each segment of the dotted string is a Struct filed name or Map key. +Each segment of the dotted string is a Struct field name or Map key. For Array elements, 1-based index is used. below are some examples From 96a41ac6bd7cec6ae08b05e16cf69209d971d4f7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 8 Jun 2023 17:10:49 +0200 Subject: [PATCH 2/5] feat(emqx_ctl): support hidden commands hidden commands do not deserve a place in the usage page --- apps/emqx_ctl/src/emqx_ctl.erl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index d2ced7268..76068d361 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -157,18 +157,21 @@ help() -> print("No commands available.~n"); Cmds -> print("Usage: ~ts~n", ["emqx ctl"]), - lists:foreach( - fun({_, {Mod, Cmd}, _}) -> - print("~110..-s~n", [""]), - apply(Mod, Cmd, [usage]) - end, - Cmds - ) + lists:foreach(fun print_usage/1, Cmds) end; false -> print("Command table is initializing.~n") end. +print_usage({_, {Mod, Cmd}, Opts}) -> + case proplists:get_bool(hidden, Opts) of + true -> + ok; + false -> + print("~110..-s~n", [""]), + apply(Mod, Cmd, [usage]) + end. + -spec print(io:format()) -> ok. print(Msg) -> io:format("~ts", [format(Msg, [])]). From 31ab973b30eedeae652d0790bbe1daddc9122a52 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 8 Jun 2023 17:54:43 +0200 Subject: [PATCH 3/5] refactor(emqx_conf_cli): rename cluster_call ctl command Renamed 'clustedr_call' ctl command to 'cluster_sync' as a sub-command of 'conf' The old 'cluster_call' command is kept, but no usage prints --- apps/emqx_conf/src/emqx_conf_cli.erl | 51 +++++++++++++++++----------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 9240d2116..c0ca18100 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -22,11 +22,12 @@ unload/0 ]). +%% kept cluster_call for compatibility -define(CLUSTER_CALL, cluster_call). -define(CONF, conf). load() -> - emqx_ctl:register_command(?CLUSTER_CALL, {?MODULE, admins}, []), + emqx_ctl:register_command(?CLUSTER_CALL, {?MODULE, admins}, [hidden]), emqx_ctl:register_command(?CONF, {?MODULE, conf}, []). unload() -> @@ -41,17 +42,10 @@ conf(["show", Key]) -> print_hocon(get_config(Key)); conf(["load", Path]) -> load_config(Path); +conf(["cluster_sync" | Args]) -> + admins(Args); conf(_) -> - emqx_ctl:usage( - [ - %% TODO add reload - %{"conf reload", "reload etc/emqx.conf on local node"}, - {"conf show --keys-only", "print all keys"}, - {"conf show", "print all running configures"}, - {"conf show ", "print a specific configuration"}, - {"conf load ", "load a hocon file to all nodes"} - ] - ). + emqx_ctl:usage(usage_conf() ++ usage_sync()). admins(["status"]) -> status(); @@ -87,14 +81,33 @@ admins(["fast_forward", Node0, ToTnxId]) -> emqx_cluster_rpc:fast_forward_to_commit(Node, TnxId), status(); admins(_) -> - emqx_ctl:usage( - [ - {"cluster_call status", "status"}, - {"cluster_call skip [node]", "increase one commit on specific node"}, - {"cluster_call tnxid ", "get detailed about TnxId"}, - {"cluster_call fast_forward [node] [tnx_id]", "fast forwards to tnx_id"} - ] - ). + emqx_ctl:usage(usage_sync()). + +usage_conf() -> + [ + %% TODO add reload + %{"conf reload", "reload etc/emqx.conf on local node"}, + {"conf show --keys-only", "Print all config keys"}, + {"conf show", "Print config in use"}, + {"conf show ", "Print configs under the given key"}, + {"conf load ", + "Load a HOCON format config file." + "The config is overlay on top of the existing configs. " + "The current node will initiate a cluster wide config change " + "transaction to sync the changes to other nodes in the cluster. " + "NOTE: do not make runtime config changes during rolling upgrade."} + ]. + +usage_sync() -> + [ + {"conf cluster_sync tatus", "Show cluster config sync status summary"}, + {"conf cluster_sync skip [node]", "Increase one commit on specific node"}, + {"conf cluster_sync tnxid ", + "Display detailed information of the config change transaction at TnxId"}, + {"conf cluster_sync fast_forward [node] [tnx_id]", + "Fast-forward config change transaction to tnx_id on the given node." + "WARNING: This results in inconsistent configs among the clustered nodes."} + ]. status() -> emqx_ctl:print("-----------------------------------------------\n"), From 83c36a3c9fc39af9d359a20ead7e26812a8ea315 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 8 Jun 2023 18:32:19 +0200 Subject: [PATCH 4/5] refactor(emqx_conf_cli): do not print hidden roots in 'conf show' cmd --- apps/emqx_conf/src/emqx_conf_cli.erl | 46 ++++++++++++++++++++++------ changes/ce/feat-10985.en.md | 2 ++ 2 files changed, 39 insertions(+), 9 deletions(-) create mode 100644 changes/ce/feat-10985.en.md diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index c0ca18100..530e4bfcb 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -22,6 +22,8 @@ unload/0 ]). +-include_lib("hocon/include/hoconsc.hrl"). + %% kept cluster_call for compatibility -define(CLUSTER_CALL, cluster_call). -define(CONF, conf). @@ -34,8 +36,8 @@ unload() -> emqx_ctl:unregister_command(?CLUSTER_CALL), emqx_ctl:unregister_command(?CONF). -conf(["show", "--keys-only"]) -> - print(emqx_config:get_root_names()); +conf(["show_keys" | _]) -> + print_keys(get_config()); conf(["show"]) -> print_hocon(get_config()); conf(["show", Key]) -> @@ -87,9 +89,10 @@ usage_conf() -> [ %% TODO add reload %{"conf reload", "reload etc/emqx.conf on local node"}, - {"conf show --keys-only", "Print all config keys"}, - {"conf show", "Print config in use"}, - {"conf show ", "Print configs under the given key"}, + {"conf show_keys", "Print all config keys"}, + {"conf show []", + "Print in-use configs (including default values) under the given key. " + "Print ALL keys if key is not provided"}, {"conf load ", "Load a HOCON format config file." "The config is overlay on top of the existing configs. " @@ -100,13 +103,13 @@ usage_conf() -> usage_sync() -> [ - {"conf cluster_sync tatus", "Show cluster config sync status summary"}, + {"conf cluster_sync status", "Show cluster config sync status summary"}, {"conf cluster_sync skip [node]", "Increase one commit on specific node"}, {"conf cluster_sync tnxid ", "Display detailed information of the config change transaction at TnxId"}, {"conf cluster_sync fast_forward [node] [tnx_id]", "Fast-forward config change transaction to tnx_id on the given node." - "WARNING: This results in inconsistent configs among the clustered nodes."} + "WARNING: This results in inconsistent configs among the clustered nodes."} ]. status() -> @@ -129,14 +132,39 @@ status() -> ), emqx_ctl:print("-----------------------------------------------\n"). +print_keys(Config) -> + print(lists:sort(maps:keys(Config))). + print(Json) -> emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Json)]). print_hocon(Hocon) -> emqx_ctl:print("~ts~n", [hocon_pp:do(Hocon, #{})]). -get_config() -> emqx_config:fill_defaults(emqx:get_raw_config([])). -get_config(Key) -> emqx_config:fill_defaults(#{Key => emqx:get_raw_config([Key])}). +get_config() -> + drop_hidden_roots(emqx_config:fill_defaults(emqx:get_raw_config([]))). + +drop_hidden_roots(Conf) -> + Hidden = hidden_roots(), + maps:without(Hidden, Conf). + +hidden_roots() -> + SchemaModule = emqx_conf:schema_module(), + Roots = hocon_schema:roots(SchemaModule), + lists:filtermap( + fun({BinName, {_RefName, Schema}}) -> + case hocon_schema:field_schema(Schema, importance) =/= ?IMPORTANCE_HIDDEN of + true -> + false; + false -> + {true, BinName} + end + end, + Roots + ). + +get_config(Key) -> + emqx_config:fill_defaults(#{Key => emqx:get_raw_config([Key])}). -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}). load_config(Path) -> diff --git a/changes/ce/feat-10985.en.md b/changes/ce/feat-10985.en.md new file mode 100644 index 000000000..89c0838a9 --- /dev/null +++ b/changes/ce/feat-10985.en.md @@ -0,0 +1,2 @@ +Renamed emqx ctl command 'cluster_call' to 'conf cluster_sync'. +The old command 'cluster_call' is still a valid command, but not included in usage info. From c9cc06b6b19f6e7a0bc6826580bcf9638b4541fd Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 9 Jun 2023 09:31:14 +0800 Subject: [PATCH 5/5] chore: compile failed --- apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl index dd340d15e..b178f77e0 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl @@ -105,7 +105,7 @@ fields("put_api_v2") -> fields("get_api_v1") -> method_fields(get, influxdb_api_v1); fields("get_api_v2") -> - method_fieds(get, influxdb_api_v2); + method_fields(get, influxdb_api_v2); fields(Type) when Type == influxdb_api_v1 orelse Type == influxdb_api_v2 ->