From a30e08d06a4e1217c103913d08f334e548b6edd7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:36:43 +0200 Subject: [PATCH 01/13] refactor: set authn config at 'low' prio instead of 'hidden' --- apps/emqx/src/emqx_schema.erl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8335d69b8..6cece7568 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2762,10 +2762,16 @@ str(S) when is_list(S) -> S. authentication(Which) -> - Desc = + {Importance, Desc} = case Which of - global -> ?DESC(global_authentication); - listener -> ?DESC(listener_authentication) + global -> + %% For root level authentication, it is recommended to configure + %% from the dashboard or API. + %% Hence it's considered a low-importance when it comes to + %% configuration importance. + {?IMPORTANCE_LOW, ?DESC(global_authentication)}; + listener -> + {?IMPORTANCE_HIDDEN, ?DESC(listener_authentication)} end, %% poor man's dependency injection %% this is due to the fact that authn is implemented outside of 'emqx' app. @@ -2781,7 +2787,7 @@ authentication(Which) -> hoconsc:mk(Type, #{ desc => Desc, converter => fun ensure_array/2, - importance => ?IMPORTANCE_HIDDEN + importance => Importance }). %% the older version schema allows individual element (instead of a chain) in config From 6bc33e86be0520b641cc0d3d26c4058dd483c9d7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:37:36 +0200 Subject: [PATCH 02/13] refactor: set rpc config at 'low' importance level instead of 'hidden' --- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 362fc1587..7aeec68f0 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -100,7 +100,7 @@ roots() -> ?R_REF("rpc"), #{ translate_to => ["gen_rpc"], - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_LOW } )} ] ++ From 4d67312baba61c2603f0b5237843247c56df16fa Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:40:32 +0200 Subject: [PATCH 03/13] refactor: set authz config at 'high' importance level and authorization.sources at 'low' level prior to this commit, the root was set to 'hidden' which is not ideal because some may still want to configure the sources from files --- apps/emqx_authz/src/emqx_authz_schema.erl | 5 ++++- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index f03ae52a8..39bbcc360 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -494,7 +494,10 @@ authz_fields() -> default => [], desc => ?DESC(sources), %% doc_lift is force a root level reference instead of nesting sub-structs - extra => #{doc_lift => true} + extra => #{doc_lift => true}, + %% it is recommended to configure authz sources from dashboard + %% hance the importance level for config is low + importance => ?IMPORTANCE_LOW } )} ]. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 7aeec68f0..8e5e6937f 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1288,7 +1288,7 @@ emqx_schema_high_prio_roots() -> ?R_REF("authorization"), #{ desc => ?DESC(authorization), - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_HIGH } )}, lists:keyreplace("authorization", 1, Roots, Authz). From d51cc750dec275460e26cc7ec3a06eb66be05858 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:44:31 +0200 Subject: [PATCH 04/13] refactor: hide psk user_lookup_fun --- apps/emqx/src/emqx_schema.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 6cece7568..f264b3e8a 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2235,6 +2235,7 @@ common_ssl_opts_schema(Defaults) -> #{ default => <<"emqx_tls_psk:lookup">>, converter => fun ?MODULE:user_lookup_fun_tr/2, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(common_ssl_opts_schema_user_lookup_fun) } )}, From 30fd9b10f78db09afe9c0f41a631ddec7a7ef5e0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:45:37 +0200 Subject: [PATCH 05/13] refactor: change bridges config importance level from hidden to low --- apps/emqx_bridge/src/schema/emqx_bridge_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index e5def2d64..4b9b7e3fe 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -137,7 +137,7 @@ namespace() -> "bridge". tags() -> [<<"Bridge">>]. -roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_HIDDEN})}]. +roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_LOW})}]. fields(bridges) -> [ From be70c7d3850e8a85de8d056f888e7ddbf5116d4b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:47:16 +0200 Subject: [PATCH 06/13] refactor: change rule_engine config importance level from hidden to low --- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 242c86c71..bc8cae07a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -38,7 +38,7 @@ namespace() -> rule_engine. tags() -> [<<"Rule Engine">>]. -roots() -> [{"rule_engine", ?HOCON(?R_REF("rule_engine"), #{importance => ?IMPORTANCE_HIDDEN})}]. +roots() -> [{"rule_engine", ?HOCON(?R_REF("rule_engine"), #{importance => ?IMPORTANCE_LOW})}]. fields("rule_engine") -> rule_engine_settings() ++ From 12e549a3a20818af2dd750fb42bbe8a6f17638c4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:48:14 +0200 Subject: [PATCH 07/13] refactor: change exhook config importance level from hidden to low --- apps/emqx_exhook/src/emqx_exhook_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 708e164fc..f6cc896f3 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -32,7 +32,7 @@ namespace() -> exhook. roots() -> - [{exhook, ?HOCON(?R_REF(exhook), #{importance => ?IMPORTANCE_HIDDEN})}]. + [{exhook, ?HOCON(?R_REF(exhook), #{importance => ?IMPORTANCE_LOW})}]. fields(exhook) -> [ From f361870ca7cd5165f644970aed671e781fbfa961 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:53:49 +0200 Subject: [PATCH 08/13] refactor: hide sysmon.top config --- apps/emqx/src/emqx_schema.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index f264b3e8a..b3c5e1778 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1633,7 +1633,9 @@ fields("sysmon") -> {"top", sc( ref("sysmon_top"), - #{} + %% Userful monitoring solution when benchmarking, + %% but hardly common enough for regular users. + #{importance => ?IMPORTANCE_HIDDEN} )} ]; fields("sysmon_vm") -> From 3e72d6c4a35d14fda9b6e260350e62a86f05f099 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 19:03:27 +0200 Subject: [PATCH 09/13] docs: sync changelogs --- changes/ce/feat-10391.en.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/changes/ce/feat-10391.en.md b/changes/ce/feat-10391.en.md index a64b01221..f33757404 100644 --- a/changes/ce/feat-10391.en.md +++ b/changes/ce/feat-10391.en.md @@ -1 +1,7 @@ -hide exhook/rewrite/topic_metric/persistent_session_store/overload_protection from the docs and configuration file. +Hide a large number of advanced options to simplify the configuration file. + +That includes `rewrite`, `topic_metric`, `persistent_session_store`, `overload_protection`, +`flapping_detect`, `conn_congestion`, `stats,auto_subscribe`, `broker_perf`, +`shared_subscription_group`, `slow_subs`, `ssl_options.user_lookup_fun` and some advance items +in `node` and `dashboard` section, [#10358](https://github.com/emqx/emqx/pull/10358), +[#10381](https://github.com/emqx/emqx/pull/10381), [#10385](https://github.com/emqx/emqx/pull/10385). From 6c352fd525ff6b45b65d433e13cd8bf1bc7d4c8f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:23:21 +0200 Subject: [PATCH 10/13] docs: update config docs --- rel/emqx_conf.template.en.md | 22 ++++++++++++++-------- rel/emqx_conf.template.zh.md | 16 ++++++++++------ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index 8740e4319..bf5120046 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -7,21 +7,27 @@ and a superset of JSON. EMQX configuration consists of two layers. From bottom up: -1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.
- Changes in this layer require a full node restart to take effect. -1. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` +1. Cluster-synced configs: `$EMQX_NODE__DATA_DIR/configs/cluster.hocon`. +2. Local node configs: `emqx.conf` + `EMQX_` prefixed environment variables. + +:::tip Tip +Prior to v5.0.23 and e5.0.3, the cluster-synced configs are stored in +`cluster-override.conf` which is applied on top of the local configs. + +If upgraded from an earlier version, as long as `cluster-override.conf` exists, +`cluster.hocon` will not be created, and `cluster-override.conf` will stay on +top of the overriding layers. +::: When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir` is used. -The `cluster-override.conf` file is overwritten at runtime when changes -are made from dashboard UI, management HTTP API, or CLI. When clustered, +The `cluster.hocon` file is overwritten at runtime when changes +are made from Dashboard, management HTTP API, or CLI. When clustered, after EMQX restarts, it copies the file from the node which has the greatest `uptime`. :::tip Tip -Some of the configs (such as `node.name`) are boot-only configs and not overridable. -Config values from `*-override.conf` are **not** mapped to boot configs for -the config fields attributed with `mapping: path.to.boot.config.key` +To avoid confusion, don't add the same keys in both `cluster.hocon` and `emqx.conf`. ::: For detailed override rules, see [Config Overlay Rules](#config-overlay-rules). diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index 9402760a2..a96c9ee48 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -5,9 +5,15 @@ HOCON（Human-Optimized Config Object Notation）是一个JSON的超集，非常 EMQX的配置文件可分为二层，自底向上依次是： -1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。
- 修改这一层的配置之后，需要重启节点来使之生效。 -1. 集群范围重载层：`$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` +1. 集群同步配置：`$EMQX_NODE__DATA_DIR/configs/cluster.hocon`。 +2. 本地节点配置：`emqx.conf` 加上 `EMQX_` 前缀的环境变量。 + +:::tip Tip +在 v5.0.23 或 e5.0.3 之前，集群同步配置保存在文件 `cluster-override.conf` 中，并且它覆盖在配置的最上层。 + +如果从之前的版本升级上来，只要 `cluster-override.conf` 文件存在， +EMQX 就不会创建 `cluster.hocon`，并且 `cluster-override.conf` 会继续覆盖在配置的最上层。 +::: 如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置，那么该目录会从 `emqx.conf` 的 `node.data_dir` 配置中读取。 @@ -16,9 +22,7 @@ EMQX的配置文件可分为二层，自底向上依次是： 当EMQX运行在集群中时，一个EMQX节点重启之后，会从集群中其他节点复制该文件内容到本地。 :::tip Tip -有些配置项是不能被重载的（例如 `node.name`）。 -配置项如果有 `mapping: path.to.boot.config.key` 这个属性， -则不能被添加到重载文件 `*-override.conf` 中。 +为避免歧义，应尽量避免让 `cluster.hocon` 和 `emqx.conf` 出现配置交集。 ::: 更多的重载规则，请参考下文 [配置重载规则](#配置重载规则)。 From bc353b0a06e39e574866cfa1447d4fa54be6318f Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 19 Apr 2023 15:56:29 +0800 Subject: [PATCH 11/13] fix(dynamo): change `database` to `table` in the schema of the DynamoDB bridge there is no term like `database` in DynamoDB, the correct concept should be `table` --- .../src/emqx_ee_bridge_dynamo.erl | 2 +- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 2 +- .../src/emqx_ee_connector_dynamo.erl | 52 ++++++++++--------- rel/i18n/emqx_ee_connector_dynamo.hocon | 6 +++ 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl index ba1fd0c70..3157db18e 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl @@ -43,7 +43,7 @@ values(_Method) -> type => dynamo, name => <<"foo">>, url => <<"http://127.0.0.1:8000">>, - database => <<"mqtt">>, + table => <<"mqtt">>, pool_size => 8, username => <<"root">>, password => <<"******">>, diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 5ebd9a89d..a63a69052 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -158,7 +158,7 @@ dynamo_config(BridgeType, Config) -> "bridges.~s.~s {\n" " enable = true\n" " url = ~p\n" - " database = ~p\n" + " table = ~p\n" " username = ~p\n" " password = ~p\n" " resource_opts = {\n" diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 85daefbb7..1e7203279 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -51,20 +51,24 @@ roots() -> fields(config) -> [ - {url, mk(binary(), #{required => true, desc => ?DESC("url")})} - | add_default_username( + {url, mk(binary(), #{required => true, desc => ?DESC("url")})}, + {table, mk(binary(), #{required => true, desc => ?DESC("table")})} + | override_schemas( emqx_connector_schema_lib:relational_db_fields() ) ]. -add_default_username(Fields) -> - lists:map( +override_schemas(Fields) -> + lists:foldr( fun - ({username, OrigUsernameFn}) -> - {username, add_default_fn(OrigUsernameFn, <<"root">>)}; - (Field) -> - Field + ({username, OrigUsernameFn}, Acc) -> + [{username, add_default_fn(OrigUsernameFn, <<"root">>)} | Acc]; + ({database, _}, Acc) -> + Acc; + (Field, Acc) -> + [Field | Acc] end, + [], Fields ). @@ -88,7 +92,7 @@ on_start( url := Url, username := Username, password := Password, - database := Database, + table := Table, pool_size := PoolSize } = Config ) -> @@ -115,7 +119,7 @@ on_start( Templates = parse_template(Config), State = #{ poolname => InstanceId, - database => Database, + table => Table, templates => Templates }, case emqx_plugin_libs_pool:start_pool(InstanceId, ?MODULE, Options) of @@ -183,7 +187,7 @@ do_query( InstanceId, Query, ApplyMode, - #{poolname := PoolName, templates := Templates, database := Database} = State + #{poolname := PoolName, templates := Templates, table := Table} = State ) -> ?TRACE( "QUERY", @@ -192,7 +196,7 @@ do_query( ), Result = ecpool:pick_and_do( PoolName, - {?MODULE, worker_do_query, [Database, Query, Templates]}, + {?MODULE, worker_do_query, [Table, Query, Templates]}, ApplyMode ), @@ -217,33 +221,33 @@ do_query( Result end. -worker_do_query(_Client, Database, Query0, Templates) -> +worker_do_query(_Client, Table, Query0, Templates) -> try Query = apply_template(Query0, Templates), - execute(Query, Database) + execute(Query, Table) catch _Type:Reason -> {error, {unrecoverable_error, {invalid_request, Reason}}} end. %% some simple query commands for authn/authz or test -execute({insert_item, Msg}, Database) -> +execute({insert_item, Msg}, Table) -> Item = convert_to_item(Msg), - erlcloud_ddb2:put_item(Database, Item); -execute({delete_item, Key}, Database) -> - erlcloud_ddb2:delete_item(Database, Key); -execute({get_item, Key}, Database) -> - erlcloud_ddb2:get_item(Database, Key); + erlcloud_ddb2:put_item(Table, Item); +execute({delete_item, Key}, Table) -> + erlcloud_ddb2:delete_item(Table, Key); +execute({get_item, Key}, Table) -> + erlcloud_ddb2:get_item(Table, Key); %% commands for data bridge query or batch query -execute({send_message, Msg}, Database) -> +execute({send_message, Msg}, Table) -> Item = convert_to_item(Msg), - erlcloud_ddb2:put_item(Database, Item); -execute([{put, _} | _] = Msgs, Database) -> + erlcloud_ddb2:put_item(Table, Item); +execute([{put, _} | _] = Msgs, Table) -> %% type of batch_write_item argument :: batch_write_item_request_items() %% batch_write_item_request_items() :: maybe_list(batch_write_item_request_item()) %% batch_write_item_request_item() :: {table_name(), list(batch_write_item_request())} %% batch_write_item_request() :: {put, item()} | {delete, key()} - erlcloud_ddb2:batch_write_item({Database, Msgs}). + erlcloud_ddb2:batch_write_item({Table, Msgs}). connect(Opts) -> #{ diff --git a/rel/i18n/emqx_ee_connector_dynamo.hocon b/rel/i18n/emqx_ee_connector_dynamo.hocon index 295929a72..6d3c69be4 100644 --- a/rel/i18n/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/emqx_ee_connector_dynamo.hocon @@ -11,4 +11,10 @@ emqx_ee_connector_dynamo { } } +table.desc: +"""DynamoDB Table.""" + +table.label: +"""DynamoDB Table""" + } From b112f544a98e4c3b77d52b9ff444d679688b32b2 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 18 Apr 2023 19:00:35 +0800 Subject: [PATCH 12/13] chore: update changes --- changes/ee/fix-10438.en.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ee/fix-10438.en.md diff --git a/changes/ee/fix-10438.en.md b/changes/ee/fix-10438.en.md new file mode 100644 index 000000000..859a35499 --- /dev/null +++ b/changes/ee/fix-10438.en.md @@ -0,0 +1,3 @@ +Fix a configuration item name error in the DynamoDB data bridge. + +Changed `database` to `table`, because there is no term like `database` in DynamoDB, the correct concept should be `table` From e89f4d45658f33865deea046d352402661d7508d Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 19 Apr 2023 15:56:46 +0800 Subject: [PATCH 13/13] fix(dynamo): fix terminology erros - Changed `username` to `aws_access_key_id` - Changed `password` to `aws_secret_access_key` --- changes/ee/fix-10438.en.md | 6 +- .../src/emqx_ee_bridge_dynamo.erl | 4 +- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 16 +++--- .../src/emqx_ee_connector_dynamo.erl | 56 ++++++++----------- rel/i18n/emqx_ee_connector_dynamo.hocon | 34 +++++++++-- 5 files changed, 68 insertions(+), 48 deletions(-) diff --git a/changes/ee/fix-10438.en.md b/changes/ee/fix-10438.en.md index 859a35499..6394bc3cf 100644 --- a/changes/ee/fix-10438.en.md +++ b/changes/ee/fix-10438.en.md @@ -1,3 +1,5 @@ -Fix a configuration item name error in the DynamoDB data bridge. +Fix some configuration item terminology errors in the DynamoDB data bridge: -Changed `database` to `table`, because there is no term like `database` in DynamoDB, the correct concept should be `table` +- Changed `database` to `table` +- Changed `username` to `aws_access_key_id` +- Changed `password` to `aws_secret_access_key` diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl index 3157db18e..cbfa5b6b1 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl @@ -45,8 +45,8 @@ values(_Method) -> url => <<"http://127.0.0.1:8000">>, table => <<"mqtt">>, pool_size => 8, - username => <<"root">>, - password => <<"******">>, + aws_access_key_id => <<"root">>, + aws_secret_access_key => <<"******">>, template => ?DEFAULT_TEMPLATE, local_topic => <<"local/topic/#">>, resource_opts => #{ diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index a63a69052..9cf7eb8f4 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -14,8 +14,8 @@ % DB defaults -define(TABLE, "mqtt"). -define(TABLE_BIN, to_bin(?TABLE)). --define(USERNAME, "root"). --define(PASSWORD, "public"). +-define(ACCESS_KEY_ID, "root"). +-define(SECRET_ACCESS_KEY, "public"). -define(HOST, "dynamo"). -define(PORT, 8000). -define(SCHEMA, "http://"). @@ -159,8 +159,8 @@ dynamo_config(BridgeType, Config) -> " enable = true\n" " url = ~p\n" " table = ~p\n" - " username = ~p\n" - " password = ~p\n" + " aws_access_key_id = ~p\n" + " aws_secret_access_key = ~p\n" " resource_opts = {\n" " request_timeout = 500ms\n" " batch_size = ~b\n" @@ -172,8 +172,8 @@ dynamo_config(BridgeType, Config) -> Name, Url, ?TABLE, - ?USERNAME, - ?PASSWORD, + ?ACCESS_KEY_ID, + ?SECRET_ACCESS_KEY, BatchSize, QueryMode ] @@ -244,10 +244,10 @@ delete_table(_Config) -> setup_dynamo(Config) -> Host = ?GET_CONFIG(host, Config), Port = ?GET_CONFIG(port, Config), - erlcloud_ddb2:configure(?USERNAME, ?PASSWORD, Host, Port, ?SCHEMA). + erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, Host, Port, ?SCHEMA). directly_setup_dynamo() -> - erlcloud_ddb2:configure(?USERNAME, ?PASSWORD, ?HOST, ?PORT, ?SCHEMA). + erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, ?HOST, ?PORT, ?SCHEMA). directly_query(Query) -> directly_setup_dynamo(), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 1e7203279..9a149b6f7 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -52,32 +52,21 @@ roots() -> fields(config) -> [ {url, mk(binary(), #{required => true, desc => ?DESC("url")})}, - {table, mk(binary(), #{required => true, desc => ?DESC("table")})} - | override_schemas( - emqx_connector_schema_lib:relational_db_fields() - ) + {table, mk(binary(), #{required => true, desc => ?DESC("table")})}, + {aws_access_key_id, + mk( + binary(), + #{required => true, desc => ?DESC("aws_access_key_id")} + )}, + {aws_secret_access_key, + mk( + binary(), + #{required => true, desc => ?DESC("aws_secret_access_key")} + )}, + {pool_size, fun emqx_connector_schema_lib:pool_size/1}, + {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} ]. -override_schemas(Fields) -> - lists:foldr( - fun - ({username, OrigUsernameFn}, Acc) -> - [{username, add_default_fn(OrigUsernameFn, <<"root">>)} | Acc]; - ({database, _}, Acc) -> - Acc; - (Field, Acc) -> - [Field | Acc] - end, - [], - Fields - ). - -add_default_fn(OrigFn, Default) -> - fun - (default) -> Default; - (Field) -> OrigFn(Field) - end. - %%======================================================================================== %% `emqx_resource' API %%======================================================================================== @@ -90,8 +79,8 @@ on_start( InstanceId, #{ url := Url, - username := Username, - password := Password, + aws_access_key_id := AccessKeyID, + aws_secret_access_key := SecretAccessKey, table := Table, pool_size := PoolSize } = Config @@ -99,7 +88,7 @@ on_start( ?SLOG(info, #{ msg => "starting_dynamo_connector", connector => InstanceId, - config => emqx_utils:redact(Config) + config => redact(Config) }), {Schema, Server} = get_host_schema(to_str(Url)), @@ -109,8 +98,8 @@ on_start( {config, #{ host => Host, port => Port, - username => to_str(Username), - password => to_str(Password), + aws_access_key_id => to_str(AccessKeyID), + aws_secret_access_key => to_str(SecretAccessKey), schema => Schema }}, {pool_size, PoolSize} @@ -251,13 +240,13 @@ execute([{put, _} | _] = Msgs, Table) -> connect(Opts) -> #{ - username := Username, - password := Password, + aws_access_key_id := AccessKeyID, + aws_secret_access_key := SecretAccessKey, host := Host, port := Port, schema := Schema } = proplists:get_value(config, Opts), - erlcloud_ddb2:configure(Username, Password, Host, Port, Schema), + erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema), %% The dynamodb driver uses caller process as its connection process %% so at here, the connection process is the ecpool worker self @@ -342,3 +331,6 @@ convert2binary(Value) when is_map(Value) -> do_async_reply(Result, {ReplyFun, [Context]}) -> ReplyFun(Context, Result). + +redact(Data) -> + emqx_utils:redact(Data, fun(Any) -> Any =:= aws_secret_access_key end). diff --git a/rel/i18n/emqx_ee_connector_dynamo.hocon b/rel/i18n/emqx_ee_connector_dynamo.hocon index 6d3c69be4..a1f3f7b02 100644 --- a/rel/i18n/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/emqx_ee_connector_dynamo.hocon @@ -11,10 +11,36 @@ emqx_ee_connector_dynamo { } } -table.desc: -"""DynamoDB Table.""" + table { + desc { + en: """DynamoDB Table.""" + zh: """DynamoDB 的表。""" + } + label: { + en: "Table " + zh: "表" + } + } -table.label: -"""DynamoDB Table""" + aws_access_key_id { + desc { + en: """Access Key ID for connecting to DynamoDB.""" + zh: """DynamoDB 的访问 ID。""" + } + label: { + en: "AWS Access Key ID" + zh: "连接访问 ID" + } + } + aws_secret_access_key { + desc { + en: """AWS Secret Access Key for connecting to DynamoDB.""" + zh: """DynamoDB 的访问密钥。""" + } + label: { + en: "AWS Secret Access Key" + zh: "连接访问密钥" + } + } }