Merge pull request #10460 from zmstone/0420-sync-release-50-to-masteer

0420 sync release 50 to masteer

apps/emqx/src/emqx_schema.erl

@@ -1633,7 +1633,9 @@ fields("sysmon") ->
         {"top",
             sc(
                 ref("sysmon_top"),
-                #{}
+                %% Userful monitoring solution when benchmarking,
+                %% but hardly common enough for regular users.
+                #{importance => ?IMPORTANCE_HIDDEN}
             )}
     ];
 fields("sysmon_vm") ->

changes/ee/fix-10438.en.md

@@ -0,0 +1,5 @@
+Fix some configuration item terminology errors in the DynamoDB data bridge:
+
+- Changed `database` to `table`
+- Changed `username` to `aws_access_key_id`
+- Changed `password` to `aws_secret_access_key`

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src

@@ -1,6 +1,6 @@
 {application, emqx_ee_bridge, [
     {description, "EMQX Enterprise data bridges"},
-    {vsn, "0.1.10"},
+    {vsn, "0.1.11"},
     {registered, []},
     {applications, [
         kernel,

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl

@@ -43,10 +43,10 @@ values(_Method) ->
         type => dynamo,
         name => <<"foo">>,
         url => <<"http://127.0.0.1:8000">>,
-        database => <<"mqtt">>,
+        table => <<"mqtt">>,
         pool_size => 8,
-        username => <<"root">>,
-        password => <<"******">>,
+        aws_access_key_id => <<"root">>,
+        aws_secret_access_key => <<"******">>,
         template => ?DEFAULT_TEMPLATE,
         local_topic => <<"local/topic/#">>,
         resource_opts => #{

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl

@@ -14,8 +14,8 @@
 % DB defaults
 -define(TABLE, "mqtt").
 -define(TABLE_BIN, to_bin(?TABLE)).
--define(USERNAME, "root").
--define(PASSWORD, "public").
+-define(ACCESS_KEY_ID, "root").
+-define(SECRET_ACCESS_KEY, "public").
 -define(HOST, "dynamo").
 -define(PORT, 8000).
 -define(SCHEMA, "http://").
@@ -158,9 +158,9 @@ dynamo_config(BridgeType, Config) ->
             "bridges.~s.~s {\n"
             "  enable = true\n"
             "  url = ~p\n"
-            "  database = ~p\n"
-            "  username = ~p\n"
-            "  password = ~p\n"
+            "  table = ~p\n"
+            "  aws_access_key_id = ~p\n"
+            "  aws_secret_access_key = ~p\n"
             "  resource_opts = {\n"
             "    request_timeout = 500ms\n"
             "    batch_size = ~b\n"
@@ -172,8 +172,8 @@ dynamo_config(BridgeType, Config) ->
                 Name,
                 Url,
                 ?TABLE,
-                ?USERNAME,
-                ?PASSWORD,
+                ?ACCESS_KEY_ID,
+                ?SECRET_ACCESS_KEY,
                 BatchSize,
                 QueryMode
             ]
@@ -244,10 +244,10 @@ delete_table(_Config) ->
 setup_dynamo(Config) ->
     Host = ?GET_CONFIG(host, Config),
     Port = ?GET_CONFIG(port, Config),
-    erlcloud_ddb2:configure(?USERNAME, ?PASSWORD, Host, Port, ?SCHEMA).
+    erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, Host, Port, ?SCHEMA).
 
 directly_setup_dynamo() ->
-    erlcloud_ddb2:configure(?USERNAME, ?PASSWORD, ?HOST, ?PORT, ?SCHEMA).
+    erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, ?HOST, ?PORT, ?SCHEMA).
 
 directly_query(Query) ->
     directly_setup_dynamo(),

lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl

@@ -51,29 +51,22 @@ roots() ->
 
 fields(config) ->
     [
-        {url, mk(binary(), #{required => true, desc => ?DESC("url")})}
-        | add_default_username(
-            emqx_connector_schema_lib:relational_db_fields()
-        )
+        {url, mk(binary(), #{required => true, desc => ?DESC("url")})},
+        {table, mk(binary(), #{required => true, desc => ?DESC("table")})},
+        {aws_access_key_id,
+            mk(
+                binary(),
+                #{required => true, desc => ?DESC("aws_access_key_id")}
+            )},
+        {aws_secret_access_key,
+            mk(
+                binary(),
+                #{required => true, desc => ?DESC("aws_secret_access_key")}
+            )},
+        {pool_size, fun emqx_connector_schema_lib:pool_size/1},
+        {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1}
     ].
 
-add_default_username(Fields) ->
-    lists:map(
-        fun
-            ({username, OrigUsernameFn}) ->
-                {username, add_default_fn(OrigUsernameFn, <<"root">>)};
-            (Field) ->
-                Field
-        end,
-        Fields
-    ).
-
-add_default_fn(OrigFn, Default) ->
-    fun
-        (default) -> Default;
-        (Field) -> OrigFn(Field)
-    end.
-
 %%========================================================================================
 %% `emqx_resource' API
 %%========================================================================================
@@ -86,16 +79,16 @@ on_start(
     InstanceId,
     #{
         url := Url,
-        username := Username,
-        password := Password,
-        database := Database,
+        aws_access_key_id := AccessKeyID,
+        aws_secret_access_key := SecretAccessKey,
+        table := Table,
         pool_size := PoolSize
     } = Config
 ) ->
     ?SLOG(info, #{
         msg => "starting_dynamo_connector",
         connector => InstanceId,
-        config => emqx_utils:redact(Config)
+        config => redact(Config)
     }),
 
     {Schema, Server} = get_host_schema(to_str(Url)),
@@ -105,8 +98,8 @@ on_start(
         {config, #{
             host => Host,
             port => Port,
-            username => to_str(Username),
-            password => to_str(Password),
+            aws_access_key_id => to_str(AccessKeyID),
+            aws_secret_access_key => to_str(SecretAccessKey),
             schema => Schema
         }},
         {pool_size, PoolSize}
@@ -115,7 +108,7 @@ on_start(
     Templates = parse_template(Config),
     State = #{
         pool_name => InstanceId,
-        database => Database,
+        table => Table,
         templates => Templates
     },
     case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of
@@ -183,7 +176,7 @@ do_query(
     InstanceId,
     Query,
     ApplyMode,
-    #{pool_name := PoolName, templates := Templates, database := Database} = State
+    #{pool_name := PoolName, templates := Templates, table := Table} = State
 ) ->
     ?TRACE(
         "QUERY",
@@ -192,7 +185,7 @@ do_query(
     ),
     Result = ecpool:pick_and_do(
         PoolName,
-        {?MODULE, worker_do_query, [Database, Query, Templates]},
+        {?MODULE, worker_do_query, [Table, Query, Templates]},
         ApplyMode
     ),
 
@@ -217,43 +210,43 @@ do_query(
             Result
     end.
 
-worker_do_query(_Client, Database, Query0, Templates) ->
+worker_do_query(_Client, Table, Query0, Templates) ->
     try
         Query = apply_template(Query0, Templates),
-        execute(Query, Database)
+        execute(Query, Table)
     catch
         _Type:Reason ->
             {error, {unrecoverable_error, {invalid_request, Reason}}}
     end.
 
 %% some simple query commands for authn/authz or test
-execute({insert_item, Msg}, Database) ->
+execute({insert_item, Msg}, Table) ->
     Item = convert_to_item(Msg),
-    erlcloud_ddb2:put_item(Database, Item);
-execute({delete_item, Key}, Database) ->
-    erlcloud_ddb2:delete_item(Database, Key);
-execute({get_item, Key}, Database) ->
-    erlcloud_ddb2:get_item(Database, Key);
+    erlcloud_ddb2:put_item(Table, Item);
+execute({delete_item, Key}, Table) ->
+    erlcloud_ddb2:delete_item(Table, Key);
+execute({get_item, Key}, Table) ->
+    erlcloud_ddb2:get_item(Table, Key);
 %% commands for data bridge query or batch query
-execute({send_message, Msg}, Database) ->
+execute({send_message, Msg}, Table) ->
     Item = convert_to_item(Msg),
-    erlcloud_ddb2:put_item(Database, Item);
-execute([{put, _} | _] = Msgs, Database) ->
+    erlcloud_ddb2:put_item(Table, Item);
+execute([{put, _} | _] = Msgs, Table) ->
     %% type of batch_write_item argument :: batch_write_item_request_items()
     %% batch_write_item_request_items() :: maybe_list(batch_write_item_request_item())
     %% batch_write_item_request_item() :: {table_name(), list(batch_write_item_request())}
     %% batch_write_item_request() :: {put, item()} | {delete, key()}
-    erlcloud_ddb2:batch_write_item({Database, Msgs}).
+    erlcloud_ddb2:batch_write_item({Table, Msgs}).
 
 connect(Opts) ->
     #{
-        username := Username,
-        password := Password,
+        aws_access_key_id := AccessKeyID,
+        aws_secret_access_key := SecretAccessKey,
         host := Host,
         port := Port,
         schema := Schema
     } = proplists:get_value(config, Opts),
-    erlcloud_ddb2:configure(Username, Password, Host, Port, Schema),
+    erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema),
 
     %% The dynamodb driver uses caller process as its connection process
     %% so at here, the connection process is the ecpool worker self
@@ -338,3 +331,6 @@ convert2binary(Value) when is_map(Value) ->
 
 do_async_reply(Result, {ReplyFun, [Context]}) ->
     ReplyFun(Context, Result).
+
+redact(Data) ->
+    emqx_utils:redact(Data, fun(Any) -> Any =:= aws_secret_access_key end).

rel/i18n/emqx_ee_connector_dynamo.hocon

@@ -1,5 +1,23 @@
 emqx_ee_connector_dynamo {
 
+aws_access_key_id.desc:
+"""Access Key ID for connecting to DynamoDB."""
+
+aws_access_key_id.label:
+"""AWS Access Key ID"""
+
+aws_secret_access_key.desc:
+"""AWS Secret Access Key for connecting to DynamoDB."""
+
+aws_secret_access_key.label:
+"""AWS Secret Access Key"""
+
+table.desc:
+"""DynamoDB Table."""
+
+table.label:
+"""Table """
+
 url.desc:
 """The url of DynamoDB endpoint."""
 

rel/i18n/zh/emqx_ee_connector_dynamo.hocon

@@ -1,5 +1,23 @@
 emqx_ee_connector_dynamo {
 
+aws_access_key_id.desc:
+"""DynamoDB 的访问 ID。"""
+
+aws_access_key_id.label:
+"""连接访问 ID"""
+
+aws_secret_access_key.desc:
+"""DynamoDB 的访问密钥。"""
+
+aws_secret_access_key.label:
+"""连接访问密钥"""
+
+table.desc:
+"""DynamoDB 的表。"""
+
+table.label:
+"""表"""
+
 url.desc:
 """DynamoDB 的地址。"""
 

scripts/split-i18n-files.escript

@@ -27,9 +27,13 @@ add_ebin(Dir) ->
 split_file(Path) ->
     {ok, DescMap} = hocon:load(Path),
     [{Module, Descs}] = maps:to_list(DescMap),
+    try
         ok = split(Path, Module, <<"en">>, Descs),
-    ok = split(Path, Module, <<"zh">>, Descs),
-    ok.
+        ok = split(Path, Module, <<"zh">>, Descs)
+    catch
+        throw : already_done ->
+            ok
+    end.
 
 split(Path, Module, Lang, Fields) when is_map(Fields) ->
     split(Path, Module, Lang, maps:to_list(Fields));
@@ -54,6 +58,8 @@ rename(FilePath, Lang) ->
     BaseName = filename:basename(FilePath),
     filename:join([Dir, Lang, BaseName]).
 
+do_split(_Path, _Name, _Lang, #{<<"desc">> := Desc}) when is_binary(Desc) ->
+    throw(already_done);
 do_split(Path, Name, Lang, #{<<"desc">> := Desc} = D) ->
     try
         Label = maps:get(<<"label">>, D, #{}),