yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/release-50' into 0420-sync-release-50-to-masteer

This commit is contained in:
Zaiming (Stone) Shi 2023-04-20 11:03:39 +02:00
parent 7d3367467a c2385f85d6
commit 415830a0a2
9 changed files with 97 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/emqx/src/emqx_schema.erl
View File

@ -1633,7 +1633,9 @@ fields("sysmon") ->
{"top",
sc(
ref("sysmon_top"),
#{}
%% Userful monitoring solution when benchmarking,
%% but hardly common enough for regular users.
#{importance => ?IMPORTANCE_HIDDEN}
)}
];
fields("sysmon_vm") ->

2
apps/emqx_rule_engine/src/emqx_rule_engine.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_rule_engine, [
{description, "EMQX Rule Engine"},
% strict semver, bump manually!
{vsn, "5.0.14"},
{vsn, "5.0.15"},
{modules, []},
{registered, [emqx_rule_engine_sup, emqx_rule_engine]},
{applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]},

5
changes/ee/fix-10438.en.md Normal file
View File

@ -0,0 +1,5 @@
Fix some configuration item terminology errors in the DynamoDB data bridge:
- Changed `database` to `table`
- Changed `username` to `aws_access_key_id`
- Changed `password` to `aws_secret_access_key`

2
lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_ee_bridge, [
{description, "EMQX Enterprise data bridges"},
{vsn, "0.1.10"},
{vsn, "0.1.11"},
{registered, []},
{applications, [
kernel,

6
lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl
View File

@ -43,10 +43,10 @@ values(_Method) ->
type => dynamo,
name => <<"foo">>,
url => <<"http://127.0.0.1:8000">>,
database => <<"mqtt">>,
table => <<"mqtt">>,
pool_size => 8,
username => <<"root">>,
password => <<"******">>,
aws_access_key_id => <<"root">>,
aws_secret_access_key => <<"******">>,
template => ?DEFAULT_TEMPLATE,
local_topic => <<"local/topic/#">>,
resource_opts => #{

18
lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl
View File

@ -14,8 +14,8 @@
% DB defaults
-define(TABLE, "mqtt").
-define(TABLE_BIN, to_bin(?TABLE)).
-define(USERNAME, "root").
-define(PASSWORD, "public").
-define(ACCESS_KEY_ID, "root").
-define(SECRET_ACCESS_KEY, "public").
-define(HOST, "dynamo").
-define(PORT, 8000).
-define(SCHEMA, "http://").
@ -158,9 +158,9 @@ dynamo_config(BridgeType, Config) ->
"bridges.~s.~s {\n"
" enable = true\n"
" url = ~p\n"
" database = ~p\n"
" username = ~p\n"
" password = ~p\n"
" table = ~p\n"
" aws_access_key_id = ~p\n"
" aws_secret_access_key = ~p\n"
" resource_opts = {\n"
" request_timeout = 500ms\n"
" batch_size = ~b\n"
@ -172,8 +172,8 @@ dynamo_config(BridgeType, Config) ->
Name,
Url,
?TABLE,
?USERNAME,
?PASSWORD,
?ACCESS_KEY_ID,
?SECRET_ACCESS_KEY,
BatchSize,
QueryMode
]
@ -244,10 +244,10 @@ delete_table(_Config) ->
setup_dynamo(Config) ->
Host = ?GET_CONFIG(host, Config),
Port = ?GET_CONFIG(port, Config),
erlcloud_ddb2:configure(?USERNAME, ?PASSWORD, Host, Port, ?SCHEMA).
erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, Host, Port, ?SCHEMA).
directly_setup_dynamo() ->
erlcloud_ddb2:configure(?USERNAME, ?PASSWORD, ?HOST, ?PORT, ?SCHEMA).
erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, ?HOST, ?PORT, ?SCHEMA).
directly_query(Query) ->
directly_setup_dynamo(),

2
lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_ee_connector, [
{description, "EMQX Enterprise connectors"},
{vsn, "0.1.10"},
{vsn, "0.1.11"},
{registered, []},
{applications, [
kernel,

88
lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl
View File

@ -51,29 +51,22 @@ roots() ->
fields(config) ->
[
{url, mk(binary(), #{required => true, desc => ?DESC("url")})}
| add_default_username(
emqx_connector_schema_lib:relational_db_fields()
)
{url, mk(binary(), #{required => true, desc => ?DESC("url")})},
{table, mk(binary(), #{required => true, desc => ?DESC("table")})},
{aws_access_key_id,
mk(
binary(),
#{required => true, desc => ?DESC("aws_access_key_id")}
)},
{aws_secret_access_key,
mk(
binary(),
#{required => true, desc => ?DESC("aws_secret_access_key")}
)},
{pool_size, fun emqx_connector_schema_lib:pool_size/1},
{auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1}
].
add_default_username(Fields) ->
lists:map(
fun
({username, OrigUsernameFn}) ->
{username, add_default_fn(OrigUsernameFn, <<"root">>)};
(Field) ->
Field
end,
Fields
).
add_default_fn(OrigFn, Default) ->
fun
(default) -> Default;
(Field) -> OrigFn(Field)
end.
%%========================================================================================
%% `emqx_resource' API
%%========================================================================================
@ -86,16 +79,16 @@ on_start(
InstanceId,
#{
url := Url,
username := Username,
password := Password,
database := Database,
aws_access_key_id := AccessKeyID,
aws_secret_access_key := SecretAccessKey,
table := Table,
pool_size := PoolSize
} = Config
) ->
?SLOG(info, #{
msg => "starting_dynamo_connector",
connector => InstanceId,
config => emqx_utils:redact(Config)
config => redact(Config)
}),
{Schema, Server} = get_host_schema(to_str(Url)),
@ -105,8 +98,8 @@ on_start(
{config, #{
host => Host,
port => Port,
username => to_str(Username),
password => to_str(Password),
aws_access_key_id => to_str(AccessKeyID),
aws_secret_access_key => to_str(SecretAccessKey),
schema => Schema
}},
{pool_size, PoolSize}
@ -114,8 +107,8 @@ on_start(
Templates = parse_template(Config),
State = #{
pool_name => InstanceId,
database => Database,
poolname => InstanceId,
table => Table,
templates => Templates
},
case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of
@ -183,7 +176,7 @@ do_query(
InstanceId,
Query,
ApplyMode,
#{pool_name := PoolName, templates := Templates, database := Database} = State
#{poolname := PoolName, templates := Templates, table := Table} = State
) ->
?TRACE(
"QUERY",
@ -192,7 +185,7 @@ do_query(
),
Result = ecpool:pick_and_do(
PoolName,
{?MODULE, worker_do_query, [Database, Query, Templates]},
{?MODULE, worker_do_query, [Table, Query, Templates]},
ApplyMode
),
@ -217,43 +210,43 @@ do_query(
Result
end.
worker_do_query(_Client, Database, Query0, Templates) ->
worker_do_query(_Client, Table, Query0, Templates) ->
try
Query = apply_template(Query0, Templates),
execute(Query, Database)
execute(Query, Table)
catch
_Type:Reason ->
{error, {unrecoverable_error, {invalid_request, Reason}}}
end.
%% some simple query commands for authn/authz or test
execute({insert_item, Msg}, Database) ->
execute({insert_item, Msg}, Table) ->
Item = convert_to_item(Msg),
erlcloud_ddb2:put_item(Database, Item);
execute({delete_item, Key}, Database) ->
erlcloud_ddb2:delete_item(Database, Key);
execute({get_item, Key}, Database) ->
erlcloud_ddb2:get_item(Database, Key);
erlcloud_ddb2:put_item(Table, Item);
execute({delete_item, Key}, Table) ->
erlcloud_ddb2:delete_item(Table, Key);
execute({get_item, Key}, Table) ->
erlcloud_ddb2:get_item(Table, Key);
%% commands for data bridge query or batch query
execute({send_message, Msg}, Database) ->
execute({send_message, Msg}, Table) ->
Item = convert_to_item(Msg),
erlcloud_ddb2:put_item(Database, Item);
execute([{put, _} | _] = Msgs, Database) ->
erlcloud_ddb2:put_item(Table, Item);
execute([{put, _} | _] = Msgs, Table) ->
%% type of batch_write_item argument :: batch_write_item_request_items()
%% batch_write_item_request_items() :: maybe_list(batch_write_item_request_item())
%% batch_write_item_request_item() :: {table_name(), list(batch_write_item_request())}
%% batch_write_item_request() :: {put, item()} | {delete, key()}
erlcloud_ddb2:batch_write_item({Database, Msgs}).
erlcloud_ddb2:batch_write_item({Table, Msgs}).
connect(Opts) ->
#{
username := Username,
password := Password,
aws_access_key_id := AccessKeyID,
aws_secret_access_key := SecretAccessKey,
host := Host,
port := Port,
schema := Schema
} = proplists:get_value(config, Opts),
erlcloud_ddb2:configure(Username, Password, Host, Port, Schema),
erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema),
%% The dynamodb driver uses caller process as its connection process
%% so at here, the connection process is the ecpool worker self
@ -338,3 +331,6 @@ convert2binary(Value) when is_map(Value) ->
do_async_reply(Result, {ReplyFun, [Context]}) ->
ReplyFun(Context, Result).
redact(Data) ->
emqx_utils:redact(Data, fun(Any) -> Any =:= aws_secret_access_key end).

32
rel/i18n/emqx_ee_connector_dynamo.hocon
View File

@ -6,4 +6,36 @@ url.desc:
url.label:
"""DynamoDB Endpoint"""
table {
desc {
en: """DynamoDB Table."""
zh: """DynamoDB 的表。"""
}
label: {
en: "Table "
zh: "表"
}
}
aws_access_key_id {
desc {
en: """Access Key ID for connecting to DynamoDB."""
zh: """DynamoDB 的访问 ID。"""
}
label: {
en: "AWS Access Key ID"
zh: "连接访问 ID"
}
}
aws_secret_access_key {
desc {
en: """AWS Secret Access Key for connecting to DynamoDB."""
zh: """DynamoDB 的访问密钥。"""
}
label: {
en: "AWS Secret Access Key"
zh: "连接访问密钥"
}
}
}