yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: move MySQL open source connector to its own app

This commit is contained in:
Kjell Winblad 2023-06-27 15:30:23 +02:00
parent d3a0b3e183
commit 85a130f68b
26 changed files with 69 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/emqx_authn/rebar.config
View File

@ -5,7 +5,8 @@
{emqx_utils, {path, "../emqx_utils"}},
{emqx_connector, {path, "../emqx_connector"}},
{emqx_mongodb, {path, "../emqx_mongodb"}},
{emqx_redis, {path, "../emqx_redis"}}
{emqx_redis, {path, "../emqx_redis"}},
{emqx_mysql, {path, "../emqx_mysql"}}
]}.
{edoc_opts, [{preprocess, true}]}.

3
apps/emqx_authn/src/emqx_authn.app.src
View File

@ -14,7 +14,8 @@
mysql,
jose,
emqx_mongodb,
emqx_redis
emqx_redis,
emqx_mysql
]},
{mod, {emqx_authn_app, []}},
{env, []},

6
apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
View File

@ -62,7 +62,7 @@ fields(mysql) ->
{query, fun query/1},
{query_timeout, fun query_timeout/1}
] ++ emqx_authn_schema:common_fields() ++
proplists:delete(prepare_statement, emqx_connector_mysql:fields(config)).
proplists:delete(prepare_statement, emqx_mysql:fields(config)).
desc(mysql) ->
?DESC(mysql);
@ -92,12 +92,12 @@ create(_AuthenticatorID, Config) ->
create(Config0) ->
ResourceId = emqx_authn_utils:make_resource_id(?MODULE),
{Config, State} = parse_config(Config0),
{ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_connector_mysql, Config),
{ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_mysql, Config),
{ok, State#{resource_id => ResourceId}}.
update(Config0, #{resource_id := ResourceId} = _State) ->
{Config, NState} = parse_config(Config0),
case emqx_authn_utils:update_resource(emqx_connector_mysql, Config, ResourceId) of
case emqx_authn_utils:update_resource(emqx_mysql, Config, ResourceId) of
{error, Reason} ->
error({load_config_error, Reason});
{ok, _} ->

2
apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
View File

@ -62,7 +62,7 @@ init_per_suite(Config) ->
{ok, _} = emqx_resource:create_local(
?MYSQL_RESOURCE,
?RESOURCE_GROUP,
emqx_connector_mysql,
emqx_mysql,
mysql_config(),
#{}
),

3
apps/emqx_authz/rebar.config
View File

@ -6,7 +6,8 @@
{emqx_utils, {path, "../emqx_utils"}},
{emqx_connector, {path, "../emqx_connector"}},
{emqx_mongodb, {path, "../emqx_mongodb"}},
{emqx_redis, {path, "../emqx_redis"}}
{emqx_redis, {path, "../emqx_redis"}},
{emqx_mysql, {path, "../emqx_mysql"}}
]}.
{shell, [

3
apps/emqx_authz/src/emqx_authz.app.src
View File

@ -11,7 +11,8 @@
emqx_resource,
emqx_connector,
emqx_mongodb,
emqx_redis
emqx_redis,
emqx_mysql
]},
{env, []},
{modules, []},

2
apps/emqx_authz/src/emqx_authz_api_schema.erl
View File

@ -70,7 +70,7 @@ fields(mongo_sharded) ->
fields(mysql) ->
authz_common_fields(mysql) ++
[{query, query()}] ++
proplists:delete(prepare_statement, emqx_connector_mysql:fields(config));
proplists:delete(prepare_statement, emqx_mysql:fields(config));
fields(postgresql) ->
authz_common_fields(postgresql) ++
[{query, query()}] ++

4
apps/emqx_authz/src/emqx_authz_mysql.erl
View File

@ -54,13 +54,13 @@ create(#{query := SQL} = Source0) ->
{PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS),
ResourceId = emqx_authz_utils:make_resource_id(?MODULE),
Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}},
{ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_mysql, Source),
{ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source),
Source#{annotations => #{id => ResourceId, tmpl_oken => TmplToken}}.
update(#{query := SQL} = Source0) ->
{PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS),
Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}},
case emqx_authz_utils:update_resource(emqx_connector_mysql, Source) of
case emqx_authz_utils:update_resource(emqx_mysql, Source) of
{error, Reason} ->
error({load_config_error, Reason});
{ok, Id} ->

2
apps/emqx_authz/src/emqx_authz_schema.erl
View File

@ -389,7 +389,7 @@ cmd() ->
connector_fields(DB) ->
connector_fields(DB, config).
connector_fields(redis = DB, Fields) ->
connector_fields(DB, Fields) when DB =:= redis; DB =:= mysql ->
connector_fields(DB, Fields, emqx);
connector_fields(DB, Fields) ->
connector_fields(DB, Fields, emqx_connector).

2
apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
View File

@ -44,7 +44,7 @@ init_per_suite(Config) ->
{ok, _} = emqx_resource:create_local(
?MYSQL_RESOURCE,
?RESOURCE_GROUP,
emqx_connector_mysql,
emqx_mysql,
mysql_config(),
#{}
),

23
apps/emqx_connector/README.md
View File

@ -3,25 +3,4 @@
This application is a collection of `connectors`.
A `connector` is a callback module of `emqx_resource` that maintains the data related to
external resources. Put all resource related callback modules in a single application is good as
we can put some util functions/modules here for reusing purpose.
For example, a MySQL connector is an emqx resource that maintains all the MySQL connection
related parameters (configs) and the TCP connections to the MySQL server.
An MySQL connector can be used as following:
```
(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose().
[#{config =>
#{cacertfile => [],certfile => [],
database => "mqtt",keyfile => [],password => "public",
pool_size => 1,
server => {{127,0,0,1},3306},
ssl => false,user => "root",verify => false},
id => <<"mysql-abc">>,mod => emqx_connector_mysql,
state => #{poolname => 'mysql-abc'},
status => connected}]
(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}).
{ok,[<<"count(1)">>],[[1]]}
```
external resources.

1
apps/emqx_connector/docker-ct
View File

@ -1,2 +1 @@
mysql
pgsql

1
apps/emqx_connector/rebar.config
View File

@ -10,7 +10,6 @@
{emqx_utils, {path, "../emqx_utils"}},
{emqx_resource, {path, "../emqx_resource"}},
{eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
{mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}},
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}}
]}.

2
apps/emqx_connector/src/emqx_connector.app.src
View File

@ -13,8 +13,6 @@
eredis,
epgsql,
eldap2,
mysql,
mongodb,
ehttpc,
jose,
emqx,

15
apps/emqx_mysql/README.md Normal file
View File

@ -0,0 +1,15 @@
# MySQL Connector
This application houses the MySQL Database connector.
It provides the APIs to connect to MySQL Databases.
It is used by the MySQL bridge to insert messages and by the emqx_authz and
emqx_authn applications to check user permissions.
## Contributing
Please see our [contributing.md](../../CONTRIBUTING.md).
## License
See [APL](../../APL.txt).

1
apps/emqx_mysql/docker-ct Normal file
View File

@ -0,0 +1 @@
mysql

9
apps/emqx_mysql/rebar.config Normal file
View File

@ -0,0 +1,9 @@
%% -*- mode: erlang; -*-
{erl_opts, [debug_info]}.
{deps, [
%% NOTE: mind ecpool version when updating eredis_cluster version
{mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}},
{emqx_connector, {path, "../../apps/emqx_connector"}},
{emqx_resource, {path, "../../apps/emqx_resource"}}
]}.

14
apps/emqx_mysql/src/emqx_mysql.app.src Normal file
View File

@ -0,0 +1,14 @@
{application, emqx_mysql, [
{description, "EMQX MySQL Database Connector"},
{vsn, "0.1.0"},
{registered, []},
{applications, [
kernel,
stdlib,
mysql
]},
{env, []},
{modules, []},
{links, []}
]}.

4
apps/emqx_connector/src/emqx_connector_mysql.erl → apps/emqx_mysql/src/emqx_mysql.erl
View File

@ -13,9 +13,9 @@
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_connector_mysql).
-module(emqx_mysql).
-include("emqx_connector.hrl").
-include_lib("emqx_connector/include/emqx_connector.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/logger.hrl").

8
apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl → apps/emqx_mysql/test/emqx_mysql_SUITE.erl
View File

@ -13,18 +13,18 @@
% %% limitations under the License.
% %%--------------------------------------------------------------------
-module(emqx_connector_mysql_SUITE).
-module(emqx_mysql_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-include("emqx_connector.hrl").
-include_lib("emqx_connector/include/emqx_connector.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("stdlib/include/assert.hrl").
-define(MYSQL_HOST, "mysql").
-define(MYSQL_RESOURCE_MOD, emqx_connector_mysql).
-define(MYSQL_RESOURCE_MOD, emqx_mysql).
all() ->
emqx_common_test_helpers:all(?MODULE).
@ -60,7 +60,7 @@ end_per_testcase(_, _Config) ->
t_lifecycle(_Config) ->
perform_lifecycle_check(
<<"emqx_connector_mysql_SUITE">>,
<<"emqx_mysql_SUITE">>,
mysql_config()
).

1
lib-ee/emqx_ee_bridge/rebar.config
View File

@ -3,6 +3,7 @@
{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}}
, {emqx_resource, {path, "../../apps/emqx_resource"}}
, {emqx_bridge, {path, "../../apps/emqx_bridge"}}
, {emqx_bridge, {path, "../../apps/emqx_mysql"}}
, {emqx_utils, {path, "../emqx_utils"}}
]}.

2
lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
View File

@ -99,7 +99,7 @@ resource_type(gcp_pubsub_consumer) -> emqx_bridge_gcp_pubsub_impl_consumer;
resource_type(mongodb_rs) -> emqx_bridge_mongodb_connector;
resource_type(mongodb_sharded) -> emqx_bridge_mongodb_connector;
resource_type(mongodb_single) -> emqx_bridge_mongodb_connector;
resource_type(mysql) -> emqx_connector_mysql;
resource_type(mysql) -> emqx_mysql;
resource_type(influxdb_api_v1) -> emqx_bridge_influxdb_connector;
resource_type(influxdb_api_v2) -> emqx_bridge_influxdb_connector;
resource_type(redis_single) -> emqx_bridge_redis_connector;

2
lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl
View File

@ -80,7 +80,7 @@ fields("config") ->
#{desc => ?DESC("local_topic"), default => undefined}
)}
] ++ emqx_resource_schema:fields("resource_opts") ++
(emqx_connector_mysql:fields(config) --
(emqx_mysql:fields(config) --
emqx_connector_schema_lib:prepare_statement_fields());
fields("post") ->
[type_field(), name_field() | fields("config")];

1
mix.exs
View File

@ -378,6 +378,7 @@ defmodule EMQXUmbrella.MixProject do
emqx_slow_subs: :permanent,
emqx_mongodb: :permanent,
emqx_redis: :permanent,
emqx_mysql: :permanent,
emqx_plugins: :permanent,
emqx_mix: :none
] ++

1
rebar.config.erl
View File

@ -441,6 +441,7 @@ relx_apps(ReleaseType, Edition) ->
emqx_slow_subs,
emqx_mongodb,
emqx_redis,
emqx_mysql,
emqx_plugins
] ++
[quicer || is_quicer_supported()] ++

2
rel/i18n/emqx_connector_mysql.hocon → rel/i18n/emqx_mysql.hocon
View File

@ -1,4 +1,4 @@
emqx_connector_mysql {
emqx_mysql {
server.desc:
"""The IPv4 or IPv6 address or the hostname to connect to.<br/>