Merge pull request #11139 from kjellwinblad/kjell/refactor/redis/EMQX-9534

refactor: redis bridge to its own application
This commit is contained in:
Kjell Winblad 2023-06-27 14:55:55 +02:00 committed by GitHub
commit d3a0b3e183
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 129 additions and 59 deletions

View File

@ -4,7 +4,8 @@
{emqx, {path, "../emqx"}},
{emqx_utils, {path, "../emqx_utils"}},
{emqx_connector, {path, "../emqx_connector"}},
{emqx_mongodb, {path, "../emqx_mongodb"}}
{emqx_mongodb, {path, "../emqx_mongodb"}},
{emqx_redis, {path, "../emqx_redis"}}
]}.
{edoc_opts, [{preprocess, true}]}.

View File

@ -5,7 +5,16 @@
{modules, []},
{registered, [emqx_authn_sup, emqx_authn_registry]},
{applications, [
kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose, emqx_mongodb
kernel,
stdlib,
emqx_resource,
emqx_connector,
ehttpc,
epgsql,
mysql,
jose,
emqx_mongodb,
emqx_redis
]},
{mod, {emqx_authn_app, []}},
{env, []},

View File

@ -60,11 +60,11 @@ roots() ->
].
fields(redis_single) ->
common_fields() ++ emqx_connector_redis:fields(single);
common_fields() ++ emqx_redis:fields(single);
fields(redis_cluster) ->
common_fields() ++ emqx_connector_redis:fields(cluster);
common_fields() ++ emqx_redis:fields(cluster);
fields(redis_sentinel) ->
common_fields() ++ emqx_connector_redis:fields(sentinel).
common_fields() ++ emqx_redis:fields(sentinel).
desc(redis_single) ->
?DESC(single);
@ -127,7 +127,7 @@ create(Config0) ->
{Config, State} ->
{ok, _Data} = emqx_authn_utils:create_resource(
ResourceId,
emqx_connector_redis,
emqx_redis,
Config
),
{ok, State#{resource_id => ResourceId}}
@ -135,7 +135,7 @@ create(Config0) ->
update(Config0, #{resource_id := ResourceId} = _State) ->
{Config, NState} = parse_config(Config0),
case emqx_authn_utils:update_resource(emqx_connector_redis, Config, ResourceId) of
case emqx_authn_utils:update_resource(emqx_redis, Config, ResourceId) of
{error, Reason} ->
error({load_config_error, Reason});
{ok, _} ->

View File

@ -67,7 +67,7 @@ init_per_suite(Config) ->
{ok, _} = emqx_resource:create_local(
?REDIS_RESOURCE,
?RESOURCE_GROUP,
emqx_connector_redis,
emqx_redis,
redis_config(),
#{}
),

View File

@ -5,7 +5,8 @@
{emqx, {path, "../emqx"}},
{emqx_utils, {path, "../emqx_utils"}},
{emqx_connector, {path, "../emqx_connector"}},
{emqx_mongodb, {path, "../emqx_mongodb"}}
{emqx_mongodb, {path, "../emqx_mongodb"}},
{emqx_redis, {path, "../emqx_redis"}}
]}.
{shell, [

View File

@ -10,7 +10,8 @@
crypto,
emqx_resource,
emqx_connector,
emqx_mongodb
emqx_mongodb,
emqx_redis
]},
{env, []},
{modules, []},

View File

@ -77,13 +77,13 @@ fields(postgresql) ->
proplists:delete(prepare_statement, emqx_connector_pgsql:fields(config));
fields(redis_single) ->
authz_redis_common_fields() ++
emqx_connector_redis:fields(single);
emqx_redis:fields(single);
fields(redis_sentinel) ->
authz_redis_common_fields() ++
emqx_connector_redis:fields(sentinel);
emqx_redis:fields(sentinel);
fields(redis_cluster) ->
authz_redis_common_fields() ++
emqx_connector_redis:fields(cluster);
emqx_redis:fields(cluster);
fields(position) ->
[
{position,

View File

@ -52,13 +52,13 @@ create(#{cmd := CmdStr} = Source) ->
Cmd = tokens(CmdStr),
ResourceId = emqx_authz_utils:make_resource_id(?MODULE),
CmdTemplate = emqx_authz_utils:parse_deep(Cmd, ?PLACEHOLDERS),
{ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_redis, Source),
{ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_redis, Source),
Source#{annotations => #{id => ResourceId}, cmd_template => CmdTemplate}.
update(#{cmd := CmdStr} = Source) ->
Cmd = tokens(CmdStr),
CmdTemplate = emqx_authz_utils:parse_deep(Cmd, ?PLACEHOLDERS),
case emqx_authz_utils:update_resource(emqx_connector_redis, Source) of
case emqx_authz_utils:update_resource(emqx_redis, Source) of
{error, Reason} ->
error({load_config_error, Reason});
{ok, Id} ->

View File

@ -389,8 +389,13 @@ cmd() ->
connector_fields(DB) ->
connector_fields(DB, config).
connector_fields(redis = DB, Fields) ->
connector_fields(DB, Fields, emqx);
connector_fields(DB, Fields) ->
Mod0 = io_lib:format("~ts_~ts", [emqx_connector, DB]),
connector_fields(DB, Fields, emqx_connector).
connector_fields(DB, Fields, Prefix) ->
Mod0 = io_lib:format("~ts_~ts", [Prefix, DB]),
Mod =
try
list_to_existing_atom(Mod0)

View File

@ -45,7 +45,7 @@ init_per_suite(Config) ->
{ok, _} = emqx_resource:create_local(
?REDIS_RESOURCE,
?RESOURCE_GROUP,
emqx_connector_redis,
emqx_redis,
redis_config(),
#{}
),

View File

@ -0,0 +1,3 @@
toxiproxy
redis
redis_cluster

View File

@ -0,0 +1,11 @@
%% -*- mode: erlang; -*-
{erl_opts, [debug_info]}.
{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}}
, {emqx_resource, {path, "../../apps/emqx_resource"}}
, {emqx_bridge, {path, "../../apps/emqx_bridge"}}
, {emqx_redis, {path, "../../apps/emqx_redis"}}
]}.
{shell, [
{apps, [emqx_bridge_redis]}
]}.

View File

@ -1,8 +1,8 @@
{application, emqx_bridge_redis, [
{description, "EMQX Enterprise Redis Bridge"},
{vsn, "0.1.0"},
{vsn, "0.1.1"},
{registered, []},
{applications, [kernel, stdlib]},
{applications, [kernel, stdlib, emqx_connector, emqx_resource, emqx_bridge, emqx_redis]},
{env, []},
{modules, []},
{links, []}

View File

@ -1,7 +1,7 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_bridge_redis).
-module(emqx_bridge_redis).
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
@ -149,7 +149,7 @@ redis_bridge_common_fields(Type) ->
connector_fields(Type) ->
RedisType = bridge_type_to_redis_conn_type(Type),
emqx_connector_redis:fields(RedisType).
emqx_redis:fields(RedisType).
bridge_type_to_redis_conn_type(redis_single) ->
single;
@ -190,11 +190,11 @@ desc("config") ->
desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
["Configuration for Redis using `", string:to_upper(Method), "` method."];
desc(redis_single) ->
?DESC(emqx_connector_redis, "single");
?DESC(emqx_redis, "single");
desc(redis_sentinel) ->
?DESC(emqx_connector_redis, "sentinel");
?DESC(emqx_redis, "sentinel");
desc(redis_cluster) ->
?DESC(emqx_connector_redis, "cluster");
?DESC(emqx_redis, "cluster");
desc("creation_opts_" ++ _Type) ->
?DESC(emqx_resource_schema, "creation_opts");
desc(_) ->

View File

@ -1,7 +1,7 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_connector_redis).
-module(emqx_bridge_redis_connector).
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
@ -25,7 +25,7 @@
callback_mode() -> always_sync.
on_start(InstId, #{command_template := CommandTemplate} = Config) ->
case emqx_connector_redis:on_start(InstId, Config) of
case emqx_redis:on_start(InstId, Config) of
{ok, RedisConnSt} ->
?tp(
redis_ee_connector_start_success,
@ -44,12 +44,12 @@ on_start(InstId, #{command_template := CommandTemplate} = Config) ->
end.
on_stop(InstId, #{conn_st := RedisConnSt}) ->
emqx_connector_redis:on_stop(InstId, RedisConnSt);
emqx_redis:on_stop(InstId, RedisConnSt);
on_stop(InstId, undefined = _State) ->
emqx_connector_redis:on_stop(InstId, undefined).
emqx_redis:on_stop(InstId, undefined).
on_get_status(InstId, #{conn_st := RedisConnSt}) ->
emqx_connector_redis:on_get_status(InstId, RedisConnSt).
emqx_redis:on_get_status(InstId, RedisConnSt).
on_query(
InstId,
@ -111,7 +111,7 @@ on_batch_query(
%% -------------------------------------------------------------------------------------------------
query(InstId, Query, RedisConnSt) ->
case emqx_connector_redis:on_query(InstId, Query, RedisConnSt) of
case emqx_redis:on_query(InstId, Query, RedisConnSt) of
{ok, _} = Ok -> Ok;
{error, no_connection} -> {error, {recoverable_error, no_connection}};
{error, _} = Error -> Error

View File

@ -1,7 +1,7 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_ee_bridge_redis_SUITE).
-module(emqx_bridge_redis_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
@ -233,7 +233,7 @@ t_check_values(_Config) ->
end,
lists:flatmap(
fun maps:to_list/1,
emqx_ee_bridge_redis:conn_bridge_examples(Method)
emqx_bridge_redis:conn_bridge_examples(Method)
)
)
end,
@ -411,7 +411,7 @@ conf_schema(StructName) ->
translations => #{},
validations => [],
namespace => undefined,
roots => [{root, hoconsc:ref(emqx_ee_bridge_redis, StructName)}]
roots => [{root, hoconsc:ref(emqx_bridge_redis, StructName)}]
}.
delete_all_rules() ->

View File

@ -1,4 +1,2 @@
redis
redis_cluster
mysql
pgsql

View File

@ -11,9 +11,7 @@
{emqx_resource, {path, "../emqx_resource"}},
{eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
{mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}},
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}},
%% NOTE: mind ecpool version when updating eredis_cluster version
{eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.8.1"}}}
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}}
]}.
{shell, [

View File

@ -11,4 +11,4 @@ Please see our [contributing.md](../../CONTRIBUTING.md).
## License
See [BSL](../../APL.txt).
See [BSL](./BSL.txt).

15
apps/emqx_redis/README.md Normal file
View File

@ -0,0 +1,15 @@
# Redis Connector
This application houses the Redis Database connector.
It provides the APIs to connect to Redis Database.
It is used by the Redis bridge to insert messages and by the emqx_authz and
emqx_authn applications to check user permissions.
## Contributing
Please see our [contributing.md](../../CONTRIBUTING.md).
## License
See [APL](../../APL.txt).

View File

@ -0,0 +1,2 @@
redis
redis_cluster

View File

@ -0,0 +1,9 @@
%% -*- mode: erlang; -*-
{erl_opts, [debug_info]}.
{deps, [
%% NOTE: mind ecpool version when updating eredis_cluster version
{eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.8.1"}}},
{emqx_connector, {path, "../../apps/emqx_connector"}},
{emqx_resource, {path, "../../apps/emqx_resource"}}
]}.

View File

@ -0,0 +1,14 @@
{application, emqx_redis, [
{description, "EMQX Redis Database Connector"},
{vsn, "0.1.0"},
{registered, []},
{applications, [
kernel,
stdlib,
eredis_cluster
]},
{env, []},
{modules, []},
{links, []}
]}.

View File

@ -13,9 +13,9 @@
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_connector_redis).
-module(emqx_redis).
-include("emqx_connector.hrl").
-include_lib("emqx_connector/include/emqx_connector.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/logger.hrl").

View File

@ -13,7 +13,7 @@
% %% limitations under the License.
% %%--------------------------------------------------------------------
-module(emqx_connector_redis_SUITE).
-module(emqx_redis_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
@ -29,7 +29,7 @@
-define(REDIS_SENTINEL_PORT, 26379).
-define(REDIS_CLUSTER_HOST, "redis-cluster-1").
-define(REDIS_CLUSTER_PORT, 6379).
-define(REDIS_RESOURCE_MOD, emqx_connector_redis).
-define(REDIS_RESOURCE_MOD, emqx_redis).
all() ->
emqx_common_test_helpers:all(?MODULE).
@ -83,21 +83,21 @@ wait_for_redis(Checks) ->
t_single_lifecycle(_Config) ->
perform_lifecycle_check(
<<"emqx_connector_redis_SUITE_single">>,
<<"emqx_redis_SUITE_single">>,
redis_config_single(),
[<<"PING">>]
).
t_cluster_lifecycle(_Config) ->
perform_lifecycle_check(
<<"emqx_connector_redis_SUITE_cluster">>,
<<"emqx_redis_SUITE_cluster">>,
redis_config_cluster(),
[<<"PING">>, <<"PONG">>]
).
t_sentinel_lifecycle(_Config) ->
perform_lifecycle_check(
<<"emqx_connector_redis_SUITE_sentinel">>,
<<"emqx_redis_SUITE_sentinel">>,
redis_config_sentinel(),
[<<"PING">>]
).

View File

@ -0,0 +1 @@
The Redis connector has been refactored to its own Erlang application to improve the code structure.

View File

@ -0,0 +1 @@
The Redis bridge has been refactored to its own Erlang application to improve the code structure and to make it easier to maintain.

View File

@ -1,4 +1,2 @@
toxiproxy
mysql
redis
redis_cluster

View File

@ -33,9 +33,9 @@ api_schemas(Method) ->
api_ref(emqx_ee_bridge_hstreamdb, <<"hstreamdb">>, Method),
api_ref(emqx_bridge_influxdb, <<"influxdb_api_v1">>, Method ++ "_api_v1"),
api_ref(emqx_bridge_influxdb, <<"influxdb_api_v2">>, Method ++ "_api_v2"),
api_ref(emqx_ee_bridge_redis, <<"redis_single">>, Method ++ "_single"),
api_ref(emqx_ee_bridge_redis, <<"redis_sentinel">>, Method ++ "_sentinel"),
api_ref(emqx_ee_bridge_redis, <<"redis_cluster">>, Method ++ "_cluster"),
api_ref(emqx_bridge_redis, <<"redis_single">>, Method ++ "_single"),
api_ref(emqx_bridge_redis, <<"redis_sentinel">>, Method ++ "_sentinel"),
api_ref(emqx_bridge_redis, <<"redis_cluster">>, Method ++ "_cluster"),
api_ref(emqx_bridge_timescale, <<"timescale">>, Method),
api_ref(emqx_bridge_matrix, <<"matrix">>, Method),
api_ref(emqx_bridge_tdengine, <<"tdengine">>, Method),
@ -59,7 +59,7 @@ schema_modules() ->
emqx_bridge_influxdb,
emqx_bridge_mongodb,
emqx_ee_bridge_mysql,
emqx_ee_bridge_redis,
emqx_bridge_redis,
emqx_bridge_pgsql,
emqx_bridge_timescale,
emqx_bridge_matrix,
@ -102,9 +102,9 @@ resource_type(mongodb_single) -> emqx_bridge_mongodb_connector;
resource_type(mysql) -> emqx_connector_mysql;
resource_type(influxdb_api_v1) -> emqx_bridge_influxdb_connector;
resource_type(influxdb_api_v2) -> emqx_bridge_influxdb_connector;
resource_type(redis_single) -> emqx_ee_connector_redis;
resource_type(redis_sentinel) -> emqx_ee_connector_redis;
resource_type(redis_cluster) -> emqx_ee_connector_redis;
resource_type(redis_single) -> emqx_bridge_redis_connector;
resource_type(redis_sentinel) -> emqx_bridge_redis_connector;
resource_type(redis_cluster) -> emqx_bridge_redis_connector;
resource_type(pgsql) -> emqx_connector_pgsql;
resource_type(timescale) -> emqx_connector_pgsql;
resource_type(matrix) -> emqx_connector_pgsql;
@ -287,7 +287,7 @@ redis_structs() ->
[
{Type,
mk(
hoconsc:map(name, ref(emqx_ee_bridge_redis, Type)),
hoconsc:map(name, ref(emqx_bridge_redis, Type)),
#{
desc => <<"Redis Bridge Config">>,
required => false

View File

@ -6,7 +6,8 @@
kernel,
stdlib,
ecpool,
hstreamdb_erl
hstreamdb_erl,
emqx_redis
]},
{env, []},
{modules, []},

View File

@ -377,6 +377,7 @@ defmodule EMQXUmbrella.MixProject do
emqx_psk: :permanent,
emqx_slow_subs: :permanent,
emqx_mongodb: :permanent,
emqx_redis: :permanent,
emqx_plugins: :permanent,
emqx_mix: :none
] ++

View File

@ -440,6 +440,7 @@ relx_apps(ReleaseType, Edition) ->
emqx_psk,
emqx_slow_subs,
emqx_mongodb,
emqx_redis,
emqx_plugins
] ++
[quicer || is_quicer_supported()] ++

View File

@ -1,4 +1,4 @@
emqx_ee_bridge_redis {
emqx_bridge_redis {
command_template.desc:
"""Redis command template used to export messages. Each list element stands for a command name or its argument.

View File

@ -1,4 +1,4 @@
emqx_connector_redis {
emqx_redis {
cluster.desc:
"""Cluster mode. Must be set to 'cluster' when Redis server is running in clustered mode."""