Merge pull request #11139 from kjellwinblad/kjell/refactor/redis/EMQX-9534

refactor: redis bridge to its own application

apps/emqx_authn/rebar.config

@@ -4,7 +4,8 @@
     {emqx, {path, "../emqx"}},
     {emqx_utils, {path, "../emqx_utils"}},
     {emqx_connector, {path, "../emqx_connector"}},
-    {emqx_mongodb, {path, "../emqx_mongodb"}}
+    {emqx_mongodb, {path, "../emqx_mongodb"}},
+    {emqx_redis, {path, "../emqx_redis"}}
 ]}.
 
 {edoc_opts, [{preprocess, true}]}.

apps/emqx_authn/src/emqx_authn.app.src

@@ -5,7 +5,16 @@
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
     {applications, [
-        kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose, emqx_mongodb
+        kernel,
+        stdlib,
+        emqx_resource,
+        emqx_connector,
+        ehttpc,
+        epgsql,
+        mysql,
+        jose,
+        emqx_mongodb,
+        emqx_redis
     ]},
     {mod, {emqx_authn_app, []}},
     {env, []},

apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl

@@ -60,11 +60,11 @@ roots() ->
     ].
 
 fields(redis_single) ->
-    common_fields() ++ emqx_connector_redis:fields(single);
+    common_fields() ++ emqx_redis:fields(single);
 fields(redis_cluster) ->
-    common_fields() ++ emqx_connector_redis:fields(cluster);
+    common_fields() ++ emqx_redis:fields(cluster);
 fields(redis_sentinel) ->
-    common_fields() ++ emqx_connector_redis:fields(sentinel).
+    common_fields() ++ emqx_redis:fields(sentinel).
 
 desc(redis_single) ->
     ?DESC(single);
@@ -127,7 +127,7 @@ create(Config0) ->
         {Config, State} ->
             {ok, _Data} = emqx_authn_utils:create_resource(
                 ResourceId,
-                emqx_connector_redis,
+                emqx_redis,
                 Config
             ),
             {ok, State#{resource_id => ResourceId}}
@@ -135,7 +135,7 @@ create(Config0) ->
 
 update(Config0, #{resource_id := ResourceId} = _State) ->
     {Config, NState} = parse_config(Config0),
-    case emqx_authn_utils:update_resource(emqx_connector_redis, Config, ResourceId) of
+    case emqx_authn_utils:update_resource(emqx_redis, Config, ResourceId) of
         {error, Reason} ->
             error({load_config_error, Reason});
         {ok, _} ->

apps/emqx_authn/test/emqx_authn_redis_SUITE.erl

@@ -67,7 +67,7 @@ init_per_suite(Config) ->
             {ok, _} = emqx_resource:create_local(
                 ?REDIS_RESOURCE,
                 ?RESOURCE_GROUP,
-                emqx_connector_redis,
+                emqx_redis,
                 redis_config(),
                 #{}
             ),

apps/emqx_authz/rebar.config

@@ -5,7 +5,8 @@
     {emqx, {path, "../emqx"}},
     {emqx_utils, {path, "../emqx_utils"}},
     {emqx_connector, {path, "../emqx_connector"}},
-    {emqx_mongodb, {path, "../emqx_mongodb"}}
+    {emqx_mongodb, {path, "../emqx_mongodb"}},
+    {emqx_redis, {path, "../emqx_redis"}}
 ]}.
 
 {shell, [

apps/emqx_authz/src/emqx_authz.app.src

@@ -10,7 +10,8 @@
         crypto,
         emqx_resource,
         emqx_connector,
-        emqx_mongodb
+        emqx_mongodb,
+        emqx_redis
     ]},
     {env, []},
     {modules, []},

apps/emqx_authz/src/emqx_authz_api_schema.erl

@@ -77,13 +77,13 @@ fields(postgresql) ->
         proplists:delete(prepare_statement, emqx_connector_pgsql:fields(config));
 fields(redis_single) ->
     authz_redis_common_fields() ++
-        emqx_connector_redis:fields(single);
+        emqx_redis:fields(single);
 fields(redis_sentinel) ->
     authz_redis_common_fields() ++
-        emqx_connector_redis:fields(sentinel);
+        emqx_redis:fields(sentinel);
 fields(redis_cluster) ->
     authz_redis_common_fields() ++
-        emqx_connector_redis:fields(cluster);
+        emqx_redis:fields(cluster);
 fields(position) ->
     [
         {position,

apps/emqx_authz/src/emqx_authz_redis.erl

@@ -52,13 +52,13 @@ create(#{cmd := CmdStr} = Source) ->
     Cmd = tokens(CmdStr),
     ResourceId = emqx_authz_utils:make_resource_id(?MODULE),
     CmdTemplate = emqx_authz_utils:parse_deep(Cmd, ?PLACEHOLDERS),
-    {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_redis, Source),
+    {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_redis, Source),
     Source#{annotations => #{id => ResourceId}, cmd_template => CmdTemplate}.
 
 update(#{cmd := CmdStr} = Source) ->
     Cmd = tokens(CmdStr),
     CmdTemplate = emqx_authz_utils:parse_deep(Cmd, ?PLACEHOLDERS),
-    case emqx_authz_utils:update_resource(emqx_connector_redis, Source) of
+    case emqx_authz_utils:update_resource(emqx_redis, Source) of
         {error, Reason} ->
             error({load_config_error, Reason});
         {ok, Id} ->

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -389,8 +389,13 @@ cmd() ->
 
 connector_fields(DB) ->
     connector_fields(DB, config).
+connector_fields(redis = DB, Fields) ->
+    connector_fields(DB, Fields, emqx);
 connector_fields(DB, Fields) ->
-    Mod0 = io_lib:format("~ts_~ts", [emqx_connector, DB]),
+    connector_fields(DB, Fields, emqx_connector).
+
+connector_fields(DB, Fields, Prefix) ->
+    Mod0 = io_lib:format("~ts_~ts", [Prefix, DB]),
     Mod =
         try
             list_to_existing_atom(Mod0)

apps/emqx_authz/test/emqx_authz_redis_SUITE.erl

@@ -45,7 +45,7 @@ init_per_suite(Config) ->
             {ok, _} = emqx_resource:create_local(
                 ?REDIS_RESOURCE,
                 ?RESOURCE_GROUP,
-                emqx_connector_redis,
+                emqx_redis,
                 redis_config(),
                 #{}
             ),

apps/emqx_bridge_redis/docker-ct

@@ -0,0 +1,3 @@
+toxiproxy
+redis
+redis_cluster

apps/emqx_bridge_redis/rebar.config

@@ -0,0 +1,11 @@
+%% -*- mode: erlang; -*-
+{erl_opts, [debug_info]}.
+{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}}
+       , {emqx_resource, {path, "../../apps/emqx_resource"}}
+       , {emqx_bridge, {path, "../../apps/emqx_bridge"}}
+       , {emqx_redis, {path, "../../apps/emqx_redis"}}
+       ]}.
+
+{shell, [
+    {apps, [emqx_bridge_redis]}
+]}.

apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src

@@ -1,8 +1,8 @@
 {application, emqx_bridge_redis, [
     {description, "EMQX Enterprise Redis Bridge"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
-    {applications, [kernel, stdlib]},
+    {applications, [kernel, stdlib, emqx_connector, emqx_resource, emqx_bridge, emqx_redis]},
     {env, []},
     {modules, []},
     {links, []}

apps/emqx_bridge_redis/src/emqx_bridge_redis.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_bridge_redis).
+-module(emqx_bridge_redis).
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
@@ -149,7 +149,7 @@ redis_bridge_common_fields(Type) ->
 
 connector_fields(Type) ->
     RedisType = bridge_type_to_redis_conn_type(Type),
-    emqx_connector_redis:fields(RedisType).
+    emqx_redis:fields(RedisType).
 
 bridge_type_to_redis_conn_type(redis_single) ->
     single;
@@ -190,11 +190,11 @@ desc("config") ->
 desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
     ["Configuration for Redis using `", string:to_upper(Method), "` method."];
 desc(redis_single) ->
-    ?DESC(emqx_connector_redis, "single");
+    ?DESC(emqx_redis, "single");
 desc(redis_sentinel) ->
-    ?DESC(emqx_connector_redis, "sentinel");
+    ?DESC(emqx_redis, "sentinel");
 desc(redis_cluster) ->
-    ?DESC(emqx_connector_redis, "cluster");
+    ?DESC(emqx_redis, "cluster");
 desc("creation_opts_" ++ _Type) ->
     ?DESC(emqx_resource_schema, "creation_opts");
 desc(_) ->

apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_connector_redis).
+-module(emqx_bridge_redis_connector).
 
 -include_lib("emqx/include/logger.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
@@ -25,7 +25,7 @@
 callback_mode() -> always_sync.
 
 on_start(InstId, #{command_template := CommandTemplate} = Config) ->
-    case emqx_connector_redis:on_start(InstId, Config) of
+    case emqx_redis:on_start(InstId, Config) of
         {ok, RedisConnSt} ->
             ?tp(
                 redis_ee_connector_start_success,
@@ -44,12 +44,12 @@ on_start(InstId, #{command_template := CommandTemplate} = Config) ->
     end.
 
 on_stop(InstId, #{conn_st := RedisConnSt}) ->
-    emqx_connector_redis:on_stop(InstId, RedisConnSt);
+    emqx_redis:on_stop(InstId, RedisConnSt);
 on_stop(InstId, undefined = _State) ->
-    emqx_connector_redis:on_stop(InstId, undefined).
+    emqx_redis:on_stop(InstId, undefined).
 
 on_get_status(InstId, #{conn_st := RedisConnSt}) ->
-    emqx_connector_redis:on_get_status(InstId, RedisConnSt).
+    emqx_redis:on_get_status(InstId, RedisConnSt).
 
 on_query(
     InstId,
@@ -111,7 +111,7 @@ on_batch_query(
 %% -------------------------------------------------------------------------------------------------
 
 query(InstId, Query, RedisConnSt) ->
-    case emqx_connector_redis:on_query(InstId, Query, RedisConnSt) of
+    case emqx_redis:on_query(InstId, Query, RedisConnSt) of
         {ok, _} = Ok -> Ok;
         {error, no_connection} -> {error, {recoverable_error, no_connection}};
         {error, _} = Error -> Error

apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_bridge_redis_SUITE).
+-module(emqx_bridge_redis_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
@@ -233,7 +233,7 @@ t_check_values(_Config) ->
                 end,
                 lists:flatmap(
                     fun maps:to_list/1,
-                    emqx_ee_bridge_redis:conn_bridge_examples(Method)
+                    emqx_bridge_redis:conn_bridge_examples(Method)
                 )
             )
         end,
@@ -411,7 +411,7 @@ conf_schema(StructName) ->
         translations => #{},
         validations => [],
         namespace => undefined,
-        roots => [{root, hoconsc:ref(emqx_ee_bridge_redis, StructName)}]
+        roots => [{root, hoconsc:ref(emqx_bridge_redis, StructName)}]
     }.
 
 delete_all_rules() ->

apps/emqx_connector/docker-ct

@@ -1,4 +1,2 @@
-redis
-redis_cluster
 mysql
 pgsql

apps/emqx_connector/rebar.config

@@ -11,9 +11,7 @@
     {emqx_resource, {path, "../emqx_resource"}},
     {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
     {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}},
-    {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}},
+    {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}}
-    %% NOTE: mind ecpool version when updating eredis_cluster version
-    {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.8.1"}}}
 ]}.
 
 {shell, [

apps/emqx_oracle/README.md

@@ -11,4 +11,4 @@ Please see our [contributing.md](../../CONTRIBUTING.md).
 
 ## License
 
-See [BSL](../../APL.txt).
+See [BSL](./BSL.txt).

apps/emqx_redis/README.md

@@ -0,0 +1,15 @@
+# Redis Connector
+
+This application houses the Redis Database connector.
+It provides the APIs to connect to Redis Database.
+
+It is used by the Redis bridge to insert messages and by the emqx_authz and
+emqx_authn applications to check user permissions.
+
+## Contributing
+
+Please see our [contributing.md](../../CONTRIBUTING.md).
+
+## License
+
+See [APL](../../APL.txt).

apps/emqx_redis/docker-ct

@@ -0,0 +1,2 @@
+redis
+redis_cluster

apps/emqx_redis/rebar.config

@@ -0,0 +1,9 @@
+%% -*- mode: erlang; -*-
+
+{erl_opts, [debug_info]}.
+{deps, [
+        %% NOTE: mind ecpool version when updating eredis_cluster version
+        {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.8.1"}}},
+        {emqx_connector, {path, "../../apps/emqx_connector"}},
+        {emqx_resource, {path, "../../apps/emqx_resource"}}
+]}.

apps/emqx_redis/src/emqx_redis.app.src

@@ -0,0 +1,14 @@
+{application, emqx_redis, [
+    {description, "EMQX Redis Database Connector"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {applications, [
+        kernel,
+        stdlib,
+        eredis_cluster
+    ]},
+    {env, []},
+    {modules, []},
+
+    {links, []}
+]}.

apps/emqx_redis/src/emqx_redis.erl

@@ -13,9 +13,9 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--module(emqx_connector_redis).
+-module(emqx_redis).
 
--include("emqx_connector.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").

apps/emqx_redis/test/emqx_redis_SUITE.erl

@@ -13,7 +13,7 @@
 % %% limitations under the License.
 % %%--------------------------------------------------------------------
 
--module(emqx_connector_redis_SUITE).
+-module(emqx_redis_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
@@ -29,7 +29,7 @@
 -define(REDIS_SENTINEL_PORT, 26379).
 -define(REDIS_CLUSTER_HOST, "redis-cluster-1").
 -define(REDIS_CLUSTER_PORT, 6379).
--define(REDIS_RESOURCE_MOD, emqx_connector_redis).
+-define(REDIS_RESOURCE_MOD, emqx_redis).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -83,21 +83,21 @@ wait_for_redis(Checks) ->
 
 t_single_lifecycle(_Config) ->
     perform_lifecycle_check(
-        <<"emqx_connector_redis_SUITE_single">>,
+        <<"emqx_redis_SUITE_single">>,
         redis_config_single(),
         [<<"PING">>]
     ).
 
 t_cluster_lifecycle(_Config) ->
     perform_lifecycle_check(
-        <<"emqx_connector_redis_SUITE_cluster">>,
+        <<"emqx_redis_SUITE_cluster">>,
         redis_config_cluster(),
         [<<"PING">>, <<"PONG">>]
     ).
 
 t_sentinel_lifecycle(_Config) ->
     perform_lifecycle_check(
-        <<"emqx_connector_redis_SUITE_sentinel">>,
+        <<"emqx_redis_SUITE_sentinel">>,
         redis_config_sentinel(),
         [<<"PING">>]
     ).

changes/ce/fix-11139.en.md

@@ -0,0 +1 @@
+The Redis connector has been refactored to its own Erlang application to improve the code structure.

changes/ee/fix-11139.en.md

@@ -0,0 +1 @@
+The Redis bridge has been refactored to its own Erlang application to improve the code structure and to make it easier to maintain.

lib-ee/emqx_ee_bridge/docker-ct

@@ -1,4 +1,2 @@
 toxiproxy
 mysql
-redis
-redis_cluster

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl

@@ -33,9 +33,9 @@ api_schemas(Method) ->
         api_ref(emqx_ee_bridge_hstreamdb, <<"hstreamdb">>, Method),
         api_ref(emqx_bridge_influxdb, <<"influxdb_api_v1">>, Method ++ "_api_v1"),
         api_ref(emqx_bridge_influxdb, <<"influxdb_api_v2">>, Method ++ "_api_v2"),
-        api_ref(emqx_ee_bridge_redis, <<"redis_single">>, Method ++ "_single"),
+        api_ref(emqx_bridge_redis, <<"redis_single">>, Method ++ "_single"),
-        api_ref(emqx_ee_bridge_redis, <<"redis_sentinel">>, Method ++ "_sentinel"),
+        api_ref(emqx_bridge_redis, <<"redis_sentinel">>, Method ++ "_sentinel"),
-        api_ref(emqx_ee_bridge_redis, <<"redis_cluster">>, Method ++ "_cluster"),
+        api_ref(emqx_bridge_redis, <<"redis_cluster">>, Method ++ "_cluster"),
         api_ref(emqx_bridge_timescale, <<"timescale">>, Method),
         api_ref(emqx_bridge_matrix, <<"matrix">>, Method),
         api_ref(emqx_bridge_tdengine, <<"tdengine">>, Method),
@@ -59,7 +59,7 @@ schema_modules() ->
         emqx_bridge_influxdb,
         emqx_bridge_mongodb,
         emqx_ee_bridge_mysql,
-        emqx_ee_bridge_redis,
+        emqx_bridge_redis,
         emqx_bridge_pgsql,
         emqx_bridge_timescale,
         emqx_bridge_matrix,
@@ -102,9 +102,9 @@ resource_type(mongodb_single) -> emqx_bridge_mongodb_connector;
 resource_type(mysql) -> emqx_connector_mysql;
 resource_type(influxdb_api_v1) -> emqx_bridge_influxdb_connector;
 resource_type(influxdb_api_v2) -> emqx_bridge_influxdb_connector;
-resource_type(redis_single) -> emqx_ee_connector_redis;
+resource_type(redis_single) -> emqx_bridge_redis_connector;
-resource_type(redis_sentinel) -> emqx_ee_connector_redis;
+resource_type(redis_sentinel) -> emqx_bridge_redis_connector;
-resource_type(redis_cluster) -> emqx_ee_connector_redis;
+resource_type(redis_cluster) -> emqx_bridge_redis_connector;
 resource_type(pgsql) -> emqx_connector_pgsql;
 resource_type(timescale) -> emqx_connector_pgsql;
 resource_type(matrix) -> emqx_connector_pgsql;
@@ -287,7 +287,7 @@ redis_structs() ->
     [
         {Type,
             mk(
-                hoconsc:map(name, ref(emqx_ee_bridge_redis, Type)),
+                hoconsc:map(name, ref(emqx_bridge_redis, Type)),
                 #{
                     desc => <<"Redis Bridge Config">>,
                     required => false

lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src

@@ -6,7 +6,8 @@
         kernel,
         stdlib,
         ecpool,
-        hstreamdb_erl
+        hstreamdb_erl,
+        emqx_redis
     ]},
     {env, []},
     {modules, []},

mix.exs

@@ -377,6 +377,7 @@ defmodule EMQXUmbrella.MixProject do
         emqx_psk: :permanent,
         emqx_slow_subs: :permanent,
         emqx_mongodb: :permanent,
+        emqx_redis: :permanent,
         emqx_plugins: :permanent,
         emqx_mix: :none
       ] ++

rebar.config.erl

@@ -440,6 +440,7 @@ relx_apps(ReleaseType, Edition) ->
             emqx_psk,
             emqx_slow_subs,
             emqx_mongodb,
+            emqx_redis,
             emqx_plugins
         ] ++
         [quicer || is_quicer_supported()] ++

rel/i18n/emqx_bridge_redis.hocon

@@ -1,4 +1,4 @@
-emqx_ee_bridge_redis {
+emqx_bridge_redis {
 
 command_template.desc:
 """Redis command template used to export messages. Each list element stands for a command name or its argument.

rel/i18n/emqx_redis.hocon

@@ -1,4 +1,4 @@
-emqx_connector_redis {
+emqx_redis {
 
 cluster.desc:
 """Cluster mode. Must be set to 'cluster' when Redis server is running in clustered mode."""