chore(authz): test Redis backend with real Redis

.ci/docker-compose-file/Makefile.local

@@ -16,10 +16,8 @@ up:
 		REDIS_TAG=6 \
 		MONGO_TAG=5 \
 		PGSQL_TAG=13 \
-		LDAP_TAG=2.4.50 \
 	docker-compose \
 		-f .ci/docker-compose-file/docker-compose.yaml \
-		-f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
 		-f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
 		-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
 		-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
@@ -29,7 +27,6 @@ up:
 down:
 	docker-compose \
 		-f .ci/docker-compose-file/docker-compose.yaml \
-		-f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
 		-f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
 		-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
 		-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \

apps/emqx_authz/src/emqx_authz_redis.erl

@@ -70,9 +70,10 @@ authorize(Client, PubSub, Topic,
 do_authorize(_Client, _PubSub, _Topic, []) ->
     nomatch;
 do_authorize(Client, PubSub, Topic, [TopicFilter, Action | Tail]) ->
-    case emqx_authz_rule:match(Client, PubSub, Topic,
+    case emqx_authz_rule:match(
-                               emqx_authz_rule:compile({allow, all, Action, [TopicFilter]})
+           Client, PubSub, Topic,
-                              )of
+           emqx_authz_rule:compile({allow, all, Action, [TopicFilter]})
+          ) of
         {matched, Permission} -> {matched, Permission};
         nomatch -> do_authorize(Client, PubSub, Topic, Tail)
     end.
@@ -81,6 +82,8 @@ replvar(Cmd, Client = #{cn := CN}) ->
     replvar(repl(Cmd, ?PH_S_CERT_CN_NAME, CN), maps:remove(cn, Client));
 replvar(Cmd, Client = #{dn := DN}) ->
     replvar(repl(Cmd, ?PH_S_CERT_SUBJECT, DN), maps:remove(dn, Client));
+replvar(Cmd, Client = #{peerhost := IpAddr}) ->
+    replvar(repl(Cmd, ?PH_S_PEERHOST, inet_parse:ntoa(IpAddr)), maps:remove(peerhost, Client));
 replvar(Cmd, Client = #{clientid := ClientId}) ->
     replvar(repl(Cmd, ?PH_S_CLIENTID, ClientId), maps:remove(clientid, Client));
 replvar(Cmd, Client = #{username := Username}) ->

apps/emqx_authz/test/emqx_authz_redis_SUITE.erl

@@ -4,7 +4,8 @@
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
 %% You may obtain a copy of the License at
-%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing, software
 %% distributed under the License is distributed on an "AS IS" BASIS,
@@ -21,8 +22,11 @@
 -include("emqx_authz.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
--include_lib("emqx/include/emqx_placeholder.hrl").
+
--define(CONF_DEFAULT, <<"authorization: {sources: []}">>).
+
+-define(REDIS_HOST, "redis").
+-define(REDIS_PORT, 6379).
+-define(REDIS_RESOURCE, <<"emqx_authz_redis_SUITE">>).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -31,86 +35,294 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
+    case emqx_authn_test_lib:is_tcp_server_available(?REDIS_HOST, ?REDIS_PORT) of
-    meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end ),
+        true ->
-    meck:expect(emqx_resource, remove, fun(_) -> ok end ),
+            ok = emqx_common_test_helpers:start_apps(
-
+                   [emqx_conf, emqx_authz],
-    ok = emqx_common_test_helpers:start_apps(
+                   fun set_special_configs/1
-           [emqx_conf, emqx_authz],
+                  ),
-           fun set_special_configs/1),
+            ok = start_apps([emqx_resource, emqx_connector]),
-
+            {ok, _} = emqx_resource:create_local(
-    Rules = [#{<<"type">> => <<"redis">>,
+              ?REDIS_RESOURCE,
-               <<"server">> => <<"127.0.0.1:27017">>,
+              emqx_connector_redis,
-               <<"pool_size">> => 1,
+              redis_config()),
-               <<"database">> => 0,
+            Config;
-               <<"password">> => <<"ee">>,
+        false ->
-               <<"auto_reconnect">> => true,
+            {skip, no_redis}
-               <<"ssl">> => #{<<"enable">> => false},
+    end.
-               <<"cmd">> => <<"HGETALL mqtt_authz:", ?PH_USERNAME/binary>>
-              }],
-    {ok, _} = emqx_authz:update(replace, Rules),
-    Config.
 
 end_per_suite(_Config) ->
-    {ok, _} = emqx:update_config(
+    ok = emqx_authz_test_lib:reset_authorizers(),
-                [authorization],
+    ok = emqx_resource:remove_local(?REDIS_RESOURCE),
-                #{<<"no_match">> => <<"allow">>,
+    ok = stop_apps([emqx_resource, emqx_connector]),
-                  <<"cache">> => #{<<"enable">> => <<"true">>},
+    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
-                  <<"sources">> => []}),
-    emqx_common_test_helpers:stop_apps([emqx_authz, emqx_resource]),
-    meck:unload(emqx_resource),
-    ok.
 
 set_special_configs(emqx_authz) ->
-    {ok, _} = emqx:update_config([authorization, cache, enable], false),
+    ok = emqx_authz_test_lib:reset_authorizers();
-    {ok, _} = emqx:update_config([authorization, no_match], deny),
+
-    {ok, _} = emqx:update_config([authorization, sources], []),
+set_special_configs(_) ->
-    ok;
-set_special_configs(_App) ->
     ok.
 
--define(SOURCE1, [<<"test/", ?PH_USERNAME/binary>>, <<"publish">>]).
--define(SOURCE2, [<<"test/", ?PH_CLIENTID/binary>>, <<"publish">>]).
--define(SOURCE3, [<<"#">>, <<"subscribe">>]).
 
 %%------------------------------------------------------------------------------
-%% Testcases
+%% Tests
 %%------------------------------------------------------------------------------
 
-t_authz(_) ->
+t_topic_rules(_Config) ->
     ClientInfo = #{clientid => <<"clientid">>,
                    username => <<"username">>,
                    peerhost => {127,0,0,1},
                    zone => default,
                    listener => {tcp, default}
-                   },
+                  },
 
-    meck:expect(emqx_resource, query, fun(_, _) -> {ok, []} end),
+    %% No rules
-    % nomatch
+
-    ?assertEqual(deny,
+    ok = setup_sample(#{}),
-                 emqx_access_control:authorize(ClientInfo, subscribe, <<"#">>)),
+    ok = setup_config(#{}),
-    ?assertEqual(deny,
+
-                 emqx_access_control:authorize(ClientInfo, publish, <<"#">>)),
+    ok = test_samples(
+           ClientInfo,
+           [{deny, subscribe, <<"#">>},
+            {deny, subscribe, <<"subs">>},
+            {deny, publish, <<"pub">>}]),
+
+    %% Publish rules
+
+    Sample0 = #{<<"mqtt_user:username">> =>
+                   #{<<"testpub1/${username}">> => <<"publish">>,
+                     <<"testpub2/${clientid}">> => <<"publish">>,
+                     <<"testpub3/#">> => <<"publish">>
+                    }
+              },
+    ok = setup_sample(Sample0),
+    ok = setup_config(#{}),
+
+    ok = test_samples(
+           ClientInfo,
+           [{allow, publish, <<"testpub1/username">>},
+            {allow, publish, <<"testpub2/clientid">>},
+            {allow, publish, <<"testpub3/foobar">>},
+
+            {deny, publish, <<"testpub2/username">>},
+            {deny, publish, <<"testpub1/clientid">>},
 
 
-    meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?SOURCE1 ++ ?SOURCE2} end),
+            {deny, subscribe, <<"testpub1/username">>},
-    % nomatch
+            {deny, subscribe, <<"testpub2/clientid">>},
-    ?assertEqual(deny,
+            {deny, subscribe, <<"testpub3/foobar">>}]),
-        emqx_access_control:authorize(ClientInfo, subscribe, <<"+">>)),
-    % nomatch
-    ?assertEqual(deny,
-        emqx_access_control:authorize(ClientInfo, subscribe, <<"test/username">>)),
 
-    ?assertEqual(allow,
+    %% Subscribe rules
-        emqx_access_control:authorize(ClientInfo, publish, <<"test/clientid">>)),
-    ?assertEqual(allow,
-        emqx_access_control:authorize(ClientInfo, publish, <<"test/clientid">>)),
 
-    meck:expect(emqx_resource, query, fun(_, _) -> {ok, ?SOURCE3} end),
+    Sample1 = #{<<"mqtt_user:username">> =>
+                   #{<<"testsub1/${username}">> => <<"subscribe">>,
+                     <<"testsub2/${clientid}">> => <<"subscribe">>,
+                     <<"testsub3/#">> => <<"subscribe">>
+                    }
+              },
+    ok = setup_sample(Sample1),
+    ok = setup_config(#{}),
 
-    ?assertEqual(allow,
+    ok = test_samples(
-        emqx_access_control:authorize(ClientInfo, subscribe, <<"#">>)),
+           ClientInfo,
-    % nomatch
+           [{allow, subscribe, <<"testsub1/username">>},
-    ?assertEqual(deny,
+            {allow, subscribe, <<"testsub2/clientid">>},
-        emqx_access_control:authorize(ClientInfo, publish, <<"#">>)),
+            {allow, subscribe, <<"testsub3/foobar">>},
+            {allow, subscribe, <<"testsub3/+/foobar">>},
+            {allow, subscribe, <<"testsub3/#">>},
+
+            {deny, subscribe, <<"testsub2/username">>},
+            {deny, subscribe, <<"testsub1/clientid">>},
+            {deny, subscribe, <<"testsub4/foobar">>},
+            {deny, publish, <<"testsub1/username">>},
+            {deny, publish, <<"testsub2/clientid">>},
+            {deny, publish, <<"testsub3/foobar">>}]),
+
+    %% All rules
+
+    Sample2 = #{<<"mqtt_user:username">> =>
+                   #{<<"testall1/${username}">> => <<"all">>,
+                     <<"testall2/${clientid}">> => <<"all">>,
+                     <<"testall3/#">> => <<"all">>
+                    }
+              },
+    ok = setup_sample(Sample2),
+    ok = setup_config(#{}),
+
+    ok = test_samples(
+           ClientInfo,
+           [{allow, subscribe, <<"testall1/username">>},
+            {allow, subscribe, <<"testall2/clientid">>},
+            {allow, subscribe, <<"testall3/foobar">>},
+            {allow, subscribe, <<"testall3/+/foobar">>},
+            {allow, subscribe, <<"testall3/#">>},
+            {allow, publish, <<"testall1/username">>},
+            {allow, publish, <<"testall2/clientid">>},
+            {allow, publish, <<"testall3/foobar">>},
+
+            {deny, subscribe, <<"testall2/username">>},
+            {deny, subscribe, <<"testall1/clientid">>},
+            {deny, subscribe, <<"testall4/foobar">>},
+            {deny, publish, <<"testall2/username">>},
+            {deny, publish, <<"testall1/clientid">>},
+            {deny, publish, <<"testall4/foobar">>}]).
+
+t_lookups(_Config) ->
+    ClientInfo = #{clientid => <<"clientid">>,
+                   cn => <<"cn">>,
+                   dn => <<"dn">>,
+                   username => <<"username">>,
+                   peerhost => {127,0,0,1},
+                   zone => default,
+                   listener => {tcp, default}
+                  },
+
+    ByClientid = #{<<"mqtt_user:clientid">> =>
+                   #{<<"a">> => <<"all">>}},
+
+    ok = setup_sample(ByClientid),
+    ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${clientid}">>}),
+
+    ok = test_samples(
+           ClientInfo,
+           [{allow, subscribe, <<"a">>},
+            {deny, subscribe, <<"b">>}]),
+
+    ByPeerhost = #{<<"mqtt_user:127.0.0.1">> =>
+                   #{<<"a">> => <<"all">>}},
+
+    ok = setup_sample(ByPeerhost),
+    ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${peerhost}">>}),
+
+    ok = test_samples(
+           ClientInfo,
+           [{allow, subscribe, <<"a">>},
+            {deny, subscribe, <<"b">>}]),
+
+    ByCN = #{<<"mqtt_user:cn">> =>
+             #{<<"a">> => <<"all">>}},
+
+    ok = setup_sample(ByCN),
+    ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${cert_common_name}">>}),
+
+    ok = test_samples(
+           ClientInfo,
+           [{allow, subscribe, <<"a">>},
+            {deny, subscribe, <<"b">>}]),
+
+
+    ByDN = #{<<"mqtt_user:dn">> =>
+             #{<<"a">> => <<"all">>}},
+
+    ok = setup_sample(ByDN),
+    ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${cert_subject}">>}),
+
+    ok = test_samples(
+           ClientInfo,
+           [{allow, subscribe, <<"a">>},
+            {deny, subscribe, <<"b">>}]).
+
+t_create_invalid(_Config) ->
+    AuthzConfig = raw_redis_authz_config(),
+
+    InvalidConfigs =
+        [maps:without([<<"server">>], AuthzConfig),
+         AuthzConfig#{<<"server">> => <<"unknownhost:3333">>},
+         AuthzConfig#{<<"password">> => <<"wrongpass">>},
+         AuthzConfig#{<<"database">> => <<"5678">>}],
+
+    lists:foreach(
+      fun(Config) ->
+            {error, _} = emqx_authz:update(?CMD_REPLACE, [Config]),
+            [] = emqx_authz:lookup()
+
+      end,
+      InvalidConfigs).
+
+t_redis_error(_Config) ->
+    ok = setup_config(#{<<"cmd">> => <<"INVALID COMMAND">>}),
+
+    ClientInfo = #{clientid => <<"clientid">>,
+                   username => <<"username">>,
+                   peerhost => {127,0,0,1},
+                   zone => default,
+                   listener => {tcp, default}
+                  },
+
+    deny = emqx_access_control:authorize(ClientInfo, subscribe, <<"a">>).
+
+%%------------------------------------------------------------------------------
+%% Helpers
+%%------------------------------------------------------------------------------
+
+test_samples(ClientInfo, Samples) ->
+    lists:foreach(
+      fun({Expected, Action, Topic}) ->
+              ct:pal(
+                "client_info: ~p, action: ~p, topic: ~p, expected: ~p",
+                [ClientInfo, Action, Topic, Expected]),
+              ?assertEqual(
+                 Expected,
+                 emqx_access_control:authorize(
+                   ClientInfo,
+                   Action,
+                   Topic))
+      end,
+      Samples).
+
+
+setup_sample(AuthzData) ->
+    {ok, _} = q(["FLUSHDB"]),
+    ok = lists:foreach(
+           fun({Key, Values}) ->
+                   lists:foreach(
+                     fun({TopicFilter, Action}) ->
+                             q(["HSET", Key, TopicFilter, Action])
+                     end,
+                     maps:to_list(Values))
+           end,
+           maps:to_list(AuthzData)).
+
+setup_config(SpecialParams) ->
+    ok = emqx_authz_test_lib:reset_authorizers(deny, false),
+    Config = maps:merge(raw_redis_authz_config(), SpecialParams),
+    {ok, _} = emqx_authz:update(?CMD_REPLACE, [Config]),
     ok.
+
+raw_redis_authz_config() ->
+    #{
+        <<"enable">> => <<"true">>,
+
+        <<"type">> => <<"redis">>,
+        <<"cmd">> => <<"HGETALL mqtt_user:${username}">>,
+        <<"database">> => <<"1">>,
+        <<"password">> => <<"public">>,
+        <<"server">> => redis_server()
+    }.
+
+redis_server() ->
+    iolist_to_binary(
+      io_lib:format(
+        "~s:~b",
+        [?REDIS_HOST, ?REDIS_PORT])).
+
+q(Command) ->
+    emqx_resource:query(
+      ?REDIS_RESOURCE,
+      {cmd, Command}).
+
+redis_config() ->
+    #{auto_reconnect => true,
+      database => 1,
+      pool_size => 8,
+      redis_type => single,
+      password => "public",
+      server => {?REDIS_HOST, ?REDIS_PORT},
+      ssl => #{enable => false}
+     }.
+
+start_apps(Apps) ->
+    lists:foreach(fun application:ensure_all_started/1, Apps).
+
+stop_apps(Apps) ->
+    lists:foreach(fun application:stop/1, Apps).

apps/emqx_authz/test/emqx_authz_test_lib.erl

@@ -0,0 +1,44 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authz_test_lib).
+
+-include("emqx_authz.hrl").
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-define(DEFAULT_CHECK_AVAIL_TIMEOUT, 1000).
+
+reset_authorizers() ->
+    reset_authorizers(allow, true).
+
+reset_authorizers(Nomatch, ChacheEnabled) ->
+    {ok, _} = emqx:update_config(
+                [authorization],
+                #{<<"no_match">> => atom_to_binary(Nomatch),
+                  <<"cache">> => #{<<"enable">> => atom_to_binary(ChacheEnabled)},
+                  <<"sources">> => []}),
+    ok.
+
+is_tcp_server_available(Host, Port) ->
+    case gen_tcp:connect(Host, Port, [], ?DEFAULT_CHECK_AVAIL_TIMEOUT) of
+        {ok, Socket} ->
+            gen_tcp:close(Socket),
+            true;
+        {error, _} ->
+            false
+    end.