Merge pull request #4453 from k32/fix-auth-mnesia

fix(emqx_auth_mnesia): add missing combinations of permissions
This commit is contained in:
k32 2021-04-06 21:37:50 +02:00 committed by GitHub
commit d54643e9e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 358 additions and 27 deletions

View File

@ -31,6 +31,7 @@
init() ->
ok = ekka_mnesia:create_table(emqx_acl, [
{type, bag},
{disc_copies, [node()]},
{attributes, record_info(fields, emqx_acl)},
{storage_properties, [{ets, [{read_concurrency, true}]}]}]),

View File

@ -39,13 +39,24 @@
-spec(add_acl(login() | all, emqx_topic:topic(), pub | sub | pubsub, allow | deny) ->
ok | {error, any()}).
add_acl(Login, Topic, Action, Access) ->
Acls = #?TABLE{
filter = {Login, Topic},
action = Action,
access = Access,
created_at = erlang:system_time(millisecond)
},
ret(mnesia:transaction(fun mnesia:write/1, [Acls])).
Filter = {Login, Topic},
Acl = #?TABLE{
filter = Filter,
action = Action,
access = Access,
created_at = erlang:system_time(millisecond)
},
ret(mnesia:transaction(
fun() ->
OldRecords = mnesia:wread({?TABLE, Filter}),
case Action of
pubsub ->
update_permission(pub, Acl, OldRecords),
update_permission(sub, Acl, OldRecords);
_ ->
update_permission(Action, Acl, OldRecords)
end
end)).
%% @doc Lookup acl by login
-spec(lookup_acl(login() | all) -> list()).
@ -233,3 +244,27 @@ print_acl({all, Topic, Action, Access, _}) ->
"Acl($all topic = ~p action = ~p access = ~p)~n",
[Topic, Action, Access]
).
update_permission(Action, Acl0, OldRecords) ->
Acl = Acl0 #?TABLE{action = Action},
maybe_delete_shadowed_records(Action, OldRecords),
mnesia:write(Acl).
maybe_delete_shadowed_records(_, []) ->
ok;
maybe_delete_shadowed_records(Action1, [Rec = #emqx_acl{action = Action2} | Rest]) ->
if Action1 =:= Action2 ->
ok = mnesia:delete_object(Rec);
Action2 =:= pubsub ->
%% Perform migration from the old data format on the
%% fly. This is needed only for the enterprise version,
%% delete this branch on 5.0
mnesia:delete_object(Rec),
mnesia:write(Rec#?TABLE{action = other_action(Action1)});
true ->
ok
end,
maybe_delete_shadowed_records(Action1, Rest).
other_action(pub) -> sub;
other_action(sub) -> pub.

View File

@ -86,11 +86,15 @@ t_management(_Config) ->
ok = emqx_acl_mnesia_cli:add_acl({username, <<"test_username">>}, <<"topic/%u">>, sub, deny),
ok = emqx_acl_mnesia_cli:add_acl({username, <<"test_username">>}, <<"topic/+">>, pub, allow),
ok = emqx_acl_mnesia_cli:add_acl(all, <<"#">>, pubsub, deny),
%% Sleeps below are needed to hide the race condition between
%% mnesia and ets dirty select in check_acl, that make this test
%% flaky
timer:sleep(100),
?assertEqual(2, length(emqx_acl_mnesia_cli:lookup_acl({clientid, <<"test_clientid">>}))),
?assertEqual(2, length(emqx_acl_mnesia_cli:lookup_acl({username, <<"test_username">>}))),
?assertEqual(1, length(emqx_acl_mnesia_cli:lookup_acl(all))),
?assertEqual(5, length(emqx_acl_mnesia_cli:all_acls())),
?assertEqual(2, length(emqx_acl_mnesia_cli:lookup_acl(all))),
?assertEqual(6, length(emqx_acl_mnesia_cli:all_acls())),
User1 = #{zone => external, clientid => <<"test_clientid">>},
User2 = #{zone => external, clientid => <<"no_exist">>, username => <<"test_username">>},
@ -105,11 +109,55 @@ t_management(_Config) ->
deny = emqx_access_control:check_acl(User3, subscribe, <<"topic/A/B">>),
deny = emqx_access_control:check_acl(User3, publish, <<"topic/A/B">>),
%% Test merging of pubsub capability:
ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pubsub, deny),
timer:sleep(100),
deny = emqx_access_control:check_acl(User1, subscribe, <<"topic/mix">>),
deny = emqx_access_control:check_acl(User1, publish, <<"topic/mix">>),
ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, allow),
timer:sleep(100),
deny = emqx_access_control:check_acl(User1, subscribe, <<"topic/mix">>),
allow = emqx_access_control:check_acl(User1, publish, <<"topic/mix">>),
ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pubsub, allow),
timer:sleep(100),
allow = emqx_access_control:check_acl(User1, subscribe, <<"topic/mix">>),
allow = emqx_access_control:check_acl(User1, publish, <<"topic/mix">>),
ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, sub, deny),
timer:sleep(100),
deny = emqx_access_control:check_acl(User1, subscribe, <<"topic/mix">>),
allow = emqx_access_control:check_acl(User1, publish, <<"topic/mix">>),
ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, deny),
timer:sleep(100),
deny = emqx_access_control:check_acl(User1, subscribe, <<"topic/mix">>),
deny = emqx_access_control:check_acl(User1, publish, <<"topic/mix">>),
%% Test implicit migration of pubsub to pub and sub:
ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>),
ok = mnesia:dirty_write(#emqx_acl{
filter = {{clientid, <<"test_clientid">>}, <<"topic/mix">>},
action = pubsub,
access = allow,
created_at = erlang:system_time(millisecond)
}),
timer:sleep(100),
allow = emqx_access_control:check_acl(User1, subscribe, <<"topic/mix">>),
allow = emqx_access_control:check_acl(User1, publish, <<"topic/mix">>),
ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, deny),
timer:sleep(100),
allow = emqx_access_control:check_acl(User1, subscribe, <<"topic/mix">>),
deny = emqx_access_control:check_acl(User1, publish, <<"topic/mix">>),
ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, sub, deny),
timer:sleep(100),
deny = emqx_access_control:check_acl(User1, subscribe, <<"topic/mix">>),
deny = emqx_access_control:check_acl(User1, publish, <<"topic/mix">>),
ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>),
ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/+">>),
ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>),
ok = emqx_acl_mnesia_cli:remove_acl({username, <<"test_username">>}, <<"topic/%u">>),
ok = emqx_acl_mnesia_cli:remove_acl({username, <<"test_username">>}, <<"topic/+">>),
ok = emqx_acl_mnesia_cli:remove_acl(all, <<"#">>),
timer:sleep(100),
?assertEqual([], emqx_acl_mnesia_cli:all_acls()).
@ -124,6 +172,7 @@ t_acl_cli(_Config) ->
?assertEqual(0, length(emqx_acl_mnesia_cli:cli(["list"]))),
emqx_acl_mnesia_cli:cli(["add", "clientid", "test_clientid", "topic/A", "pub", "deny"]),
emqx_acl_mnesia_cli:cli(["add", "clientid", "test_clientid", "topic/A", "pub", "allow"]),
R1 = emqx_ctl:format("Acl(clientid = ~p topic = ~p action = ~p access = ~p)~n",
[<<"test_clientid">>, <<"topic/A">>, pub, allow]),
@ -136,11 +185,14 @@ t_acl_cli(_Config) ->
?assertEqual([R2], emqx_acl_mnesia_cli:cli(["show", "username", "test_username"])),
?assertEqual([R2], emqx_acl_mnesia_cli:cli(["list", "username"])),
emqx_acl_mnesia_cli:cli(["add", "_all", "#", "pub", "allow"]),
emqx_acl_mnesia_cli:cli(["add", "_all", "#", "pubsub", "deny"]),
?assertMatch(["Acl($all topic = <<\"#\">> action = pubsub access = deny)\n"],
emqx_acl_mnesia_cli:cli(["list", "_all"])
?assertMatch(["",
"Acl($all topic = <<\"#\">> action = pub access = deny)",
"Acl($all topic = <<\"#\">> action = sub access = deny)"],
lists:sort(string:split(emqx_acl_mnesia_cli:cli(["list", "_all"]), "\n", all))
),
?assertEqual(3, length(emqx_acl_mnesia_cli:cli(["list"]))),
?assertEqual(4, length(emqx_acl_mnesia_cli:cli(["list"]))),
emqx_acl_mnesia_cli:cli(["del", "clientid", "test_clientid", "topic/A"]),
emqx_acl_mnesia_cli:cli(["del", "username", "test_username", "topic/B"]),
@ -169,7 +221,7 @@ t_rest_api(_Config) ->
}],
{ok, _} = request_http_rest_add([], Params1),
{ok, Re1} = request_http_rest_list(["clientid", "test_clientid"]),
?assertMatch(3, length(get_http_data(Re1))),
?assertMatch(4, length(get_http_data(Re1))),
{ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/A"]),
{ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/B"]),
{ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/C"]),
@ -193,7 +245,7 @@ t_rest_api(_Config) ->
}],
{ok, _} = request_http_rest_add([], Params2),
{ok, Re2} = request_http_rest_list(["username", "test_username"]),
?assertMatch(3, length(get_http_data(Re2))),
?assertMatch(4, length(get_http_data(Re2))),
{ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/A"]),
{ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/B"]),
{ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/C"]),
@ -214,7 +266,7 @@ t_rest_api(_Config) ->
}],
{ok, _} = request_http_rest_add([], Params3),
{ok, Re3} = request_http_rest_list(["$all"]),
?assertMatch(3, length(get_http_data(Re3))),
?assertMatch(4, length(get_http_data(Re3))),
{ok, _} = request_http_rest_delete(["$all", "topic", "topic/A"]),
{ok, _} = request_http_rest_delete(["$all", "topic", "topic/B"]),
{ok, _} = request_http_rest_delete(["$all", "topic", "topic/C"]),

View File

@ -127,7 +127,7 @@ import(_Bindings, Params) ->
do_import(Filename) ->
FullFilename = filename:join([emqx:get_env(data_dir), Filename]),
emqx_mgmt_data_backup:import(FullFilename).
emqx_mgmt_data_backup:import(FullFilename, "{}").
download(#{filename := Filename}, _Params) ->
FullFilename = filename:join([emqx:get_env(data_dir), Filename]),

View File

@ -562,7 +562,9 @@ data(["export"]) ->
end;
data(["import", Filename]) ->
case emqx_mgmt_data_backup:import(Filename) of
data(["import", Filename, "--env", "{}"]);
data(["import", Filename, "--env", Env]) ->
case emqx_mgmt_data_backup:import(Filename, Env) of
ok ->
emqx_ctl:print("The emqx data has been imported successfully.~n");
{error, import_failed} ->
@ -574,8 +576,9 @@ data(["import", Filename]) ->
end;
data(_) ->
emqx_ctl:usage([{"data import <File>", "Import data from the specified file"},
{"data export", "Export data"}]).
emqx_ctl:usage([{"data import <File> [--env '<json>']",
"Import data from the specified file, possibly with overrides"},
{"data export", "Export data"}]).
%%--------------------------------------------------------------------
%% @doc acl Command

View File

@ -52,7 +52,7 @@
]).
-export([ export/0
, import/1
, import/2
]).
%%--------------------------------------------------------------------
@ -441,9 +441,11 @@ import_acl_mnesia(Acls, _) ->
do_import_acl_mnesia(Acls).
-else.
import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse
FromVersion =:= "4.1" orelse
FromVersion =:= "4.2" ->
FromVersion =:= "4.1" ->
do_import_auth_mnesia_by_old_data(Auths);
import_auth_mnesia(Auths, "4.2") ->
%% 4.2 contains a bug where password is not base64-encoded
do_import_auth_mnesia_4_2(Auths);
import_auth_mnesia(Auths, _) ->
do_import_auth_mnesia(Auths).
@ -454,6 +456,17 @@ import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse
import_acl_mnesia(Acls, _) ->
do_import_acl_mnesia(Acls).
do_import_auth_mnesia_4_2(Auths) ->
case ets:info(emqx_user) of
undefined -> ok;
_ ->
CreatedAt = erlang:system_time(millisecond),
lists:foreach(fun(#{<<"login">> := Login,
<<"password">> := Password}) ->
mnesia:dirty_write({emqx_user, {get_old_type(), Login}, Password, CreatedAt})
end, Auths)
end.
-endif.
do_import_auth_mnesia_by_old_data(Auths) ->
@ -463,9 +476,11 @@ do_import_auth_mnesia_by_old_data(Auths) ->
CreatedAt = erlang:system_time(millisecond),
lists:foreach(fun(#{<<"login">> := Login,
<<"password">> := Password}) ->
mnesia:dirty_write({emqx_user, {username, Login}, base64:decode(Password), CreatedAt})
mnesia:dirty_write({emqx_user, {get_old_type(), Login}, base64:decode(Password), CreatedAt})
end, Auths)
end.
do_import_auth_mnesia(Auths) ->
case ets:info(emqx_user) of
undefined -> ok;
@ -491,7 +506,7 @@ do_import_acl_mnesia_by_old_data(Acls) ->
true -> allow;
false -> deny
end,
mnesia:dirty_write({emqx_acl, {{username, Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt})
mnesia:dirty_write({emqx_acl, {{get_old_type(), Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt})
end, Acls)
end.
do_import_acl_mnesia(Acls) ->
@ -599,11 +614,14 @@ do_export_extra_data() ->
do_export_extra_data() -> [].
-endif.
import(Filename) ->
import(Filename, OverridesJson) ->
case file:read_file(Filename) of
{ok, Json} ->
Data = emqx_json:decode(Json, [return_maps]),
Imported = emqx_json:decode(Json, [return_maps]),
Overrides = emqx_json:decode(OverridesJson, [return_maps]),
Data = maps:merge(Imported, Overrides),
Version = to_version(maps:get(<<"version">>, Data)),
read_global_auth_type(Data, Version),
case lists:member(Version, ?VERSIONS) of
true ->
try
@ -648,4 +666,28 @@ covert_empty_headers(Headers) ->
[] -> #{};
Other -> Other
end.
-endif.
-endif.
read_global_auth_type(Data, Version) when Version =:= "4.0" orelse
Version =:= "4.1" orelse
Version =:= "4.2" ->
case Data of
#{<<"auth.mnesia.as">> := <<"username">>} -> application:set_env(emqx_auth_mnesia, as, username);
#{<<"auth.mnesia.as">> := <<"clientid">>} -> application:set_env(emqx_auth_mnesia, as, clientid);
_ ->
logger:error("While importing data from EMQX versions prior to 4.3 "
"it is necessary to specify the value of \"auth.mnesia.as\" parameter "
"as it was configured in etc/plugins/emqx_auth_mnesia.conf.\n"
"Use the following command to import data:\n"
" $ emqx_ctl data import <filename> --env '{\"auth.mnesia.as\":\"username\"}'\n"
"or\n"
" $ emqx_ctl data import <filename> --env '{\"auth.mnesia.as\":\"clientid\"}'",
[]),
error(import_failed)
end;
read_global_auth_type(_Data, _Version) ->
ok.
get_old_type() ->
{ok, Type} = application:get_env(emqx_auth_mnesia, as),
Type.

View File

@ -0,0 +1,97 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_auth_mnesia_migration_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx_auth_mnesia/include/emqx_auth_mnesia.hrl").
all() ->
[{group, Id} || {Id, _, _} <- groups()].
groups() ->
[{username, [], cases()}, {clientid, [], cases()}].
cases() ->
[t_import_4_2, t_import_4_1].
init_per_suite(Config) ->
emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard, emqx_auth_mnesia]),
ekka_mnesia:start(),
emqx_mgmt_auth:mnesia(boot),
Config.
end_per_suite(_Config) ->
emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_dashboard, emqx_management, emqx_auth_mnesia]),
ekka_mnesia:ensure_stopped().
init_per_group(username, Config) ->
[{cred_type, username} | Config];
init_per_group(clientid, Config) ->
[{cred_type, clientid} | Config].
end_per_group(_, Config) ->
Config.
init_per_testcase(_, Config) ->
Config.
end_per_testcase(_, _Config) ->
mnesia:clear_table(emqx_acl),
mnesia:clear_table(emqx_user),
ok.
t_import_4_2(Config) ->
test_import(Config, "v4.2.json").
t_import_4_1(Config) ->
test_import(Config, "v4.1.json").
test_import(Config, File) ->
Type = proplists:get_value(cred_type, Config),
mnesia:clear_table(emqx_acl),
mnesia:clear_table(emqx_user),
Filename = filename:join(proplists:get_value(data_dir, Config), File),
Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(Type)}),
?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, Overrides)),
Records = lists:sort(ets:tab2list(emqx_acl)),
%% Check importing of records related to emqx_auth_mnesia
?assertMatch([#emqx_acl{
filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>},
action = pub,
access = allow
},
#emqx_acl{
filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>},
action = sub,
access = allow
}],
lists:sort(Records)),
?assertMatch([#emqx_user{
login = {Type, <<"emqx_c">>}
}], ets:tab2list(emqx_user)),
Req = #{clientid => <<"blah">>}
#{Type => <<"emqx_c">>,
password => "emqx_p"
},
?assertMatch({stop, #{auth_result := success}},
emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})).

View File

@ -0,0 +1,48 @@
{
"acl_mnesia": [
{
"action": "sub",
"allow": true,
"login": "emqx_c",
"topic": "Topic/A"
},
{
"action": "pub",
"allow": true,
"login": "emqx_c",
"topic": "Topic/A"
}
],
"apps": [
{
"desc": "Application user",
"expired": "undefined",
"id": "admin",
"name": "Default",
"secret": "public",
"status": true
}
],
"auth_clientid": [],
"auth_mnesia": [
{
"is_superuser": false,
"login": "emqx_c",
"password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ=="
}
],
"auth_username": [],
"blacklist": [],
"date": "2021-03-30 09:11:29",
"resources": [],
"rules": [],
"schemas": [],
"users": [
{
"password": "t89PhgOb15rSCdpxm7Obp7QGcyY=",
"tags": "administrator",
"username": "admin"
}
],
"version": "4.1"
}

View File

@ -0,0 +1,53 @@
{
"schemas": [],
"acl_mnesia": [
{
"allow": true,
"action": "sub",
"topic": "Topic/A",
"login": "emqx_c"
},
{
"allow": true,
"action": "pub",
"topic": "Topic/A",
"login": "emqx_c"
}
],
"auth_mnesia": [
{
"is_superuser": false,
"password": "ceb5e917f7930ae8f0dc3ceb496a428f7e644736eebca36a2b8f6bbac756171a",
"login": "emqx_c"
}
],
"auth_username": [],
"auth_clientid": [],
"users": [
{
"tags": "viewer",
"password": "oVqjR1wOi2u4DtsuXNctYt6+SKE=",
"username": "test"
},
{
"tags": "administrator",
"password": "9SO4rEEZ6rNwA4vAwp3cnXgQsAM=",
"username": "admin"
}
],
"apps": [
{
"expired": "undefined",
"status": true,
"desc": "Application user",
"name": "Default",
"secret": "public",
"id": "admin"
}
],
"blacklist": [],
"resources": [],
"rules": [],
"date": "2021-03-26 09:51:38",
"version": "4.2"
}