diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index e3e1f9fce..53698dcea 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -32,4 +32,4 @@ -define(ERROR14, 114). %% OldPassword error -define(ERROR15, 115). %% bad topic --define(VERSIONS, ["1", "3.2", "3.4", "4.0", "4.1", "4.2", "4.3"]). \ No newline at end of file +-define(VERSIONS, ["4.0", "4.1", "4.2", "4.3"]). \ No newline at end of file diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index 855e09525..e2c9a1fbd 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -112,8 +112,10 @@ import(_Bindings, Params) -> undefined -> minirest:return({error, missing_required_params}); Filename -> - Result = case proplists:get_value(<<"node">>, Params) of - undefined -> do_import(Filename); + case proplists:get_value(<<"node">>, Params) of + undefined -> + Result = do_import(Filename), + minirest:return(Result); Node -> case lists:member(Node, [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ] @@ -121,8 +123,7 @@ import(_Bindings, Params) -> true -> minirest:return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename])); false -> minirest:return({error, no_existent_node}) end - end, - minirest:return(Result) + end end. do_import(Filename) -> diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 90dbebcac..7ef0b3daf 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -439,33 +439,16 @@ import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse import_acl_mnesia(Acls, _) -> do_import_acl_mnesia(Acls). -else. -import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> - do_import_auth_mnesia_by_old_data(Auths); -import_auth_mnesia(Auths, "4.2") -> - %% 4.2 contains a bug where password is not base64-encoded - do_import_auth_mnesia_4_2(Auths); -import_auth_mnesia(Auths, _) -> - do_import_auth_mnesia(Auths). +import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.3" -> + do_import_auth_mnesia(Auths); +import_auth_mnesia(Auths, _FromVersion) -> + do_import_auth_mnesia_by_old_data(Auths). -import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" orelse - FromVersion =:= "4.2" -> - do_import_acl_mnesia_by_old_data(Acls); +import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.3" -> + do_import_acl_mnesia(Acls); +import_acl_mnesia(Acls, _FromVersion) -> + do_import_acl_mnesia_by_old_data(Acls). -import_acl_mnesia(Acls, _) -> - do_import_acl_mnesia(Acls). - -do_import_auth_mnesia_4_2(Auths) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - CreatedAt = erlang:system_time(millisecond), - lists:foreach(fun(#{<<"login">> := Login, - <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {get_old_type(), Login}, Password, CreatedAt}) - end, Auths) - end. -endif. do_import_auth_mnesia_by_old_data(Auths) -> @@ -620,8 +603,8 @@ import(Filename, OverridesJson) -> Overrides = emqx_json:decode(OverridesJson, [return_maps]), Data = maps:merge(Imported, Overrides), Version = to_version(maps:get(<<"version">>, Data)), - read_global_auth_type(Data, Version), - case lists:member(Version, ?VERSIONS) of + read_global_auth_type(Data), + case is_version_supported(Data, Version) of true -> try do_import_data(Data, Version), @@ -668,18 +651,40 @@ flag_to_boolean(<<"off">>) -> false; flag_to_boolean(Other) -> Other. -endif. -read_global_auth_type(Data, Version) when Version =:= "4.0" orelse - Version =:= "4.1" orelse - Version =:= "4.2" -> +is_version_supported(Data, Version) -> + case { maps:get(<<"auth_clientid">>, Data, []) + , maps:get(<<"auth_username">>, Data, []) + , maps:get(<<"auth_mnesia">>, Data, [])} of + {[], [], []} -> lists:member(Version, ?VERSIONS); + _ -> is_version_supported2(Version) + end. + +is_version_supported2("4.1") -> + true; +is_version_supported2("4.3") -> + true; +is_version_supported2(Version) -> + case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of + match -> + try lists:map(fun erlang:list_to_integer/1, string:tokens(Version, ".")) of + [4, 2, N] -> N >= 11; + [4, 0, N] -> N >= 13; + _ -> false + catch + _ : _ -> false + end; + nomatch -> + false + end. + +read_global_auth_type(Data) -> case {maps:get(<<"auth_mnesia">>, Data, []), maps:get(<<"acl_mnesia">>, Data, [])} of {[], []} -> %% Auth mnesia plugin is not used: ok; _ -> do_read_global_auth_type(Data) - end; -read_global_auth_type(_Data, _Version) -> - ok. + end. do_read_global_auth_type(Data) -> case Data of diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl index 26f4cccb6..0a492cbe7 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl @@ -43,7 +43,7 @@ matrix() -> , Version <- ["v4.2.10", "v4.1.5"]]. all() -> - [t_matrix, t_import_4_0, t_import_no_auth]. + [t_import_4_0, t_import_4_1, t_import_4_2]. -endif. %% EMQX_ENTERPRISE @@ -71,41 +71,55 @@ end_per_testcase(_, _Config) -> {atomic,ok} = mnesia:clear_table(emqx_user), ok. -t_matrix(Config) -> - [begin - ct:pal("Testing import of ~p from ~p", [ImportAs, FromVersion]), - do_import(Config, ImportAs, FromVersion), - test_clientid_import(), - ct:pal("ok") - end - || {ImportAs, FromVersion} <- matrix()]. - -%% This version is special, since it doesn't have mnesia ACL plugin t_import_4_0(Config) -> - do_import_no_auth("v4.0.11.json", Config). - -t_import_no_auth(Config) -> - do_import_no_auth("v4.2.10-no-auth.json", Config). - -%% Test that importing configs that don't contain any mnesia ACL data -%% doesn't require additional overrides: -do_import_no_auth(File, Config) -> - mnesia:clear_table(emqx_acl), - mnesia:clear_table(emqx_user), - Filename = filename:join(proplists:get_value(data_dir, Config), File), - ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, "{}")), + ?assertMatch(ok, do_import("v4.0.11-no-auth.json", Config)), timer:sleep(100), - test_clientid_import(). + ?assertMatch(0, ets:info(emqx_user, size)), + + ?assertMatch({error, unsupported_version, "4.0"}, do_import("v4.0.11.json", Config)), + + ?assertMatch(ok, do_import("v4.0.13.json", Config)), + timer:sleep(100), + test_import(clientid, {<<"client_for_test">>, <<"public">>}), + test_import(username, {<<"user_for_test">>, <<"public">>}). + +t_import_4_1(Config) -> + Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), + ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides)), + timer:sleep(100), + test_import(clientid, {<<"user_mnesia">>, <<"public">>}), + test_import(clientid, {<<"client_for_test">>, <<"public">>}), + test_import(username, {<<"user_for_test">>, <<"public">>}), + + Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), + ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides1)), + timer:sleep(100), + test_import(username, {<<"user_mnesia">>, <<"public">>}), + test_import(clientid, {<<"client_for_test">>, <<"public">>}), + test_import(username, {<<"user_for_test">>, <<"public">>}). + +t_import_4_2(Config) -> + ?assertMatch(ok, do_import("v4.2.10-no-auth.json", Config)), + timer:sleep(100), + ?assertMatch(0, ets:info(emqx_user, size)), + + Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), + ?assertMatch({error, unsupported_version, "4.2"}, do_import("v4.2.10.json", Config, Overrides)), + + Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), + ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides1)), + timer:sleep(100), + test_import(clientid, {<<"user_mnesia">>, <<"public">>}), + test_import(clientid, {<<"client_for_test">>, <<"public">>}), + test_import(username, {<<"user_for_test">>, <<"public">>}), + + Overrides2 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), + ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides2)), + timer:sleep(100), + test_import(username, {<<"user_mnesia">>, <<"public">>}), + test_import(clientid, {<<"client_for_test">>, <<"public">>}), + test_import(username, {<<"user_for_test">>, <<"public">>}), -do_import(Config, Type, V) -> - File = V ++ ".json", - mnesia:clear_table(emqx_acl), - mnesia:clear_table(emqx_user), - Filename = filename:join(proplists:get_value(data_dir, Config), File), - Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(Type)}), - ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, Overrides)), - Records = lists:sort(ets:tab2list(emqx_acl)), - %% Check importing of records related to emqx_auth_mnesia ?assertMatch([#emqx_acl{ filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>}, action = pub, @@ -116,21 +130,27 @@ do_import(Config, Type, V) -> action = sub, access = allow }], - lists:sort(Records)), - ?assertMatch([_, _], ets:tab2list(emqx_user)), - ?assertMatch([_], ets:lookup(emqx_user, {Type, <<"emqx_c">>})), - Req = #{clientid => <<"blah">>} - #{Type => <<"emqx_c">>, - password => <<"emqx_p">> - }, - ?assertMatch({stop, #{auth_result := success}}, - emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})). + lists:sort(ets:tab2list(emqx_acl))). -test_clientid_import() -> - [#emqx_user{password = _Pass}] = ets:lookup(emqx_user, {clientid, <<"emqx_clientid">>}), - Req = #{clientid => <<"emqx_clientid">>, - password => <<"emqx_p">> - }, +do_import(File, Config) -> + do_import(File, Config, "{}"). + +do_import(File, Config, Overrides) -> + mnesia:clear_table(emqx_acl), + mnesia:clear_table(emqx_user), + Filename = filename:join(proplists:get_value(data_dir, Config), File), + emqx_mgmt_data_backup:import(Filename, Overrides). + +test_import(username, {Username, Password}) -> + [#emqx_user{password = _}] = ets:lookup(emqx_user, {username, Username}), + Req = #{clientid => <<"anyname">>, + username => Username, + password => Password}, ?assertMatch({stop, #{auth_result := success}}, - emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})), - ok. + emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})); +test_import(clientid, {ClientID, Password}) -> + [#emqx_user{password = _}] = ets:lookup(emqx_user, {clientid, ClientID}), + Req = #{clientid => ClientID, + password => Password}, + ?assertMatch({stop, #{auth_result := success}}, + emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})). \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json new file mode 100644 index 000000000..8dad197dd --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json @@ -0,0 +1,23 @@ +{ + "version": "4.0", + "users": [], + "schemas": [], + "rules": [], + "resources": [], + "date": "2021-04-10 11:45:26", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [], + "auth_clientid": [], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json new file mode 100644 index 000000000..9c602a0e3 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json @@ -0,0 +1,37 @@ +{ + "version":"4.0.13", + "users":[ + { + "username":"admin", + "tags":"administrator", + "password":"p6C65OF0BQhvPmCziM2yRa8JN5o=" + } + ], + "schemas":[], + "rules":[], + "resources":[], + "date":"2021-04-16 11:20:00", + "blacklist":[], + "auth_username":[ + { + "username":"user_for_test", + "password":"ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" + } + ], + "auth_clientid":[ + { + "password":"JBgSnzIxOWNiMDU1ZWFiNDAwMjVhOTQzZThlZjkxN2JlZWE4MGE4YzlmM2I5MjQ4OGI1NjllY2Q4NGQ4NjhjYzQ1NDM=", + "clientid":"client_for_test" + } + ], + "apps":[ + { + "status":true, + "secret":"public", + "name":"Default", + "id":"admin", + "expired":"undefined", + "desc":"Application user" + } + ] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json index 324d8188a..3d3b9aa7e 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json @@ -12,18 +12,23 @@ "resources": [], "date": "2021-04-07 14:28:58", "blacklist": [], - "auth_username": [], + "auth_username": [ + { + "username":"user_for_test", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" + } + ], "auth_mnesia": [ { - "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ==", - "login": "emqx_c", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "login": "user_mnesia", "is_superuser": true } ], "auth_clientid": [ { - "password": "MctXdjZkYzRhMDUwMTc4MDM0OWY4YTg1NTg4Y2ZlOThjYWIyMDk3M2UzNjgzYzYyZWYwOTAzMTk2N2E4OWVjZDk4Mjk=", - "clientid": "emqx_clientid" + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "clientid": "client_for_test" } ], "apps": [ diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json index 10e5c7078..446a46f07 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json @@ -21,12 +21,7 @@ "tags": "administrator" } ], - "auth_clientid": [ - { - "clientid": "emqx_clientid", - "password": "uAP84TgyMjAyNGFhY2NlMWVlNDI2NTk1MzFiZjA4YzBjY2RjNjViZmZhNjkzYjhkMDE4NTg0ZWExYjFkZGY0MTBjYWM=" - } - ], + "auth_clientid": [], "auth_username": [], "auth_mnesia": [], "acl_mnesia": [], diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json new file mode 100644 index 000000000..52a91ac68 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json @@ -0,0 +1,58 @@ +{ + "version": "4.2.11", + "date": "2021-04-12 10:40:58", + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "test", + "password": "8Vd7+gVg2J3nE1Xjyxqd59sA5mo=", + "tags": "administrator" + } + ], + "auth_clientid": [ + { + "clientid": "client_for_test", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" + } + ], + "auth_username": [ + { + "username": "user_for_test", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" + } + ], + "auth_mnesia": [ + { + "login": "user_mnesia", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "is_superuser": true + } + ], + "acl_mnesia": [ + { + "login": "emqx_c", + "topic": "Topic/A", + "action": "sub", + "allow": true + }, + { + "login": "emqx_c", + "topic": "Topic/A", + "action": "pub", + "allow": true + } + ], + "schemas": [] +}