diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index e8b235be7..bee65781a 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.10"}, % strict semver, bump manually! + {vsn, "4.3.11"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index d17167785..9623c6682 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -46,8 +46,8 @@ , import_users/1 , import_auth_clientid/1 %% BACKW: 4.1.x , import_auth_username/1 %% BACKW: 4.1.x - , import_auth_mnesia/2 - , import_acl_mnesia/2 + , import_auth_mnesia/1 + , import_acl_mnesia/1 , to_version/1 ]). @@ -422,87 +422,103 @@ import_auth_username(Lists) -> end, Lists) end. --ifdef(EMQX_ENTERPRISE). -import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> - do_import_auth_mnesia_by_old_data(Auths); -import_auth_mnesia(Auths, _) -> - do_import_auth_mnesia(Auths). +import_auth_mnesia(Auths) -> + case validate_auth(Auths) of + ignore -> ok; + old -> do_import_auth_mnesia_by_old_data(Auths); + new -> do_import_auth_mnesia(Auths) + end. -import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> - do_import_acl_mnesia_by_old_data(Acls); +validate_auth(Auths) -> + case ets:info(emqx_user) of + undefined -> ignore; + _ -> + case lists:all(fun is_new_auth_data/1, Auths) of + true -> new; + false -> + case lists:all(fun is_old_auth_data/1, Auths) of + true -> + _ = get_old_type(), + old; + false -> error({auth_mnesia_data_error, Auths}) + end + end + end. -import_acl_mnesia(Acls, _) -> - do_import_acl_mnesia(Acls). --else. -import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.3" -> - do_import_auth_mnesia(Auths); -import_auth_mnesia(Auths, _FromVersion) -> - do_import_auth_mnesia_by_old_data(Auths). +is_new_auth_data(#{<<"type">> := _, <<"login">> := _, <<"password">> := _}) -> true; +is_new_auth_data(_) -> false. -import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.3" -> - do_import_acl_mnesia(Acls); -import_acl_mnesia(Acls, _FromVersion) -> - do_import_acl_mnesia_by_old_data(Acls). - --endif. +is_old_auth_data(#{<<"login">> := _, <<"password">> := _} = Auth) -> + not maps:is_key(<<"type">>, Auth); +is_old_auth_data(_) -> false. do_import_auth_mnesia_by_old_data(Auths) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - CreatedAt = erlang:system_time(millisecond), - lists:foreach(fun(#{<<"login">> := Login, - <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {get_old_type(), Login}, base64:decode(Password), CreatedAt}) - end, Auths) - end. - + CreatedAt = erlang:system_time(millisecond), + Type = get_old_type(), + lists:foreach(fun(#{<<"login">> := Login, <<"password">> := Password}) -> + mnesia:dirty_write({emqx_user, {Type, Login}, base64:decode(Password), CreatedAt}) + end, Auths). do_import_auth_mnesia(Auths) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - lists:foreach(fun(#{<<"login">> := Login, - <<"type">> := Type, - <<"password">> := Password } = Map) -> - CreatedAt = maps:get(<<"created_at">>, Map, erlang:system_time(millisecond)), - mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt}) - end, Auths) + CreatedAt0 = erlang:system_time(millisecond), + lists:foreach(fun(#{<<"login">> := Login, + <<"type">> := Type, <<"password">> := Password } = Map) -> + CreatedAt = maps:get(<<"created_at">>, Map, CreatedAt0), + mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt}) + end, Auths). + +import_acl_mnesia(Acls) -> + case validate_acl(Acls) of + ignore -> ok; + old -> do_import_acl_mnesia_by_old_data(Acls); + new -> do_import_acl_mnesia(Acls) end. +validate_acl(Acls) -> + case ets:info(emqx_acl2) of + undefined -> ignore; + _ -> + case lists:all(fun is_new_acl_data/1, Acls) of + true -> new; + false -> + case lists:all(fun is_old_acl_data/1, Acls) of + true -> + _ = get_old_type(), + old; + false -> error({acl_mnesia_data_error, Acls}) + end + end + end. + +is_new_acl_data(#{<<"action">> := _, <<"access">> := _, + <<"topic">> := _, <<"type">> := _}) -> true; +is_new_acl_data(_) -> false. + +is_old_acl_data(#{<<"login">> := _, <<"topic">> := _, + <<"allow">> := Allow, <<"action">> := _}) -> is_boolean(any_to_atom(Allow)); +is_old_acl_data(_) -> false. + do_import_acl_mnesia_by_old_data(Acls) -> - case ets:info(emqx_acl2) of - undefined -> ok; - _ -> - lists:foreach(fun(#{<<"login">> := Login, - <<"topic">> := Topic, - <<"allow">> := Allow, - <<"action">> := Action}) -> - Allow1 = case any_to_atom(Allow) of - true -> allow; - false -> deny - end, - emqx_acl_mnesia_db:add_acl({get_old_type(), Login}, Topic, any_to_atom(Action), Allow1) - end, Acls) - end. + lists:foreach(fun(#{<<"login">> := Login, + <<"topic">> := Topic, + <<"allow">> := Allow, + <<"action">> := Action}) -> + Allow1 = case any_to_atom(Allow) of + true -> allow; + false -> deny + end, + emqx_acl_mnesia_db:add_acl({get_old_type(), Login}, Topic, any_to_atom(Action), Allow1) + end, Acls). + do_import_acl_mnesia(Acls) -> - case ets:info(emqx_acl2) of - undefined -> ok; - _ -> - lists:foreach(fun(Map = #{<<"action">> := Action, - <<"access">> := Access}) -> - Topic = maps:get(<<"topic">>, Map), - Login = case maps:get(<<"type_value">>, Map, undefined) of - undefined -> - all; - Value -> - {any_to_atom(maps:get(<<"type">>, Map)), Value} - end, - emqx_acl_mnesia_db:add_acl(Login, Topic, any_to_atom(Action), any_to_atom(Access)) - end, Acls) - end. + lists:foreach(fun(Map = #{<<"action">> := Action, + <<"access">> := Access, <<"topic">> := Topic}) -> + Login = case maps:get(<<"type_value">>, Map, undefined) of + undefined -> all; + Value -> {any_to_atom(maps:get(<<"type">>, Map)), Value} + end, + emqx_acl_mnesia_db:add_acl(Login, Topic, any_to_atom(Action), any_to_atom(Access)) + end, Acls). -ifdef(EMQX_ENTERPRISE). -dialyzer({nowarn_function, [import_modules/1]}). @@ -648,8 +664,8 @@ do_import_data(Data, Version) -> import_users(maps:get(<<"users">>, Data, [])), import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), - import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), - import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version). + import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])), + import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])). -ifdef(EMQX_ENTERPRISE). do_import_extra_data(Data, _Version) -> @@ -709,9 +725,9 @@ read_global_auth_type(Data, Version) -> do_read_global_auth_type(Data, _Version) -> case Data of #{<<"auth.mnesia.as">> := <<"username">>} -> - application:set_env(emqx_auth_mnesia, as, username); + set_old_type(username); #{<<"auth.mnesia.as">> := <<"clientid">>} -> - application:set_env(emqx_auth_mnesia, as, clientid); + set_old_type(clientid); _ -> ok end. @@ -720,9 +736,9 @@ do_read_global_auth_type(Data, _Version) -> do_read_global_auth_type(Data, FromVersion) -> case Data of #{<<"auth.mnesia.as">> := <<"username">>} -> - application:set_env(emqx_auth_mnesia, as, username); + set_old_type(username); #{<<"auth.mnesia.as">> := <<"clientid">>} -> - application:set_env(emqx_auth_mnesia, as, clientid); + set_old_type(clientid); _ when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse FromVersion =:= "4.2"-> @@ -734,7 +750,7 @@ do_read_global_auth_type(Data, FromVersion) -> "or\n" " $ emqx_ctl data import --env '{\"auth.mnesia.as\":\"clientid\"}'", []), - error(import_failed); + error({import_failed, FromVersion}); _ -> ok end. @@ -743,3 +759,6 @@ do_read_global_auth_type(Data, FromVersion) -> get_old_type() -> {ok, Type} = application:get_env(emqx_auth_mnesia, as), Type. + +set_old_type(Type) -> + application:set_env(emqx_auth_mnesia, as, Type). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl index 7ccba161b..19ec8d61e 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl @@ -39,13 +39,13 @@ cases() -> [t_import]. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard, emqx_auth_mnesia]), + emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia]), ekka_mnesia:start(), emqx_mgmt_auth:mnesia(boot), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_dashboard, emqx_auth_mnesia]), + emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_auth_mnesia]), ekka_mnesia:ensure_stopped(). init_per_testcase(_, Config) -> @@ -167,7 +167,7 @@ t_export_import(_Config) -> ?assertEqual([], emqx_acl_mnesia_db:all_acls()), - emqx_mgmt_data_backup:import_acl_mnesia(emqx_json:decode(AclData, [return_maps]), "4.3"), + emqx_mgmt_data_backup:import_acl_mnesia(emqx_json:decode(AclData, [return_maps])), timer:sleep(100), ?assertMatch([