diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index da551782d..855e09525 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -127,7 +127,7 @@ import(_Bindings, Params) -> do_import(Filename) -> FullFilename = filename:join([emqx:get_env(data_dir), Filename]), - emqx_mgmt_data_backup:import(FullFilename). + emqx_mgmt_data_backup:import(FullFilename, "{}"). download(#{filename := Filename}, _Params) -> FullFilename = filename:join([emqx:get_env(data_dir), Filename]), diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index e4f443cee..e342cf532 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -562,7 +562,9 @@ data(["export"]) -> end; data(["import", Filename]) -> - case emqx_mgmt_data_backup:import(Filename) of + data(["import", Filename, "--env", "{}"]); +data(["import", Filename, "--env", Env]) -> + case emqx_mgmt_data_backup:import(Filename, Env) of ok -> emqx_ctl:print("The emqx data has been imported successfully.~n"); {error, import_failed} -> @@ -574,8 +576,9 @@ data(["import", Filename]) -> end; data(_) -> - emqx_ctl:usage([{"data import ", "Import data from the specified file"}, - {"data export", "Export data"}]). + emqx_ctl:usage([{"data import [--env '']", + "Import data from the specified file, possibly with overrides"}, + {"data export", "Export data"}]). %%-------------------------------------------------------------------- %% @doc acl Command diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index a455e18c8..2d6447bde 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -52,7 +52,7 @@ ]). -export([ export/0 - , import/1 + , import/2 ]). %%-------------------------------------------------------------------- @@ -464,7 +464,7 @@ do_import_auth_mnesia_4_2(Auths) -> CreatedAt = erlang:system_time(millisecond), lists:foreach(fun(#{<<"login">> := Login, <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {username, Login}, Password, CreatedAt}) + mnesia:dirty_write({emqx_user, {get_old_type(), Login}, Password, CreatedAt}) end, Auths) end. -endif. @@ -476,7 +476,7 @@ do_import_auth_mnesia_by_old_data(Auths) -> CreatedAt = erlang:system_time(millisecond), lists:foreach(fun(#{<<"login">> := Login, <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {username, Login}, base64:decode(Password), CreatedAt}) + mnesia:dirty_write({emqx_user, {get_old_type(), Login}, base64:decode(Password), CreatedAt}) end, Auths) end. @@ -506,7 +506,7 @@ do_import_acl_mnesia_by_old_data(Acls) -> true -> allow; false -> deny end, - mnesia:dirty_write({emqx_acl, {{username, Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt}) + mnesia:dirty_write({emqx_acl, {{get_old_type(), Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt}) end, Acls) end. do_import_acl_mnesia(Acls) -> @@ -614,11 +614,14 @@ do_export_extra_data() -> do_export_extra_data() -> []. -endif. -import(Filename) -> +import(Filename, OverridesJson) -> case file:read_file(Filename) of {ok, Json} -> - Data = emqx_json:decode(Json, [return_maps]), + Imported = emqx_json:decode(Json, [return_maps]), + Overrides = emqx_json:decode(OverridesJson, [return_maps]), + Data = maps:merge(Imported, Overrides), Version = to_version(maps:get(<<"version">>, Data)), + read_global_auth_type(Data, Version), case lists:member(Version, ?VERSIONS) of true -> try @@ -664,3 +667,27 @@ covert_empty_headers(Headers) -> Other -> Other end. -endif. + +read_global_auth_type(Data, Version) when Version =:= "4.0" orelse + Version =:= "4.1" orelse + Version =:= "4.2" -> + case Data of + #{<<"auth.mnesia.as">> := <<"username">>} -> application:set_env(emqx_auth_mnesia, as, username); + #{<<"auth.mnesia.as">> := <<"clientid">>} -> application:set_env(emqx_auth_mnesia, as, clientid); + _ -> + logger:error("While importing data from EMQX versions prior to 4.3 " + "it is necessary to specify the value of \"auth.mnesia.as\" parameter " + "as it was configured in etc/plugins/emqx_auth_mnesia.conf.\n" + "Use the following command to import data:\n" + " $ emqx_ctl data import --env '{\"auth.mnesia.as\":\"username\"}'\n" + "or\n" + " $ emqx_ctl data import --env '{\"auth.mnesia.as\":\"clientid\"}'", + []), + error(import_failed) + end; +read_global_auth_type(_Data, _Version) -> + ok. + +get_old_type() -> + {ok, Type} = application:get_env(emqx_auth_mnesia, as), + Type. diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl index 42ab0d907..39312cc40 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl @@ -26,20 +26,32 @@ -include_lib("emqx_auth_mnesia/include/emqx_auth_mnesia.hrl"). all() -> + [{group, Id} || {Id, _, _} <- groups()]. + +groups() -> + [{username, [], cases()}, {clientid, [], cases()}]. + +cases() -> [t_import_4_2, t_import_4_1]. init_per_suite(Config) -> emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard, emqx_auth_mnesia]), ekka_mnesia:start(), emqx_mgmt_auth:mnesia(boot), - mnesia:clear_table(emqx_acl), - mnesia:clear_table(emqx_user), Config. end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_dashboard, emqx_management, emqx_auth_mnesia]), ekka_mnesia:ensure_stopped(). +init_per_group(username, Config) -> + [{cred_type, username} | Config]; +init_per_group(clientid, Config) -> + [{cred_type, clientid} | Config]. + +end_per_group(_, Config) -> + Config. + init_per_testcase(_, Config) -> Config. @@ -55,26 +67,30 @@ t_import_4_1(Config) -> test_import(Config, "v4.1.json"). test_import(Config, File) -> + Type = proplists:get_value(cred_type, Config), + mnesia:clear_table(emqx_acl), + mnesia:clear_table(emqx_user), Filename = filename:join(proplists:get_value(data_dir, Config), File), - ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename)), + Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(Type)}), + ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, Overrides)), Records = lists:sort(ets:tab2list(emqx_acl)), %% Check importing of records related to emqx_auth_mnesia ?assertMatch([#emqx_acl{ - filter = {{username,<<"emqx_c">>}, <<"Topic/A">>}, + filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>}, action = pub, access = allow }, #emqx_acl{ - filter = {{username,<<"emqx_c">>}, <<"Topic/A">>}, + filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>}, action = sub, access = allow }], lists:sort(Records)), ?assertMatch([#emqx_user{ - login = {username, <<"emqx_c">>} + login = {Type, <<"emqx_c">>} }], ets:tab2list(emqx_user)), - Req = #{clientid => "blah", - username => <<"emqx_c">>, + Req = #{clientid => <<"blah">>} + #{Type => <<"emqx_c">>, password => "emqx_p" }, ?assertMatch({stop, #{auth_result := success}},