From 8304a96e1de10b968a55ba1375fed091bbf709b6 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Sat, 10 Apr 2021 14:55:02 +0200 Subject: [PATCH] fix(emqx_management): Don't fix broken emqx_auth_mnesia data --- .../src/emqx_mgmt_data_backup.erl | 35 ++++++++++--------- .../test/emqx_auth_mnesia_migration_SUITE.erl | 27 ++++++++------ .../make_data.sh | 22 +++++++++--- .../{v4.0.7.json => v4.0.11.json} | 4 +-- .../v4.2.10-no-auth.json | 34 ++++++++++++++++++ .../{v4.2.9.json => v4.2.10.json} | 6 ++-- 6 files changed, 91 insertions(+), 37 deletions(-) rename apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/{v4.0.7.json => v4.0.11.json} (74%) create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json rename apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/{v4.2.9.json => v4.2.10.json} (81%) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index cfab5d23d..b09337c19 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -44,7 +44,7 @@ , import_blacklist/1 , import_applications/1 , import_users/1 - , import_auth_clientid/2 %% BACKW: 4.1.x + , import_auth_clientid/1 %% BACKW: 4.1.x , import_auth_username/1 %% BACKW: 4.1.x , import_auth_mnesia/2 , import_acl_mnesia/2 @@ -405,17 +405,13 @@ import_users(Users) -> emqx_dashboard_admin:force_add_user(Username, NPassword, Tags) end, Users). -import_auth_clientid(Lists, Version) -> +import_auth_clientid(Lists) -> case ets:info(emqx_user) of undefined -> ok; _ -> - lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password0}) -> - Password = case Version of - "4.1" -> base64:decode(Password0); - _ -> ensure_binary(Password0) - end, + lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password}) -> mnesia:dirty_write({emqx_user, {clientid, Clientid} - , Password + , base64:decode(Password) , erlang:system_time(millisecond)}) end, Lists) end. @@ -648,7 +644,7 @@ do_import_data(Data, Version) -> import_blacklist(maps:get(<<"blacklist">>, Data, [])), import_applications(maps:get(<<"apps">>, Data, [])), import_users(maps:get(<<"users">>, Data, [])), - import_auth_clientid(maps:get(<<"auth_clientid">>, Data, []), Version), + import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version). @@ -675,6 +671,18 @@ flag_to_boolean(Other) -> Other. read_global_auth_type(Data, Version) when Version =:= "4.0" orelse Version =:= "4.1" orelse Version =:= "4.2" -> + ct:print("|>=> :~p~n", [Data]), + case {maps:get(<<"auth_mnesia">>, Data, []), maps:get(<<"acl_mnesia">>, Data, [])} of + {[], []} -> + %% Auth mnesia plugin is not used: + ok; + _ -> + do_read_global_auth_type(Data) + end; +read_global_auth_type(_Data, _Version) -> + ok. + +do_read_global_auth_type(Data) -> case Data of #{<<"auth.mnesia.as">> := <<"username">>} -> application:set_env(emqx_auth_mnesia, as, username); @@ -690,15 +698,8 @@ read_global_auth_type(Data, Version) when Version =:= "4.0" orelse " $ emqx_ctl data import --env '{\"auth.mnesia.as\":\"clientid\"}'", []), error(import_failed) - end; -read_global_auth_type(_Data, _Version) -> - ok. + end. get_old_type() -> {ok, Type} = application:get_env(emqx_auth_mnesia, as), Type. - -ensure_binary(A) when is_binary(A) -> - A; -ensure_binary(A) -> - list_to_binary(A). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl index ad98150b3..563dd33ae 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl @@ -40,10 +40,10 @@ all() -> matrix() -> [{ImportAs, Version} || ImportAs <- [clientid, username] - , Version <- ["v4.2.9", "v4.1.5"]]. + , Version <- ["v4.2.10", "v4.1.5"]]. all() -> - [t_matrix, t_import_4_0]. + [t_matrix, t_import_4_0, t_import_no_auth]. -endif. %% EMQX_ENTERPRISE @@ -82,11 +82,18 @@ t_matrix(Config) -> %% This version is special, since it doesn't have mnesia ACL plugin t_import_4_0(Config) -> + do_import_no_auth("v4.0.11.json", Config). + +t_import_no_auth(Config) -> + do_import_no_auth("v4.2.10-no-auth.json", Config). + +%% Test that importing configs that don't contain any mnesia ACL data +%% doesn't require additional overrides: +do_import_no_auth(File, Config) -> mnesia:clear_table(emqx_acl), mnesia:clear_table(emqx_user), - Filename = filename:join(proplists:get_value(data_dir, Config), "v4.0.7.json"), - Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), - ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, Overrides)), + Filename = filename:join(proplists:get_value(data_dir, Config), File), + ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, "{}")), timer:sleep(100), test_clientid_import(). @@ -121,9 +128,9 @@ do_import(Config, Type, V) -> test_clientid_import() -> [#emqx_user{password = _Pass}] = ets:lookup(emqx_user, {clientid, <<"emqx_clientid">>}), - %% Req = #{clientid => <<"emqx_clientid">>, - %% password => <<"emqx_p">> - %% }, - %% ?assertMatch({stop, #{auth_result := success}}, - %% emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})), + Req = #{clientid => <<"emqx_clientid">>, + password => <<"emqx_p">> + }, + ?assertMatch({stop, #{auth_result := success}}, + emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})), ok. diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh index ab1743cb1..e31729784 100755 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh @@ -56,17 +56,28 @@ export_data() { cat "${filename}.json" } + +collect_4_2_no_mnesia_auth () { + container "4.2.10" + + # Add clientid + docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p + + export_data "v4.2.10-no-auth" +} + collect_4_2 () { - container "4.2.9" + container "4.2.10" create_acls "api/v4/mqtt_acl" create_user mqtt_user # Add clientid docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - export_data "v4.2.9" + export_data "v4.2.10" } + collect_e4_2 () { container "4.2.5" "ee" # Add ACLs: @@ -77,7 +88,7 @@ collect_e4_2 () { # Add clientid docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - export_data "e4.2.9" + export_data "e4.2.5" } collect_e4_1 () { @@ -105,17 +116,18 @@ collect_4_1 () { } collect_4_0 () { - container "v4.0.7" + container "v4.0.11" # Add clientid docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - export_data "v4.0.7" + export_data "v4.0.11" } collect_4_0 collect_4_1 collect_4_2 +collect_4_2_no_mnesia_auth collect_e4_2 collect_e4_1 diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json similarity index 74% rename from apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json rename to apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json index 6de74dc48..a701d0944 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json @@ -4,13 +4,13 @@ "schemas": [], "rules": [], "resources": [], - "date": "2021-04-07 14:28:49", + "date": "2021-04-10 11:45:26", "blacklist": [], "auth_username": [], "auth_mnesia": [], "auth_clientid": [ { - "password": "��Pd56c0fcdcd7636dcf8ed1ea48cd3d58acab74030157551f7f7f8684804b9239e", + "password": "9Sv2tzJlNDlmNWZhYWQ5Yzc4MWUwNmFhZWI4NjFlMDM2OWEzYmE1OTkxOTBhOGQ4N2Y3MzExY2ZiZmIxNTFkMTdkZmY=", "clientid": "emqx_clientid" } ], diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json new file mode 100644 index 000000000..10e5c7078 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json @@ -0,0 +1,34 @@ +{ + "version": "4.2", + "date": "2021-04-12 10:41:10", + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "e5M8oWEwQVqjdqceQIthC+3cPoY=", + "tags": "administrator" + } + ], + "auth_clientid": [ + { + "clientid": "emqx_clientid", + "password": "uAP84TgyMjAyNGFhY2NlMWVlNDI2NTk1MzFiZjA4YzBjY2RjNjViZmZhNjkzYjhkMDE4NTg0ZWExYjFkZGY0MTBjYWM=" + } + ], + "auth_username": [], + "auth_mnesia": [], + "acl_mnesia": [], + "schemas": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json similarity index 81% rename from apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json rename to apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json index f1ae8c9f0..1ccc6ce9d 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json @@ -1,6 +1,6 @@ { "version": "4.2", - "date": "2021-04-07 14:29:08", + "date": "2021-04-12 10:40:58", "rules": [], "resources": [], "blacklist": [], @@ -17,14 +17,14 @@ "users": [ { "username": "admin", - "password": "gCBXISkivpaeKetFcPSm+Eaxyxs=", + "password": "8Vd7+gVg2J3nE1Xjyxqd59sA5mo=", "tags": "administrator" } ], "auth_clientid": [ { "clientid": "emqx_clientid", - "password": "z�7d413fee461607065c161072f3707dc0a01bd1fc8476eb7be703a74a66701bb14" + "password": "UNb0e2RhNDc3NWIyNjg5Yjg4ZDExOTVhNWFkY2MzNGFmNzY2OTNmNmRlYzE4Y2ZiZjRjNzIyMWZlZTljZmEyZDE5Yzc=" } ], "auth_username": [],