From 5009ec0c34262914ffe3f8d4b3482f45bcb2656c Mon Sep 17 00:00:00 2001 From: Turtle Date: Fri, 16 Apr 2021 19:11:00 +0800 Subject: [PATCH] fix(tests): fix ee import auth mnesia fail --- .../src/emqx_mgmt_data_backup.erl | 40 +++++++- .../test/emqx_auth_mnesia_migration_SUITE.erl | 49 +++++++--- .../e4.0.10.json | 52 ++++++++++ .../e4.1.1.json | 53 +++++++++++ .../e4.2.9.json | 94 +++++++++++++++++++ 5 files changed, 271 insertions(+), 17 deletions(-) create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.0.10.json create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.1.1.json create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.2.9.json diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 7ef0b3daf..a89f08da7 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -596,6 +596,26 @@ do_export_extra_data() -> do_export_extra_data() -> []. -endif. +-ifdef(EMQX_ENTERPRISE). +import(Filename, OverridesJson) -> + case file:read_file(Filename) of + {ok, Json} -> + Imported = emqx_json:decode(Json, [return_maps]), + Overrides = emqx_json:decode(OverridesJson, [return_maps]), + Data = maps:merge(Imported, Overrides), + Version = to_version(maps:get(<<"version">>, Data)), + read_global_auth_type(Data), + try + do_import_data(Data, Version), + logger:debug("The emqx data has been imported successfully"), + ok + catch Class:Reason:Stack -> + logger:error("The emqx data import failed: ~0p", [{Class, Reason, Stack}]), + {error, import_failed} + end; + Error -> Error + end. +-else. import(Filename, OverridesJson) -> case file:read_file(Filename) of {ok, Json} -> @@ -620,6 +640,7 @@ import(Filename, OverridesJson) -> end; Error -> Error end. +-endif. do_import_data(Data, Version) -> do_import_extra_data(Data, Version), @@ -651,6 +672,7 @@ flag_to_boolean(<<"off">>) -> false; flag_to_boolean(Other) -> Other. -endif. +-ifndef(EMQX_ENTERPRISE). is_version_supported(Data, Version) -> case { maps:get(<<"auth_clientid">>, Data, []) , maps:get(<<"auth_username">>, Data, []) @@ -662,7 +684,7 @@ is_version_supported(Data, Version) -> is_version_supported2("4.1") -> true; is_version_supported2("4.3") -> - true; + true; is_version_supported2(Version) -> case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of match -> @@ -670,12 +692,13 @@ is_version_supported2(Version) -> [4, 2, N] -> N >= 11; [4, 0, N] -> N >= 13; _ -> false - catch + catch _ : _ -> false end; nomatch -> false end. +-endif. read_global_auth_type(Data) -> case {maps:get(<<"auth_mnesia">>, Data, []), maps:get(<<"acl_mnesia">>, Data, [])} of @@ -686,6 +709,18 @@ read_global_auth_type(Data) -> do_read_global_auth_type(Data) end. +-ifdef(EMQX_ENTERPRISE). +do_read_global_auth_type(Data) -> + case Data of + #{<<"auth.mnesia.as">> := <<"username">>} -> + application:set_env(emqx_auth_mnesia, as, username); + #{<<"auth.mnesia.as">> := <<"clientid">>} -> + application:set_env(emqx_auth_mnesia, as, clientid); + _ -> + ok + end. + +-else. do_read_global_auth_type(Data) -> case Data of #{<<"auth.mnesia.as">> := <<"username">>} -> @@ -703,6 +738,7 @@ do_read_global_auth_type(Data) -> []), error(import_failed) end. +-endif. get_old_type() -> {ok, Type} = application:get_env(emqx_auth_mnesia, as), diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl index 0a492cbe7..838529f03 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl @@ -25,19 +25,6 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_auth_mnesia/include/emqx_auth_mnesia.hrl"). --ifdef(EMQX_ENTERPRISE). - -matrix() -> - [ {username, "e4.2.9"} - , {clientid, "e4.1.1"} - , {username, "e4.1.1"} - ]. - -all() -> - [t_matrix]. - --else. %% ! EMQX_ENTERPRISE - matrix() -> [{ImportAs, Version} || ImportAs <- [clientid, username] , Version <- ["v4.2.10", "v4.1.5"]]. @@ -45,8 +32,6 @@ matrix() -> all() -> [t_import_4_0, t_import_4_1, t_import_4_2]. --endif. %% EMQX_ENTERPRISE - groups() -> [{username, [], cases()}, {clientid, [], cases()}]. @@ -70,7 +55,40 @@ end_per_testcase(_, _Config) -> {atomic,ok} = mnesia:clear_table(emqx_acl), {atomic,ok} = mnesia:clear_table(emqx_user), ok. +-ifdef(EMQX_ENTERPRISE). +t_import_4_0(Config) -> + Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), + ?assertMatch(ok, do_import("e4.0.10.json", Config, Overrides)), + timer:sleep(100), + ct:pal("---~p~n", [ets:tab2list(emqx_user)]), + test_import(username, {<<"emqx_username">>, <<"public">>}), + test_import(clientid, {<<"emqx_c">>, <<"public">>}), + Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), + ?assertMatch(ok, do_import("e4.0.10.json", Config, Overrides1)), + timer:sleep(100), + test_import(username, {<<"emqx_c">>, <<"public">>}), + test_import(username, {<<"emqx_username">>, <<"public">>}). +t_import_4_1(Config) -> + Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), + ?assertMatch(ok, do_import("e4.1.1.json", Config, Overrides)), + timer:sleep(100), + test_import(clientid, {<<"emqx_c">>, <<"public">>}), + test_import(clientid, {<<"emqx_c">>, <<"public">>}), + + Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), + ?assertMatch(ok, do_import("e4.1.1.json", Config, Overrides1)), + timer:sleep(100), + test_import(username, {<<"emqx_c">>, <<"public">>}), + test_import(clientid, {<<"emqx_clientid">>, <<"public">>}). + +t_import_4_2(Config) -> + ?assertMatch(ok, do_import("e4.2.9.json", Config, "{}")), + timer:sleep(100), + test_import(username, {<<"emqx_c">>, <<"public">>}), + test_import(clientid, {<<"emqx_clientid">>, <<"public">>}). + +-else. t_import_4_0(Config) -> ?assertMatch(ok, do_import("v4.0.11-no-auth.json", Config)), timer:sleep(100), @@ -131,6 +149,7 @@ t_import_4_2(Config) -> access = allow }], lists:sort(ets:tab2list(emqx_acl))). +-endif. do_import(File, Config) -> do_import(File, Config, "{}"). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.0.10.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.0.10.json new file mode 100644 index 000000000..b2bf22fe0 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.0.10.json @@ -0,0 +1,52 @@ +{ + "version":"4.0", + "users":[ + { + "username":"admin", + "tags":"administrator", + "password":"Tc7LLR/wbleSDfRkENwtN2CxYbU=" + } + ], + "schemas":[ + + ], + "rules":[ + + ], + "resources":[ + + ], + "date":"2021-04-16 18:35:21", + "blacklist":[ + + ], + "auth_clientid":[ + + ], + "auth_mnesia":[ + { + "password":"ZWZhMWYzNzVkNzYxOTRmYTUxYTM1NTZhOTdlNjQxZTYxNjg1ZjkxNGQ0NDY5NzlkYTUwYTU1MWE0MzMzZmZkNw==", + "login":"emqx_c", + "is_superuser":true + } + ], + "auth_username": [ + { + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "username": "emqx_username" + } + ], + "apps":[ + { + "status":true, + "secret":"public", + "name":"Default", + "id":"admin", + "expired":"undefined", + "desc":"Application user" + } + ], + "acl_mnesia":[ + + ] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.1.1.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.1.1.json new file mode 100644 index 000000000..3afe0eb9e --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.1.1.json @@ -0,0 +1,53 @@ +{ + "version": "4.1", + "users": [ + { + "username": "admin", + "tags": "administrator", + "password": "m2grkhqiXmsYb0MtOHSi+JXsmck=" + } + ], + "schemas": [], + "rules": [], + "resources": [], + "date": "2021-04-08 12:12:02", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [ + { + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "login": "emqx_c", + "is_superuser": true + } + ], + "auth_clientid": [ + { + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "clientid": "emqx_clientid" + } + ], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [ + { + "topic": "Topic/A", + "login": "emqx_c", + "allow": true, + "action": "sub" + }, + { + "topic": "Topic/A", + "login": "emqx_c", + "allow": true, + "action": "pub" + } + ] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.2.9.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.2.9.json new file mode 100644 index 000000000..064db3e22 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.2.9.json @@ -0,0 +1,94 @@ +{ + "version": "4.2", + "date": "2021-04-08 12:11:53", + "modules": [ + { + "id": "module:d32ee5e4", + "type": "internal_acl", + "config": { + "acl_rule_file": "etc/acl.conf" + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:ea216644", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:f7deaca2", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:8f63640c", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + } + ], + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "uZGCSOEcLJIbJWgtZn8igEsnlCE=", + "tags": "administrator" + } + ], + "auth_mnesia": [ + { + "login": "emqx_c", + "type": "username", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "created_at": 1617883912202 + }, + { + "login": "emqx_clientid", + "type": "clientid", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "created_at": 1617883912629 + } + ], + "acl_mnesia": [ + { + "type": "username", + "type_value": "emqx_c", + "topic": "Topic/A", + "action": "pubsub", + "access": "allow", + "created_at": 1617883911747 + } + ], + "schemas": [], + "configs": [], + "listeners_state": [] +}