From 664b9c79fc8a68df9013b62d8a379f6e1eb8ab97 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Wed, 7 Apr 2021 11:41:02 +0200 Subject: [PATCH] fix(emqx_management): Add clientid data to the migration suite --- .../src/emqx_mgmt_data_backup.erl | 23 +++- .../test/emqx_auth_mnesia_migration_SUITE.erl | 35 ++++- .../e4.1.1.json | 53 ++++++++ .../e4.2.9.json | 94 ++++++++++++++ .../make_data.sh | 122 ++++++++++++++++++ .../v4.0.7.json | 28 ++++ .../v4.1.5.json | 53 ++++++++ .../v4.1.json | 48 ------- .../v4.2.9.json | 53 ++++++++ .../v4.2.json | 53 -------- 10 files changed, 448 insertions(+), 114 deletions(-) create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.1.1.json create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.2.9.json create mode 100755 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json delete mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.json create mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json delete mode 100644 apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.json diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 2b1791c1d..d1f2d03f2 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -44,7 +44,7 @@ , import_blacklist/1 , import_applications/1 , import_users/1 - , import_auth_clientid/1 %% BACKW: 4.1.x + , import_auth_clientid/2 %% BACKW: 4.1.x , import_auth_username/1 %% BACKW: 4.1.x , import_auth_mnesia/2 , import_acl_mnesia/2 @@ -408,12 +408,18 @@ import_users(Users) -> emqx_dashboard_admin:force_add_user(Username, NPassword, Tags) end, Users). -import_auth_clientid(Lists) -> +import_auth_clientid(Lists, Version) -> case ets:info(emqx_user) of undefined -> ok; _ -> - lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {clientid, Clientid}, base64:decode(Password), erlang:system_time(millisecond)}) + lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password0}) -> + Password = case Version of + "4.1" -> base64:decode(Password0); + _ -> ensure_binary(Password0) + end, + mnesia:dirty_write({emqx_user, {clientid, Clientid} + , Password + , erlang:system_time(millisecond)}) end, Lists) end. @@ -522,7 +528,7 @@ do_import_acl_mnesia(Acls) -> Value -> {{any_to_atom(maps:get(<<"type">>, Map)), Value}, maps:get(<<"topic">>, Map)} end, - mnesia:dirty_write({emqx_acl ,Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt}) + mnesia:dirty_write({emqx_acl, Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt}) end, Acls) end. @@ -645,7 +651,7 @@ do_import_data(Data, Version) -> import_blacklist(maps:get(<<"blacklist">>, Data, [])), import_applications(maps:get(<<"apps">>, Data, [])), import_users(maps:get(<<"users">>, Data, [])), - import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), + import_auth_clientid(maps:get(<<"auth_clientid">>, Data, []), Version), import_auth_username(maps:get(<<"auth_username">>, Data, [])), import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version). @@ -691,3 +697,8 @@ read_global_auth_type(_Data, _Version) -> get_old_type() -> {ok, Type} = application:get_env(emqx_auth_mnesia, as), Type. + +ensure_binary(A) when is_binary(A) -> + A; +ensure_binary(A) -> + list_to_binary(A). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl index 8db0c68d7..3a89c01f3 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl @@ -26,13 +26,13 @@ -include_lib("emqx_auth_mnesia/include/emqx_auth_mnesia.hrl"). -ifdef(EMQX_ENTERPRISE). --define(VERSIONS, ["e4.1", "e4.2"]). +-define(VERSIONS, ["e4.1.1", "e4.2.9"]). -else. --define(VERSIONS, ["v4.1", "v4.2"]). +-define(VERSIONS, ["v4.1.5", "v4.2.9"]). -endif. all() -> - [{group, Id} || {Id, _, _} <- groups()]. + [{group, Id} || {Id, _, _} <- groups()] ++ [t_import_4_0]. groups() -> [{username, [], cases()}, {clientid, [], cases()}]. @@ -53,7 +53,9 @@ end_per_suite(_Config) -> init_per_group(username, Config) -> [{cred_type, username} | Config]; init_per_group(clientid, Config) -> - [{cred_type, clientid} | Config]. + [{cred_type, clientid} | Config]; +init_per_group(_, Config) -> + Config. end_per_group(_, Config) -> Config. @@ -71,9 +73,20 @@ t_import(Config) -> test_import(Config, [V | Versions]) -> do_import(Config, V), + test_clientid_import(), test_import(Config, Versions); test_import(_Config, []) -> ok. +%% This version is special, since it doesn't have mnesia ACL plugin +t_import_4_0(Config) -> + mnesia:clear_table(emqx_acl), + mnesia:clear_table(emqx_user), + Filename = filename:join(proplists:get_value(data_dir, Config), "v4.0.7.json"), + Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), + ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, Overrides)), + timer:sleep(100), + test_clientid_import(). + do_import(Config, V) -> File = V ++ ".json", Type = proplists:get_value(cred_type, Config), @@ -95,12 +108,20 @@ do_import(Config, V) -> access = allow }], lists:sort(Records)), - ?assertMatch([#emqx_user{ - login = {Type, <<"emqx_c">>} - }], ets:tab2list(emqx_user)), + ?assertMatch([_, _], ets:tab2list(emqx_user)), + ?assertMatch([_], ets:lookup(emqx_user, {Type, <<"emqx_c">>})), Req = #{clientid => <<"blah">>} #{Type => <<"emqx_c">>, password => "emqx_p" }, ?assertMatch({stop, #{auth_result := success}}, emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})). + +test_clientid_import() -> + [#emqx_user{password = _Pass}] = ets:lookup(emqx_user, {clientid, <<"emqx_clientid">>}), + Req = #{clientid => <<"emqx_clientid">>, + password => <<"emqx_p">> + }, + catch %% TODO currently broken on some releases. + ?assertMatch({stop, #{auth_result := success}}, + emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.1.1.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.1.1.json new file mode 100644 index 000000000..c3e3e9b1c --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.1.1.json @@ -0,0 +1,53 @@ +{ + "version": "4.1", + "users": [ + { + "username": "admin", + "tags": "administrator", + "password": "gqKZGCdl/Mj0LUwaIhyXG6UT7eE=" + } + ], + "schemas": [], + "rules": [], + "resources": [], + "date": "2021-04-07 12:41:33", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [ + { + "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ==", + "login": "emqx_c", + "is_superuser": true + } + ], + "auth_clientid": [ + { + "password": "mtXuNzViN2M1YzEwNzNkMDUzZTcyNmE1ZWQ4NTMyMzMxNDJiY2RmYTVmNDJiN2YyMjQ1OWQ5ZDE5OTBkZmJhMWE3OTY=", + "clientid": "emqx_clientid" + } + ], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [ + { + "topic": "Topic/A", + "login": "emqx_c", + "allow": true, + "action": "sub" + }, + { + "topic": "Topic/A", + "login": "emqx_c", + "allow": true, + "action": "pub" + } + ] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.2.9.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.2.9.json new file mode 100644 index 000000000..48b65c674 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.2.9.json @@ -0,0 +1,94 @@ +{ + "version": "4.2", + "date": "2021-04-07 11:27:10", + "modules": [ + { + "id": "module:b8d9cb41", + "type": "internal_acl", + "config": { + "acl_rule_file": "etc/acl.conf" + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:0c9501cf", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:1292cc54", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:146f7ae4", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + } + ], + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "AdzLv1Gqo3caj/jqu5EbINEahTo=", + "tags": "administrator" + } + ], + "auth_mnesia": [ + { + "login": "emqx_c", + "type": "username", + "password": "tTwXNjVjNGExNTQyMzc4YTU1Y2MyMjYxZjFmNmU3ZjY5ODYyMDE5MTUzMDU2NTQ5ZjVkNGNmN2E3MTM0MWQwMTdjM2U=", + "created_at": 1617794829294 + }, + { + "login": "emqx_clientid", + "type": "clientid", + "password": "vTDjrzdjODIzNGExZmMwM2Q4MzU5N2FiYjYwODI4ZjM4YmIyZDQ5YjVjMjFkYjBkOTg4ZjhmODY2MTIzZTQ4NDU2M2E=", + "created_at": 1617794829697 + } + ], + "acl_mnesia": [ + { + "type": "username", + "type_value": "emqx_c", + "topic": "Topic/A", + "action": "pubsub", + "access": "allow", + "created_at": 1617794828874 + } + ], + "schemas": [], + "configs": [], + "listeners_state": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh new file mode 100755 index 000000000..ab1743cb1 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh @@ -0,0 +1,122 @@ +#!/bin/bash +set -eux pipefail +# Helper script for creating data export files + +container() { + version="${1}" + if [ -z ${2+x} ]; then + ee="" + else + ee="-ee" + fi + container="emqx/emqx${ee}:${version}" + docker rm -f emqx || true + docker run "$container" true # Make sure the image is cached locally + docker run --rm -e EMQX_LOADED_PLUGINS="emqx_auth_mnesia emqx_auth_clientid emqx_management" \ + --name emqx -p 8081:8081 "$container" emqx foreground & + sleep 7 +} + +create_acls () { + url="${1}" + curl -f -v "http://localhost:8081/$url" -u admin:public -d@- < "$filename.json" + cat "${filename}.json" +} + +collect_4_2 () { + container "4.2.9" + create_acls "api/v4/mqtt_acl" + create_user mqtt_user + + # Add clientid + docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p + + export_data "v4.2.9" +} + +collect_e4_2 () { + container "4.2.5" "ee" + # Add ACLs: + docker exec emqx emqx_ctl acl add username emqx_c Topic/A pubsub allow + # Create users + docker exec emqx emqx_ctl user add emqx_c emqx_p + + # Add clientid + docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p + + export_data "e4.2.9" +} + +collect_e4_1 () { + container "4.1.1" "ee" + # Add ACLs: + create_acls "api/v4/emqx_acl" + # Create users + create_user "auth_user" + + # Add clientid + docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p + + export_data "e4.1.1" +} + +collect_4_1 () { + container "v4.1.5" + create_acls "api/v4/emqx_acl" + create_user auth_user + + # Add clientid + docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p + + export_data "v4.1.5" +} + +collect_4_0 () { + container "v4.0.7" + + # Add clientid + docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p + + export_data "v4.0.7" +} + +collect_4_0 +collect_4_1 +collect_4_2 +collect_e4_2 +collect_e4_1 + +docker kill emqx diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json new file mode 100644 index 000000000..96e3e9d37 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json @@ -0,0 +1,28 @@ +{ + "version": "4.0", + "users": [], + "schemas": [], + "rules": [], + "resources": [], + "date": "2021-04-06 15:32:45", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [], + "auth_clientid": [ + { + "password": "*Y¨cabaca59dfcd78d3622f493a3cb8c9e8a02890ee06e8ba264d362930f40b0ce1", + "clientid": "emqx_clientid" + } + ], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json new file mode 100644 index 000000000..c7cf13d1a --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json @@ -0,0 +1,53 @@ +{ + "version": "4.1", + "users": [ + { + "username": "admin", + "tags": "administrator", + "password": "C5J9Qhy+/NYRnAnrK0E6Z5Aqp3I=" + } + ], + "schemas": [], + "rules": [], + "resources": [], + "date": "2021-04-06 15:32:54", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [ + { + "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ==", + "login": "emqx_c", + "is_superuser": true + } + ], + "auth_clientid": [ + { + "password": "8vxdUDM2YTgzMGJjYTlkYjEzMmI2NTQ1MzE0ZGYxMGNjZDMxNmI0ZDQyOTA1NjRlZTNkODAzYTI0MWQyY2Y0Yjg5MTQ=", + "clientid": "emqx_clientid" + } + ], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [ + { + "topic": "Topic/A", + "login": "emqx_c", + "allow": true, + "action": "sub" + }, + { + "topic": "Topic/A", + "login": "emqx_c", + "allow": true, + "action": "pub" + } + ] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.json deleted file mode 100644 index 04a7a273f..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "acl_mnesia": [ - { - "action": "sub", - "allow": true, - "login": "emqx_c", - "topic": "Topic/A" - }, - { - "action": "pub", - "allow": true, - "login": "emqx_c", - "topic": "Topic/A" - } - ], - "apps": [ - { - "desc": "Application user", - "expired": "undefined", - "id": "admin", - "name": "Default", - "secret": "public", - "status": true - } - ], - "auth_clientid": [], - "auth_mnesia": [ - { - "is_superuser": false, - "login": "emqx_c", - "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ==" - } - ], - "auth_username": [], - "blacklist": [], - "date": "2021-03-30 09:11:29", - "resources": [], - "rules": [], - "schemas": [], - "users": [ - { - "password": "t89PhgOb15rSCdpxm7Obp7QGcyY=", - "tags": "administrator", - "username": "admin" - } - ], - "version": "4.1" -} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json new file mode 100644 index 000000000..e20ef9f48 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json @@ -0,0 +1,53 @@ +{ + "version": "4.2", + "date": "2021-04-06 15:33:05", + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "cZv6y9wEMhK3kpUV4gz/MJOKUWI=", + "tags": "administrator" + } + ], + "auth_clientid": [ + { + "clientid": "emqx_clientid", + "password": "��a�03bfa8dc2c7a1e58ee990e71f419d2fa40f8e7b25b8f8406cbc65fc531ed344d" + } + ], + "auth_username": [], + "auth_mnesia": [ + { + "login": "emqx_c", + "password": "ceb5e917f7930ae8f0dc3ceb496a428f7e644736eebca36a2b8f6bbac756171a", + "is_superuser": true + } + ], + "acl_mnesia": [ + { + "login": "emqx_c", + "topic": "Topic/A", + "action": "sub", + "allow": true + }, + { + "login": "emqx_c", + "topic": "Topic/A", + "action": "pub", + "allow": true + } + ], + "schemas": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.json deleted file mode 100644 index 57958aa58..000000000 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "schemas": [], - "acl_mnesia": [ - { - "allow": true, - "action": "sub", - "topic": "Topic/A", - "login": "emqx_c" - }, - { - "allow": true, - "action": "pub", - "topic": "Topic/A", - "login": "emqx_c" - } - ], - "auth_mnesia": [ - { - "is_superuser": false, - "password": "ceb5e917f7930ae8f0dc3ceb496a428f7e644736eebca36a2b8f6bbac756171a", - "login": "emqx_c" - } - ], - "auth_username": [], - "auth_clientid": [], - "users": [ - { - "tags": "viewer", - "password": "oVqjR1wOi2u4DtsuXNctYt6+SKE=", - "username": "test" - }, - { - "tags": "administrator", - "password": "9SO4rEEZ6rNwA4vAwp3cnXgQsAM=", - "username": "admin" - } - ], - "apps": [ - { - "expired": "undefined", - "status": true, - "desc": "Application user", - "name": "Default", - "secret": "public", - "id": "admin" - } - ], - "blacklist": [], - "resources": [], - "rules": [], - "date": "2021-03-26 09:51:38", - "version": "4.2" -}