fix(emqx_management): Import ACLs from the enterprise version

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -493,8 +493,8 @@ do_import_auth_mnesia(Auths) ->
         _ ->
             lists:foreach(fun(#{<<"login">> := Login,
                                 <<"type">> := Type,
-                                <<"password">> := Password,
+                                <<"password">> := Password } = Map) ->
-                                <<"created_at">> := CreatedAt }) ->
+                            CreatedAt = maps:get(<<"created_at">>, Map, erlang:system_time(millisecond)),
                             mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt})
                           end, Auths)
     end.
@@ -520,15 +520,15 @@ do_import_acl_mnesia(Acls) ->
         undefined -> ok;
         _ ->
             lists:foreach(fun(Map = #{<<"action">> := Action,
-                                      <<"access">> := Access,
+                                      <<"access">> := Access}) ->
-                                      <<"created_at">> := CreatedAt}) ->
+                            Topic = maps:get(<<"topic">>, Map),
-                            Filter = case maps:get(<<"type_value">>, Map, undefined) of
+                            Login = case maps:get(<<"type_value">>, Map, undefined) of
                                 undefined ->
-                                    {any_to_atom(maps:get(<<"type">>, Map)), maps:get(<<"topic">>, Map)};
+                                    all;
                                 Value ->
-                                    {{any_to_atom(maps:get(<<"type">>, Map)), Value}, maps:get(<<"topic">>, Map)}
+                                    {any_to_atom(maps:get(<<"type">>, Map)), Value}
                             end,
-                            mnesia:dirty_write({emqx_acl, Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt})
+                            emqx_acl_mnesia_cli:add_acl(Login, Topic, any_to_atom(Action), any_to_atom(Access))
                           end, Acls)
     end.
 

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl

@@ -26,13 +26,26 @@
 -include_lib("emqx_auth_mnesia/include/emqx_auth_mnesia.hrl").
 
 -ifdef(EMQX_ENTERPRISE).
--define(VERSIONS, ["e4.1.1", "e4.2.9"]).
+
--else.
+matrix() ->
--define(VERSIONS, ["v4.1.5", "v4.2.9"]).
+    [ {username, "e4.2.9"}
--endif.
+    , {clientid, "e4.1.1"}
+    , {username, "e4.1.1"}
+    ].
 
 all() ->
-    [{group, Id} || {Id, _, _} <- groups()] ++ [t_import_4_0].
+    [t_matrix].
+
+-else. %% ! EMQX_ENTERPRISE
+
+matrix() ->
+    [{ImportAs, Version} || ImportAs <- [clientid, username]
+                          , Version <- ["v4.2.9", "v4.1.5"]].
+
+all() ->
+    [t_matrix, t_import_4_0].
+
+-endif. %% EMQX_ENTERPRISE
 
 groups() ->
     [{username, [], cases()}, {clientid, [], cases()}].
@@ -50,16 +63,6 @@ end_per_suite(_Config) ->
     emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_dashboard, emqx_management, emqx_auth_mnesia]),
     ekka_mnesia:ensure_stopped().
 
-init_per_group(username, Config) ->
-    [{cred_type, username} | Config];
-init_per_group(clientid, Config) ->
-    [{cred_type, clientid} | Config];
-init_per_group(_, Config) ->
-    Config.
-
-end_per_group(_, Config) ->
-    Config.
-
 init_per_testcase(_, Config) ->
     Config.
 
@@ -68,14 +71,14 @@ end_per_testcase(_, _Config) ->
     mnesia:clear_table(emqx_user),
     ok.
 
-t_import(Config) ->
+t_matrix(Config) ->
-    test_import(Config, ?VERSIONS).
+    [begin
-
+         ct:pal("Testing import of ~p from ~p", [ImportAs, FromVersion]),
-test_import(Config, [V | Versions]) ->
+         do_import(Config, ImportAs, FromVersion),
-    do_import(Config, V),
+         test_clientid_import(),
-    test_clientid_import(),
+         ct:pal("ok")
-    test_import(Config, Versions);
+     end
-test_import(_Config, []) -> ok.
+     || {ImportAs, FromVersion} <- matrix()].
 
 %% This version is special, since it doesn't have mnesia ACL plugin
 t_import_4_0(Config) ->
@@ -87,9 +90,8 @@ t_import_4_0(Config) ->
     timer:sleep(100),
     test_clientid_import().
 
-do_import(Config, V) ->
+do_import(Config, Type, V) ->
     File = V ++ ".json",
-    Type = proplists:get_value(cred_type, Config),
     mnesia:clear_table(emqx_acl),
     mnesia:clear_table(emqx_user),
     Filename = filename:join(proplists:get_value(data_dir, Config), File),
@@ -112,16 +114,16 @@ do_import(Config, V) ->
     ?assertMatch([_], ets:lookup(emqx_user, {Type, <<"emqx_c">>})),
     Req = #{clientid => <<"blah">>}
           #{Type => <<"emqx_c">>,
-            password => "emqx_p"
+            password => <<"emqx_p">>
            },
     ?assertMatch({stop, #{auth_result := success}},
                  emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})).
 
 test_clientid_import() ->
     [#emqx_user{password = _Pass}] = ets:lookup(emqx_user, {clientid, <<"emqx_clientid">>}),
-    Req = #{clientid => <<"emqx_clientid">>,
+    %% Req = #{clientid => <<"emqx_clientid">>,
-            password => <<"emqx_p">>
+    %%         password => <<"emqx_p">>
-           },
+    %%        },
-    catch %% TODO currently broken on some releases.
+    %% ?assertMatch({stop, #{auth_result := success}},
-    ?assertMatch({stop, #{auth_result := success}},
+    %%              emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})),
-                 emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})).
+    ok.

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.1.1.json

@@ -1,53 +0,0 @@
-{
-  "version": "4.1",
-  "users": [
-    {
-      "username": "admin",
-      "tags": "administrator",
-      "password": "gqKZGCdl/Mj0LUwaIhyXG6UT7eE="
-    }
-  ],
-  "schemas": [],
-  "rules": [],
-  "resources": [],
-  "date": "2021-04-07 12:41:33",
-  "blacklist": [],
-  "auth_username": [],
-  "auth_mnesia": [
-    {
-      "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ==",
-      "login": "emqx_c",
-      "is_superuser": true
-    }
-  ],
-  "auth_clientid": [
-    {
-      "password": "mtXuNzViN2M1YzEwNzNkMDUzZTcyNmE1ZWQ4NTMyMzMxNDJiY2RmYTVmNDJiN2YyMjQ1OWQ5ZDE5OTBkZmJhMWE3OTY=",
-      "clientid": "emqx_clientid"
-    }
-  ],
-  "apps": [
-    {
-      "status": true,
-      "secret": "public",
-      "name": "Default",
-      "id": "admin",
-      "expired": "undefined",
-      "desc": "Application user"
-    }
-  ],
-  "acl_mnesia": [
-    {
-      "topic": "Topic/A",
-      "login": "emqx_c",
-      "allow": true,
-      "action": "sub"
-    },
-    {
-      "topic": "Topic/A",
-      "login": "emqx_c",
-      "allow": true,
-      "action": "pub"
-    }
-  ]
-}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.2.9.json

@@ -1,94 +0,0 @@
-{
-  "version": "4.2",
-  "date": "2021-04-07 11:27:10",
-  "modules": [
-    {
-      "id": "module:b8d9cb41",
-      "type": "internal_acl",
-      "config": {
-        "acl_rule_file": "etc/acl.conf"
-      },
-      "enabled": true,
-      "created_at": "undefined",
-      "description": ""
-    },
-    {
-      "id": "module:0c9501cf",
-      "type": "recon",
-      "config": {},
-      "enabled": true,
-      "created_at": "undefined",
-      "description": ""
-    },
-    {
-      "id": "module:1292cc54",
-      "type": "presence",
-      "config": {
-        "qos": 0
-      },
-      "enabled": true,
-      "created_at": "undefined",
-      "description": ""
-    },
-    {
-      "id": "module:146f7ae4",
-      "type": "retainer",
-      "config": {
-        "storage_type": "ram",
-        "max_retained_messages": 0,
-        "max_payload_size": "1MB",
-        "expiry_interval": 0
-      },
-      "enabled": true,
-      "created_at": "undefined",
-      "description": ""
-    }
-  ],
-  "rules": [],
-  "resources": [],
-  "blacklist": [],
-  "apps": [
-    {
-      "id": "admin",
-      "secret": "public",
-      "name": "Default",
-      "desc": "Application user",
-      "status": true,
-      "expired": "undefined"
-    }
-  ],
-  "users": [
-    {
-      "username": "admin",
-      "password": "AdzLv1Gqo3caj/jqu5EbINEahTo=",
-      "tags": "administrator"
-    }
-  ],
-  "auth_mnesia": [
-    {
-      "login": "emqx_c",
-      "type": "username",
-      "password": "tTwXNjVjNGExNTQyMzc4YTU1Y2MyMjYxZjFmNmU3ZjY5ODYyMDE5MTUzMDU2NTQ5ZjVkNGNmN2E3MTM0MWQwMTdjM2U=",
-      "created_at": 1617794829294
-    },
-    {
-      "login": "emqx_clientid",
-      "type": "clientid",
-      "password": "vTDjrzdjODIzNGExZmMwM2Q4MzU5N2FiYjYwODI4ZjM4YmIyZDQ5YjVjMjFkYjBkOTg4ZjhmODY2MTIzZTQ4NDU2M2E=",
-      "created_at": 1617794829697
-    }
-  ],
-  "acl_mnesia": [
-    {
-      "type": "username",
-      "type_value": "emqx_c",
-      "topic": "Topic/A",
-      "action": "pubsub",
-      "access": "allow",
-      "created_at": 1617794828874
-    }
-  ],
-  "schemas": [],
-  "configs": [],
-  "listeners_state": []
-}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json

@@ -4,13 +4,13 @@
   "schemas": [],
   "rules": [],
   "resources": [],
-  "date": "2021-04-06 15:32:45",
+  "date": "2021-04-07 14:28:49",
   "blacklist": [],
   "auth_username": [],
   "auth_mnesia": [],
   "auth_clientid": [
     {
-      "password": "*Y¨cabaca59dfcd78d3622f493a3cb8c9e8a02890ee06e8ba264d362930f40b0ce1",
+      "password": "<EFBFBD><EFBFBD>Pd56c0fcdcd7636dcf8ed1ea48cd3d58acab74030157551f7f7f8684804b9239e",
       "clientid": "emqx_clientid"
     }
   ],

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json

@@ -4,13 +4,13 @@
     {
       "username": "admin",
       "tags": "administrator",
-      "password": "C5J9Qhy+/NYRnAnrK0E6Z5Aqp3I="
+      "password": "R0TpDmJtE/d5rIXAm6YY61RI0mg="
     }
   ],
   "schemas": [],
   "rules": [],
   "resources": [],
-  "date": "2021-04-06 15:32:54",
+  "date": "2021-04-07 14:28:58",
   "blacklist": [],
   "auth_username": [],
   "auth_mnesia": [
@@ -22,7 +22,7 @@
   ],
   "auth_clientid": [
     {
-      "password": "8vxdUDM2YTgzMGJjYTlkYjEzMmI2NTQ1MzE0ZGYxMGNjZDMxNmI0ZDQyOTA1NjRlZTNkODAzYTI0MWQyY2Y0Yjg5MTQ=",
+      "password": "MctXdjZkYzRhMDUwMTc4MDM0OWY4YTg1NTg4Y2ZlOThjYWIyMDk3M2UzNjgzYzYyZWYwOTAzMTk2N2E4OWVjZDk4Mjk=",
       "clientid": "emqx_clientid"
     }
   ],

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json

@@ -1,6 +1,6 @@
 {
   "version": "4.2",
-  "date": "2021-04-06 15:33:05",
+  "date": "2021-04-07 14:29:08",
   "rules": [],
   "resources": [],
   "blacklist": [],
@@ -17,14 +17,14 @@
   "users": [
     {
       "username": "admin",
-      "password": "cZv6y9wEMhK3kpUV4gz/MJOKUWI=",
+      "password": "gCBXISkivpaeKetFcPSm+Eaxyxs=",
       "tags": "administrator"
     }
   ],
   "auth_clientid": [
     {
       "clientid": "emqx_clientid",
-      "password": "<EFBFBD><EFBFBD>a<EFBFBD>03bfa8dc2c7a1e58ee990e71f419d2fa40f8e7b25b8f8406cbc65fc531ed344d"
+      "password": "z<EFBFBD>7d413fee461607065c161072f3707dc0a01bd1fc8476eb7be703a74a66701bb14"
     }
   ],
   "auth_username": [],