fix(backup): support minimum version number when import (#4582)

apps/emqx_management/include/emqx_mgmt.hrl

@@ -32,4 +32,4 @@
 -define(ERROR14, 114). %% OldPassword error
 -define(ERROR15, 115). %% bad topic
 
--define(VERSIONS, ["1", "3.2", "3.4", "4.0", "4.1", "4.2", "4.3"]).
+-define(VERSIONS, ["4.0", "4.1", "4.2", "4.3"]).

apps/emqx_management/src/emqx_mgmt_api_data.erl

@@ -112,8 +112,10 @@ import(_Bindings, Params) ->
         undefined ->
             minirest:return({error, missing_required_params});
         Filename ->
-            Result = case proplists:get_value(<<"node">>, Params) of
+            case proplists:get_value(<<"node">>, Params) of
-                undefined -> do_import(Filename);
+                undefined ->
+                    Result = do_import(Filename),
+                    minirest:return(Result);
                 Node ->
                     case lists:member(Node,
                           [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ]
@@ -121,8 +123,7 @@ import(_Bindings, Params) ->
                         true -> minirest:return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename]));
                         false -> minirest:return({error, no_existent_node})
                     end
-            end,
+            end
-            minirest:return(Result)
     end.
 
 do_import(Filename) ->

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -439,33 +439,16 @@ import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse
 import_acl_mnesia(Acls, _) ->
     do_import_acl_mnesia(Acls).
 -else.
-import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse
+import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.3" ->
-                                            FromVersion =:= "4.1" ->
+    do_import_auth_mnesia(Auths);
-    do_import_auth_mnesia_by_old_data(Auths);
+import_auth_mnesia(Auths, _FromVersion) ->
-import_auth_mnesia(Auths, "4.2") ->
+    do_import_auth_mnesia_by_old_data(Auths).
-    %% 4.2 contains a bug where password is not base64-encoded
-    do_import_auth_mnesia_4_2(Auths);
-import_auth_mnesia(Auths, _) ->
-    do_import_auth_mnesia(Auths).
 
-import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse
+import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.3" ->
-                                          FromVersion =:= "4.1" orelse
+    do_import_acl_mnesia(Acls);
-                                          FromVersion =:= "4.2" ->
+import_acl_mnesia(Acls, _FromVersion) ->
-    do_import_acl_mnesia_by_old_data(Acls);
+    do_import_acl_mnesia_by_old_data(Acls).
 
-import_acl_mnesia(Acls, _) ->
-    do_import_acl_mnesia(Acls).
-
-do_import_auth_mnesia_4_2(Auths) ->
-    case ets:info(emqx_user) of
-        undefined -> ok;
-        _ ->
-            CreatedAt = erlang:system_time(millisecond),
-            lists:foreach(fun(#{<<"login">> := Login,
-                                <<"password">> := Password}) ->
-                            mnesia:dirty_write({emqx_user, {get_old_type(), Login}, Password, CreatedAt})
-                          end, Auths)
-    end.
 -endif.
 
 do_import_auth_mnesia_by_old_data(Auths) ->
@@ -620,8 +603,8 @@ import(Filename, OverridesJson) ->
             Overrides = emqx_json:decode(OverridesJson, [return_maps]),
             Data = maps:merge(Imported, Overrides),
             Version = to_version(maps:get(<<"version">>, Data)),
-            read_global_auth_type(Data, Version),
+            read_global_auth_type(Data),
-            case lists:member(Version, ?VERSIONS) of
+            case is_version_supported(Data, Version) of
                 true  ->
                     try
                         do_import_data(Data, Version),
@@ -668,18 +651,40 @@ flag_to_boolean(<<"off">>) -> false;
 flag_to_boolean(Other) -> Other.
 -endif.
 
-read_global_auth_type(Data, Version) when Version =:= "4.0" orelse
+is_version_supported(Data, Version) ->
-                                          Version =:= "4.1" orelse
+    case { maps:get(<<"auth_clientid">>, Data, [])
-                                          Version =:= "4.2" ->
+         , maps:get(<<"auth_username">>, Data, [])
+         , maps:get(<<"auth_mnesia">>, Data, [])} of
+        {[], [], []} -> lists:member(Version, ?VERSIONS);
+        _ -> is_version_supported2(Version)
+    end.
+
+is_version_supported2("4.1") ->
+    true;
+is_version_supported2("4.3") ->
+    true; 
+is_version_supported2(Version) ->
+    case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of
+        match ->
+            try lists:map(fun erlang:list_to_integer/1, string:tokens(Version, ".")) of 
+                [4, 2, N] -> N >= 11;
+                [4, 0, N] -> N >= 13;
+                _ -> false
+            catch 
+                _ : _ -> false
+            end;
+        nomatch ->
+            false
+    end.
+
+read_global_auth_type(Data) ->
     case {maps:get(<<"auth_mnesia">>, Data, []), maps:get(<<"acl_mnesia">>, Data, [])} of
         {[], []} ->
             %% Auth mnesia plugin is not used:
             ok;
         _ ->
             do_read_global_auth_type(Data)
-    end;
+    end.
-read_global_auth_type(_Data, _Version) ->
-    ok.
 
 do_read_global_auth_type(Data) ->
     case Data of

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl

@@ -43,7 +43,7 @@ matrix() ->
                           , Version <- ["v4.2.10", "v4.1.5"]].
 
 all() ->
-    [t_matrix, t_import_4_0, t_import_no_auth].
+    [t_import_4_0, t_import_4_1, t_import_4_2].
 
 -endif. %% EMQX_ENTERPRISE
 
@@ -71,41 +71,55 @@ end_per_testcase(_, _Config) ->
     {atomic,ok} = mnesia:clear_table(emqx_user),
     ok.
 
-t_matrix(Config) ->
-    [begin
-         ct:pal("Testing import of ~p from ~p", [ImportAs, FromVersion]),
-         do_import(Config, ImportAs, FromVersion),
-         test_clientid_import(),
-         ct:pal("ok")
-     end
-     || {ImportAs, FromVersion} <- matrix()].
-
-%% This version is special, since it doesn't have mnesia ACL plugin
 t_import_4_0(Config) ->
-    do_import_no_auth("v4.0.11.json", Config).
+    ?assertMatch(ok, do_import("v4.0.11-no-auth.json", Config)),
-
-t_import_no_auth(Config) ->
-    do_import_no_auth("v4.2.10-no-auth.json", Config).
-
-%% Test that importing configs that don't contain any mnesia ACL data
-%% doesn't require additional overrides:
-do_import_no_auth(File, Config) ->
-    mnesia:clear_table(emqx_acl),
-    mnesia:clear_table(emqx_user),
-    Filename = filename:join(proplists:get_value(data_dir, Config), File),
-    ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, "{}")),
     timer:sleep(100),
-    test_clientid_import().
+    ?assertMatch(0, ets:info(emqx_user, size)),
+
+    ?assertMatch({error, unsupported_version, "4.0"}, do_import("v4.0.11.json", Config)),
+
+    ?assertMatch(ok, do_import("v4.0.13.json", Config)),
+    timer:sleep(100),
+    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
+    test_import(username, {<<"user_for_test">>, <<"public">>}).
+
+t_import_4_1(Config) ->
+    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
+    ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides)),
+    timer:sleep(100),
+    test_import(clientid, {<<"user_mnesia">>, <<"public">>}),
+    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
+    test_import(username, {<<"user_for_test">>, <<"public">>}),
+
+    Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}),
+    ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides1)),
+    timer:sleep(100),
+    test_import(username, {<<"user_mnesia">>, <<"public">>}),
+    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
+    test_import(username, {<<"user_for_test">>, <<"public">>}).
+
+t_import_4_2(Config) ->
+    ?assertMatch(ok, do_import("v4.2.10-no-auth.json", Config)),
+    timer:sleep(100),
+    ?assertMatch(0, ets:info(emqx_user, size)),
+
+    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
+    ?assertMatch({error, unsupported_version, "4.2"}, do_import("v4.2.10.json", Config, Overrides)),
+
+    Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
+    ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides1)),
+    timer:sleep(100),
+    test_import(clientid, {<<"user_mnesia">>, <<"public">>}),
+    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
+    test_import(username, {<<"user_for_test">>, <<"public">>}),
+
+    Overrides2 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}),
+    ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides2)),
+    timer:sleep(100),
+    test_import(username, {<<"user_mnesia">>, <<"public">>}),
+    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
+    test_import(username, {<<"user_for_test">>, <<"public">>}),
 
-do_import(Config, Type, V) ->
-    File = V ++ ".json",
-    mnesia:clear_table(emqx_acl),
-    mnesia:clear_table(emqx_user),
-    Filename = filename:join(proplists:get_value(data_dir, Config), File),
-    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(Type)}),
-    ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, Overrides)),
-    Records = lists:sort(ets:tab2list(emqx_acl)),
-    %% Check importing of records related to emqx_auth_mnesia
     ?assertMatch([#emqx_acl{
                      filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>},
                      action = pub,
@@ -116,21 +130,27 @@ do_import(Config, Type, V) ->
                      action = sub,
                      access = allow
                     }],
-                 lists:sort(Records)),
+                 lists:sort(ets:tab2list(emqx_acl))).
-    ?assertMatch([_, _], ets:tab2list(emqx_user)),
+
-    ?assertMatch([_], ets:lookup(emqx_user, {Type, <<"emqx_c">>})),
+do_import(File, Config) ->
-    Req = #{clientid => <<"blah">>}
+    do_import(File, Config, "{}").
-          #{Type => <<"emqx_c">>,
+
-            password => <<"emqx_p">>
+do_import(File, Config, Overrides) ->
-           },
+    mnesia:clear_table(emqx_acl),
+    mnesia:clear_table(emqx_user),
+    Filename = filename:join(proplists:get_value(data_dir, Config), File),
+    emqx_mgmt_data_backup:import(Filename, Overrides).
+
+test_import(username, {Username, Password}) ->
+    [#emqx_user{password = _}] = ets:lookup(emqx_user, {username, Username}),
+    Req = #{clientid => <<"anyname">>,
+            username => Username,
+            password => Password},
+    ?assertMatch({stop, #{auth_result := success}},
+                 emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256}));
+test_import(clientid, {ClientID, Password}) ->
+    [#emqx_user{password = _}] = ets:lookup(emqx_user, {clientid, ClientID}),
+    Req = #{clientid => ClientID,
+            password => Password},
     ?assertMatch({stop, #{auth_result := success}},
                  emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})).
-
-test_clientid_import() ->
-    [#emqx_user{password = _Pass}] = ets:lookup(emqx_user, {clientid, <<"emqx_clientid">>}),
-    Req = #{clientid => <<"emqx_clientid">>,
-            password => <<"emqx_p">>
-           },
-    ?assertMatch({stop, #{auth_result := success}},
-                 emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})),
-    ok.

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json

@@ -0,0 +1,23 @@
+{
+  "version": "4.0",
+  "users": [],
+  "schemas": [],
+  "rules": [],
+  "resources": [],
+  "date": "2021-04-10 11:45:26",
+  "blacklist": [],
+  "auth_username": [],
+  "auth_mnesia": [],
+  "auth_clientid": [],
+  "apps": [
+    {
+      "status": true,
+      "secret": "public",
+      "name": "Default",
+      "id": "admin",
+      "expired": "undefined",
+      "desc": "Application user"
+    }
+  ],
+  "acl_mnesia": []
+}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json

@@ -0,0 +1,37 @@
+{
+    "version":"4.0.13",
+    "users":[
+        {
+            "username":"admin",
+            "tags":"administrator",
+            "password":"p6C65OF0BQhvPmCziM2yRa8JN5o="
+        }
+    ],
+    "schemas":[],
+    "rules":[],
+    "resources":[],
+    "date":"2021-04-16 11:20:00",
+    "blacklist":[],
+    "auth_username":[
+        {
+            "username":"user_for_test",
+            "password":"ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
+        }
+    ],
+    "auth_clientid":[
+        {
+            "password":"JBgSnzIxOWNiMDU1ZWFiNDAwMjVhOTQzZThlZjkxN2JlZWE4MGE4YzlmM2I5MjQ4OGI1NjllY2Q4NGQ4NjhjYzQ1NDM=",
+            "clientid":"client_for_test"
+        }
+    ],
+    "apps":[
+        {
+            "status":true,
+            "secret":"public",
+            "name":"Default",
+            "id":"admin",
+            "expired":"undefined",
+            "desc":"Application user"
+        }
+    ]
+}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json

@@ -12,18 +12,23 @@
   "resources": [],
   "date": "2021-04-07 14:28:58",
   "blacklist": [],
-  "auth_username": [],
+  "auth_username": [
+    {
+      "username":"user_for_test",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
+    }
+  ],
   "auth_mnesia": [
     {
-      "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ==",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=",
-      "login": "emqx_c",
+      "login": "user_mnesia",
       "is_superuser": true
     }
   ],
   "auth_clientid": [
     {
-      "password": "MctXdjZkYzRhMDUwMTc4MDM0OWY4YTg1NTg4Y2ZlOThjYWIyMDk3M2UzNjgzYzYyZWYwOTAzMTk2N2E4OWVjZDk4Mjk=",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=",
-      "clientid": "emqx_clientid"
+      "clientid": "client_for_test"
     }
   ],
   "apps": [

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json

@@ -21,12 +21,7 @@
       "tags": "administrator"
     }
   ],
-  "auth_clientid": [
+  "auth_clientid": [],
-    {
-      "clientid": "emqx_clientid",
-      "password": "uAP84TgyMjAyNGFhY2NlMWVlNDI2NTk1MzFiZjA4YzBjY2RjNjViZmZhNjkzYjhkMDE4NTg0ZWExYjFkZGY0MTBjYWM="
-    }
-  ],
   "auth_username": [],
   "auth_mnesia": [],
   "acl_mnesia": [],

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json

@@ -0,0 +1,58 @@
+{
+  "version": "4.2.11",
+  "date": "2021-04-12 10:40:58",
+  "rules": [],
+  "resources": [],
+  "blacklist": [],
+  "apps": [
+    {
+      "id": "admin",
+      "secret": "public",
+      "name": "Default",
+      "desc": "Application user",
+      "status": true,
+      "expired": "undefined"
+    }
+  ],
+  "users": [
+    {
+      "username": "test",
+      "password": "8Vd7+gVg2J3nE1Xjyxqd59sA5mo=",
+      "tags": "administrator"
+    }
+  ],
+  "auth_clientid": [
+    {
+      "clientid": "client_for_test",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
+    }
+  ],
+  "auth_username": [
+    {
+      "username": "user_for_test",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
+    }
+  ],
+  "auth_mnesia": [
+    {
+      "login": "user_mnesia",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=",
+      "is_superuser": true
+    }
+  ],
+  "acl_mnesia": [
+    {
+      "login": "emqx_c",
+      "topic": "Topic/A",
+      "action": "sub",
+      "allow": true
+    },
+    {
+      "login": "emqx_c",
+      "topic": "Topic/A",
+      "action": "pub",
+      "allow": true
+    }
+  ],
+  "schemas": []
+}