fix(backup): support minimum version number when import (#4582)

2021-04-16 15:51:36 +08:00 · 2021-04-16 15:51:36 +08:00 · 1a630a308a
parent a09434acd2
commit 1a630a308a
9 changed files with 242 additions and 98 deletions
--- a/apps/emqx_management/include/emqx_mgmt.hrl
+++ b/apps/emqx_management/include/emqx_mgmt.hrl
@ -32,4 +32,4 @@
 -define(ERROR14, 114). %% OldPassword error
 -define(ERROR15, 115). %% bad topic

-define(VERSIONS, ["1", "3.2", "3.4", "4.0", "4.1", "4.2", "4.3"]).
+-define(VERSIONS, ["4.0", "4.1", "4.2", "4.3"]).
--- a/apps/emqx_management/src/emqx_mgmt_api_data.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl
@ -112,8 +112,10 @@ import(_Bindings, Params) ->
        undefined ->
            minirest:return({error, missing_required_params});
        Filename ->
-            Result = case proplists:get_value(<<"node">>, Params) of
-                undefined -> do_import(Filename);
+            case proplists:get_value(<<"node">>, Params) of
+                undefined ->
+                    Result = do_import(Filename),
+                    minirest:return(Result);
                Node ->
                    case lists:member(Node,
                          [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ]
@ -121,8 +123,7 @@ import(_Bindings, Params) ->
                        true -> minirest:return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename]));
                        false -> minirest:return({error, no_existent_node})
                    end
-            end,
-            minirest:return(Result)
+            end
    end.

 do_import(Filename) ->
--- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl
+++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl
@ -439,33 +439,16 @@ import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse
 import_acl_mnesia(Acls, _) ->
    do_import_acl_mnesia(Acls).
 -else.
-import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse
-                                            FromVersion =:= "4.1" ->
-    do_import_auth_mnesia_by_old_data(Auths);
-import_auth_mnesia(Auths, "4.2") ->
-    %% 4.2 contains a bug where password is not base64-encoded
-    do_import_auth_mnesia_4_2(Auths);
-import_auth_mnesia(Auths, _) ->
-    do_import_auth_mnesia(Auths).
+import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.3" ->
+    do_import_auth_mnesia(Auths);
+import_auth_mnesia(Auths, _FromVersion) ->
+    do_import_auth_mnesia_by_old_data(Auths).

-import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse
-                                          FromVersion =:= "4.1" orelse
-                                          FromVersion =:= "4.2" ->
-    do_import_acl_mnesia_by_old_data(Acls);
+import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.3" ->
+    do_import_acl_mnesia(Acls);
+import_acl_mnesia(Acls, _FromVersion) ->
+    do_import_acl_mnesia_by_old_data(Acls).

-import_acl_mnesia(Acls, _) ->
-    do_import_acl_mnesia(Acls).
-
-do_import_auth_mnesia_4_2(Auths) ->
-    case ets:info(emqx_user) of
-        undefined -> ok;
-        _ ->
-            CreatedAt = erlang:system_time(millisecond),
-            lists:foreach(fun(#{<<"login">> := Login,
-                                <<"password">> := Password}) ->
-                            mnesia:dirty_write({emqx_user, {get_old_type(), Login}, Password, CreatedAt})
-                          end, Auths)
-    end.
 -endif.

 do_import_auth_mnesia_by_old_data(Auths) ->
@ -620,8 +603,8 @@ import(Filename, OverridesJson) ->
            Overrides = emqx_json:decode(OverridesJson, [return_maps]),
            Data = maps:merge(Imported, Overrides),
            Version = to_version(maps:get(<<"version">>, Data)),
-            read_global_auth_type(Data, Version),
-            case lists:member(Version, ?VERSIONS) of
+            read_global_auth_type(Data),
+            case is_version_supported(Data, Version) of
                true  ->
                    try
                        do_import_data(Data, Version),
@ -668,18 +651,40 @@ flag_to_boolean(<<"off">>) -> false;
 flag_to_boolean(Other) -> Other.
 -endif.

-read_global_auth_type(Data, Version) when Version =:= "4.0" orelse
-                                          Version =:= "4.1" orelse
-                                          Version =:= "4.2" ->
+is_version_supported(Data, Version) ->
+    case { maps:get(<<"auth_clientid">>, Data, [])
+         , maps:get(<<"auth_username">>, Data, [])
+         , maps:get(<<"auth_mnesia">>, Data, [])} of
+        {[], [], []} -> lists:member(Version, ?VERSIONS);
+        _ -> is_version_supported2(Version)
+    end.
+
+is_version_supported2("4.1") ->
+    true;
+is_version_supported2("4.3") ->
+    true; 
+is_version_supported2(Version) ->
+    case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of
+        match ->
+            try lists:map(fun erlang:list_to_integer/1, string:tokens(Version, ".")) of 
+                [4, 2, N] -> N >= 11;
+                [4, 0, N] -> N >= 13;
+                _ -> false
+            catch 
+                _ : _ -> false
+            end;
+        nomatch ->
+            false
+    end.
+
+read_global_auth_type(Data) ->
    case {maps:get(<<"auth_mnesia">>, Data, []), maps:get(<<"acl_mnesia">>, Data, [])} of
        {[], []} ->
            %% Auth mnesia plugin is not used:
            ok;
        _ ->
            do_read_global_auth_type(Data)
-    end;
-read_global_auth_type(_Data, _Version) ->
-    ok.
+    end.

 do_read_global_auth_type(Data) ->
    case Data of
--- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl
+++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl
@ -43,7 +43,7 @@ matrix() ->
                          , Version <- ["v4.2.10", "v4.1.5"]].

 all() ->
-    [t_matrix, t_import_4_0, t_import_no_auth].
+    [t_import_4_0, t_import_4_1, t_import_4_2].

 -endif. %% EMQX_ENTERPRISE

@ -71,41 +71,55 @@ end_per_testcase(_, _Config) ->
    {atomic,ok} = mnesia:clear_table(emqx_user),
    ok.

-t_matrix(Config) ->
-    [begin
-         ct:pal("Testing import of ~p from ~p", [ImportAs, FromVersion]),
-         do_import(Config, ImportAs, FromVersion),
-         test_clientid_import(),
-         ct:pal("ok")
-     end
-     || {ImportAs, FromVersion} <- matrix()].
-
-%% This version is special, since it doesn't have mnesia ACL plugin
 t_import_4_0(Config) ->
-    do_import_no_auth("v4.0.11.json", Config).
-
-t_import_no_auth(Config) ->
-    do_import_no_auth("v4.2.10-no-auth.json", Config).
-
-%% Test that importing configs that don't contain any mnesia ACL data
-%% doesn't require additional overrides:
-do_import_no_auth(File, Config) ->
-    mnesia:clear_table(emqx_acl),
-    mnesia:clear_table(emqx_user),
-    Filename = filename:join(proplists:get_value(data_dir, Config), File),
-    ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, "{}")),
+    ?assertMatch(ok, do_import("v4.0.11-no-auth.json", Config)),
    timer:sleep(100),
-    test_clientid_import().
+    ?assertMatch(0, ets:info(emqx_user, size)),
+
+    ?assertMatch({error, unsupported_version, "4.0"}, do_import("v4.0.11.json", Config)),
+
+    ?assertMatch(ok, do_import("v4.0.13.json", Config)),
+    timer:sleep(100),
+    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
+    test_import(username, {<<"user_for_test">>, <<"public">>}).
+
+t_import_4_1(Config) ->
+    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
+    ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides)),
+    timer:sleep(100),
+    test_import(clientid, {<<"user_mnesia">>, <<"public">>}),
+    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
+    test_import(username, {<<"user_for_test">>, <<"public">>}),
+
+    Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}),
+    ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides1)),
+    timer:sleep(100),
+    test_import(username, {<<"user_mnesia">>, <<"public">>}),
+    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
+    test_import(username, {<<"user_for_test">>, <<"public">>}).
+
+t_import_4_2(Config) ->
+    ?assertMatch(ok, do_import("v4.2.10-no-auth.json", Config)),
+    timer:sleep(100),
+    ?assertMatch(0, ets:info(emqx_user, size)),
+
+    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
+    ?assertMatch({error, unsupported_version, "4.2"}, do_import("v4.2.10.json", Config, Overrides)),
+
+    Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
+    ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides1)),
+    timer:sleep(100),
+    test_import(clientid, {<<"user_mnesia">>, <<"public">>}),
+    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
+    test_import(username, {<<"user_for_test">>, <<"public">>}),
+
+    Overrides2 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}),
+    ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides2)),
+    timer:sleep(100),
+    test_import(username, {<<"user_mnesia">>, <<"public">>}),
+    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
+    test_import(username, {<<"user_for_test">>, <<"public">>}),

-do_import(Config, Type, V) ->
-    File = V ++ ".json",
-    mnesia:clear_table(emqx_acl),
-    mnesia:clear_table(emqx_user),
-    Filename = filename:join(proplists:get_value(data_dir, Config), File),
-    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(Type)}),
-    ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, Overrides)),
-    Records = lists:sort(ets:tab2list(emqx_acl)),
-    %% Check importing of records related to emqx_auth_mnesia
    ?assertMatch([#emqx_acl{
                     filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>},
                     action = pub,
@ -116,21 +130,27 @@ do_import(Config, Type, V) ->
                     action = sub,
                     access = allow
                    }],
-                 lists:sort(Records)),
-    ?assertMatch([_, _], ets:tab2list(emqx_user)),
-    ?assertMatch([_], ets:lookup(emqx_user, {Type, <<"emqx_c">>})),
-    Req = #{clientid => <<"blah">>}
-          #{Type => <<"emqx_c">>,
-            password => <<"emqx_p">>
-           },
+                 lists:sort(ets:tab2list(emqx_acl))).
+
+do_import(File, Config) ->
+    do_import(File, Config, "{}").
+
+do_import(File, Config, Overrides) ->
+    mnesia:clear_table(emqx_acl),
+    mnesia:clear_table(emqx_user),
+    Filename = filename:join(proplists:get_value(data_dir, Config), File),
+    emqx_mgmt_data_backup:import(Filename, Overrides).
+
+test_import(username, {Username, Password}) ->
+    [#emqx_user{password = _}] = ets:lookup(emqx_user, {username, Username}),
+    Req = #{clientid => <<"anyname">>,
+            username => Username,
+            password => Password},
+    ?assertMatch({stop, #{auth_result := success}},
+                 emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256}));
+test_import(clientid, {ClientID, Password}) ->
+    [#emqx_user{password = _}] = ets:lookup(emqx_user, {clientid, ClientID}),
+    Req = #{clientid => ClientID,
+            password => Password},
    ?assertMatch({stop, #{auth_result := success}},
                 emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})).
-
-test_clientid_import() ->
-    [#emqx_user{password = _Pass}] = ets:lookup(emqx_user, {clientid, <<"emqx_clientid">>}),
-    Req = #{clientid => <<"emqx_clientid">>,
-            password => <<"emqx_p">>
-           },
-    ?assertMatch({stop, #{auth_result := success}},
-                 emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})),
-    ok.
--- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json
+++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json
@ -0,0 +1,23 @@
+{
+  "version": "4.0",
+  "users": [],
+  "schemas": [],
+  "rules": [],
+  "resources": [],
+  "date": "2021-04-10 11:45:26",
+  "blacklist": [],
+  "auth_username": [],
+  "auth_mnesia": [],
+  "auth_clientid": [],
+  "apps": [
+    {
+      "status": true,
+      "secret": "public",
+      "name": "Default",
+      "id": "admin",
+      "expired": "undefined",
+      "desc": "Application user"
+    }
+  ],
+  "acl_mnesia": []
+}
--- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json
+++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json
@ -0,0 +1,37 @@
+{
+    "version":"4.0.13",
+    "users":[
+        {
+            "username":"admin",
+            "tags":"administrator",
+            "password":"p6C65OF0BQhvPmCziM2yRa8JN5o="
+        }
+    ],
+    "schemas":[],
+    "rules":[],
+    "resources":[],
+    "date":"2021-04-16 11:20:00",
+    "blacklist":[],
+    "auth_username":[
+        {
+            "username":"user_for_test",
+            "password":"ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
+        }
+    ],
+    "auth_clientid":[
+        {
+            "password":"JBgSnzIxOWNiMDU1ZWFiNDAwMjVhOTQzZThlZjkxN2JlZWE4MGE4YzlmM2I5MjQ4OGI1NjllY2Q4NGQ4NjhjYzQ1NDM=",
+            "clientid":"client_for_test"
+        }
+    ],
+    "apps":[
+        {
+            "status":true,
+            "secret":"public",
+            "name":"Default",
+            "id":"admin",
+            "expired":"undefined",
+            "desc":"Application user"
+        }
+    ]
+}
--- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json
+++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json
@ -12,18 +12,23 @@
  "resources": [],
  "date": "2021-04-07 14:28:58",
  "blacklist": [],
-  "auth_username": [],
+  "auth_username": [
+    {
+      "username":"user_for_test",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
+    }
+  ],
  "auth_mnesia": [
    {
-      "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ==",
-      "login": "emqx_c",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=",
+      "login": "user_mnesia",
      "is_superuser": true
    }
  ],
  "auth_clientid": [
    {
-      "password": "MctXdjZkYzRhMDUwMTc4MDM0OWY4YTg1NTg4Y2ZlOThjYWIyMDk3M2UzNjgzYzYyZWYwOTAzMTk2N2E4OWVjZDk4Mjk=",
-      "clientid": "emqx_clientid"
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=",
+      "clientid": "client_for_test"
    }
  ],
  "apps": [
--- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json
+++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json
@ -21,12 +21,7 @@
      "tags": "administrator"
    }
  ],
-  "auth_clientid": [
-    {
-      "clientid": "emqx_clientid",
-      "password": "uAP84TgyMjAyNGFhY2NlMWVlNDI2NTk1MzFiZjA4YzBjY2RjNjViZmZhNjkzYjhkMDE4NTg0ZWExYjFkZGY0MTBjYWM="
-    }
-  ],
+  "auth_clientid": [],
  "auth_username": [],
  "auth_mnesia": [],
  "acl_mnesia": [],
--- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json
+++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json
@ -0,0 +1,58 @@
+{
+  "version": "4.2.11",
+  "date": "2021-04-12 10:40:58",
+  "rules": [],
+  "resources": [],
+  "blacklist": [],
+  "apps": [
+    {
+      "id": "admin",
+      "secret": "public",
+      "name": "Default",
+      "desc": "Application user",
+      "status": true,
+      "expired": "undefined"
+    }
+  ],
+  "users": [
+    {
+      "username": "test",
+      "password": "8Vd7+gVg2J3nE1Xjyxqd59sA5mo=",
+      "tags": "administrator"
+    }
+  ],
+  "auth_clientid": [
+    {
+      "clientid": "client_for_test",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
+    }
+  ],
+  "auth_username": [
+    {
+      "username": "user_for_test",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
+    }
+  ],
+  "auth_mnesia": [
+    {
+      "login": "user_mnesia",
+      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=",
+      "is_superuser": true
+    }
+  ],
+  "acl_mnesia": [
+    {
+      "login": "emqx_c",
+      "topic": "Topic/A",
+      "action": "sub",
+      "allow": true
+    },
+    {
+      "login": "emqx_c",
+      "topic": "Topic/A",
+      "action": "pub",
+      "allow": true
+    }
+  ],
+  "schemas": []
+}