fix(auth_mnesia): import auth acls data according to different data format

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -46,8 +46,8 @@
         , import_users/1
         , import_auth_clientid/1 %% BACKW: 4.1.x
         , import_auth_username/1 %% BACKW: 4.1.x
-        , import_auth_mnesia/2
+        , import_auth_mnesia/1
-        , import_acl_mnesia/2
+        , import_acl_mnesia/1
         , to_version/1
         ]).
 
@@ -422,60 +422,83 @@ import_auth_username(Lists) ->
                           end, Lists)
     end.
 
--ifdef(EMQX_ENTERPRISE).
+import_auth_mnesia(Auths) ->
-import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse
+    case validate_auth(Auths) of
-                                            FromVersion =:= "4.1" ->
+        ignore -> ok;
-    do_import_auth_mnesia_by_old_data(Auths);
+        old -> do_import_auth_mnesia_by_old_data(Auths);
-import_auth_mnesia(Auths, _) ->
+        new -> do_import_auth_mnesia(Auths)
-    do_import_auth_mnesia(Auths).
+    end.
 
-import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse
+validate_auth(Auths) ->
-                                          FromVersion =:= "4.1" ->
+    case ets:info(emqx_user) of
-    do_import_acl_mnesia_by_old_data(Acls);
+        undefined -> ignore;
+        _ ->
+            case lists:all(fun is_new_auth_data/1, Auths) of
+                true -> new;
+                false ->
+                    case lists:all(fun is_old_auth_data/1, Auths) of
+                        true ->
+                            _ = get_old_type(),
+                            old;
+                        false -> error({auth_mnesia_data_error, Auths})
+                    end
+            end
+    end.
 
-import_acl_mnesia(Acls, _) ->
+is_new_auth_data(#{<<"type">> := _, <<"login">> := _, <<"password">> := _}) -> true;
-    do_import_acl_mnesia(Acls).
+is_new_auth_data(_) -> false.
--else.
-import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.3" ->
-    do_import_auth_mnesia(Auths);
-import_auth_mnesia(Auths, _FromVersion) ->
-    do_import_auth_mnesia_by_old_data(Auths).
 
-import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.3" ->
+is_old_auth_data(#{<<"login">> := _, <<"password">> := _} = Auth) ->
-    do_import_acl_mnesia(Acls);
+    not maps:is_key(<<"type">>, Auth);
-import_acl_mnesia(Acls, _FromVersion) ->
+is_old_auth_data(_) -> false.
-    do_import_acl_mnesia_by_old_data(Acls).
-
--endif.
 
 do_import_auth_mnesia_by_old_data(Auths) ->
-    case ets:info(emqx_user) of
-        undefined -> ok;
-        _ ->
     CreatedAt = erlang:system_time(millisecond),
-            lists:foreach(fun(#{<<"login">> := Login,
+    Type = get_old_type(),
-                                <<"password">> := Password}) ->
+    lists:foreach(fun(#{<<"login">> := Login, <<"password">> := Password}) ->
-                            mnesia:dirty_write({emqx_user, {get_old_type(), Login}, base64:decode(Password), CreatedAt})
+        mnesia:dirty_write({emqx_user, {Type, Login}, base64:decode(Password), CreatedAt})
-                          end, Auths)
+                  end, Auths).
-    end.
-
 
 do_import_auth_mnesia(Auths) ->
-    case ets:info(emqx_user) of
+    CreatedAt0 = erlang:system_time(millisecond),
-        undefined -> ok;
-        _ ->
     lists:foreach(fun(#{<<"login">> := Login,
-                                <<"type">> := Type,
+        <<"type">> := Type, <<"password">> := Password } = Map) ->
-                                <<"password">> := Password } = Map) ->
+        CreatedAt = maps:get(<<"created_at">>, Map, CreatedAt0),
-                            CreatedAt = maps:get(<<"created_at">>, Map, erlang:system_time(millisecond)),
         mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt})
-                          end, Auths)
+                  end, Auths).
+
+import_acl_mnesia(Acls) ->
+    case validate_acl(Acls) of
+        ignore -> ok;
+        old -> do_import_acl_mnesia_by_old_data(Acls);
+        new -> do_import_acl_mnesia(Acls)
     end.
 
-do_import_acl_mnesia_by_old_data(Acls) ->
+validate_acl(Acls) ->
     case ets:info(emqx_acl2) of
-        undefined -> ok;
+        undefined -> ignore;
         _ ->
+            case lists:all(fun is_new_acl_data/1, Acls) of
+                true -> new;
+                false ->
+                    case lists:all(fun is_old_acl_data/1, Acls) of
+                        true ->
+                            _ = get_old_type(),
+                            old;
+                        false -> error({acl_mnesia_data_error, Acls})
+                    end
+            end
+    end.
+
+is_new_acl_data(#{<<"action">> := _, <<"access">> := _,
+    <<"topic">> := _, <<"type">> := _}) -> true;
+is_new_acl_data(_) -> false.
+
+is_old_acl_data(#{<<"login">> := _, <<"topic">> := _,
+    <<"allow">> := Allow, <<"action">> := _}) -> is_boolean(Allow);
+is_old_acl_data(_) -> false.
+
+do_import_acl_mnesia_by_old_data(Acls) ->
     lists:foreach(fun(#{<<"login">> := Login,
         <<"topic">> := Topic,
         <<"allow">> := Allow,
@@ -485,24 +508,17 @@ do_import_acl_mnesia_by_old_data(Acls) ->
                      false -> deny
                  end,
         emqx_acl_mnesia_db:add_acl({get_old_type(), Login}, Topic, any_to_atom(Action), Allow1)
-                          end, Acls)
+                  end, Acls).
-    end.
+
 do_import_acl_mnesia(Acls) ->
-    case ets:info(emqx_acl2) of
-        undefined -> ok;
-        _ ->
     lists:foreach(fun(Map = #{<<"action">> := Action,
-                                      <<"access">> := Access}) ->
+        <<"access">> := Access, <<"topic">> := Topic}) ->
-                            Topic = maps:get(<<"topic">>, Map),
         Login = case maps:get(<<"type_value">>, Map, undefined) of
-                                undefined ->
+                    undefined -> all;
-                                    all;
+                    Value -> {any_to_atom(maps:get(<<"type">>, Map)), Value}
-                                Value ->
-                                    {any_to_atom(maps:get(<<"type">>, Map)), Value}
                 end,
         emqx_acl_mnesia_db:add_acl(Login, Topic, any_to_atom(Action), any_to_atom(Access))
-                          end, Acls)
+                  end, Acls).
-    end.
 
 -ifdef(EMQX_ENTERPRISE).
 -dialyzer({nowarn_function, [import_modules/1]}).
@@ -648,8 +664,8 @@ do_import_data(Data, Version) ->
     import_users(maps:get(<<"users">>, Data, [])),
     import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])),
     import_auth_username(maps:get(<<"auth_username">>, Data, [])),
-    import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version),
+    import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])),
-    import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version).
+    import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])).
 
 -ifdef(EMQX_ENTERPRISE).
 do_import_extra_data(Data, _Version) ->
@@ -709,9 +725,9 @@ read_global_auth_type(Data, Version) ->
 do_read_global_auth_type(Data, _Version) ->
     case Data of
         #{<<"auth.mnesia.as">> := <<"username">>} ->
-            application:set_env(emqx_auth_mnesia, as, username);
+            set_old_type(username);
         #{<<"auth.mnesia.as">> := <<"clientid">>} ->
-            application:set_env(emqx_auth_mnesia, as, clientid);
+            set_old_type(clientid);
         _ ->
             ok
     end.
@@ -720,9 +736,9 @@ do_read_global_auth_type(Data, _Version) ->
 do_read_global_auth_type(Data, FromVersion) ->
     case Data of
         #{<<"auth.mnesia.as">> := <<"username">>} ->
-            application:set_env(emqx_auth_mnesia, as, username);
+            set_old_type(username);
         #{<<"auth.mnesia.as">> := <<"clientid">>} ->
-            application:set_env(emqx_auth_mnesia, as, clientid);
+            set_old_type(clientid);
         _ when FromVersion =:= "4.0" orelse
             FromVersion =:= "4.1" orelse
             FromVersion =:= "4.2"->
@@ -734,7 +750,7 @@ do_read_global_auth_type(Data, FromVersion) ->
                          "or\n"
                          "  $ emqx_ctl data import <filename> --env '{\"auth.mnesia.as\":\"clientid\"}'",
                          []),
-            error(import_failed);
+            error({import_failed, FromVersion});
         _ ->
             ok
     end.
@@ -743,3 +759,6 @@ do_read_global_auth_type(Data, FromVersion) ->
 get_old_type() ->
     {ok, Type} = application:get_env(emqx_auth_mnesia, as),
     Type.
+
+set_old_type(Type) ->
+    application:set_env(emqx_auth_mnesia, as, Type).

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl

@@ -39,13 +39,13 @@ cases() ->
     [t_import].
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard, emqx_auth_mnesia]),
+    emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia]),
     ekka_mnesia:start(),
     emqx_mgmt_auth:mnesia(boot),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_dashboard, emqx_auth_mnesia]),
+    emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_auth_mnesia]),
     ekka_mnesia:ensure_stopped().
 
 init_per_testcase(_, Config) ->
@@ -167,7 +167,7 @@ t_export_import(_Config) ->
 
     ?assertEqual([], emqx_acl_mnesia_db:all_acls()),
 
-    emqx_mgmt_data_backup:import_acl_mnesia(emqx_json:decode(AclData, [return_maps]), "4.3"),
+    emqx_mgmt_data_backup:import_acl_mnesia(emqx_json:decode(AclData, [return_maps])),
     timer:sleep(100),
 
     ?assertMatch([