feat(authentication): require header in csv file and use new json format

apps/emqx_authentication/data/user-credentials.csv

@@ -1,2 +1,3 @@
+user_id,password_hash
 myuser3,8d41233e39c95b5da13361e354e1c9e639f07b27d397463a8f91b71ee07ccfb2
-myuser4,5809df0154f3cb4ac5c3a5572eaca0c5f7f9d858e887fc675b2becab9feb19d1
+myuser4,5809df0154f3cb4ac5c3a5572eaca0c5f7f9d858e887fc675b2becab9feb19d1

apps/emqx_authentication/data/user-credentials.json

@@ -1,4 +1,10 @@
-{
+[
-    "myuser1": "09343625c6c123d3434932fe1ce08bae5ac00a8f95bd746e10491b0bafdd1817",
+    {
-    "myuser2": "8767a7d316ad68cb607c7c805b859ffa78277dda13b7a3e2e8b53cad3cabbc6e"
+        "user_id":"myuser1",
-}
+        "password_hash":"09343625c6c123d3434932fe1ce08bae5ac00a8f95bd746e10491b0bafdd1817"
+    },
+    {
+        "user_id":"myuser2",
+        "password_hash":"8767a7d316ad68cb607c7c805b859ffa78277dda13b7a3e2e8b53cad3cabbc6e"
+    }
+]

apps/emqx_authentication/src/emqx_authentication_mnesia.erl

@@ -52,6 +52,8 @@
     }
 }).
 
+
+
 -record(user_info,
         { user_id :: {user_group(), user_id()}
         , password_hash :: binary()
@@ -188,17 +190,13 @@ list_users(#{user_group := UserGroup}) ->
 %% Internal functions
 %%------------------------------------------------------------------------------
 
-%% Example:
+%% Example: data/user-credentials.json
-%% {
-%%     "myuser1":"password_hash1",
-%%     "myuser2":"password_hash2"
-%% }
 import_users_from_json(Filename, #{user_group := UserGroup}) ->
     case file:read_file(Filename) of
         {ok, Bin} ->
-            case emqx_json:safe_decode(Bin) of
+            case emqx_json:safe_decode(Bin, [return_maps]) of
                 {ok, List} ->
-                    import(UserGroup, List);
+                    trans(fun import/2, [UserGroup, List]);
                 {error, Reason} ->
                     {error, Reason}
             end;
@@ -206,44 +204,43 @@ import_users_from_json(Filename, #{user_group := UserGroup}) ->
             {error, Reason}
     end.
 
-%% Example:
+%% Example: data/user-credentials.csv
-%% myuser1,password_hash1
-%% myuser2,password_hash2
 import_users_from_csv(Filename, #{user_group := UserGroup}) ->
     case file:open(Filename, [read, binary]) of
         {ok, File} ->
-            Result = import(UserGroup, File),
+            case get_csv_header(File) of
-            file:close(File),
+                {ok, Seq} ->
-            Result;
+                    Result = trans(fun import/3, [UserGroup, File, Seq]),
+                    file:close(File),
+                    Result;
+                {error, Reason} ->
+                    {error, Reason}
+            end;
         {error, Reason} ->
             {error, Reason}
     end.
 
-import(UserGroup, ListOrFile) ->
+import(_UserGroup, []) ->
-    trans(fun do_import/2, [UserGroup, ListOrFile]).
-
-do_import(_UserGroup, []) ->
     ok;
-do_import(UserGroup, [{UserID, PasswordHash} | More])
+import(UserGroup, [#{<<"user_id">> := UserID, <<"password_hash">> := PasswordHash} | More])
   when is_binary(UserID) andalso is_binary(PasswordHash) ->
     import_user(UserGroup, UserID, PasswordHash),
-    do_import(UserGroup, More);
+    import(UserGroup, More);
-do_import(_UserGroup, [_ | _More]) ->
+import(_UserGroup, [_ | _More]) ->
-    {error, bad_format};
+    {error, bad_format}.
 
 %% Importing 5w users needs 1.7 seconds 
-do_import(UserGroup, File)  ->
+import(UserGroup, File, Seq) ->
     case file:read_line(File) of
         {ok, Line} ->
-            case binary:split(Line, [<<",">>, <<"\n">>], [global]) of
+            Fields = binary:split(Line, [<<",">>, <<" ">>, <<"\n">>], [global, trim_all]),
-                [UserID, PasswordHash, <<>>] ->
+            case get_user_info_by_seq(Fields, Seq) of
-                    import_user(UserGroup, UserID, PasswordHash),
+                {ok, #{user_id := UserID,
-                    do_import(UserGroup, File);
+                       password_hash := PasswordHash}} ->
-                [UserID, PasswordHash] ->
                     import_user(UserGroup, UserID, PasswordHash),
-                    do_import(UserGroup, File);
+                    import(UserGroup, File, Seq);
-                _ ->
+                {error, Reason} ->
-                    {error, bad_format}
+                    {error, Reason}
             end;
         eof ->
             ok;
@@ -251,6 +248,31 @@ do_import(UserGroup, File)  ->
             {error, Reason}
     end.
 
+get_csv_header(File) ->
+    case file:read_line(File) of
+        {ok, Line} ->
+            Seq = binary:split(Line, [<<",">>, <<" ">>, <<"\n">>], [global, trim_all]),
+            {ok, Seq};
+        eof ->
+            {error, empty_file};
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+get_user_info_by_seq(Fields, Seq) ->
+    get_user_info_by_seq(Fields, Seq, #{}).
+
+get_user_info_by_seq([], [], #{user_id := _, password_hash := _} = Acc) ->
+    {ok, Acc};
+get_user_info_by_seq(_, [], _) ->
+    {error, bad_format};
+get_user_info_by_seq([UserID | More1], [<<"user_id">> | More2], Acc) ->
+    get_user_info_by_seq(More1, More2, Acc#{user_id => UserID});
+get_user_info_by_seq([PasswordHash | More1], [<<"password_hash">> | More2], Acc) ->
+    get_user_info_by_seq(More1, More2, Acc#{password_hash => PasswordHash});
+get_user_info_by_seq(_, _, _) ->
+    {error, bad_format}.
+
 -compile({inline, [add/4]}).
 add(UserGroup, UserID, Password, Algorithm) ->
     Credential = #user_info{user_id = {UserGroup, UserID},

apps/emqx_authentication/test/data/user-credentials.csv

@@ -1,2 +1,3 @@
+user_id,password_hash
 myuser3,8d41233e39c95b5da13361e354e1c9e639f07b27d397463a8f91b71ee07ccfb2
-myuser4,5809df0154f3cb4ac5c3a5572eaca0c5f7f9d858e887fc675b2becab9feb19d1
+myuser4,5809df0154f3cb4ac5c3a5572eaca0c5f7f9d858e887fc675b2becab9feb19d1

apps/emqx_authentication/test/data/user-credentials.json

@@ -1,4 +1,10 @@
-{
+[
-    "myuser1": "09343625c6c123d3434932fe1ce08bae5ac00a8f95bd746e10491b0bafdd1817",
+    {
-    "myuser2": "8767a7d316ad68cb607c7c805b859ffa78277dda13b7a3e2e8b53cad3cabbc6e"
+        "user_id":"myuser1",
-}
+        "password_hash":"09343625c6c123d3434932fe1ce08bae5ac00a8f95bd746e10491b0bafdd1817"
+    },
+    {
+        "user_id":"myuser2",
+        "password_hash":"8767a7d316ad68cb607c7c805b859ffa78277dda13b7a3e2e8b53cad3cabbc6e"
+    }
+]