diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 35f4139c4..abfa55c36 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -166,11 +166,11 @@ when when State :: state(). --callback import_users(Filename, State) -> +-callback import_users({Filename, FileData}, State) -> ok | {error, term()} when - Filename :: binary(), State :: state(). + Filename :: binary(), FileData :: binary(), State :: state(). -callback add_user(UserInfo, State) -> {ok, User} @@ -385,7 +385,8 @@ list_authenticators(ChainName) -> move_authenticator(ChainName, AuthenticatorID, Position) -> call({move_authenticator, ChainName, AuthenticatorID, Position}). --spec import_users(chain_name(), authenticator_id(), binary()) -> ok | {error, term()}. +-spec import_users(chain_name(), authenticator_id(), {binary(), binary()}) -> + ok | {error, term()}. import_users(ChainName, AuthenticatorID, Filename) -> call({import_users, ChainName, AuthenticatorID, Filename}). diff --git a/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf b/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf index 44725e935..a31a335d5 100644 --- a/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf +++ b/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf @@ -105,20 +105,6 @@ emqx_authn_api { } } - authentication_id_import_users_post { - desc { - en: """Import users into authenticator in global authentication chain.""" - zh: """为全局认证链上的指定认证器导入用户数据。""" - } - } - - listeners_listener_id_authentication_id_import_users_post { - desc { - en: """Import users into authenticator in listener authentication chain.""" - zh: """为监听器认证链上的指定认证器导入用户数据。""" - } - } - authentication_id_users_post { desc { en: """Create users for authenticator in global authentication chain.""" diff --git a/apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf b/apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf new file mode 100644 index 000000000..294897ec1 --- /dev/null +++ b/apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf @@ -0,0 +1,17 @@ +emqx_authn_user_import_api { + + authentication_id_import_users_post { + desc { + en: """Import users into authenticator in global authentication chain.""" + zh: """为全局认证链上的指定认证器导入用户数据。""" + } + } + + listeners_listener_id_authentication_id_import_users_post { + desc { + en: """Import users into authenticator in listener authentication chain.""" + zh: """为监听器认证链上的指定认证器导入用户数据。""" + } + } + +} diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index f20a2e5ad..b01327ff8 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -62,8 +62,6 @@ listener_authenticator_status/2, authenticator_move/2, listener_authenticator_move/2, - authenticator_import_users/2, - listener_authenticator_import_users/2, authenticator_users/2, authenticator_user/2, listener_authenticator_users/2, @@ -75,7 +73,6 @@ -export([ authenticator_examples/0, request_move_examples/0, - request_import_users_examples/0, request_user_create_examples/0, request_user_update_examples/0, response_user_examples/0, @@ -90,7 +87,11 @@ find_user/3, update_user/4, serialize_error/1, - aggregate_metrics/1 + aggregate_metrics/1, + + with_chain/2, + param_auth_id/0, + param_listener_id/0 ]). -elvis([{elvis_style, god_modules, disable}]). @@ -104,7 +105,6 @@ paths() -> "/authentication/:id", "/authentication/:id/status", "/authentication/:id/move", - "/authentication/:id/import_users", "/authentication/:id/users", "/authentication/:id/users/:user_id", @@ -112,7 +112,6 @@ paths() -> "/listeners/:listener_id/authentication/:id", "/listeners/:listener_id/authentication/:id/status", "/listeners/:listener_id/authentication/:id/move", - "/listeners/:listener_id/authentication/:id/import_users", "/listeners/:listener_id/authentication/:id/users", "/listeners/:listener_id/authentication/:id/users/:user_id" ]. @@ -122,7 +121,6 @@ roots() -> request_user_create, request_user_update, request_move, - request_import_users, response_user, response_users ]. @@ -139,9 +137,6 @@ fields(request_user_update) -> ]; fields(request_move) -> [{position, mk(binary(), #{required => true})}]; -fields(request_import_users) -> - %% TODO: add file update - [{filename, mk(binary(), #{required => true})}]; fields(response_user) -> [ {user_id, mk(binary(), #{required => true})}, @@ -375,42 +370,6 @@ schema("/listeners/:listener_id/authentication/:id/move") -> } } }; -schema("/authentication/:id/import_users") -> - #{ - 'operationId' => authenticator_import_users, - post => #{ - tags => ?API_TAGS_GLOBAL, - description => ?DESC(authentication_id_import_users_post), - parameters => [param_auth_id()], - 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - ref(request_import_users), - request_import_users_examples() - ), - responses => #{ - 204 => <<"Users imported">>, - 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), - 404 => error_codes([?NOT_FOUND], <<"Not Found">>) - } - } - }; -schema("/listeners/:listener_id/authentication/:id/import_users") -> - #{ - 'operationId' => listener_authenticator_import_users, - post => #{ - tags => ?API_TAGS_SINGLE, - description => ?DESC(listeners_listener_id_authentication_id_import_users_post), - parameters => [param_listener_id(), param_auth_id()], - 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - ref(request_import_users), - request_import_users_examples() - ), - responses => #{ - 204 => <<"Users imported">>, - 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), - 404 => error_codes([?NOT_FOUND], <<"Not Found">>) - } - } - }; schema("/authentication/:id/users") -> #{ 'operationId' => authenticator_users, @@ -747,39 +706,6 @@ listener_authenticator_move( listener_authenticator_move(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> serialize_error({missing_parameter, position}). -authenticator_import_users( - post, - #{ - bindings := #{id := AuthenticatorID}, - body := #{<<"filename">> := Filename} - } -) -> - case emqx_authentication:import_users(?GLOBAL, AuthenticatorID, Filename) of - ok -> {204}; - {error, Reason} -> serialize_error(Reason) - end; -authenticator_import_users(post, #{bindings := #{id := _}, body := _}) -> - serialize_error({missing_parameter, filename}). - -listener_authenticator_import_users( - post, - #{ - bindings := #{listener_id := ListenerID, id := AuthenticatorID}, - body := #{<<"filename">> := Filename} - } -) -> - with_chain( - ListenerID, - fun(ChainName) -> - case emqx_authentication:import_users(ChainName, AuthenticatorID, Filename) of - ok -> {204}; - {error, Reason} -> serialize_error(Reason) - end - end - ); -listener_authenticator_import_users(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> - serialize_error({missing_parameter, filename}). - authenticator_users(post, #{bindings := #{id := AuthenticatorID}, body := UserInfo}) -> add_user(?GLOBAL, AuthenticatorID, UserInfo); authenticator_users(get, #{bindings := #{id := AuthenticatorID}, query_string := QueryString}) -> @@ -1579,22 +1505,6 @@ request_move_examples() -> } }. -request_import_users_examples() -> - #{ - import_csv => #{ - summary => <<"Import users from CSV file">>, - value => #{ - filename => <<"/path/to/user/data.csv">> - } - }, - import_json => #{ - summary => <<"Import users from JSON file">>, - value => #{ - filename => <<"/path/to/user/data.json">> - } - } - }. - response_user_examples() -> #{ regular_user => #{ diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl new file mode 100644 index 000000000..30417acf7 --- /dev/null +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -0,0 +1,144 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_user_import_api). + +-behaviour(minirest_api). + +-include("emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(emqx_dashboard_swagger, [error_codes/2]). + +-define(BAD_REQUEST, 'BAD_REQUEST'). +-define(NOT_FOUND, 'NOT_FOUND'). + +% Swagger + +-define(API_TAGS_GLOBAL, [ + ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(global)">> +]). +-define(API_TAGS_SINGLE, [ + ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(single listener)">> +]). + +-export([ + api_spec/0, + paths/0, + schema/1 +]). + +-export([ + authenticator_import_users/2, + listener_authenticator_import_users/2 +]). + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}). + +paths() -> + [ + "/authentication/:id/import_users", + "/listeners/:listener_id/authentication/:id/import_users" + ]. + +schema("/authentication/:id/import_users") -> + #{ + 'operationId' => authenticator_import_users, + post => #{ + tags => ?API_TAGS_GLOBAL, + description => ?DESC(authentication_id_import_users_post), + parameters => [emqx_authn_api:param_auth_id()], + 'requestBody' => #{ + content => #{ + 'multipart/form-data' => #{ + schema => #{ + filename => file + } + } + } + }, + responses => #{ + 204 => <<"Users imported">>, + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }; +schema("/listeners/:listener_id/authentication/:id/import_users") -> + #{ + 'operationId' => listener_authenticator_import_users, + post => #{ + tags => ?API_TAGS_SINGLE, + description => ?DESC(listeners_listener_id_authentication_id_import_users_post), + parameters => [emqx_authn_api:param_listener_id(), emqx_authn_api:param_auth_id()], + 'requestBody' => #{ + content => #{ + 'multipart/form-data' => #{ + schema => #{ + filename => file + } + } + } + }, + responses => #{ + 204 => <<"Users imported">>, + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }. + +authenticator_import_users( + post, + #{ + bindings := #{id := AuthenticatorID}, + body := #{<<"filename">> := #{type := _} = File} + } +) -> + [{FileName, FileData}] = maps:to_list(maps:without([type], File)), + case emqx_authentication:import_users(?GLOBAL, AuthenticatorID, {FileName, FileData}) of + ok -> {204}; + {error, Reason} -> emqx_authn_api:serialize_error(Reason) + end; +authenticator_import_users(post, #{bindings := #{id := _}, body := _}) -> + emqx_authn_api:serialize_error({missing_parameter, filename}). + +listener_authenticator_import_users( + post, + #{ + bindings := #{listener_id := ListenerID, id := AuthenticatorID}, + body := #{<<"filename">> := #{type := _} = File} + } +) -> + [{FileName, FileData}] = maps:to_list(maps:without([type], File)), + emqx_authn_api:with_chain( + ListenerID, + fun(ChainName) -> + case + emqx_authentication:import_users(ChainName, AuthenticatorID, {FileName, FileData}) + of + ok -> {204}; + {error, Reason} -> emqx_authn_api:serialize_error(Reason) + end + end + ); +listener_authenticator_import_users(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> + emqx_authn_api:serialize_error({missing_parameter, filename}). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index bdcca33f8..81dc89c54 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -182,13 +182,14 @@ destroy(#{user_group := UserGroup}) -> end ). -import_users(Filename0, State) -> +import_users({Filename0, FileData}, State) -> Filename = to_binary(Filename0), case filename:extension(Filename) of <<".json">> -> - import_users_from_json(Filename, State); + import_users_from_json(FileData, State); <<".csv">> -> - import_users_from_csv(Filename, State); + CSV = csv_data(FileData), + import_users_from_csv(CSV, State); <<>> -> {error, unknown_file_format}; Extension -> @@ -327,31 +328,19 @@ run_fuzzy_filter( %%------------------------------------------------------------------------------ %% Example: data/user-credentials.json -import_users_from_json(Filename, #{user_group := UserGroup}) -> - case file:read_file(Filename) of - {ok, Bin} -> - case emqx_json:safe_decode(Bin, [return_maps]) of - {ok, List} -> - trans(fun import/2, [UserGroup, List]); - {error, Reason} -> - {error, Reason} - end; +import_users_from_json(Bin, #{user_group := UserGroup}) -> + case emqx_json:safe_decode(Bin, [return_maps]) of + {ok, List} -> + trans(fun import/2, [UserGroup, List]); {error, Reason} -> {error, Reason} end. %% Example: data/user-credentials.csv -import_users_from_csv(Filename, #{user_group := UserGroup}) -> - case file:open(Filename, [read, binary]) of - {ok, File} -> - case get_csv_header(File) of - {ok, Seq} -> - Result = trans(fun import/3, [UserGroup, File, Seq]), - _ = file:close(File), - Result; - {error, Reason} -> - {error, Reason} - end; +import_users_from_csv(CSV, #{user_group := UserGroup}) -> + case get_csv_header(CSV) of + {ok, Seq, NewCSV} -> + trans(fun import_csv/3, [UserGroup, NewCSV, Seq]); {error, Reason} -> {error, Reason} end. @@ -375,9 +364,9 @@ import(_UserGroup, [_ | _More]) -> {error, bad_format}. %% Importing 5w users needs 1.7 seconds -import(UserGroup, File, Seq) -> - case file:read_line(File) of - {ok, Line} -> +import_csv(UserGroup, CSV, Seq) -> + case csv_read_line(CSV) of + {ok, Line, NewCSV} -> Fields = binary:split(Line, [<<",">>, <<" ">>, <<"\n">>], [global, trim_all]), case get_user_info_by_seq(Fields, Seq) of {ok, @@ -388,25 +377,21 @@ import(UserGroup, File, Seq) -> Salt = maps:get(salt, UserInfo, <<>>), IsSuperuser = maps:get(is_superuser, UserInfo, false), insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), - import(UserGroup, File, Seq); + import_csv(UserGroup, NewCSV, Seq); {error, Reason} -> {error, Reason} end; eof -> - ok; - {error, Reason} -> - {error, Reason} + ok end. -get_csv_header(File) -> - case file:read_line(File) of - {ok, Line} -> +get_csv_header(CSV) -> + case csv_read_line(CSV) of + {ok, Line, NewCSV} -> Seq = binary:split(Line, [<<",">>, <<" ">>, <<"\n">>], [global, trim_all]), - {ok, Seq}; + {ok, Seq, NewCSV}; eof -> - {error, empty_file}; - {error, Reason} -> - {error, Reason} + {error, empty_file} end. get_user_info_by_seq(Fields, Seq) -> @@ -487,3 +472,12 @@ group_match_spec(UserGroup, QString) -> User end) end. + +csv_data(Data) -> + Lines = binary:split(Data, [<<"\r">>, <<"\n">>], [global, trim_all]), + {csv_data, Lines}. + +csv_read_line({csv_data, [Line | Lines]}) -> + {ok, Line, {csv_data, Lines}}; +csv_read_line({csv_data, []}) -> + eof. diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index c61f52119..648fb8a0f 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_dashboard_api_test_helpers, [request/3, uri/1]). +-import(emqx_dashboard_api_test_helpers, [request/3, uri/1, multipart_formdata_request/3]). -include("emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -643,19 +643,24 @@ test_authenticator_import_users(PathPrefix) -> emqx_authn_test_lib:built_in_database_example() ), - {ok, 400, _} = request(post, ImportUri, #{}), - - {ok, 400, _} = request(post, ImportUri, #{filename => <<"/etc/passwd">>}), - - {ok, 400, _} = request(post, ImportUri, #{filename => <<"/not_exists.csv">>}), + {ok, 400, _} = multipart_formdata_request(ImportUri, [], []), + {ok, 400, _} = multipart_formdata_request(ImportUri, [], [ + {filenam, "user-credentials.json", <<>>} + ]), Dir = code:lib_dir(emqx_authn, test), JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), - {ok, 204, _} = request(post, ImportUri, #{filename => JSONFileName}), + {ok, JSONData} = file:read_file(JSONFileName), + {ok, 204, _} = multipart_formdata_request(ImportUri, [], [ + {filename, "user-credentials.json", JSONData} + ]), - {ok, 204, _} = request(post, ImportUri, #{filename => CSVFileName}). + {ok, CSVData} = file:read_file(CSVFileName), + {ok, 204, _} = multipart_formdata_request(ImportUri, [], [ + {filename, "user-credentials.csv", CSVData} + ]). t_switch_to_global_chain(_) -> {ok, 200, _} = request( diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index 693bb9eba..2ab9efb1d 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -228,54 +228,75 @@ t_import_users(_) -> Config = Config0#{password_hash_algorithm => #{name => sha256}}, {ok, State} = emqx_authn_mnesia:create(?AUTHN_ID, Config), - ok = emqx_authn_mnesia:import_users( - data_filename(<<"user-credentials.json">>), - State + ?assertEqual( + ok, + emqx_authn_mnesia:import_users( + sample_filename_and_data(<<"user-credentials.json">>), + State + ) ), - ok = emqx_authn_mnesia:import_users( - data_filename(<<"user-credentials.csv">>), - State + ?assertEqual( + ok, + emqx_authn_mnesia:import_users( + sample_filename_and_data(<<"user-credentials.csv">>), + State + ) ), - {error, {unsupported_file_format, _}} = emqx_authn_mnesia:import_users( - <<"/file/with/unknown.extension">>, - State + ?assertMatch( + {error, {unsupported_file_format, _}}, + emqx_authn_mnesia:import_users( + {<<"/file/with/unknown.extension">>, <<>>}, + State + ) ), - {error, unknown_file_format} = emqx_authn_mnesia:import_users( - <<"/file/with/no/extension">>, - State + ?assertEqual( + {error, unknown_file_format}, + emqx_authn_mnesia:import_users( + {<<"/file/with/no/extension">>, <<>>}, + State + ) ), - {error, enoent} = emqx_authn_mnesia:import_users( - <<"/file/that/not/exist.json">>, - State + ?assertEqual( + {error, bad_format}, + emqx_authn_mnesia:import_users( + sample_filename_and_data(<<"user-credentials-malformed-0.json">>), + State + ) ), - {error, bad_format} = emqx_authn_mnesia:import_users( - data_filename(<<"user-credentials-malformed-0.json">>), - State + ?assertMatch( + {error, {_, invalid_json}}, + emqx_authn_mnesia:import_users( + sample_filename_and_data(<<"user-credentials-malformed-1.json">>), + State + ) ), - {error, {_, invalid_json}} = emqx_authn_mnesia:import_users( - data_filename(<<"user-credentials-malformed-1.json">>), - State - ), - - {error, bad_format} = emqx_authn_mnesia:import_users( - data_filename(<<"user-credentials-malformed.csv">>), - State + ?assertEqual( + {error, bad_format}, + emqx_authn_mnesia:import_users( + sample_filename_and_data(<<"user-credentials-malformed.csv">>), + State + ) ). %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ -data_filename(Name) -> +sample_filename(Name) -> Dir = code:lib_dir(emqx_authn, test), filename:join([Dir, <<"data">>, Name]). +sample_filename_and_data(Name) -> + Filename = sample_filename(Name), + {ok, Data} = file:read_file(Filename), + {Filename, Data}. + config() -> #{ user_id_type => username, diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index 0f3405b57..eacca6aa2 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -22,6 +22,8 @@ request/2, request/3, request/4, + multipart_formdata_request/3, + multipart_formdata_request/4, uri/0, uri/1 ]). @@ -97,3 +99,67 @@ auth_header(Username) -> Password = <<"public">>, {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), {"Authorization", "Bearer " ++ binary_to_list(Token)}. + +multipart_formdata_request(Url, Fields, Files) -> + multipart_formdata_request(Url, <<"admin">>, Fields, Files). + +multipart_formdata_request(Url, Username, Fields, Files) -> + Boundary = + "------------" ++ integer_to_list(rand:uniform(99999999999999999)) ++ + integer_to_list(erlang:system_time(millisecond)), + Body = format_multipart_formdata(Boundary, Fields, Files), + ContentType = lists:concat(["multipart/form-data; boundary=", Boundary]), + Headers = + [ + auth_header(Username), + {"Content-Length", integer_to_list(length(Body))} + ], + case httpc:request(post, {Url, Headers, ContentType, Body}, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return}} -> + {ok, Code, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +format_multipart_formdata(Boundary, Fields, Files) -> + FieldParts = lists:map( + fun({FieldName, FieldContent}) -> + [ + lists:concat(["--", Boundary]), + lists:concat([ + "Content-Disposition: form-data; name=\"", atom_to_list(FieldName), "\"" + ]), + "", + to_list(FieldContent) + ] + end, + Fields + ), + FieldParts2 = lists:append(FieldParts), + FileParts = lists:map( + fun({FieldName, FileName, FileContent}) -> + [ + lists:concat(["--", Boundary]), + lists:concat([ + "Content-Disposition: form-data; name=\"", + atom_to_list(FieldName), + "\"; filename=\"", + FileName, + "\"" + ]), + lists:concat(["Content-Type: ", "application/octet-stream"]), + "", + to_list(FileContent) + ] + end, + Files + ), + FileParts2 = lists:append(FileParts), + EndingParts = [lists:concat(["--", Boundary, "--"]), ""], + Parts = lists:append([FieldParts2, FileParts2, EndingParts]), + string:join(Parts, "\r\n"). + +to_list(Bin) when is_binary(Bin) -> binary_to_list(Bin); +to_list(Str) when is_list(Str) -> Str. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index 3c4b77f97..c49b69e1c 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -46,8 +46,7 @@ -export([ authn/2, users/2, - users_insta/2, - import_users/2 + users_insta/2 ]). %% internal export for emqx_gateway_api_listeners module @@ -64,8 +63,7 @@ paths() -> [ "/gateway/:name/authentication", "/gateway/:name/authentication/users", - "/gateway/:name/authentication/users/:uid", - "/gateway/:name/authentication/import_users" + "/gateway/:name/authentication/users/:uid" ]. %%-------------------------------------------------------------------- @@ -160,32 +158,6 @@ users_insta(delete, #{bindings := #{name := Name0, uid := UserId}}) -> emqx_authn_api:delete_user(ChainName, AuthId, UserId) end). -import_users(post, #{ - bindings := #{name := Name0}, - body := Body -}) -> - with_authn(Name0, fun( - _GwName, - #{ - id := AuthId, - chain_name := ChainName - } - ) -> - case maps:get(<<"filename">>, Body, undefined) of - undefined -> - emqx_authn_api:serialize_error({missing_parameter, filename}); - Filename -> - case - emqx_authentication:import_users( - ChainName, AuthId, Filename - ) - of - ok -> {204}; - {error, Reason} -> emqx_authn_api:serialize_error(Reason) - end - end - end). - %%-------------------------------------------------------------------- %% Utils @@ -326,21 +298,6 @@ schema("/gateway/:name/authentication/users/:uid") -> responses => ?STANDARD_RESP(#{204 => <<"User Deleted">>}) } - }; -schema("/gateway/:name/authentication/import_users") -> - #{ - 'operationId' => import_users, - post => - #{ - desc => ?DESC(import_users), - parameters => params_gateway_name_in_path(), - 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - ref(emqx_authn_api, request_import_users), - emqx_authn_api:request_import_users_examples() - ), - responses => - ?STANDARD_RESP(#{204 => <<"Imported">>}) - } }. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl new file mode 100644 index 000000000..b86dc4510 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl @@ -0,0 +1,190 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_api_authn_user_import). + +-behaviour(minirest_api). + +-include("emqx_gateway_http.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-import(emqx_dashboard_swagger, [error_codes/2]). +-import(hoconsc, [mk/2, ref/2]). +-import( + emqx_gateway_http, + [ + with_authn/2, + with_listener_authn/3 + ] +). + +%% minirest/dashboard_swagger behaviour callbacks +-export([ + api_spec/0, + paths/0, + schema/1 +]). + +%% http handlers +-export([ + import_users/2, + import_listener_users/2 +]). + +%%-------------------------------------------------------------------- +%% minirest behaviour callbacks +%%-------------------------------------------------------------------- + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}). + +paths() -> + [ + "/gateway/:name/authentication/import_users", + "/gateway/:name/listeners/:id/authentication/import_users" + ]. + +%%-------------------------------------------------------------------- +%% http handlers + +import_users(post, #{ + bindings := #{name := Name0}, + body := Body +}) -> + with_authn(Name0, fun( + _GwName, + #{ + id := AuthId, + chain_name := ChainName + } + ) -> + case maps:get(<<"filename">>, Body, undefined) of + undefined -> + emqx_authn_api:serialize_error({missing_parameter, filename}); + File -> + [{FileName, FileData}] = maps:to_list(maps:without([type], File)), + case + emqx_authentication:import_users( + ChainName, AuthId, {FileName, FileData} + ) + of + ok -> {204}; + {error, Reason} -> emqx_authn_api:serialize_error(Reason) + end + end + end). + +import_listener_users(post, #{ + bindings := #{name := Name0, id := Id}, + body := Body +}) -> + with_listener_authn( + Name0, + Id, + fun(_GwName, #{id := AuthId, chain_name := ChainName}) -> + case maps:get(<<"filename">>, Body, undefined) of + undefined -> + emqx_authn_api:serialize_error({missing_parameter, filename}); + File -> + [{FileName, FileData}] = maps:to_list(maps:without([type], File)), + case + emqx_authentication:import_users( + ChainName, AuthId, {FileName, FileData} + ) + of + ok -> {204}; + {error, Reason} -> emqx_authn_api:serialize_error(Reason) + end + end + end + ). + +%%-------------------------------------------------------------------- +%% Swagger defines +%%-------------------------------------------------------------------- + +schema("/gateway/:name/authentication/import_users") -> + #{ + 'operationId' => import_users, + post => + #{ + desc => ?DESC(emqx_gateway_api_authn, import_users), + parameters => params_gateway_name_in_path(), + 'requestBody' => #{ + content => #{ + 'multipart/form-data' => #{ + schema => #{ + filename => file + } + } + } + }, + responses => + ?STANDARD_RESP(#{204 => <<"Imported">>}) + } + }; +schema("/gateway/:name/listeners/:id/authentication/import_users") -> + #{ + 'operationId' => import_listener_users, + post => + #{ + desc => ?DESC(emqx_gateway_api_listeners, import_users), + parameters => params_gateway_name_in_path() ++ + params_listener_id_in_path(), + 'requestBody' => #{ + content => #{ + 'multipart/form-data' => #{ + schema => #{ + filename => file + } + } + } + }, + responses => + ?STANDARD_RESP(#{204 => <<"Imported">>}) + } + }. + +%%-------------------------------------------------------------------- +%% params defines +%%-------------------------------------------------------------------- + +params_gateway_name_in_path() -> + [ + {name, + mk( + binary(), + #{ + in => path, + desc => ?DESC(emqx_gateway_api, gateway_name), + example => <<"stomp">> + } + )} + ]. + +params_listener_id_in_path() -> + [ + {id, + mk( + binary(), + #{ + in => path, + desc => ?DESC(emqx_gateway_api_listeners, listener_id), + example => <<"stomp:tcp:def">> + } + )} + ]. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 4064af05b..d0778717a 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -54,8 +54,7 @@ listeners_insta/2, listeners_insta_authn/2, users/2, - users_insta/2, - import_users/2 + users_insta/2 ]). %% RPC @@ -74,8 +73,7 @@ paths() -> "/gateway/:name/listeners/:id", "/gateway/:name/listeners/:id/authentication", "/gateway/:name/listeners/:id/authentication/users", - "/gateway/:name/listeners/:id/authentication/users/:uid", - "/gateway/:name/listeners/:id/authentication/import_users" + "/gateway/:name/listeners/:id/authentication/users/:uid" ]. %%-------------------------------------------------------------------- @@ -239,30 +237,6 @@ users_insta(delete, #{bindings := #{name := Name0, id := Id, uid := UserId}}) -> end ). -import_users(post, #{ - bindings := #{name := Name0, id := Id}, - body := Body -}) -> - with_listener_authn( - Name0, - Id, - fun(_GwName, #{id := AuthId, chain_name := ChainName}) -> - case maps:get(<<"filename">>, Body, undefined) of - undefined -> - emqx_authn_api:serialize_error({missing_parameter, filename}); - Filename -> - case - emqx_authentication:import_users( - ChainName, AuthId, Filename - ) - of - ok -> {204}; - {error, Reason} -> emqx_authn_api:serialize_error(Reason) - end - end - end - ). - %%-------------------------------------------------------------------- %% Utils @@ -549,22 +523,6 @@ schema("/gateway/:name/listeners/:id/authentication/users/:uid") -> responses => ?STANDARD_RESP(#{204 => <<"Deleted">>}) } - }; -schema("/gateway/:name/listeners/:id/authentication/import_users") -> - #{ - 'operationId' => import_users, - post => - #{ - desc => ?DESC(import_users), - parameters => params_gateway_name_in_path() ++ - params_listener_id_in_path(), - 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - ref(emqx_authn_api, request_import_users), - emqx_authn_api:request_import_users_examples() - ), - responses => - ?STANDARD_RESP(#{204 => <<"Imported">>}) - } }. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index d2f967c68..db1bcdbca 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -312,6 +312,23 @@ t_authn_data_mgmt(_) -> "/gateway/stomp/authentication/users" ), + ImportUri = emqx_dashboard_api_test_helpers:uri( + ["gateway", "stomp", "authentication", "import_users"] + ), + + Dir = code:lib_dir(emqx_authn, test), + JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), + {ok, JSONData} = file:read_file(JSONFileName), + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(ImportUri, [], [ + {filename, "user-credentials.json", JSONData} + ]), + + CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), + {ok, CSVData} = file:read_file(CSVFileName), + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(ImportUri, [], [ + {filename, "user-credentials.csv", CSVData} + ]), + {204, _} = request(delete, "/gateway/stomp/authentication"), {204, _} = request(get, "/gateway/stomp/authentication"), {204, _} = request(delete, "/gateway/stomp"). @@ -451,6 +468,24 @@ t_listeners_authn_data_mgmt(_) -> get, Path ++ "/users" ), + + ImportUri = emqx_dashboard_api_test_helpers:uri( + ["gateway", "stomp", "listeners", "stomp:tcp:def", "authentication", "import_users"] + ), + + Dir = code:lib_dir(emqx_authn, test), + JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), + {ok, JSONData} = file:read_file(JSONFileName), + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(ImportUri, [], [ + {filename, "user-credentials.json", JSONData} + ]), + + CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), + {ok, CSVData} = file:read_file(CSVFileName), + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(ImportUri, [], [ + {filename, "user-credentials.csv", CSVData} + ]), + {204, _} = request(delete, "/gateway/stomp"). t_authn_fuzzy_search(_) ->