From 3be617cf400566c01c3c1a0b399f1cadc939769e Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 31 May 2022 00:20:22 +0300 Subject: [PATCH 1/2] feat(authn api): add method for user file upload --- apps/emqx/src/emqx_authentication.erl | 3 +- .../i18n/emqx_authn_user_upload_api_i18n.conf | 18 ++ apps/emqx_authn/src/emqx_authn_api.erl | 6 +- .../src/emqx_authn_user_upload_api.erl | 144 +++++++++++++ .../src/simple_authn/emqx_authn_mnesia.erl | 102 +++++++--- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 38 +++- .../test/emqx_authn_mnesia_SUITE.erl | 135 ++++++++++--- .../test/emqx_dashboard_api_test_helpers.erl | 66 ++++++ ...qx_gateway_api_authn_user_upload_i18n.conf | 17 ++ .../emqx_gateway_api_authn_user_upload.erl | 190 ++++++++++++++++++ .../test/emqx_gateway_api_SUITE.erl | 35 ++++ 11 files changed, 696 insertions(+), 58 deletions(-) create mode 100644 apps/emqx_authn/i18n/emqx_authn_user_upload_api_i18n.conf create mode 100644 apps/emqx_authn/src/emqx_authn_user_upload_api.erl create mode 100644 apps/emqx_gateway/i18n/emqx_gateway_api_authn_user_upload_i18n.conf create mode 100644 apps/emqx_gateway/src/emqx_gateway_api_authn_user_upload.erl diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 35f4139c4..778d2d4cb 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -385,7 +385,8 @@ list_authenticators(ChainName) -> move_authenticator(ChainName, AuthenticatorID, Position) -> call({move_authenticator, ChainName, AuthenticatorID, Position}). --spec import_users(chain_name(), authenticator_id(), binary()) -> ok | {error, term()}. +-spec import_users(chain_name(), authenticator_id(), binary() | {binary(), binary()}) -> + ok | {error, term()}. import_users(ChainName, AuthenticatorID, Filename) -> call({import_users, ChainName, AuthenticatorID, Filename}). diff --git a/apps/emqx_authn/i18n/emqx_authn_user_upload_api_i18n.conf b/apps/emqx_authn/i18n/emqx_authn_user_upload_api_i18n.conf new file mode 100644 index 000000000..877b951fc --- /dev/null +++ b/apps/emqx_authn/i18n/emqx_authn_user_upload_api_i18n.conf @@ -0,0 +1,18 @@ +emqx_authn_user_upload_api { + + authentication_id_upload_users_post { + desc { + en: """Upload file with users into authenticator in global authentication chain.""" + zh: """将带有用户的文件上传到全局身份验证链中的身份验证器。""" + } + } + + listeners_listener_id_authentication_id_upload_users_post { + desc { + en: """Upload file with users into authenticator in listener-specific authentication chain.""" + zh: """将带有用户的文件上传到特定于侦听器的身份验证链中的身份验证器。""" + } + } + + +} diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 1f08cf1f2..0369ab541 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -90,7 +90,11 @@ find_user/3, update_user/4, serialize_error/1, - aggregate_metrics/1 + aggregate_metrics/1, + + with_chain/2, + param_auth_id/0, + param_listener_id/0 ]). -elvis([{elvis_style, god_modules, disable}]). diff --git a/apps/emqx_authn/src/emqx_authn_user_upload_api.erl b/apps/emqx_authn/src/emqx_authn_user_upload_api.erl new file mode 100644 index 000000000..19930a77d --- /dev/null +++ b/apps/emqx_authn/src/emqx_authn_user_upload_api.erl @@ -0,0 +1,144 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_user_upload_api). + +-behaviour(minirest_api). + +-include("emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(emqx_dashboard_swagger, [error_codes/2]). + +-define(BAD_REQUEST, 'BAD_REQUEST'). +-define(NOT_FOUND, 'NOT_FOUND'). + +% Swagger + +-define(API_TAGS_GLOBAL, [ + ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(global)">> +]). +-define(API_TAGS_SINGLE, [ + ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(single listener)">> +]). + +-export([ + api_spec/0, + paths/0, + schema/1 +]). + +-export([ + authenticator_upload_users/2, + listener_authenticator_upload_users/2 +]). + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}). + +paths() -> + [ + "/authentication/:id/upload_users", + "/listeners/:listener_id/authentication/:id/upload_users" + ]. + +schema("/authentication/:id/upload_users") -> + #{ + 'operationId' => authenticator_upload_users, + post => #{ + tags => ?API_TAGS_GLOBAL, + description => ?DESC(authentication_id_upload_users_post), + parameters => [emqx_authn_api:param_auth_id()], + 'requestBody' => #{ + content => #{ + 'multipart/form-data' => #{ + schema => #{ + filename => file + } + } + } + }, + responses => #{ + 204 => <<"Users imported">>, + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }; +schema("/listeners/:listener_id/authentication/:id/upload_users") -> + #{ + 'operationId' => listener_authenticator_upload_users, + post => #{ + tags => ?API_TAGS_SINGLE, + description => ?DESC(listeners_listener_id_authentication_id_upload_users_post), + parameters => [emqx_authn_api:param_listener_id(), emqx_authn_api:param_auth_id()], + 'requestBody' => #{ + content => #{ + 'multipart/form-data' => #{ + schema => #{ + filename => file + } + } + } + }, + responses => #{ + 204 => <<"Users imported">>, + 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), + 404 => error_codes([?NOT_FOUND], <<"Not Found">>) + } + } + }. + +authenticator_upload_users( + post, + #{ + bindings := #{id := AuthenticatorID}, + body := #{<<"filename">> := #{type := _} = File} + } +) -> + [{FileName, FileData}] = maps:to_list(maps:without([type], File)), + case emqx_authentication:import_users(?GLOBAL, AuthenticatorID, {FileName, FileData}) of + ok -> {204}; + {error, Reason} -> emqx_authn_api:serialize_error(Reason) + end; +authenticator_upload_users(post, #{bindings := #{id := _}, body := _}) -> + emqx_authn_api:serialize_error({missing_parameter, filename}). + +listener_authenticator_upload_users( + post, + #{ + bindings := #{listener_id := ListenerID, id := AuthenticatorID}, + body := #{<<"filename">> := #{type := _} = File} + } +) -> + [{FileName, FileData}] = maps:to_list(maps:without([type], File)), + emqx_authn_api:with_chain( + ListenerID, + fun(ChainName) -> + case + emqx_authentication:import_users(ChainName, AuthenticatorID, {FileName, FileData}) + of + ok -> {204}; + {error, Reason} -> emqx_authn_api:serialize_error(Reason) + end + end + ); +listener_authenticator_upload_users(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> + emqx_authn_api:serialize_error({missing_parameter, filename}). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index bdcca33f8..d7585eb40 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -182,13 +182,34 @@ destroy(#{user_group := UserGroup}) -> end ). +import_users({Filename0, FileData}, State) -> + Filename = to_binary(Filename0), + case filename:extension(Filename) of + <<".json">> -> + import_users_from_json(FileData, State); + <<".csv">> -> + {ok, CSV} = csv_data_reader(FileData), + import_users_from_csv(CSV, State); + <<>> -> + {error, unknown_file_format}; + Extension -> + {error, {unsupported_file_format, Extension}} + end; import_users(Filename0, State) -> Filename = to_binary(Filename0), case filename:extension(Filename) of <<".json">> -> - import_users_from_json(Filename, State); + case file:read_file(Filename) of + {ok, Data} -> import_users_from_json(Data, State); + {error, _} = Error -> Error + end; <<".csv">> -> - import_users_from_csv(Filename, State); + case csv_file_reader(Filename) of + {ok, CSV} -> + import_users_from_csv(CSV, State); + {error, _} = Error -> + Error + end; <<>> -> {error, unknown_file_format}; Extension -> @@ -327,31 +348,21 @@ run_fuzzy_filter( %%------------------------------------------------------------------------------ %% Example: data/user-credentials.json -import_users_from_json(Filename, #{user_group := UserGroup}) -> - case file:read_file(Filename) of - {ok, Bin} -> - case emqx_json:safe_decode(Bin, [return_maps]) of - {ok, List} -> - trans(fun import/2, [UserGroup, List]); - {error, Reason} -> - {error, Reason} - end; +import_users_from_json(Bin, #{user_group := UserGroup}) -> + case emqx_json:safe_decode(Bin, [return_maps]) of + {ok, List} -> + trans(fun import/2, [UserGroup, List]); {error, Reason} -> {error, Reason} end. %% Example: data/user-credentials.csv -import_users_from_csv(Filename, #{user_group := UserGroup}) -> - case file:open(Filename, [read, binary]) of - {ok, File} -> - case get_csv_header(File) of - {ok, Seq} -> - Result = trans(fun import/3, [UserGroup, File, Seq]), - _ = file:close(File), - Result; - {error, Reason} -> - {error, Reason} - end; +import_users_from_csv(CSV, #{user_group := UserGroup}) -> + case get_csv_header(CSV) of + {ok, Seq, NewCSV} -> + Result = trans(fun import_csv/3, [UserGroup, NewCSV, Seq]), + _ = csv_close(CSV), + Result; {error, Reason} -> {error, Reason} end. @@ -375,9 +386,9 @@ import(_UserGroup, [_ | _More]) -> {error, bad_format}. %% Importing 5w users needs 1.7 seconds -import(UserGroup, File, Seq) -> - case file:read_line(File) of - {ok, Line} -> +import_csv(UserGroup, CSV, Seq) -> + case csv_read_line(CSV) of + {ok, Line, NewCSV} -> Fields = binary:split(Line, [<<",">>, <<" ">>, <<"\n">>], [global, trim_all]), case get_user_info_by_seq(Fields, Seq) of {ok, @@ -388,7 +399,7 @@ import(UserGroup, File, Seq) -> Salt = maps:get(salt, UserInfo, <<>>), IsSuperuser = maps:get(is_superuser, UserInfo, false), insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), - import(UserGroup, File, Seq); + import_csv(UserGroup, NewCSV, Seq); {error, Reason} -> {error, Reason} end; @@ -398,11 +409,11 @@ import(UserGroup, File, Seq) -> {error, Reason} end. -get_csv_header(File) -> - case file:read_line(File) of - {ok, Line} -> +get_csv_header(CSV) -> + case csv_read_line(CSV) of + {ok, Line, NewCSV} -> Seq = binary:split(Line, [<<",">>, <<" ">>, <<"\n">>], [global, trim_all]), - {ok, Seq}; + {ok, Seq, NewCSV}; eof -> {error, empty_file}; {error, Reason} -> @@ -487,3 +498,34 @@ group_match_spec(UserGroup, QString) -> User end) end. + +csv_file_reader(Filename) -> + case file:open(Filename, [read, binary]) of + {ok, File} -> + {ok, {csv_file_reader, File}}; + {error, Reason} -> + {error, Reason} + end. + +csv_data_reader(Data) -> + Lines = binary:split(Data, [<<"\r">>, <<"\n">>], [global, trim_all]), + {ok, {csv_data_reader, Lines}}. + +csv_read_line({csv_file_reader, File} = CSV) -> + case file:read_line(File) of + {ok, Line} -> + {ok, Line, CSV}; + eof -> + eof; + {error, Reason} -> + {error, Reason} + end; +csv_read_line({csv_data_reader, [Line | Lines]}) -> + {ok, Line, {csv_data_reader, Lines}}; +csv_read_line({csv_data_reader, []}) -> + eof. + +csv_close({csv_file_reader, File}) -> + file:close(File); +csv_close({csv_data_reader, _}) -> + ok. diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 86fc1f66c..d5f049836 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_dashboard_api_test_helpers, [request/3, uri/1]). +-import(emqx_dashboard_api_test_helpers, [request/3, uri/1, multipart_formdata_request/3]). -include("emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -102,6 +102,9 @@ t_authenticator_move(_) -> t_authenticator_import_users(_) -> test_authenticator_import_users([]). +t_authenticator_upload_users(_) -> + test_authenticator_upload_users([]). + t_listener_authenticators(_) -> test_authenticators(["listeners", ?TCP_DEFAULT]). @@ -120,6 +123,9 @@ t_listener_authenticator_move(_) -> t_listener_authenticator_import_users(_) -> test_authenticator_import_users(["listeners", ?TCP_DEFAULT]). +t_listener_authenticator_upload_users(_) -> + test_authenticator_upload_users(["listeners", ?TCP_DEFAULT]). + t_aggregate_metrics(_) -> Metrics = #{ 'emqx@node1.emqx.io' => #{ @@ -657,6 +663,36 @@ test_authenticator_import_users(PathPrefix) -> {ok, 204, _} = request(post, ImportUri, #{filename => CSVFileName}). +test_authenticator_upload_users(PathPrefix) -> + UploadUri = uri( + PathPrefix ++ + [?CONF_NS, "password_based:built_in_database", "upload_users"] + ), + + {ok, 200, _} = request( + post, + uri(PathPrefix ++ [?CONF_NS]), + emqx_authn_test_lib:built_in_database_example() + ), + + {ok, 400, _} = multipart_formdata_request(UploadUri, [], [ + {filenam, "user-credentials.json", <<>>} + ]), + + Dir = code:lib_dir(emqx_authn, test), + JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), + CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), + + {ok, JSONData} = file:read_file(JSONFileName), + {ok, 204, _} = multipart_formdata_request(UploadUri, [], [ + {filename, "user-credentials.json", JSONData} + ]), + + {ok, CSVData} = file:read_file(CSVFileName), + {ok, 204, _} = multipart_formdata_request(UploadUri, [], [ + {filename, "user-credentials.csv", CSVData} + ]). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index 693bb9eba..83929be80 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -228,54 +228,139 @@ t_import_users(_) -> Config = Config0#{password_hash_algorithm => #{name => sha256}}, {ok, State} = emqx_authn_mnesia:create(?AUTHN_ID, Config), - ok = emqx_authn_mnesia:import_users( - data_filename(<<"user-credentials.json">>), - State + ?assertEqual( + ok, + emqx_authn_mnesia:import_users( + sample_filename(<<"user-credentials.json">>), + State + ) ), - ok = emqx_authn_mnesia:import_users( - data_filename(<<"user-credentials.csv">>), - State + ?assertEqual( + ok, + emqx_authn_mnesia:import_users( + sample_filename_and_data(<<"user-credentials.json">>), + State + ) ), - {error, {unsupported_file_format, _}} = emqx_authn_mnesia:import_users( - <<"/file/with/unknown.extension">>, - State + ?assertEqual( + ok, + emqx_authn_mnesia:import_users( + sample_filename(<<"user-credentials.csv">>), + State + ) ), - {error, unknown_file_format} = emqx_authn_mnesia:import_users( - <<"/file/with/no/extension">>, - State + ?assertEqual( + ok, + emqx_authn_mnesia:import_users( + sample_filename_and_data(<<"user-credentials.csv">>), + State + ) ), - {error, enoent} = emqx_authn_mnesia:import_users( - <<"/file/that/not/exist.json">>, - State + ?assertMatch( + {error, {unsupported_file_format, _}}, + emqx_authn_mnesia:import_users( + <<"/file/with/unknown.extension">>, + State + ) ), - {error, bad_format} = emqx_authn_mnesia:import_users( - data_filename(<<"user-credentials-malformed-0.json">>), - State + ?assertMatch( + {error, {unsupported_file_format, _}}, + emqx_authn_mnesia:import_users( + {<<"/file/with/unknown.extension">>, <<>>}, + State + ) ), - {error, {_, invalid_json}} = emqx_authn_mnesia:import_users( - data_filename(<<"user-credentials-malformed-1.json">>), - State + ?assertEqual( + {error, unknown_file_format}, + emqx_authn_mnesia:import_users( + <<"/file/with/no/extension">>, + State + ) ), - {error, bad_format} = emqx_authn_mnesia:import_users( - data_filename(<<"user-credentials-malformed.csv">>), - State + ?assertEqual( + {error, unknown_file_format}, + emqx_authn_mnesia:import_users( + {<<"/file/with/no/extension">>, <<>>}, + State + ) + ), + + ?assertEqual( + {error, enoent}, + emqx_authn_mnesia:import_users( + <<"/file/that/not/exist.json">>, + State + ) + ), + + ?assertEqual( + {error, bad_format}, + emqx_authn_mnesia:import_users( + sample_filename(<<"user-credentials-malformed-0.json">>), + State + ) + ), + + ?assertEqual( + {error, bad_format}, + emqx_authn_mnesia:import_users( + sample_filename_and_data(<<"user-credentials-malformed-0.json">>), + State + ) + ), + + ?assertMatch( + {error, {_, invalid_json}}, + emqx_authn_mnesia:import_users( + sample_filename(<<"user-credentials-malformed-1.json">>), + State + ) + ), + + ?assertMatch( + {error, {_, invalid_json}}, + emqx_authn_mnesia:import_users( + sample_filename_and_data(<<"user-credentials-malformed-1.json">>), + State + ) + ), + + ?assertEqual( + {error, bad_format}, + emqx_authn_mnesia:import_users( + sample_filename(<<"user-credentials-malformed.csv">>), + State + ) + ), + + ?assertEqual( + {error, bad_format}, + emqx_authn_mnesia:import_users( + sample_filename_and_data(<<"user-credentials-malformed.csv">>), + State + ) ). %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ -data_filename(Name) -> +sample_filename(Name) -> Dir = code:lib_dir(emqx_authn, test), filename:join([Dir, <<"data">>, Name]). +sample_filename_and_data(Name) -> + Filename = sample_filename(Name), + {ok, Data} = file:read_file(Filename), + {Filename, Data}. + config() -> #{ user_id_type => username, diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index 0f3405b57..eacca6aa2 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -22,6 +22,8 @@ request/2, request/3, request/4, + multipart_formdata_request/3, + multipart_formdata_request/4, uri/0, uri/1 ]). @@ -97,3 +99,67 @@ auth_header(Username) -> Password = <<"public">>, {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), {"Authorization", "Bearer " ++ binary_to_list(Token)}. + +multipart_formdata_request(Url, Fields, Files) -> + multipart_formdata_request(Url, <<"admin">>, Fields, Files). + +multipart_formdata_request(Url, Username, Fields, Files) -> + Boundary = + "------------" ++ integer_to_list(rand:uniform(99999999999999999)) ++ + integer_to_list(erlang:system_time(millisecond)), + Body = format_multipart_formdata(Boundary, Fields, Files), + ContentType = lists:concat(["multipart/form-data; boundary=", Boundary]), + Headers = + [ + auth_header(Username), + {"Content-Length", integer_to_list(length(Body))} + ], + case httpc:request(post, {Url, Headers, ContentType, Body}, [], []) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return}} -> + {ok, Code, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +format_multipart_formdata(Boundary, Fields, Files) -> + FieldParts = lists:map( + fun({FieldName, FieldContent}) -> + [ + lists:concat(["--", Boundary]), + lists:concat([ + "Content-Disposition: form-data; name=\"", atom_to_list(FieldName), "\"" + ]), + "", + to_list(FieldContent) + ] + end, + Fields + ), + FieldParts2 = lists:append(FieldParts), + FileParts = lists:map( + fun({FieldName, FileName, FileContent}) -> + [ + lists:concat(["--", Boundary]), + lists:concat([ + "Content-Disposition: form-data; name=\"", + atom_to_list(FieldName), + "\"; filename=\"", + FileName, + "\"" + ]), + lists:concat(["Content-Type: ", "application/octet-stream"]), + "", + to_list(FileContent) + ] + end, + Files + ), + FileParts2 = lists:append(FileParts), + EndingParts = [lists:concat(["--", Boundary, "--"]), ""], + Parts = lists:append([FieldParts2, FileParts2, EndingParts]), + string:join(Parts, "\r\n"). + +to_list(Bin) when is_binary(Bin) -> binary_to_list(Bin); +to_list(Str) when is_list(Str) -> Str. diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_user_upload_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_user_upload_i18n.conf new file mode 100644 index 000000000..90a7cd378 --- /dev/null +++ b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_user_upload_i18n.conf @@ -0,0 +1,17 @@ +emqx_gateway_api_authn_user_upload { + + upload_users { + desc { + en: """Upload file with users into the gateway authentication""" + zh: """将带有用户的文件上传到网关身份验证中。""" + } + } + + upload_listener_users { + desc { + en: """Upload file with users into listener-specific authentication""" + zh: """将带有用户的文件上传到特定于侦听器的身份验证中。""" + } + } + +} diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_upload.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_upload.erl new file mode 100644 index 000000000..a3b5035f3 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_upload.erl @@ -0,0 +1,190 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_api_authn_user_upload). + +-behaviour(minirest_api). + +-include("emqx_gateway_http.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-import(emqx_dashboard_swagger, [error_codes/2]). +-import(hoconsc, [mk/2, ref/2]). +-import( + emqx_gateway_http, + [ + with_authn/2, + with_listener_authn/3 + ] +). + +%% minirest/dashbaord_swagger behaviour callbacks +-export([ + api_spec/0, + paths/0, + schema/1 +]). + +%% http handlers +-export([ + upload_users/2, + upload_listener_users/2 +]). + +%%-------------------------------------------------------------------- +%% minirest behaviour callbacks +%%-------------------------------------------------------------------- + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}). + +paths() -> + [ + "/gateway/:name/authentication/upload_users", + "/gateway/:name/listeners/:id/authentication/upload_users" + ]. + +%%-------------------------------------------------------------------- +%% http handlers + +upload_users(post, #{ + bindings := #{name := Name0}, + body := Body +}) -> + with_authn(Name0, fun( + _GwName, + #{ + id := AuthId, + chain_name := ChainName + } + ) -> + case maps:get(<<"filename">>, Body, undefined) of + undefined -> + emqx_authn_api:serialize_error({missing_parameter, filename}); + File -> + [{FileName, FileData}] = maps:to_list(maps:without([type], File)), + case + emqx_authentication:import_users( + ChainName, AuthId, {FileName, FileData} + ) + of + ok -> {204}; + {error, Reason} -> emqx_authn_api:serialize_error(Reason) + end + end + end). + +upload_listener_users(post, #{ + bindings := #{name := Name0, id := Id}, + body := Body +}) -> + with_listener_authn( + Name0, + Id, + fun(_GwName, #{id := AuthId, chain_name := ChainName}) -> + case maps:get(<<"filename">>, Body, undefined) of + undefined -> + emqx_authn_api:serialize_error({missing_parameter, filename}); + File -> + [{FileName, FileData}] = maps:to_list(maps:without([type], File)), + case + emqx_authentication:import_users( + ChainName, AuthId, {FileName, FileData} + ) + of + ok -> {204}; + {error, Reason} -> emqx_authn_api:serialize_error(Reason) + end + end + end + ). + +%%-------------------------------------------------------------------- +%% Swagger defines +%%-------------------------------------------------------------------- + +schema("/gateway/:name/authentication/upload_users") -> + #{ + 'operationId' => upload_users, + post => + #{ + desc => ?DESC(upload_users), + parameters => params_gateway_name_in_path(), + 'requestBody' => #{ + content => #{ + 'multipart/form-data' => #{ + schema => #{ + filename => file + } + } + } + }, + responses => + ?STANDARD_RESP(#{204 => <<"Imported">>}) + } + }; +schema("/gateway/:name/listeners/:id/authentication/upload_users") -> + #{ + 'operationId' => upload_listener_users, + post => + #{ + desc => ?DESC(upload_listener_users), + parameters => params_gateway_name_in_path() ++ + params_listener_id_in_path(), + 'requestBody' => #{ + content => #{ + 'multipart/form-data' => #{ + schema => #{ + filename => file + } + } + } + }, + responses => + ?STANDARD_RESP(#{204 => <<"Imported">>}) + } + }. + +%%-------------------------------------------------------------------- +%% params defines +%%-------------------------------------------------------------------- + +params_gateway_name_in_path() -> + [ + {name, + mk( + binary(), + #{ + in => path, + desc => ?DESC(emqx_gateway_api, gateway_name), + example => <<"stomp">> + } + )} + ]. + +params_listener_id_in_path() -> + [ + {id, + mk( + binary(), + #{ + in => path, + desc => ?DESC(emqx_gateway_api_listeners, listener_id), + example => <<"stomp:tcp:def">> + } + )} + ]. diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index d2f967c68..fff8e64e9 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -312,6 +312,23 @@ t_authn_data_mgmt(_) -> "/gateway/stomp/authentication/users" ), + UploadUri = emqx_dashboard_api_test_helpers:uri( + ["gateway", "stomp", "authentication", "upload_users"] + ), + + Dir = code:lib_dir(emqx_authn, test), + JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), + {ok, JSONData} = file:read_file(JSONFileName), + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(UploadUri, [], [ + {filename, "user-credentials.json", JSONData} + ]), + + CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), + {ok, CSVData} = file:read_file(CSVFileName), + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(UploadUri, [], [ + {filename, "user-credentials.csv", CSVData} + ]), + {204, _} = request(delete, "/gateway/stomp/authentication"), {204, _} = request(get, "/gateway/stomp/authentication"), {204, _} = request(delete, "/gateway/stomp"). @@ -451,6 +468,24 @@ t_listeners_authn_data_mgmt(_) -> get, Path ++ "/users" ), + + UploadUri = emqx_dashboard_api_test_helpers:uri( + ["gateway", "stomp", "listeners", "stomp:tcp:def", "authentication", "upload_users"] + ), + + Dir = code:lib_dir(emqx_authn, test), + JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), + {ok, JSONData} = file:read_file(JSONFileName), + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(UploadUri, [], [ + {filename, "user-credentials.json", JSONData} + ]), + + CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), + {ok, CSVData} = file:read_file(CSVFileName), + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(UploadUri, [], [ + {filename, "user-credentials.csv", CSVData} + ]), + {204, _} = request(delete, "/gateway/stomp"). t_authn_fuzzy_search(_) -> From 2d53c21d45632b22d3e24f2b00cf63b457cbde69 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 1 Jun 2022 00:36:09 +0300 Subject: [PATCH 2/2] chore(authn api): replace user import api with upload api --- apps/emqx/src/emqx_authentication.erl | 6 +- apps/emqx_authn/i18n/emqx_authn_api_i18n.conf | 14 --- .../i18n/emqx_authn_user_import_api_i18n.conf | 17 ++++ .../i18n/emqx_authn_user_upload_api_i18n.conf | 18 ---- apps/emqx_authn/src/emqx_authn_api.erl | 94 ------------------- ...api.erl => emqx_authn_user_import_api.erl} | 30 +++--- .../src/simple_authn/emqx_authn_mnesia.erl | 66 ++----------- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 39 +------- .../test/emqx_authn_mnesia_SUITE.erl | 64 ------------- ...qx_gateway_api_authn_user_upload_i18n.conf | 17 ---- .../src/emqx_gateway_api_authn.erl | 47 +--------- ...=> emqx_gateway_api_authn_user_import.erl} | 28 +++--- .../src/emqx_gateway_api_listeners.erl | 46 +-------- .../test/emqx_gateway_api_SUITE.erl | 16 ++-- 14 files changed, 74 insertions(+), 428 deletions(-) create mode 100644 apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf delete mode 100644 apps/emqx_authn/i18n/emqx_authn_user_upload_api_i18n.conf rename apps/emqx_authn/src/{emqx_authn_user_upload_api.erl => emqx_authn_user_import_api.erl} (84%) delete mode 100644 apps/emqx_gateway/i18n/emqx_gateway_api_authn_user_upload_i18n.conf rename apps/emqx_gateway/src/{emqx_gateway_api_authn_user_upload.erl => emqx_gateway_api_authn_user_import.erl} (88%) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 778d2d4cb..abfa55c36 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -166,11 +166,11 @@ when when State :: state(). --callback import_users(Filename, State) -> +-callback import_users({Filename, FileData}, State) -> ok | {error, term()} when - Filename :: binary(), State :: state(). + Filename :: binary(), FileData :: binary(), State :: state(). -callback add_user(UserInfo, State) -> {ok, User} @@ -385,7 +385,7 @@ list_authenticators(ChainName) -> move_authenticator(ChainName, AuthenticatorID, Position) -> call({move_authenticator, ChainName, AuthenticatorID, Position}). --spec import_users(chain_name(), authenticator_id(), binary() | {binary(), binary()}) -> +-spec import_users(chain_name(), authenticator_id(), {binary(), binary()}) -> ok | {error, term()}. import_users(ChainName, AuthenticatorID, Filename) -> call({import_users, ChainName, AuthenticatorID, Filename}). diff --git a/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf b/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf index 44725e935..a31a335d5 100644 --- a/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf +++ b/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf @@ -105,20 +105,6 @@ emqx_authn_api { } } - authentication_id_import_users_post { - desc { - en: """Import users into authenticator in global authentication chain.""" - zh: """为全局认证链上的指定认证器导入用户数据。""" - } - } - - listeners_listener_id_authentication_id_import_users_post { - desc { - en: """Import users into authenticator in listener authentication chain.""" - zh: """为监听器认证链上的指定认证器导入用户数据。""" - } - } - authentication_id_users_post { desc { en: """Create users for authenticator in global authentication chain.""" diff --git a/apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf b/apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf new file mode 100644 index 000000000..294897ec1 --- /dev/null +++ b/apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf @@ -0,0 +1,17 @@ +emqx_authn_user_import_api { + + authentication_id_import_users_post { + desc { + en: """Import users into authenticator in global authentication chain.""" + zh: """为全局认证链上的指定认证器导入用户数据。""" + } + } + + listeners_listener_id_authentication_id_import_users_post { + desc { + en: """Import users into authenticator in listener authentication chain.""" + zh: """为监听器认证链上的指定认证器导入用户数据。""" + } + } + +} diff --git a/apps/emqx_authn/i18n/emqx_authn_user_upload_api_i18n.conf b/apps/emqx_authn/i18n/emqx_authn_user_upload_api_i18n.conf deleted file mode 100644 index 877b951fc..000000000 --- a/apps/emqx_authn/i18n/emqx_authn_user_upload_api_i18n.conf +++ /dev/null @@ -1,18 +0,0 @@ -emqx_authn_user_upload_api { - - authentication_id_upload_users_post { - desc { - en: """Upload file with users into authenticator in global authentication chain.""" - zh: """将带有用户的文件上传到全局身份验证链中的身份验证器。""" - } - } - - listeners_listener_id_authentication_id_upload_users_post { - desc { - en: """Upload file with users into authenticator in listener-specific authentication chain.""" - zh: """将带有用户的文件上传到特定于侦听器的身份验证链中的身份验证器。""" - } - } - - -} diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index ae0dffb01..12ad8d1d9 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -62,8 +62,6 @@ listener_authenticator_status/2, authenticator_move/2, listener_authenticator_move/2, - authenticator_import_users/2, - listener_authenticator_import_users/2, authenticator_users/2, authenticator_user/2, listener_authenticator_users/2, @@ -75,7 +73,6 @@ -export([ authenticator_examples/0, request_move_examples/0, - request_import_users_examples/0, request_user_create_examples/0, request_user_update_examples/0, response_user_examples/0, @@ -108,7 +105,6 @@ paths() -> "/authentication/:id", "/authentication/:id/status", "/authentication/:id/move", - "/authentication/:id/import_users", "/authentication/:id/users", "/authentication/:id/users/:user_id", @@ -116,7 +112,6 @@ paths() -> "/listeners/:listener_id/authentication/:id", "/listeners/:listener_id/authentication/:id/status", "/listeners/:listener_id/authentication/:id/move", - "/listeners/:listener_id/authentication/:id/import_users", "/listeners/:listener_id/authentication/:id/users", "/listeners/:listener_id/authentication/:id/users/:user_id" ]. @@ -126,7 +121,6 @@ roots() -> request_user_create, request_user_update, request_move, - request_import_users, response_user, response_users ]. @@ -143,9 +137,6 @@ fields(request_user_update) -> ]; fields(request_move) -> [{position, mk(binary(), #{required => true})}]; -fields(request_import_users) -> - %% TODO: add file update - [{filename, mk(binary(), #{required => true})}]; fields(response_user) -> [ {user_id, mk(binary(), #{required => true})}, @@ -379,42 +370,6 @@ schema("/listeners/:listener_id/authentication/:id/move") -> } } }; -schema("/authentication/:id/import_users") -> - #{ - 'operationId' => authenticator_import_users, - post => #{ - tags => ?API_TAGS_GLOBAL, - description => ?DESC(authentication_id_import_users_post), - parameters => [param_auth_id()], - 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - ref(request_import_users), - request_import_users_examples() - ), - responses => #{ - 204 => <<"Users imported">>, - 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), - 404 => error_codes([?NOT_FOUND], <<"Not Found">>) - } - } - }; -schema("/listeners/:listener_id/authentication/:id/import_users") -> - #{ - 'operationId' => listener_authenticator_import_users, - post => #{ - tags => ?API_TAGS_SINGLE, - description => ?DESC(listeners_listener_id_authentication_id_import_users_post), - parameters => [param_listener_id(), param_auth_id()], - 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - ref(request_import_users), - request_import_users_examples() - ), - responses => #{ - 204 => <<"Users imported">>, - 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), - 404 => error_codes([?NOT_FOUND], <<"Not Found">>) - } - } - }; schema("/authentication/:id/users") -> #{ 'operationId' => authenticator_users, @@ -751,39 +706,6 @@ listener_authenticator_move( listener_authenticator_move(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> serialize_error({missing_parameter, position}). -authenticator_import_users( - post, - #{ - bindings := #{id := AuthenticatorID}, - body := #{<<"filename">> := Filename} - } -) -> - case emqx_authentication:import_users(?GLOBAL, AuthenticatorID, Filename) of - ok -> {204}; - {error, Reason} -> serialize_error(Reason) - end; -authenticator_import_users(post, #{bindings := #{id := _}, body := _}) -> - serialize_error({missing_parameter, filename}). - -listener_authenticator_import_users( - post, - #{ - bindings := #{listener_id := ListenerID, id := AuthenticatorID}, - body := #{<<"filename">> := Filename} - } -) -> - with_chain( - ListenerID, - fun(ChainName) -> - case emqx_authentication:import_users(ChainName, AuthenticatorID, Filename) of - ok -> {204}; - {error, Reason} -> serialize_error(Reason) - end - end - ); -listener_authenticator_import_users(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> - serialize_error({missing_parameter, filename}). - authenticator_users(post, #{bindings := #{id := AuthenticatorID}, body := UserInfo}) -> add_user(?GLOBAL, AuthenticatorID, UserInfo); authenticator_users(get, #{bindings := #{id := AuthenticatorID}, query_string := QueryString}) -> @@ -1583,22 +1505,6 @@ request_move_examples() -> } }. -request_import_users_examples() -> - #{ - import_csv => #{ - summary => <<"Import users from CSV file">>, - value => #{ - filename => <<"/path/to/user/data.csv">> - } - }, - import_json => #{ - summary => <<"Import users from JSON file">>, - value => #{ - filename => <<"/path/to/user/data.json">> - } - } - }. - response_user_examples() -> #{ regular_user => #{ diff --git a/apps/emqx_authn/src/emqx_authn_user_upload_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl similarity index 84% rename from apps/emqx_authn/src/emqx_authn_user_upload_api.erl rename to apps/emqx_authn/src/emqx_authn_user_import_api.erl index 19930a77d..30417acf7 100644 --- a/apps/emqx_authn/src/emqx_authn_user_upload_api.erl +++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_authn_user_upload_api). +-module(emqx_authn_user_import_api). -behaviour(minirest_api). @@ -46,8 +46,8 @@ ]). -export([ - authenticator_upload_users/2, - listener_authenticator_upload_users/2 + authenticator_import_users/2, + listener_authenticator_import_users/2 ]). api_spec() -> @@ -55,16 +55,16 @@ api_spec() -> paths() -> [ - "/authentication/:id/upload_users", - "/listeners/:listener_id/authentication/:id/upload_users" + "/authentication/:id/import_users", + "/listeners/:listener_id/authentication/:id/import_users" ]. -schema("/authentication/:id/upload_users") -> +schema("/authentication/:id/import_users") -> #{ - 'operationId' => authenticator_upload_users, + 'operationId' => authenticator_import_users, post => #{ tags => ?API_TAGS_GLOBAL, - description => ?DESC(authentication_id_upload_users_post), + description => ?DESC(authentication_id_import_users_post), parameters => [emqx_authn_api:param_auth_id()], 'requestBody' => #{ content => #{ @@ -82,12 +82,12 @@ schema("/authentication/:id/upload_users") -> } } }; -schema("/listeners/:listener_id/authentication/:id/upload_users") -> +schema("/listeners/:listener_id/authentication/:id/import_users") -> #{ - 'operationId' => listener_authenticator_upload_users, + 'operationId' => listener_authenticator_import_users, post => #{ tags => ?API_TAGS_SINGLE, - description => ?DESC(listeners_listener_id_authentication_id_upload_users_post), + description => ?DESC(listeners_listener_id_authentication_id_import_users_post), parameters => [emqx_authn_api:param_listener_id(), emqx_authn_api:param_auth_id()], 'requestBody' => #{ content => #{ @@ -106,7 +106,7 @@ schema("/listeners/:listener_id/authentication/:id/upload_users") -> } }. -authenticator_upload_users( +authenticator_import_users( post, #{ bindings := #{id := AuthenticatorID}, @@ -118,10 +118,10 @@ authenticator_upload_users( ok -> {204}; {error, Reason} -> emqx_authn_api:serialize_error(Reason) end; -authenticator_upload_users(post, #{bindings := #{id := _}, body := _}) -> +authenticator_import_users(post, #{bindings := #{id := _}, body := _}) -> emqx_authn_api:serialize_error({missing_parameter, filename}). -listener_authenticator_upload_users( +listener_authenticator_import_users( post, #{ bindings := #{listener_id := ListenerID, id := AuthenticatorID}, @@ -140,5 +140,5 @@ listener_authenticator_upload_users( end end ); -listener_authenticator_upload_users(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> +listener_authenticator_import_users(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> emqx_authn_api:serialize_error({missing_parameter, filename}). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index d7585eb40..81dc89c54 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -188,32 +188,12 @@ import_users({Filename0, FileData}, State) -> <<".json">> -> import_users_from_json(FileData, State); <<".csv">> -> - {ok, CSV} = csv_data_reader(FileData), + CSV = csv_data(FileData), import_users_from_csv(CSV, State); <<>> -> {error, unknown_file_format}; Extension -> {error, {unsupported_file_format, Extension}} - end; -import_users(Filename0, State) -> - Filename = to_binary(Filename0), - case filename:extension(Filename) of - <<".json">> -> - case file:read_file(Filename) of - {ok, Data} -> import_users_from_json(Data, State); - {error, _} = Error -> Error - end; - <<".csv">> -> - case csv_file_reader(Filename) of - {ok, CSV} -> - import_users_from_csv(CSV, State); - {error, _} = Error -> - Error - end; - <<>> -> - {error, unknown_file_format}; - Extension -> - {error, {unsupported_file_format, Extension}} end. add_user( @@ -360,9 +340,7 @@ import_users_from_json(Bin, #{user_group := UserGroup}) -> import_users_from_csv(CSV, #{user_group := UserGroup}) -> case get_csv_header(CSV) of {ok, Seq, NewCSV} -> - Result = trans(fun import_csv/3, [UserGroup, NewCSV, Seq]), - _ = csv_close(CSV), - Result; + trans(fun import_csv/3, [UserGroup, NewCSV, Seq]); {error, Reason} -> {error, Reason} end. @@ -404,9 +382,7 @@ import_csv(UserGroup, CSV, Seq) -> {error, Reason} end; eof -> - ok; - {error, Reason} -> - {error, Reason} + ok end. get_csv_header(CSV) -> @@ -415,9 +391,7 @@ get_csv_header(CSV) -> Seq = binary:split(Line, [<<",">>, <<" ">>, <<"\n">>], [global, trim_all]), {ok, Seq, NewCSV}; eof -> - {error, empty_file}; - {error, Reason} -> - {error, Reason} + {error, empty_file} end. get_user_info_by_seq(Fields, Seq) -> @@ -499,33 +473,11 @@ group_match_spec(UserGroup, QString) -> end) end. -csv_file_reader(Filename) -> - case file:open(Filename, [read, binary]) of - {ok, File} -> - {ok, {csv_file_reader, File}}; - {error, Reason} -> - {error, Reason} - end. - -csv_data_reader(Data) -> +csv_data(Data) -> Lines = binary:split(Data, [<<"\r">>, <<"\n">>], [global, trim_all]), - {ok, {csv_data_reader, Lines}}. + {csv_data, Lines}. -csv_read_line({csv_file_reader, File} = CSV) -> - case file:read_line(File) of - {ok, Line} -> - {ok, Line, CSV}; - eof -> - eof; - {error, Reason} -> - {error, Reason} - end; -csv_read_line({csv_data_reader, [Line | Lines]}) -> - {ok, Line, {csv_data_reader, Lines}}; -csv_read_line({csv_data_reader, []}) -> +csv_read_line({csv_data, [Line | Lines]}) -> + {ok, Line, {csv_data, Lines}}; +csv_read_line({csv_data, []}) -> eof. - -csv_close({csv_file_reader, File}) -> - file:close(File); -csv_close({csv_data_reader, _}) -> - ok. diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 668a91798..648fb8a0f 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -102,9 +102,6 @@ t_authenticator_move(_) -> t_authenticator_import_users(_) -> test_authenticator_import_users([]). -t_authenticator_upload_users(_) -> - test_authenticator_upload_users([]). - t_listener_authenticators(_) -> test_authenticators(["listeners", ?TCP_DEFAULT]). @@ -123,9 +120,6 @@ t_listener_authenticator_move(_) -> t_listener_authenticator_import_users(_) -> test_authenticator_import_users(["listeners", ?TCP_DEFAULT]). -t_listener_authenticator_upload_users(_) -> - test_authenticator_upload_users(["listeners", ?TCP_DEFAULT]). - t_aggregate_metrics(_) -> Metrics = #{ 'emqx@node1.emqx.io' => #{ @@ -649,33 +643,8 @@ test_authenticator_import_users(PathPrefix) -> emqx_authn_test_lib:built_in_database_example() ), - {ok, 400, _} = request(post, ImportUri, #{}), - - {ok, 400, _} = request(post, ImportUri, #{filename => <<"/etc/passwd">>}), - - {ok, 400, _} = request(post, ImportUri, #{filename => <<"/not_exists.csv">>}), - - Dir = code:lib_dir(emqx_authn, test), - JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), - CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), - - {ok, 204, _} = request(post, ImportUri, #{filename => JSONFileName}), - - {ok, 204, _} = request(post, ImportUri, #{filename => CSVFileName}). - -test_authenticator_upload_users(PathPrefix) -> - UploadUri = uri( - PathPrefix ++ - [?CONF_NS, "password_based:built_in_database", "upload_users"] - ), - - {ok, 200, _} = request( - post, - uri(PathPrefix ++ [?CONF_NS]), - emqx_authn_test_lib:built_in_database_example() - ), - - {ok, 400, _} = multipart_formdata_request(UploadUri, [], [ + {ok, 400, _} = multipart_formdata_request(ImportUri, [], []), + {ok, 400, _} = multipart_formdata_request(ImportUri, [], [ {filenam, "user-credentials.json", <<>>} ]), @@ -684,12 +653,12 @@ test_authenticator_upload_users(PathPrefix) -> CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), {ok, JSONData} = file:read_file(JSONFileName), - {ok, 204, _} = multipart_formdata_request(UploadUri, [], [ + {ok, 204, _} = multipart_formdata_request(ImportUri, [], [ {filename, "user-credentials.json", JSONData} ]), {ok, CSVData} = file:read_file(CSVFileName), - {ok, 204, _} = multipart_formdata_request(UploadUri, [], [ + {ok, 204, _} = multipart_formdata_request(ImportUri, [], [ {filename, "user-credentials.csv", CSVData} ]). diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index 83929be80..2ab9efb1d 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -228,14 +228,6 @@ t_import_users(_) -> Config = Config0#{password_hash_algorithm => #{name => sha256}}, {ok, State} = emqx_authn_mnesia:create(?AUTHN_ID, Config), - ?assertEqual( - ok, - emqx_authn_mnesia:import_users( - sample_filename(<<"user-credentials.json">>), - State - ) - ), - ?assertEqual( ok, emqx_authn_mnesia:import_users( @@ -244,14 +236,6 @@ t_import_users(_) -> ) ), - ?assertEqual( - ok, - emqx_authn_mnesia:import_users( - sample_filename(<<"user-credentials.csv">>), - State - ) - ), - ?assertEqual( ok, emqx_authn_mnesia:import_users( @@ -260,14 +244,6 @@ t_import_users(_) -> ) ), - ?assertMatch( - {error, {unsupported_file_format, _}}, - emqx_authn_mnesia:import_users( - <<"/file/with/unknown.extension">>, - State - ) - ), - ?assertMatch( {error, {unsupported_file_format, _}}, emqx_authn_mnesia:import_users( @@ -276,14 +252,6 @@ t_import_users(_) -> ) ), - ?assertEqual( - {error, unknown_file_format}, - emqx_authn_mnesia:import_users( - <<"/file/with/no/extension">>, - State - ) - ), - ?assertEqual( {error, unknown_file_format}, emqx_authn_mnesia:import_users( @@ -292,22 +260,6 @@ t_import_users(_) -> ) ), - ?assertEqual( - {error, enoent}, - emqx_authn_mnesia:import_users( - <<"/file/that/not/exist.json">>, - State - ) - ), - - ?assertEqual( - {error, bad_format}, - emqx_authn_mnesia:import_users( - sample_filename(<<"user-credentials-malformed-0.json">>), - State - ) - ), - ?assertEqual( {error, bad_format}, emqx_authn_mnesia:import_users( @@ -316,14 +268,6 @@ t_import_users(_) -> ) ), - ?assertMatch( - {error, {_, invalid_json}}, - emqx_authn_mnesia:import_users( - sample_filename(<<"user-credentials-malformed-1.json">>), - State - ) - ), - ?assertMatch( {error, {_, invalid_json}}, emqx_authn_mnesia:import_users( @@ -332,14 +276,6 @@ t_import_users(_) -> ) ), - ?assertEqual( - {error, bad_format}, - emqx_authn_mnesia:import_users( - sample_filename(<<"user-credentials-malformed.csv">>), - State - ) - ), - ?assertEqual( {error, bad_format}, emqx_authn_mnesia:import_users( diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_user_upload_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_user_upload_i18n.conf deleted file mode 100644 index 90a7cd378..000000000 --- a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_user_upload_i18n.conf +++ /dev/null @@ -1,17 +0,0 @@ -emqx_gateway_api_authn_user_upload { - - upload_users { - desc { - en: """Upload file with users into the gateway authentication""" - zh: """将带有用户的文件上传到网关身份验证中。""" - } - } - - upload_listener_users { - desc { - en: """Upload file with users into listener-specific authentication""" - zh: """将带有用户的文件上传到特定于侦听器的身份验证中。""" - } - } - -} diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index 3c4b77f97..c49b69e1c 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -46,8 +46,7 @@ -export([ authn/2, users/2, - users_insta/2, - import_users/2 + users_insta/2 ]). %% internal export for emqx_gateway_api_listeners module @@ -64,8 +63,7 @@ paths() -> [ "/gateway/:name/authentication", "/gateway/:name/authentication/users", - "/gateway/:name/authentication/users/:uid", - "/gateway/:name/authentication/import_users" + "/gateway/:name/authentication/users/:uid" ]. %%-------------------------------------------------------------------- @@ -160,32 +158,6 @@ users_insta(delete, #{bindings := #{name := Name0, uid := UserId}}) -> emqx_authn_api:delete_user(ChainName, AuthId, UserId) end). -import_users(post, #{ - bindings := #{name := Name0}, - body := Body -}) -> - with_authn(Name0, fun( - _GwName, - #{ - id := AuthId, - chain_name := ChainName - } - ) -> - case maps:get(<<"filename">>, Body, undefined) of - undefined -> - emqx_authn_api:serialize_error({missing_parameter, filename}); - Filename -> - case - emqx_authentication:import_users( - ChainName, AuthId, Filename - ) - of - ok -> {204}; - {error, Reason} -> emqx_authn_api:serialize_error(Reason) - end - end - end). - %%-------------------------------------------------------------------- %% Utils @@ -326,21 +298,6 @@ schema("/gateway/:name/authentication/users/:uid") -> responses => ?STANDARD_RESP(#{204 => <<"User Deleted">>}) } - }; -schema("/gateway/:name/authentication/import_users") -> - #{ - 'operationId' => import_users, - post => - #{ - desc => ?DESC(import_users), - parameters => params_gateway_name_in_path(), - 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - ref(emqx_authn_api, request_import_users), - emqx_authn_api:request_import_users_examples() - ), - responses => - ?STANDARD_RESP(#{204 => <<"Imported">>}) - } }. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_upload.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl similarity index 88% rename from apps/emqx_gateway/src/emqx_gateway_api_authn_user_upload.erl rename to apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl index a3b5035f3..b86dc4510 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_upload.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_gateway_api_authn_user_upload). +-module(emqx_gateway_api_authn_user_import). -behaviour(minirest_api). @@ -32,7 +32,7 @@ ] ). -%% minirest/dashbaord_swagger behaviour callbacks +%% minirest/dashboard_swagger behaviour callbacks -export([ api_spec/0, paths/0, @@ -41,8 +41,8 @@ %% http handlers -export([ - upload_users/2, - upload_listener_users/2 + import_users/2, + import_listener_users/2 ]). %%-------------------------------------------------------------------- @@ -54,14 +54,14 @@ api_spec() -> paths() -> [ - "/gateway/:name/authentication/upload_users", - "/gateway/:name/listeners/:id/authentication/upload_users" + "/gateway/:name/authentication/import_users", + "/gateway/:name/listeners/:id/authentication/import_users" ]. %%-------------------------------------------------------------------- %% http handlers -upload_users(post, #{ +import_users(post, #{ bindings := #{name := Name0}, body := Body }) -> @@ -88,7 +88,7 @@ upload_users(post, #{ end end). -upload_listener_users(post, #{ +import_listener_users(post, #{ bindings := #{name := Name0, id := Id}, body := Body }) -> @@ -117,12 +117,12 @@ upload_listener_users(post, #{ %% Swagger defines %%-------------------------------------------------------------------- -schema("/gateway/:name/authentication/upload_users") -> +schema("/gateway/:name/authentication/import_users") -> #{ - 'operationId' => upload_users, + 'operationId' => import_users, post => #{ - desc => ?DESC(upload_users), + desc => ?DESC(emqx_gateway_api_authn, import_users), parameters => params_gateway_name_in_path(), 'requestBody' => #{ content => #{ @@ -137,12 +137,12 @@ schema("/gateway/:name/authentication/upload_users") -> ?STANDARD_RESP(#{204 => <<"Imported">>}) } }; -schema("/gateway/:name/listeners/:id/authentication/upload_users") -> +schema("/gateway/:name/listeners/:id/authentication/import_users") -> #{ - 'operationId' => upload_listener_users, + 'operationId' => import_listener_users, post => #{ - desc => ?DESC(upload_listener_users), + desc => ?DESC(emqx_gateway_api_listeners, import_users), parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => #{ diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 4064af05b..d0778717a 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -54,8 +54,7 @@ listeners_insta/2, listeners_insta_authn/2, users/2, - users_insta/2, - import_users/2 + users_insta/2 ]). %% RPC @@ -74,8 +73,7 @@ paths() -> "/gateway/:name/listeners/:id", "/gateway/:name/listeners/:id/authentication", "/gateway/:name/listeners/:id/authentication/users", - "/gateway/:name/listeners/:id/authentication/users/:uid", - "/gateway/:name/listeners/:id/authentication/import_users" + "/gateway/:name/listeners/:id/authentication/users/:uid" ]. %%-------------------------------------------------------------------- @@ -239,30 +237,6 @@ users_insta(delete, #{bindings := #{name := Name0, id := Id, uid := UserId}}) -> end ). -import_users(post, #{ - bindings := #{name := Name0, id := Id}, - body := Body -}) -> - with_listener_authn( - Name0, - Id, - fun(_GwName, #{id := AuthId, chain_name := ChainName}) -> - case maps:get(<<"filename">>, Body, undefined) of - undefined -> - emqx_authn_api:serialize_error({missing_parameter, filename}); - Filename -> - case - emqx_authentication:import_users( - ChainName, AuthId, Filename - ) - of - ok -> {204}; - {error, Reason} -> emqx_authn_api:serialize_error(Reason) - end - end - end - ). - %%-------------------------------------------------------------------- %% Utils @@ -549,22 +523,6 @@ schema("/gateway/:name/listeners/:id/authentication/users/:uid") -> responses => ?STANDARD_RESP(#{204 => <<"Deleted">>}) } - }; -schema("/gateway/:name/listeners/:id/authentication/import_users") -> - #{ - 'operationId' => import_users, - post => - #{ - desc => ?DESC(import_users), - parameters => params_gateway_name_in_path() ++ - params_listener_id_in_path(), - 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - ref(emqx_authn_api, request_import_users), - emqx_authn_api:request_import_users_examples() - ), - responses => - ?STANDARD_RESP(#{204 => <<"Imported">>}) - } }. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index fff8e64e9..db1bcdbca 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -312,20 +312,20 @@ t_authn_data_mgmt(_) -> "/gateway/stomp/authentication/users" ), - UploadUri = emqx_dashboard_api_test_helpers:uri( - ["gateway", "stomp", "authentication", "upload_users"] + ImportUri = emqx_dashboard_api_test_helpers:uri( + ["gateway", "stomp", "authentication", "import_users"] ), Dir = code:lib_dir(emqx_authn, test), JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), {ok, JSONData} = file:read_file(JSONFileName), - {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(UploadUri, [], [ + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(ImportUri, [], [ {filename, "user-credentials.json", JSONData} ]), CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), {ok, CSVData} = file:read_file(CSVFileName), - {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(UploadUri, [], [ + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(ImportUri, [], [ {filename, "user-credentials.csv", CSVData} ]), @@ -469,20 +469,20 @@ t_listeners_authn_data_mgmt(_) -> Path ++ "/users" ), - UploadUri = emqx_dashboard_api_test_helpers:uri( - ["gateway", "stomp", "listeners", "stomp:tcp:def", "authentication", "upload_users"] + ImportUri = emqx_dashboard_api_test_helpers:uri( + ["gateway", "stomp", "listeners", "stomp:tcp:def", "authentication", "import_users"] ), Dir = code:lib_dir(emqx_authn, test), JSONFileName = filename:join([Dir, <<"data/user-credentials.json">>]), {ok, JSONData} = file:read_file(JSONFileName), - {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(UploadUri, [], [ + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(ImportUri, [], [ {filename, "user-credentials.json", JSONData} ]), CSVFileName = filename:join([Dir, <<"data/user-credentials.csv">>]), {ok, CSVData} = file:read_file(CSVFileName), - {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(UploadUri, [], [ + {ok, 204, _} = emqx_dashboard_api_test_helpers:multipart_formdata_request(ImportUri, [], [ {filename, "user-credentials.csv", CSVData} ]),