From 5265c3cc1fe281d26298fe582b494974006bcb68 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 26 Jun 2024 09:07:10 +0800 Subject: [PATCH 1/7] feat: support bootstrap_file on build-in-db authn --- .../src/emqx_auth_mnesia.app.src | 2 +- .../src/emqx_authn_mnesia.erl | 24 ++++++- .../src/emqx_authn_mnesia_schema.erl | 23 ++++++- .../test/emqx_authn_mnesia_SUITE.erl | 69 ++++++++++++++++++- rel/i18n/emqx_authn_mnesia_schema.hocon | 10 +++ 5 files changed, 124 insertions(+), 4 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src index 9a3c51002..7fcdda1d3 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_mnesia, [ {description, "EMQX Buitl-in Database Authentication and Authorization"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {mod, {emqx_auth_mnesia_app, []}}, {applications, [ diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl index 0c9631896..85230537f 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl @@ -116,7 +116,7 @@ create( user_id_type := Type, password_hash_algorithm := Algorithm, user_group := UserGroup - } + } = Config ) -> ok = emqx_authn_password_hashing:init(Algorithm), State = #{ @@ -124,6 +124,7 @@ create( user_id_type => Type, password_hash_algorithm => Algorithm }, + ok = boostrap_user_from_file(Config, State), {ok, State}. update(Config, _State) -> @@ -537,3 +538,24 @@ find_password_hash(_, _, _) -> is_superuser(#{<<"is_superuser">> := <<"true">>}) -> true; is_superuser(#{<<"is_superuser">> := true}) -> true; is_superuser(_) -> false. + +boostrap_user_from_file(Config, State) -> + case maps:get(boostrap_file, Config, <<>>) of + <<>> -> + ok; + FileName -> + #{boostrap_type := Type} = Config, + case file:read_file(FileName) of + {ok, FileData} -> + %% if there is a key conflict, override with the key which from the bootstrap file + _ = import_users({Type, FileName, FileData}, State), + ok; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "boostrap_authn(built_in_database)_failed", + boostrap_file => FileName, + boostrap_type => Type, + reason => Reason + }) + end + end. diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl index 7ad6616d4..6544874dc 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl @@ -46,7 +46,7 @@ select_union_member(_Kind, _Value) -> fields(builtin_db) -> [ {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1} - ] ++ common_fields(); + ] ++ common_fields() ++ bootstrap_fields(); fields(builtin_db_api) -> [ {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw_api/1} @@ -69,3 +69,24 @@ common_fields() -> {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, {user_id_type, fun user_id_type/1} ] ++ emqx_authn_schema:common_fields(). + +bootstrap_fields() -> + [ + {bootstrap_file, + ?HOCON( + binary(), + #{ + desc => ?DESC(bootstrap_file), + required => false, + default => <<>> + } + )}, + {bootstrap_type, + ?HOCON( + ?ENUM([hash, plain]), #{ + desc => ?DESC(bootstrap_type), + required => false, + default => <<"plain">> + } + )} + ]. diff --git a/apps/emqx_auth_mnesia/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_authn_mnesia_SUITE.erl index 54409a73f..666b4a628 100644 --- a/apps/emqx_auth_mnesia/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_authn_mnesia_SUITE.erl @@ -54,7 +54,74 @@ t_create(_) -> {ok, _} = emqx_authn_mnesia:create(?AUTHN_ID, Config0), Config1 = Config0#{password_hash_algorithm => #{name => sha256}}, - {ok, _} = emqx_authn_mnesia:create(?AUTHN_ID, Config1). + {ok, _} = emqx_authn_mnesia:create(?AUTHN_ID, Config1), + ok. +t_bootstrap_file(_) -> + Config = config(), + %% hash to hash + HashConfig = Config#{password_hash_algorithm => #{name => sha256, salt_position => suffix}}, + ?assertMatch( + [ + {user_info, {_, <<"myuser1">>}, _, _, true}, + {user_info, {_, <<"myuser2">>}, _, _, false} + ], + test_bootstrap_file(HashConfig, hash, <<"user-credentials.json">>) + ), + ?assertMatch( + [ + {user_info, {_, <<"myuser3">>}, _, _, true}, + {user_info, {_, <<"myuser4">>}, _, _, false} + ], + test_bootstrap_file(HashConfig, hash, <<"user-credentials.csv">>) + ), + + %% plain to plain + PlainConfig = Config#{ + password_hash_algorithm => + #{name => plain, salt_position => disable} + }, + ?assertMatch( + [ + {user_info, {_, <<"myuser1">>}, <<"password1">>, _, true}, + {user_info, {_, <<"myuser2">>}, <<"password2">>, _, false} + ], + test_bootstrap_file(PlainConfig, plain, <<"user-credentials-plain.json">>) + ), + ?assertMatch( + [ + {user_info, {_, <<"myuser3">>}, <<"password3">>, _, true}, + {user_info, {_, <<"myuser4">>}, <<"password4">>, _, false} + ], + test_bootstrap_file(PlainConfig, plain, <<"user-credentials-plain.csv">>) + ), + %% plain to hash + ?assertMatch( + [ + {user_info, {_, <<"myuser1">>}, _, _, true}, + {user_info, {_, <<"myuser2">>}, _, _, false} + ], + test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain.json">>) + ), + ?assertMatch( + [ + {user_info, {_, <<"myuser3">>}, _, _, true}, + {user_info, {_, <<"myuser4">>}, _, _, false} + ], + test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain.csv">>) + ), + ok. + +test_bootstrap_file(Config0, Type, File) -> + {Type, Filename, _FileData} = sample_filename_and_data(Type, File), + Config2 = Config0#{ + boostrap_file => Filename, + boostrap_type => Type + }, + {ok, State0} = emqx_authn_mnesia:create(?AUTHN_ID, Config2), + Result = ets:tab2list(emqx_authn_mnesia), + ok = emqx_authn_mnesia:destroy(State0), + ?assertMatch([], ets:tab2list(emqx_authn_mnesia)), + Result. t_update(_) -> Config0 = config(), diff --git a/rel/i18n/emqx_authn_mnesia_schema.hocon b/rel/i18n/emqx_authn_mnesia_schema.hocon index b0d1a8517..e32f1f772 100644 --- a/rel/i18n/emqx_authn_mnesia_schema.hocon +++ b/rel/i18n/emqx_authn_mnesia_schema.hocon @@ -9,4 +9,14 @@ user_id_type.desc: user_id_type.label: """Authentication ID Type""" +bootstrap_file.desc: +"""The bootstrap file imports users into the built-in database.""" + +bootstrap_file.label: +"""Bootstrap File Path""" + +bootstrap_type.desc: +"""plain: bootstrap_file.cvs should be `user_id,password,is_superuser`. +hash: bootstrap_file.cvs should be `user_id,password_hash,salt,is_superuser`""" + } From 29076f7eb8d7d4ab8cf5f4074656d3f7a99b3fa0 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 26 Jun 2024 11:40:34 +0800 Subject: [PATCH 2/7] chore: changelog for bootstrap_file/type --- changes/ce/feat-13336.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-13336.en.md diff --git a/changes/ce/feat-13336.en.md b/changes/ce/feat-13336.en.md new file mode 100644 index 000000000..85223311c --- /dev/null +++ b/changes/ce/feat-13336.en.md @@ -0,0 +1 @@ +Added support for configuring the bootstrap_file and bootstrap_type when using the built-in db authentication method. From 24d2534641e345708d9866250370bf0d30069241 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 26 Jun 2024 16:55:32 +0800 Subject: [PATCH 3/7] chore: apply review suggestion --- apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl | 7 ++++--- changes/ce/feat-13336.en.md | 2 +- rel/i18n/emqx_authn_mnesia_schema.hocon | 8 +++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl index 85230537f..dd4ae4845 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl @@ -543,8 +543,9 @@ boostrap_user_from_file(Config, State) -> case maps:get(boostrap_file, Config, <<>>) of <<>> -> ok; - FileName -> + FileName0 -> #{boostrap_type := Type} = Config, + FileName = emqx_schema:naive_env_interpolation(FileName0), case file:read_file(FileName) of {ok, FileData} -> %% if there is a key conflict, override with the key which from the bootstrap file @@ -552,10 +553,10 @@ boostrap_user_from_file(Config, State) -> ok; {error, Reason} -> ?SLOG(warning, #{ - msg => "boostrap_authn(built_in_database)_failed", + msg => "boostrap_authn_built_in_database_failed", boostrap_file => FileName, boostrap_type => Type, - reason => Reason + reason => emqx_utils:explain_posix(Reason) }) end end. diff --git a/changes/ce/feat-13336.en.md b/changes/ce/feat-13336.en.md index 85223311c..ff09f624b 100644 --- a/changes/ce/feat-13336.en.md +++ b/changes/ce/feat-13336.en.md @@ -1 +1 @@ -Added support for configuring the bootstrap_file and bootstrap_type when using the built-in db authentication method. +Added new configs `bootstrap_file` and `bootstrap_type` for built-in database for authentication to support bootstrapping the table with csv and json file. diff --git a/rel/i18n/emqx_authn_mnesia_schema.hocon b/rel/i18n/emqx_authn_mnesia_schema.hocon index e32f1f772..010d32f61 100644 --- a/rel/i18n/emqx_authn_mnesia_schema.hocon +++ b/rel/i18n/emqx_authn_mnesia_schema.hocon @@ -10,13 +10,15 @@ user_id_type.label: """Authentication ID Type""" bootstrap_file.desc: -"""The bootstrap file imports users into the built-in database.""" +"""The bootstrap file imports users into the built-in database. +the file content format is determined by `bootstrap_type`.""" bootstrap_file.label: """Bootstrap File Path""" bootstrap_type.desc: -"""plain: bootstrap_file.cvs should be `user_id,password,is_superuser`. -hash: bootstrap_file.cvs should be `user_id,password_hash,salt,is_superuser`""" +"""plain: bootstrap_file.cvs should have lines of format `{user_id},{password},{is_superuser}` where `user_id` can be either clientid or username depending on `user_id_type`. +hash: bootstrap_file.cvs should have line of format `{user_id},{password_hash},{salt},{is_superuser}.` +All file format support is the same as `authentication/password_based:built_in_database/import_users` API.""" } From 9594b6df32687026b31edc56bcf86e19efc40c59 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 26 Jun 2024 17:34:16 +0800 Subject: [PATCH 4/7] chore: warning overrided when restart authn --- .../src/emqx_authn_mnesia.erl | 20 +++++++++++++++++-- rel/i18n/emqx_authn_mnesia_schema.hocon | 8 +++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl index dd4ae4845..028718aa4 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl @@ -339,8 +339,24 @@ run_fuzzy_filter( %%------------------------------------------------------------------------------ insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) -> - UserInfoRecord = user_info_record(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), - insert_user(UserInfoRecord). + UserInfoRecord = + #user_info{user_id = DBUserID} = + user_info_record(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), + case mnesia:read(?TAB, DBUserID, write) of + [] -> + insert_user(UserInfoRecord); + [UserInfoRecord] -> + ok; + [_] -> + ?SLOG(warning, #{ + msg => "bootstrap_authentication_overridden_in_the_built_in_database", + user_id => UserID, + group_id => UserGroup, + suggestion => + "If you have made changes in other way, remove the user_id from the bootstrap file." + }), + insert_user(UserInfoRecord) + end. insert_user(#user_info{} = UserInfoRecord) -> mnesia:write(?TAB, UserInfoRecord, write). diff --git a/rel/i18n/emqx_authn_mnesia_schema.hocon b/rel/i18n/emqx_authn_mnesia_schema.hocon index 010d32f61..ad90dbe19 100644 --- a/rel/i18n/emqx_authn_mnesia_schema.hocon +++ b/rel/i18n/emqx_authn_mnesia_schema.hocon @@ -11,14 +11,16 @@ user_id_type.label: bootstrap_file.desc: """The bootstrap file imports users into the built-in database. -the file content format is determined by `bootstrap_type`.""" +The file content format is determined by `bootstrap_type`. +Remove the item from the bootstrap file when you have made changes in other way, +otherwise, after restarting, the bootstrap item will be overridden again.""" bootstrap_file.label: """Bootstrap File Path""" bootstrap_type.desc: -"""plain: bootstrap_file.cvs should have lines of format `{user_id},{password},{is_superuser}` where `user_id` can be either clientid or username depending on `user_id_type`. -hash: bootstrap_file.cvs should have line of format `{user_id},{password_hash},{salt},{is_superuser}.` +"""`plain`: bootstrap_file.csv should have lines of format `{user_id},{password},{is_superuser}` where `user_id` can be either clientid or username depending on `user_id_type`. +`hash`: bootstrap_file.csv should have line of format `{user_id},{password_hash},{salt},{is_superuser}.` All file format support is the same as `authentication/password_based:built_in_database/import_users` API.""" } From cbaa0b0be0da3fbd0344c37c908d51c84ced99d0 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 27 Jun 2024 11:35:33 +0800 Subject: [PATCH 5/7] docs: add password_hash generate document --- rel/i18n/emqx_authn_mnesia_schema.hocon | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/rel/i18n/emqx_authn_mnesia_schema.hocon b/rel/i18n/emqx_authn_mnesia_schema.hocon index ad90dbe19..ad4082a13 100644 --- a/rel/i18n/emqx_authn_mnesia_schema.hocon +++ b/rel/i18n/emqx_authn_mnesia_schema.hocon @@ -19,8 +19,22 @@ bootstrap_file.label: """Bootstrap File Path""" bootstrap_type.desc: -"""`plain`: bootstrap_file.csv should have lines of format `{user_id},{password},{is_superuser}` where `user_id` can be either clientid or username depending on `user_id_type`. -`hash`: bootstrap_file.csv should have line of format `{user_id},{password_hash},{salt},{is_superuser}.` -All file format support is the same as `authentication/password_based:built_in_database/import_users` API.""" +"""- **`plain`**: + - Format: `{user_id},{password},{is_superuser}` + - `user_id`: Can be `clientid` or `username`, based on `user_id_type`. + - `password`: User's plaintext password. + - `is_superuser`: Boolean, user's administrative status. + + - **`hash`**: + - Format: `{user_id},{password_hash},{salt},{is_superuser}` + - Definitions similar to `plain` type, with `password_hash` and `salt` added for security. + +All file format support is the same as `authentication/password_based:built_in_database/import_users` API. +Json file plain format example:`[{"user_id": "my_user","password": "s3cr3tp@ssw0rd","is_superuser": true}]`. +CSV file hash format example :`user_id,password_hash,salt,is_superuser\nmy_user,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235,true` + +Formula for `password_hash`: +If configured as `password_hash_algorithm {name = sha256, salt_position = suffix}`, +the Python code to calculate the `password_hash` is `hashlib.sha256(password + salt).hexdigest()`.""" } From d9086139eb51dc5428a3220bfec92220006fa457 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 27 Jun 2024 19:31:35 +0800 Subject: [PATCH 6/7] chore: apply suggestions from code review Co-authored-by: zmstone --- rel/i18n/emqx_authn_mnesia_schema.hocon | 30 +++++++++++++------------ 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/rel/i18n/emqx_authn_mnesia_schema.hocon b/rel/i18n/emqx_authn_mnesia_schema.hocon index ad4082a13..5ebfeaf38 100644 --- a/rel/i18n/emqx_authn_mnesia_schema.hocon +++ b/rel/i18n/emqx_authn_mnesia_schema.hocon @@ -19,22 +19,24 @@ bootstrap_file.label: """Bootstrap File Path""" bootstrap_type.desc: -"""- **`plain`**: - - Format: `{user_id},{password},{is_superuser}` - - `user_id`: Can be `clientid` or `username`, based on `user_id_type`. - - `password`: User's plaintext password. - - `is_superuser`: Boolean, user's administrative status. +"""Specify which type of content the bootstrap file has. - - **`hash`**: - - Format: `{user_id},{password_hash},{salt},{is_superuser}` - - Definitions similar to `plain` type, with `password_hash` and `salt` added for security. +- **`plain`**: + - Expected data fields: `user_id`, `password`, `is_superuser` + - `user_id`: Can be Client ID or username, depending on built-in database authentication's `user_id_type` config. + - `password`: User's plaintext password. + - `is_superuser`: Boolean, user's administrative status. -All file format support is the same as `authentication/password_based:built_in_database/import_users` API. -Json file plain format example:`[{"user_id": "my_user","password": "s3cr3tp@ssw0rd","is_superuser": true}]`. -CSV file hash format example :`user_id,password_hash,salt,is_superuser\nmy_user,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235,true` +- **`hash`**: + - Expected data fields: `user_id`,`password_hash`,`salt`,`is_superuser` + - Definitions similar to `plain` type, with `password_hash` and `salt` added for security. -Formula for `password_hash`: -If configured as `password_hash_algorithm {name = sha256, salt_position = suffix}`, -the Python code to calculate the `password_hash` is `hashlib.sha256(password + salt).hexdigest()`.""" +The content can be either in CSV, or JSON format. + +Here is a CSV example: `user_id,password_hash,salt,is_superuser\nmy_user,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235,true` + +And JSON content should be decoded into an array of objects, for example: `[{"user_id": "my_user","password": "s3cr3tp@ssw0rd","is_superuser": true}]`. + +The hash string for password_hash depends on how password_hash_algorithm is configured for the built-in database authentication mechanism. For example, if it's configured as `password_hash_algorithm {name = sha256, salt_position = suffix}`, then the salt is appended to the password before hashed. Here is the equivalent Python expression: `hashlib.sha256(password + salt).hexdigest()`.""" } From cf8dbdf0a03f12f9440305d04b6b375d88cae84c Mon Sep 17 00:00:00 2001 From: zmstone Date: Thu, 27 Jun 2024 14:16:40 +0200 Subject: [PATCH 7/7] docs: Update rel/i18n/emqx_authn_mnesia_schema.hocon --- rel/i18n/emqx_authn_mnesia_schema.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/i18n/emqx_authn_mnesia_schema.hocon b/rel/i18n/emqx_authn_mnesia_schema.hocon index 5ebfeaf38..3c9a24d2c 100644 --- a/rel/i18n/emqx_authn_mnesia_schema.hocon +++ b/rel/i18n/emqx_authn_mnesia_schema.hocon @@ -37,6 +37,6 @@ Here is a CSV example: `user_id,password_hash,salt,is_superuser\nmy_user,b6c7435 And JSON content should be decoded into an array of objects, for example: `[{"user_id": "my_user","password": "s3cr3tp@ssw0rd","is_superuser": true}]`. -The hash string for password_hash depends on how password_hash_algorithm is configured for the built-in database authentication mechanism. For example, if it's configured as `password_hash_algorithm {name = sha256, salt_position = suffix}`, then the salt is appended to the password before hashed. Here is the equivalent Python expression: `hashlib.sha256(password + salt).hexdigest()`.""" +The hash string for `password_hash` depends on how `password_hash_algorithm` is configured for the built-in database authentication mechanism. For example, if it's configured as `password_hash_algorithm {name = sha256, salt_position = suffix}`, then the salt is appended to the password before hashed. Here is the equivalent Python expression: `hashlib.sha256(password + salt).hexdigest()`.""" }