From 5521b7fa71a6070709a28fa073f9009aa5f92d4f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 12 Jan 2022 17:18:12 +0800 Subject: [PATCH 01/13] fix(auth): force update default mqtt_user when password or hashtype changed. --- .../emqx_auth_mnesia/src/emqx_auth_mnesia.erl | 4 +- .../src/emqx_auth_mnesia_cli.erl | 27 ++++++++- .../test/emqx_auth_mnesia_SUITE.erl | 56 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl index 905bcaaf0..74c7c71ee 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl @@ -32,6 +32,8 @@ , description/0 ]). +-export([match_password/3]). + init(#{clientid_list := ClientidList, username_list := UsernameList}) -> ok = ekka_mnesia:create_table(?TABLE, [ {disc_copies, [node()]}, @@ -45,7 +47,7 @@ init(#{clientid_list := ClientidList, username_list := UsernameList}) -> %% @private add_default_user({Login, Password}) when is_tuple(Login) -> - emqx_auth_mnesia_cli:add_user(Login, Password). + emqx_auth_mnesia_cli:force_add_user(Login, Password). -spec(register_metrics() -> ok). register_metrics() -> diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl index d89e6836c..72a932aa1 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl @@ -22,6 +22,7 @@ -define(TABLE, emqx_user). %% Auth APIs -export([ add_user/2 + , force_add_user/2 , update_user/2 , remove_user/1 , lookup_user/1 @@ -56,6 +57,28 @@ insert_user(User = #emqx_user{login = Login}) -> [_|_] -> mnesia:abort(existed) end. +force_add_user(Login, Password) -> + User = #emqx_user{ + login = Login, + password = encrypted_data(Password), + created_at = erlang:system_time(millisecond) + }, + ret(mnesia:transaction(fun insert_or_update_user/2, [Password, User])). + +insert_or_update_user(NewPwd, User = #emqx_user{login = Login}) -> + case mnesia:read(?TABLE, Login) of + [] -> mnesia:write(User); + [#emqx_user{password = Pwd}] -> + case emqx_auth_mnesia:match_password(NewPwd, hash_type(), [Pwd]) of + true -> ok; + false -> + Res = mnesia:write(User), + ?LOG(warning, "[Mnesia] (~p)'s password has be updated.", [Login]), + Res + end + end. + + %% @doc Update User -spec(update_user(tuple(), binary()) -> ok | {error, any()}). update_user(Login, NewPassword) -> @@ -109,7 +132,7 @@ ret({atomic, ok}) -> ok; ret({aborted, Error}) -> {error, Error}. encrypted_data(Password) -> - HashType = application:get_env(emqx_auth_mnesia, password_hash, sha256), + HashType = hash_type(), SaltBin = salt(), <>. @@ -192,3 +215,5 @@ auth_username_cli(_) -> {"user add ", "Add username auth rule"}, {"user update ", "Update username auth rule"}, {"user delete ", "Delete username auth rule"}]). +hash_type() -> + application:get_env(emqx_auth_mnesia, password_hash, sha256). diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index c5c0eb727..01dff1488 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -46,11 +46,15 @@ all() -> groups() -> []. +init_per_suite(t_boot) -> + ok; init_per_suite(Config) -> ok = emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia], fun set_special_configs/1), create_default_app(), Config. +end_per_suite(t_boot) -> + ok; end_per_suite(_Config) -> delete_default_app(), emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia]). @@ -65,10 +69,62 @@ set_special_configs(emqx) -> set_special_configs(_App) -> ok. +set_default(ClientId, UserName, Pwd, HashType) -> + application:set_env(emqx_auth_mnesia, clientid_list, [{ClientId, Pwd}]), + application:set_env(emqx_auth_mnesia, username_list, [{UserName, Pwd}]), + application:set_env(emqx_auth_mnesia, password_hash, HashType), + ok. %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ +t_boot(_Config) -> + clean_all_users(), + emqx_ct_helpers:stop_apps([emqx_auth_mnesia]), + ClientId = <<"clientid-test">>, + UserName = <<"username-test">>, + Pwd = <<"emqx123456">>, + ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia], + fun(_) -> set_default(ClientId, UserName, Pwd, sha256) end), + Ok = {stop, #{anonymous => false, auth_result => success}}, + Failed = {stop, #{anonymous => false, auth_result => password_error}}, + ?assertEqual(Ok, + emqx_auth_mnesia:check(#{clientid => ClientId, password => Pwd}, #{}, #{hash_type => sha256})), + ?assertEqual(Ok, + emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => Pwd}, + #{}, #{hash_type => sha256})), + ?assertEqual(Failed, + emqx_auth_mnesia:check(#{clientid => ClientId, password => <>}, + #{}, #{hash_type => sha256})), + ?assertEqual(Failed, + emqx_auth_mnesia:check(#{clientid => ClientId, username => UserName, password => <>}, + #{}, #{hash_type => sha256})), + emqx_ct_helpers:stop_apps([emqx_auth_mnesia]), + + %% change default pwd + NewPwd = <<"emqx654321">>, + ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia], + fun(_) -> set_default(ClientId, UserName, NewPwd, sha256) end), + ?assertEqual(Ok, + emqx_auth_mnesia:check(#{clientid => ClientId, password => NewPwd}, + #{}, #{hash_type => sha256})), + ?assertEqual(Ok, + emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd}, + #{}, #{hash_type => sha256})), + emqx_ct_helpers:stop_apps([emqx_auth_mnesia]), + + %% change hash_type + NewPwd2 = <<"emqx6543210">>, + ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia], + fun(_) -> set_default(ClientId, UserName, NewPwd2, plain) end), + ?assertEqual(Ok, + emqx_auth_mnesia:check(#{clientid => ClientId, password => NewPwd2}, + #{}, #{hash_type => plain})), + ?assertEqual(Ok, + emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd2}, + #{}, #{hash_type => plain})), + ok. + t_management(_Config) -> clean_all_users(), From 88060c0f9b634e2fa80ec4414c628079008edd30 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 12 Jan 2022 17:34:32 +0800 Subject: [PATCH 02/13] chore(auth): bump emqx_auth_mnesia version:4.3.5 --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src | 2 +- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src index b15c7fdd3..d782f0272 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mnesia, [{description, "EMQ X Authentication with Mnesia"}, - {vsn, "4.3.4"}, % strict semver, bump manually + {vsn, "4.3.5"}, % strict semver, bump manually {modules, []}, {registered, []}, {applications, [kernel,stdlib,mnesia]}, diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index 82df99b3a..2a6353760 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -7,10 +7,14 @@ {update, emqx_auth_mnesia_sup, supervisor}, {apply, {emqx_acl_mnesia_migrator, start_supervised, []}}, {load_module,emqx_auth_mnesia_api, brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_mnesia, brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia, brutal_purge, soft_purge,[]}, {load_module,emqx_acl_mnesia_api, brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_cli, brutal_purge,soft_purge,[]} ]}, + {<<"4.3.4">>, [ + {load_module,emqx_auth_mnesia, brutal_purge, soft_purge,[]}, + {load_module,emqx_auth_mnesia_cli, brutal_purge,soft_purge,[]}, + ]}, {<<".*">>, [ ]} ], @@ -25,6 +29,10 @@ {delete_module,emqx_acl_mnesia_migrator}, {delete_module,emqx_acl_mnesia_db} ]}, + {<<"4.3.4">>, [ + {load_module,emqx_auth_mnesia, brutal_purge, soft_purge,[]}, + {load_module,emqx_auth_mnesia_cli, brutal_purge,soft_purge,[]}, + ]}, {<<".*">>, [ ]} ] From 85d568be602c94d6e99a00f63b9fc30c7429f4b5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 12 Jan 2022 18:46:05 +0800 Subject: [PATCH 03/13] chore(test): clean up auth_mnesia_SUITE:t_boot config --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src | 10 +++++----- apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index 2a6353760..95a0c1877 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -7,13 +7,13 @@ {update, emqx_auth_mnesia_sup, supervisor}, {apply, {emqx_acl_mnesia_migrator, start_supervised, []}}, {load_module,emqx_auth_mnesia_api, brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_mnesia, brutal_purge, soft_purge,[]}, + {load_module,emqx_acl_mnesia, brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_api, brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_cli, brutal_purge,soft_purge,[]} ]}, {<<"4.3.4">>, [ - {load_module,emqx_auth_mnesia, brutal_purge, soft_purge,[]}, - {load_module,emqx_auth_mnesia_cli, brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia, brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia_cli, brutal_purge,soft_purge,[]} ]}, {<<".*">>, [ ]} @@ -30,8 +30,8 @@ {delete_module,emqx_acl_mnesia_db} ]}, {<<"4.3.4">>, [ - {load_module,emqx_auth_mnesia, brutal_purge, soft_purge,[]}, - {load_module,emqx_auth_mnesia_cli, brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia, brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia_cli, brutal_purge,soft_purge,[]} ]}, {<<".*">>, [ ]} diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index 01dff1488..4246965d9 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -73,6 +73,7 @@ set_default(ClientId, UserName, Pwd, HashType) -> application:set_env(emqx_auth_mnesia, clientid_list, [{ClientId, Pwd}]), application:set_env(emqx_auth_mnesia, username_list, [{UserName, Pwd}]), application:set_env(emqx_auth_mnesia, password_hash, HashType), + application:set_env(emqx_auth_mnesia, password_hash, HashType), ok. %%------------------------------------------------------------------------------ %% Testcases @@ -123,6 +124,7 @@ t_boot(_Config) -> ?assertEqual(Ok, emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd2}, #{}, #{hash_type => plain})), + clean_all_users(), ok. t_management(_Config) -> From 27f5e765b584e51f5476bd602610a7c4d1e479f7 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 13 Jan 2022 13:53:37 +0800 Subject: [PATCH 04/13] fix(export): emqx_auth_mnesia import failed after 4.3.x --- .../test/emqx_auth_mnesia_SUITE.erl | 1 - .../src/emqx_mgmt_data_backup.erl | 20 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index 4246965d9..12b473815 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -73,7 +73,6 @@ set_default(ClientId, UserName, Pwd, HashType) -> application:set_env(emqx_auth_mnesia, clientid_list, [{ClientId, Pwd}]), application:set_env(emqx_auth_mnesia, username_list, [{UserName, Pwd}]), application:set_env(emqx_auth_mnesia, password_hash, HashType), - application:set_env(emqx_auth_mnesia, password_hash, HashType), ok. %%------------------------------------------------------------------------------ %% Testcases diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 6e467a8ba..d17167785 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -602,7 +602,7 @@ import(Filename, OverridesJson) -> Overrides = emqx_json:decode(OverridesJson, [return_maps]), Data = maps:merge(Imported, Overrides), Version = to_version(maps:get(<<"version">>, Data)), - read_global_auth_type(Data), + read_global_auth_type(Data, Version), try do_import_data(Data, Version), logger:debug("The emqx data has been imported successfully"), @@ -621,7 +621,7 @@ import(Filename, OverridesJson) -> Overrides = emqx_json:decode(OverridesJson, [return_maps]), Data = maps:merge(Imported, Overrides), Version = to_version(maps:get(<<"version">>, Data)), - read_global_auth_type(Data), + read_global_auth_type(Data, Version), case is_version_supported(Data, Version) of true -> try @@ -696,17 +696,17 @@ is_version_supported2(Version) -> end. -endif. -read_global_auth_type(Data) -> +read_global_auth_type(Data, Version) -> case {maps:get(<<"auth_mnesia">>, Data, []), maps:get(<<"acl_mnesia">>, Data, [])} of {[], []} -> %% Auth mnesia plugin is not used: ok; _ -> - do_read_global_auth_type(Data) + do_read_global_auth_type(Data, Version) end. -ifdef(EMQX_ENTERPRISE). -do_read_global_auth_type(Data) -> +do_read_global_auth_type(Data, _Version) -> case Data of #{<<"auth.mnesia.as">> := <<"username">>} -> application:set_env(emqx_auth_mnesia, as, username); @@ -717,13 +717,15 @@ do_read_global_auth_type(Data) -> end. -else. -do_read_global_auth_type(Data) -> +do_read_global_auth_type(Data, FromVersion) -> case Data of #{<<"auth.mnesia.as">> := <<"username">>} -> application:set_env(emqx_auth_mnesia, as, username); #{<<"auth.mnesia.as">> := <<"clientid">>} -> application:set_env(emqx_auth_mnesia, as, clientid); - _ -> + _ when FromVersion =:= "4.0" orelse + FromVersion =:= "4.1" orelse + FromVersion =:= "4.2"-> logger:error("While importing data from EMQX versions prior to 4.3 " "it is necessary to specify the value of \"auth.mnesia.as\" parameter " "as it was configured in etc/plugins/emqx_auth_mnesia.conf.\n" @@ -732,7 +734,9 @@ do_read_global_auth_type(Data) -> "or\n" " $ emqx_ctl data import --env '{\"auth.mnesia.as\":\"clientid\"}'", []), - error(import_failed) + error(import_failed); + _ -> + ok end. -endif. From e96c9ada52049b8210f39eec2939a9956e768280 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 13 Jan 2022 16:26:32 +0800 Subject: [PATCH 05/13] chore(test): fix typo error --- apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index e45acfd42..bbec02722 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -583,7 +583,7 @@ t_data(_) -> ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename, <<"node">> => Node})), ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename})), application:stop(emqx_rule_engine), - application:stop(emqx_dahboard), + application:stop(emqx_dashboard), ok. t_data_import_content(_) -> @@ -598,7 +598,7 @@ t_data_import_content(_) -> Content = emqx_json:decode(Bin), ?assertMatch({ok, "{\"code\":0}"}, request_api(post, api_path(["data","import"]), [], auth_header_(), Content)), application:stop(emqx_rule_engine), - application:stop(emqx_dahboard). + application:stop(emqx_dashboard). request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). From 9a17bcfcc9ac64327e036cea52786d835ca0f33b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Jan 2022 11:03:27 +0800 Subject: [PATCH 06/13] chore(appup): update eqmx.appup.src --- src/emqx.appup.src | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index c4885b7b1..f50a52e74 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- {VSN, [{"4.3.12", - [{load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -199,7 +200,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.12", - [{load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, From 4ba43d3aeacb54485ea7a29ecdd1680db700cf22 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Jan 2022 11:37:47 +0800 Subject: [PATCH 07/13] fix(auth): move log outside of transaction --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl index 72a932aa1..55e3e4966 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl @@ -63,7 +63,12 @@ force_add_user(Login, Password) -> password = encrypted_data(Password), created_at = erlang:system_time(millisecond) }, - ret(mnesia:transaction(fun insert_or_update_user/2, [Password, User])). + case ret(mnesia:transaction(fun insert_or_update_user/2, [Password, User])) of + {ok, override} -> + ?LOG(warning, "[Mnesia] (~p)'s password has be updated.", [Login]), + ok; + Other -> Other + end. insert_or_update_user(NewPwd, User = #emqx_user{login = Login}) -> case mnesia:read(?TABLE, Login) of @@ -72,9 +77,8 @@ insert_or_update_user(NewPwd, User = #emqx_user{login = Login}) -> case emqx_auth_mnesia:match_password(NewPwd, hash_type(), [Pwd]) of true -> ok; false -> - Res = mnesia:write(User), - ?LOG(warning, "[Mnesia] (~p)'s password has be updated.", [Login]), - Res + ok = mnesia:write(User), + {ok, override} end end. @@ -128,7 +132,7 @@ comparing({?TABLE, _, _, CreatedAt1}, {?TABLE, _, _, CreatedAt2}) -> CreatedAt1 >= CreatedAt2. -ret({atomic, ok}) -> ok; +ret({atomic, Res}) -> Res; ret({aborted, Error}) -> {error, Error}. encrypted_data(Password) -> From 5481723513d5d899fee4ef9f8b1c9efd2d223c8a Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Jan 2022 14:55:31 +0800 Subject: [PATCH 08/13] chore(docs): update changes-4.3.md --- CHANGES-4.3.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index a442ff6c8..0e293d898 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,6 +10,13 @@ File format: - One list item per change topic Change log ends with a list of github PRs +## v4.3.12 +### Important changes + +### Minor changes +* Fix updating `emqx_auth_mnesia.conf` password and restarting the new password does not take effect [#6717] +* Fix import data crash when emqx_auth_mnesia's record is not empty [#6717] + ## v4.3.11 Important notes: From eb003e33054d4883975b0e6bfdbe36ee99c265dd Mon Sep 17 00:00:00 2001 From: lafirest Date: Tue, 18 Jan 2022 14:06:26 +0800 Subject: [PATCH 09/13] feat(emqx_limiter): add support for update overall limiter --- src/emqx_limiter.erl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/emqx_limiter.erl b/src/emqx_limiter.erl index 181e5c6bf..52b830c8e 100644 --- a/src/emqx_limiter.erl +++ b/src/emqx_limiter.erl @@ -23,6 +23,7 @@ , init/4 %% XXX: Compatible with before 4.2 version , info/1 , check/2 + , update_overall_limiter/4 ]). -record(limiter, { @@ -152,3 +153,15 @@ is_message_limiter(conn_messages_in) -> true; is_message_limiter(conn_messages_routing) -> true; is_message_limiter(overall_messages_routing) -> true; is_message_limiter(_) -> false. + +update_overall_limiter(Zone, Name, Capacity, Interval) -> + case is_overall_limiter(Name) of + false -> false; + _ -> + try + esockd_limiter:update({Zone, Name}, Capacity, Interval), + true + catch _:_:_ -> + false + end + end. From deada0ea44a0708d2eae0631e06d616b25deeee1 Mon Sep 17 00:00:00 2001 From: lafirest Date: Tue, 18 Jan 2022 14:46:43 +0800 Subject: [PATCH 10/13] chore(appup): update appup file and esockd version --- rebar.config | 2 +- src/emqx.appup.src | 76 +++++++++++++++++++++++++++++++--------------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/rebar.config b/rebar.config index fb60981b2..f76404052 100644 --- a/rebar.config +++ b/rebar.config @@ -42,7 +42,7 @@ , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.4"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.7"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 4dc667da2..459696e3b 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -4,7 +4,8 @@ [ {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} - , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} + , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} + , {load_module,emqx_limiter,brutal_purge,soft_purge,[]} ]}, {"4.3.11", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -15,7 +16,8 @@ {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, @@ -26,7 +28,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -42,7 +45,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -58,7 +62,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -76,7 +81,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -95,7 +101,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -115,7 +122,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -136,7 +144,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -158,7 +167,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -180,7 +190,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -206,7 +217,8 @@ {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -235,13 +247,15 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.12", [ {load_module,emqx_channel,brutal_purge,soft_purge,[]} , {load_module,emqx_metrics,brutal_purge,soft_purge,[]} , {load_module,emqx_session,brutal_purge,soft_purge,[]} , {load_module,emqx_alarm,brutal_purge,soft_purge,[]} + , {load_module,emqx_limiter,brutal_purge,soft_purge,[]} ]}, {"4.3.11", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -252,7 +266,8 @@ {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, @@ -263,7 +278,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -279,7 +295,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -295,7 +312,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -313,7 +331,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -332,7 +351,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -352,7 +372,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -373,7 +394,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -395,7 +417,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -417,7 +440,8 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -443,7 +467,8 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -471,5 +496,6 @@ {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. From 19fc143f415a85dc2d00b3ee47e3fedf891032e1 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 19 Jan 2022 00:08:53 +0800 Subject: [PATCH 11/13] fix(auth_mnesia): import auth acls data according to different data format --- .../src/emqx_mgmt_data_backup.erl | 177 ++++++++++-------- .../test/emqx_auth_mnesia_migration_SUITE.erl | 6 +- 2 files changed, 101 insertions(+), 82 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index d17167785..119906c7e 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -46,8 +46,8 @@ , import_users/1 , import_auth_clientid/1 %% BACKW: 4.1.x , import_auth_username/1 %% BACKW: 4.1.x - , import_auth_mnesia/2 - , import_acl_mnesia/2 + , import_auth_mnesia/1 + , import_acl_mnesia/1 , to_version/1 ]). @@ -422,87 +422,103 @@ import_auth_username(Lists) -> end, Lists) end. --ifdef(EMQX_ENTERPRISE). -import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> - do_import_auth_mnesia_by_old_data(Auths); -import_auth_mnesia(Auths, _) -> - do_import_auth_mnesia(Auths). +import_auth_mnesia(Auths) -> + case validate_auth(Auths) of + ignore -> ok; + old -> do_import_auth_mnesia_by_old_data(Auths); + new -> do_import_auth_mnesia(Auths) + end. -import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> - do_import_acl_mnesia_by_old_data(Acls); +validate_auth(Auths) -> + case ets:info(emqx_user) of + undefined -> ignore; + _ -> + case lists:all(fun is_new_auth_data/1, Auths) of + true -> new; + false -> + case lists:all(fun is_old_auth_data/1, Auths) of + true -> + _ = get_old_type(), + old; + false -> error({auth_mnesia_data_error, Auths}) + end + end + end. -import_acl_mnesia(Acls, _) -> - do_import_acl_mnesia(Acls). --else. -import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.3" -> - do_import_auth_mnesia(Auths); -import_auth_mnesia(Auths, _FromVersion) -> - do_import_auth_mnesia_by_old_data(Auths). +is_new_auth_data(#{<<"type">> := _, <<"login">> := _, <<"password">> := _}) -> true; +is_new_auth_data(_) -> false. -import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.3" -> - do_import_acl_mnesia(Acls); -import_acl_mnesia(Acls, _FromVersion) -> - do_import_acl_mnesia_by_old_data(Acls). - --endif. +is_old_auth_data(#{<<"login">> := _, <<"password">> := _} = Auth) -> + not maps:is_key(<<"type">>, Auth); +is_old_auth_data(_) -> false. do_import_auth_mnesia_by_old_data(Auths) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - CreatedAt = erlang:system_time(millisecond), - lists:foreach(fun(#{<<"login">> := Login, - <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {get_old_type(), Login}, base64:decode(Password), CreatedAt}) - end, Auths) - end. - + CreatedAt = erlang:system_time(millisecond), + Type = get_old_type(), + lists:foreach(fun(#{<<"login">> := Login, <<"password">> := Password}) -> + mnesia:dirty_write({emqx_user, {Type, Login}, base64:decode(Password), CreatedAt}) + end, Auths). do_import_auth_mnesia(Auths) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - lists:foreach(fun(#{<<"login">> := Login, - <<"type">> := Type, - <<"password">> := Password } = Map) -> - CreatedAt = maps:get(<<"created_at">>, Map, erlang:system_time(millisecond)), - mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt}) - end, Auths) + CreatedAt0 = erlang:system_time(millisecond), + lists:foreach(fun(#{<<"login">> := Login, + <<"type">> := Type, <<"password">> := Password } = Map) -> + CreatedAt = maps:get(<<"created_at">>, Map, CreatedAt0), + mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt}) + end, Auths). + +import_acl_mnesia(Acls) -> + case validate_acl(Acls) of + ignore -> ok; + old -> do_import_acl_mnesia_by_old_data(Acls); + new -> do_import_acl_mnesia(Acls) end. +validate_acl(Acls) -> + case ets:info(emqx_acl2) of + undefined -> ignore; + _ -> + case lists:all(fun is_new_acl_data/1, Acls) of + true -> new; + false -> + case lists:all(fun is_old_acl_data/1, Acls) of + true -> + _ = get_old_type(), + old; + false -> error({acl_mnesia_data_error, Acls}) + end + end + end. + +is_new_acl_data(#{<<"action">> := _, <<"access">> := _, + <<"topic">> := _, <<"type">> := _}) -> true; +is_new_acl_data(_) -> false. + +is_old_acl_data(#{<<"login">> := _, <<"topic">> := _, + <<"allow">> := Allow, <<"action">> := _}) -> is_boolean(Allow); +is_old_acl_data(_) -> false. + do_import_acl_mnesia_by_old_data(Acls) -> - case ets:info(emqx_acl2) of - undefined -> ok; - _ -> - lists:foreach(fun(#{<<"login">> := Login, - <<"topic">> := Topic, - <<"allow">> := Allow, - <<"action">> := Action}) -> - Allow1 = case any_to_atom(Allow) of - true -> allow; - false -> deny - end, - emqx_acl_mnesia_db:add_acl({get_old_type(), Login}, Topic, any_to_atom(Action), Allow1) - end, Acls) - end. + lists:foreach(fun(#{<<"login">> := Login, + <<"topic">> := Topic, + <<"allow">> := Allow, + <<"action">> := Action}) -> + Allow1 = case any_to_atom(Allow) of + true -> allow; + false -> deny + end, + emqx_acl_mnesia_db:add_acl({get_old_type(), Login}, Topic, any_to_atom(Action), Allow1) + end, Acls). + do_import_acl_mnesia(Acls) -> - case ets:info(emqx_acl2) of - undefined -> ok; - _ -> - lists:foreach(fun(Map = #{<<"action">> := Action, - <<"access">> := Access}) -> - Topic = maps:get(<<"topic">>, Map), - Login = case maps:get(<<"type_value">>, Map, undefined) of - undefined -> - all; - Value -> - {any_to_atom(maps:get(<<"type">>, Map)), Value} - end, - emqx_acl_mnesia_db:add_acl(Login, Topic, any_to_atom(Action), any_to_atom(Access)) - end, Acls) - end. + lists:foreach(fun(Map = #{<<"action">> := Action, + <<"access">> := Access, <<"topic">> := Topic}) -> + Login = case maps:get(<<"type_value">>, Map, undefined) of + undefined -> all; + Value -> {any_to_atom(maps:get(<<"type">>, Map)), Value} + end, + emqx_acl_mnesia_db:add_acl(Login, Topic, any_to_atom(Action), any_to_atom(Access)) + end, Acls). -ifdef(EMQX_ENTERPRISE). -dialyzer({nowarn_function, [import_modules/1]}). @@ -648,8 +664,8 @@ do_import_data(Data, Version) -> import_users(maps:get(<<"users">>, Data, [])), import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), - import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), - import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version). + import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])), + import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])). -ifdef(EMQX_ENTERPRISE). do_import_extra_data(Data, _Version) -> @@ -709,9 +725,9 @@ read_global_auth_type(Data, Version) -> do_read_global_auth_type(Data, _Version) -> case Data of #{<<"auth.mnesia.as">> := <<"username">>} -> - application:set_env(emqx_auth_mnesia, as, username); + set_old_type(username); #{<<"auth.mnesia.as">> := <<"clientid">>} -> - application:set_env(emqx_auth_mnesia, as, clientid); + set_old_type(clientid); _ -> ok end. @@ -720,9 +736,9 @@ do_read_global_auth_type(Data, _Version) -> do_read_global_auth_type(Data, FromVersion) -> case Data of #{<<"auth.mnesia.as">> := <<"username">>} -> - application:set_env(emqx_auth_mnesia, as, username); + set_old_type(username); #{<<"auth.mnesia.as">> := <<"clientid">>} -> - application:set_env(emqx_auth_mnesia, as, clientid); + set_old_type(clientid); _ when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse FromVersion =:= "4.2"-> @@ -734,7 +750,7 @@ do_read_global_auth_type(Data, FromVersion) -> "or\n" " $ emqx_ctl data import --env '{\"auth.mnesia.as\":\"clientid\"}'", []), - error(import_failed); + error({import_failed, FromVersion}); _ -> ok end. @@ -743,3 +759,6 @@ do_read_global_auth_type(Data, FromVersion) -> get_old_type() -> {ok, Type} = application:get_env(emqx_auth_mnesia, as), Type. + +set_old_type(Type) -> + application:set_env(emqx_auth_mnesia, as, Type). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl index 7ccba161b..19ec8d61e 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl @@ -39,13 +39,13 @@ cases() -> [t_import]. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard, emqx_auth_mnesia]), + emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia]), ekka_mnesia:start(), emqx_mgmt_auth:mnesia(boot), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_dashboard, emqx_auth_mnesia]), + emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_auth_mnesia]), ekka_mnesia:ensure_stopped(). init_per_testcase(_, Config) -> @@ -167,7 +167,7 @@ t_export_import(_Config) -> ?assertEqual([], emqx_acl_mnesia_db:all_acls()), - emqx_mgmt_data_backup:import_acl_mnesia(emqx_json:decode(AclData, [return_maps]), "4.3"), + emqx_mgmt_data_backup:import_acl_mnesia(emqx_json:decode(AclData, [return_maps])), timer:sleep(100), ?assertMatch([ From 506be21aa220e8f8231fef66c92624c2b16c5805 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 19 Jan 2022 10:36:12 +0800 Subject: [PATCH 12/13] fix(auth_mnesia): bump up emqx_mgmt to 4.3.10 --- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt_data_backup.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index e8b235be7..bee65781a 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.10"}, % strict semver, bump manually! + {vsn, "4.3.11"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 119906c7e..9623c6682 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -495,7 +495,7 @@ is_new_acl_data(#{<<"action">> := _, <<"access">> := _, is_new_acl_data(_) -> false. is_old_acl_data(#{<<"login">> := _, <<"topic">> := _, - <<"allow">> := Allow, <<"action">> := _}) -> is_boolean(Allow); + <<"allow">> := Allow, <<"action">> := _}) -> is_boolean(any_to_atom(Allow)); is_old_acl_data(_) -> false. do_import_acl_mnesia_by_old_data(Acls) -> From 50606a7eab3e50656e2dc4c260ce9f89d09aa6dd Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 19 Jan 2022 11:55:39 +0800 Subject: [PATCH 13/13] fix(data_import): support v4.4 --- apps/emqx_management/src/emqx_mgmt_data_backup.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 7cf72886d..07bd3d27c 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -733,6 +733,8 @@ is_version_supported2("4.1") -> true; is_version_supported2("4.3") -> true; +is_version_supported2("4.4") -> + true; is_version_supported2(Version) -> case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of match ->