From c2c089d0e94271bf45e5b0d3d2676e6e1dace576 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 28 Oct 2022 17:21:24 +0800 Subject: [PATCH 01/15] chore: remove useless information from the dashboard listener failure log --- changes/v4.3.22-en.md | 2 ++ changes/v4.3.22-zh.md | 2 ++ .../test/emqx_dashboard_SUITE.erl | 17 ++++++++++++++++- rebar.config | 2 +- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index eeed46202..61e918a81 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -2,6 +2,8 @@ ## Enhancements +- Remove useless information from the dashboard listener failure log [#9260](https://github.com/emqx/emqx/pull/9260). + - Support to use placeholders like `${var}` in the HTTP `Headers` of rule-engine's Webhook actions [#9239](https://github.com/emqx/emqx/pull/9239). - Asynchronously refresh the resources and rules during emqx boot-up [#9199](https://github.com/emqx/emqx/pull/9199). diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index a3526b146..5aa72400e 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -2,6 +2,8 @@ ## 增强 +- 删除 Dashboard 监听器失败时日志中的无用信息 [#9260](https://github.com/emqx/emqx/pull/9260). + - 支持在规则引擎的 Webhook 动作的 HTTP Headers 里使用 `${var}` 格式的占位符 [#9239](https://github.com/emqx/emqx/pull/9239)。 - 在 emqx 启动时，异步地刷新资源和规则 [#9199](https://github.com/emqx/emqx/pull/9199)。 diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl index b1a8c0194..113a315ab 100644 --- a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -54,7 +54,7 @@ groups() -> {overview, [sequence], [t_overview]}, {admins, [sequence], [t_admins_add_delete, t_admins_persist_default_password, t_default_password_persists_after_leaving_cluster]}, {rest, [sequence], [t_rest_api]}, - {cli, [sequence], [t_cli]} + {cli, [sequence], [t_cli, t_start_listener_failed_log]} ]. init_per_suite(Config) -> @@ -236,6 +236,21 @@ t_cli(_Config) -> AdminList = emqx_dashboard_admin:all_users(), ?assertEqual(2, length(AdminList)). +t_start_listener_failed_log({init, Config}) -> + _ = application:stop(emqx_dashboard), + Config; +t_start_listener_failed_log({'end', _Config}) -> + _ = application:start(emqx_dashboard), + ok; +t_start_listener_failed_log(_Config) -> + ct:capture_start(), + Options = [{num_acceptors,4}, {max_connections,512}, {inet6,false}, {ipv6_v6only,false}], + ?assertError(_, emqx_dashboard:start_listener({http, {"1.1.1.1", 8080}, Options})), + ct:capture_stop(), + I0 = ct:capture_get(), + ?assertNotMatch(nomatch, re:run(iolist_to_binary(I0), "eaddrnotavail", [])), + ok. + %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ diff --git a/rebar.config b/rebar.config index 86436101b..26fbf7f8e 100644 --- a/rebar.config +++ b/rebar.config @@ -50,7 +50,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.11"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.2"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.9"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.10"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.4"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} From 844ebf5ed67bc305a270fc7957938ac28319f663 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 28 Oct 2022 20:34:19 +0800 Subject: [PATCH 02/15] chore: fix ci typo --- .github/workflows/build_slim_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index a13c11f11..da63fd5fb 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -39,7 +39,7 @@ jobs: with: # dialyzer PLTs path: ~/.cache/rebar3/ - key: dialyer-${{ matrix.otp }} + key: dialyzer-${{ matrix.otp }} - name: make xref run: make xref - name: make dialyzer From 04c0caefac55a27051fe462d204be3bb331f98b0 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 28 Oct 2022 09:31:45 +0800 Subject: [PATCH 03/15] feat: bootstrap dashboard users from dashboard.bootstrap_users_file --- changes/v4.3.22-en.md | 3 + changes/v4.3.22-zh.md | 3 + lib-ce/emqx_dashboard/etc/emqx_dashboard.conf | 9 ++ .../emqx_dashboard/priv/emqx_dashboard.schema | 5 + .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- .../src/emqx_dashboard_admin.erl | 95 +++++++++++++++++-- 6 files changed, 106 insertions(+), 11 deletions(-) diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index bac959ca9..7bb224925 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -17,6 +17,9 @@ - Enhanced log security in ACL modules, sensitive data will be obscured. [#9242](https://github.com/emqx/emqx/pull/9242). +- Add `dashboard.bootstrap_users_file` configuration to bulk import default user&password when EMQX first starts. + + ## Bug fixes - Fix that after uploading a backup file with an UTF8 filename, HTTP API `GET /data/export` fails with status code 500 [#9224](https://github.com/emqx/emqx/pull/9224). diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 286b2a2f0..bc5ea5607 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -17,6 +17,9 @@ - 增强 ACL 模块中的日志安全性，敏感数据将被模糊化。[#9242](https://github.com/emqx/emqx/pull/9242)。 +- 增加 `dashboard.bootstrap_users_file` 配置，可以在EMQX第一次启动时批量导入默认的用户/密码。 + + ## 修复 - 修复若上传的备份文件名中包含 UTF8 字符，`GET /data/export` HTTP 接口返回 500 错误 [#9224](https://github.com/emqx/emqx/pull/9224)。 diff --git a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf index f59f27a47..7de3dbbf4 100644 --- a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf +++ b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf @@ -17,6 +17,15 @@ dashboard.default_user.login = admin ## Value: String dashboard.default_user.password = public +## Initialize users file +## Is used to add an administrative user to Dashboard when emqx is first launched, +## the format is: +## ``` +##username1:password1 +##username2:password2 +## ``` +dashboard.bootstrap_users_file = {{ platform_etc_dir }}/bootstrap_users.txt + ##-------------------------------------------------------------------- ## HTTP Listener diff --git a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema index 7ef39ac8d..93607b61b 100644 --- a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema +++ b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema @@ -10,6 +10,11 @@ {override_env, "ADMIN_PASSWORD"} ]}. +{mapping, "dashboard.bootstrap_users_file", "emqx_dashboard.bootstrap_users_file", [ + {datatype, string}, + hidden +]}. + {mapping, "dashboard.listener.http", "emqx_dashboard.listeners", [ {datatype, [integer, ip]} ]}. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 07c67545b..cb36b99a8 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.18"}, % strict semver, bump manually! + {vsn, "4.3.19"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl index a76ed9cff..5ebe18221 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -23,6 +23,7 @@ -include("emqx_dashboard.hrl"). -include_lib("emqx/include/logger.hrl"). -define(DEFAULT_PASSWORD, <<"public">>). +-define(BOOTSTRAP_USER_TAG, <<"bootstrap user">>). -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). @@ -43,6 +44,7 @@ , change_password/3 , all_users/0 , check/2 + , add_bootstrap_users/0 ]). %% gen_server Function Exports @@ -195,6 +197,75 @@ check(Username, Password) -> {error, <<"Username/Password error">>} end. +add_bootstrap_users() -> + Bootstrap = application:get_env(emqx_dashboard, bootstrap_users_file, undefined), + Size = mnesia:table_info(mqtt_admin, size), + add_bootstrap_users(Bootstrap, Size). + +add_bootstrap_users(undefined, _) -> ok; +add_bootstrap_users(_File, Size)when Size > 0 -> ok; +add_bootstrap_users(File, 0) -> + case file:open(File, [read, binary]) of + {ok, Dev} -> + {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), + case add_bootstrap_users(File, Dev, MP) of + ok -> ok; + Error -> + %% if failed add bootstrap users, we should clear all bootstrap users + mnesia:transaction(fun clear_bootstrap_users/0, []), + Error + end; + {error, Reason} = Error -> + ?LOG(error, + "failed to open the dashboard bootstrap users file(~s) for ~p", + [File, Reason] + ), + Error + end. + +add_bootstrap_users(File, Dev, MP) -> + try + add_bootstrap_user(File, Dev, MP, 1) + catch + throw:Error -> {error, Error}; + Type:Reason:Stacktrace -> + {error, {Type, Reason, Stacktrace}} + after + file:close(Dev) + end. + +add_bootstrap_user(File, Dev, MP, Line) -> + case file:read_line(Dev) of + {ok, Bin} -> + case re:run(Bin, MP, [global, {capture, all_but_first, binary}]) of + {match, [[Username, Password]]} -> + case add_user(Username, Password, ?BOOTSTRAP_USER_TAG) of + ok -> + add_bootstrap_user(File, Dev, MP, Line + 1); + Reason -> + throw(#{file => File, line => Line, content => Bin, reason => Reason}) + end; + _ -> + ?LOG(error, + "failed to bootstrap users file(~s) for Line(~w): ~ts", + [File, Line, Bin] + ), + throw(#{file => File, line => Line, content => Bin, reason => "invalid format"}) + end; + eof -> + ok; + Error -> + throw(#{file => File, line => Line, reason => Error}) + end. + +clear_bootstrap_users() -> + FoldFun = + fun(#mqtt_admin{tags = ?BOOTSTRAP_USER_TAG} = User, Acc) -> + mnesia:delete_object(User), Acc; + (_, Acc) -> Acc + end, + mnesia:foldl(FoldFun, ok, mqtt_admin). + bad_login_penalty() -> timer:sleep(2000), ok. @@ -207,16 +278,20 @@ is_valid_pwd(<>, Password) -> %%-------------------------------------------------------------------- init([]) -> - case binenv(default_user_username) of - <<>> -> ok; - UserName -> - %% Add default admin user - {ok, _} = mnesia:subscribe({table, mqtt_admin, simple}), - PasswordHash = ensure_default_user_in_db(UserName), - ok = ensure_default_user_passwd_hashed_in_pt(PasswordHash), - ok = maybe_warn_default_pwd() - end, - {ok, state}. + case add_bootstrap_users() of + ok -> + case binenv(default_user_username) of + <<>> -> ok; + UserName -> + %% Add default admin user + {ok, _} = mnesia:subscribe({table, mqtt_admin, simple}), + PasswordHash = ensure_default_user_in_db(UserName), + ok = ensure_default_user_passwd_hashed_in_pt(PasswordHash), + ok = maybe_warn_default_pwd() + end, + {ok, state}; + Error -> {stop, Error} + end. handle_call(_Req, _From, State) -> {reply, error, State}. From ec426df0a747e3345d32283768240b3cb081d679 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 28 Oct 2022 09:51:09 +0800 Subject: [PATCH 04/15] fix: typo error --- apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl | 4 ++-- changes/v4.3.22-en.md | 3 ++- changes/v4.3.22-zh.md | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl index 34f9777b9..c4b11dbe9 100644 --- a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl @@ -122,8 +122,8 @@ cli(_) -> , {"acl list ", "List all acls"} , {"acl show clientid ", "Lookup clientid acl detail"} , {"acl show username ", "Lookup username acl detail"} - , {"acl aad clientid ", "Add clientid acl"} - , {"acl add Username ", "Add username acl"} + , {"acl add clientid ", "Add clientid acl"} + , {"acl add username ", "Add username acl"} , {"acl add _all ", "Add $all acl"} , {"acl delete clientid ", "Delete clientid acl"} , {"acl delete username ", "Delete username acl"} diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 7bb224925..05d8f05b1 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -17,7 +17,8 @@ - Enhanced log security in ACL modules, sensitive data will be obscured. [#9242](https://github.com/emqx/emqx/pull/9242). -- Add `dashboard.bootstrap_users_file` configuration to bulk import default user&password when EMQX first starts. +- Add `dashboard.bootstrap_users_file` configuration to bulk import default user&password when EMQX first starts [#9256](https://github.com/emqx/emqx/pull/9256). + ## Bug fixes diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index bc5ea5607..fc7d1a435 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -17,7 +17,7 @@ - 增强 ACL 模块中的日志安全性，敏感数据将被模糊化。[#9242](https://github.com/emqx/emqx/pull/9242)。 -- 增加 `dashboard.bootstrap_users_file` 配置，可以在EMQX第一次启动时批量导入默认的用户/密码。 +- 增加 `dashboard.bootstrap_users_file` 配置，可以在EMQX第一次启动时批量导入默认的用户/密码 [#9256](https://github.com/emqx/emqx/pull/9256)。 ## 修复 From ab51684b36eccb5c7dca4cd8df7ce29bad999582 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 28 Oct 2022 14:41:19 +0800 Subject: [PATCH 05/15] chore: add more test for emqx_dashboard --- lib-ce/emqx_dashboard/etc/emqx_dashboard.conf | 2 +- .../src/emqx_dashboard_admin.erl | 2 +- .../emqx_dashboard_admin_bootstrap_user.erl | 89 +++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 lib-ce/emqx_dashboard/test/emqx_dashboard_admin_bootstrap_user.erl diff --git a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf index 7de3dbbf4..5bcebfc4d 100644 --- a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf +++ b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf @@ -24,7 +24,7 @@ dashboard.default_user.password = public ##username1:password1 ##username2:password2 ## ``` -dashboard.bootstrap_users_file = {{ platform_etc_dir }}/bootstrap_users.txt +# dashboard.bootstrap_users_file = {{ platform_etc_dir }}/bootstrap_users.txt ##-------------------------------------------------------------------- ## HTTP Listener diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl index 5ebe18221..402f3e304 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -212,7 +212,7 @@ add_bootstrap_users(File, 0) -> ok -> ok; Error -> %% if failed add bootstrap users, we should clear all bootstrap users - mnesia:transaction(fun clear_bootstrap_users/0, []), + {atomic, ok} = mnesia:transaction(fun clear_bootstrap_users/0, []), Error end; {error, Reason} = Error -> diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_admin_bootstrap_user.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_admin_bootstrap_user.erl new file mode 100644 index 000000000..452cfc25a --- /dev/null +++ b/lib-ce/emqx_dashboard/test/emqx_dashboard_admin_bootstrap_user.erl @@ -0,0 +1,89 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_dashboard_admin_bootstrap_user). + +-compile(export_all). +-compile(nowarn_export_all). +-import(emqx_dashboard_SUITE, [http_post/2]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + Config. + +end_per_suite(_) -> + ok. + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_load_ok(_) -> + Bin = <<"test-1:password-1\ntest-2:password-2">>, + File = "./bootstrap_users.txt", + ok = file:write_file(File, Bin), + _ = mnesia:clear_table(emqx_admin), + application:set_env(emqx_dashboard, bootstrap_users_file, File), + emqx_ct_helpers:start_apps([emqx_dashboard]), + ?assertEqual(#{<<"code">> => 0}, check_auth(<<"test-1">>, <<"password-1">>)), + ?assertEqual(#{<<"code">> => 0}, check_auth(<<"test-2">>, <<"password-2">>)), + ?assertEqual(#{<<"message">> => <<"Username/Password error">>}, + check_auth(<<"test-2">>, <<"password-1">>)), + emqx_ct_helpers:stop_apps([emqx_dashboard]). + +t_bootstrap_user_file_not_found(_) -> + File = "./bootstrap_users_not_exist.txt", + check_load_failed(File), + ok. + +t_load_invalid_username_failed(_) -> + Bin = <<"test-1:password-1\ntest&2:password-2">>, + File = "./bootstrap_users.txt", + ok = file:write_file(File, Bin), + check_load_failed(File), + ok. + +t_load_invalid_format_failed(_) -> + Bin = <<"test-1:password-1\ntest-2password-2">>, + File = "./bootstrap_users.txt", + ok = file:write_file(File, Bin), + check_load_failed(File), + ok. + +check_load_failed(File) -> + _ = mnesia:clear_table(emqx_admin), + application:set_env(emqx_dashboard, bootstrap_users_file, File), + ?assertError(_, emqx_ct_helpers:start_apps([emqx_dashboard])), + ?assertNot(lists:member(emqx_dashboard, application:which_applications())), + ?assertEqual(0, mnesia:table_info(mqtt_admin, size)). + + +check_auth(Username, Password) -> + {ok, Res} = http_post("auth", #{<<"username">> => Username, <<"password">> => Password}), + json(Res). + +json(Data) -> + {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. From 47d88186d17543f065d97443859328351e8d41c4 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 28 Oct 2022 14:50:20 +0800 Subject: [PATCH 06/15] chore: update emqx_auth_mnesia appup --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src | 2 +- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src index 6dd1dcdfc..3bce055f6 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mnesia, [{description, "EMQ X Authentication with Mnesia"}, - {vsn, "4.3.9"}, % strict semver, bump manually + {vsn, "4.3.10"}, % strict semver, bump manually {modules, []}, {registered, []}, {applications, [kernel,stdlib,mnesia]}, diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index 7906449db..5cf05d34d 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.7", + [{<<"4\\.3\\.[8-9]">>, + [{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, + {"4.3.7", [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[5-6]">>, @@ -33,7 +35,9 @@ {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.7", + [{<<"4\\.3\\.[8-9]">>, + [{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, + {"4.3.7", [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[5-6]">>, From d37ad38e7a25f885ca03dd535c2e55074fba063e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 28 Oct 2022 15:43:02 +0800 Subject: [PATCH 07/15] chore: clear_table mqtt_admin when failed --- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index cb36b99a8..07c67545b 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.19"}, % strict semver, bump manually! + {vsn, "4.3.18"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl index 402f3e304..a0f62e7bd 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -212,9 +212,9 @@ add_bootstrap_users(File, 0) -> ok -> ok; Error -> %% if failed add bootstrap users, we should clear all bootstrap users - {atomic, ok} = mnesia:transaction(fun clear_bootstrap_users/0, []), + {atomic, ok} = mnesia:clear_table(mqtt_admin), Error - end; + end; {error, Reason} = Error -> ?LOG(error, "failed to open the dashboard bootstrap users file(~s) for ~p", @@ -258,14 +258,6 @@ add_bootstrap_user(File, Dev, MP, Line) -> throw(#{file => File, line => Line, reason => Error}) end. -clear_bootstrap_users() -> - FoldFun = - fun(#mqtt_admin{tags = ?BOOTSTRAP_USER_TAG} = User, Acc) -> - mnesia:delete_object(User), Acc; - (_, Acc) -> Acc - end, - mnesia:foldl(FoldFun, ok, mqtt_admin). - bad_login_penalty() -> timer:sleep(2000), ok. From 93924f567f9dea880234f06d09793c72e45d8735 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 28 Oct 2022 21:39:45 +0800 Subject: [PATCH 08/15] chore: apply suggestions from code review Co-authored-by: Zaiming (Stone) Shi --- changes/v4.3.22-en.md | 3 +-- changes/v4.3.22-zh.md | 2 +- lib-ce/emqx_dashboard/etc/emqx_dashboard.conf | 5 +++-- lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 05d8f05b1..6b5ed74b7 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -17,8 +17,7 @@ - Enhanced log security in ACL modules, sensitive data will be obscured. [#9242](https://github.com/emqx/emqx/pull/9242). -- Add `dashboard.bootstrap_users_file` configuration to bulk import default user&password when EMQX first starts [#9256](https://github.com/emqx/emqx/pull/9256). - +- Add `dashboard.bootstrap_users_file` configuration to bulk import default administrative username and password when EMQX initializes the database [#9256](https://github.com/emqx/emqx/pull/9256). ## Bug fixes diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index fc7d1a435..8fb86bc22 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -17,7 +17,7 @@ - 增强 ACL 模块中的日志安全性，敏感数据将被模糊化。[#9242](https://github.com/emqx/emqx/pull/9242)。 -- 增加 `dashboard.bootstrap_users_file` 配置，可以在EMQX第一次启动时批量导入默认的用户/密码 [#9256](https://github.com/emqx/emqx/pull/9256)。 +- 增加 `dashboard.bootstrap_users_file` 配置，可以让 EMQX 初始化数据库时，从该文件批量导入一些控制台用户的用户名 / 密码 [#9256](https://github.com/emqx/emqx/pull/9256)。 ## 修复 diff --git a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf index 5bcebfc4d..18756f06a 100644 --- a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf +++ b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf @@ -18,8 +18,9 @@ dashboard.default_user.login = admin dashboard.default_user.password = public ## Initialize users file -## Is used to add an administrative user to Dashboard when emqx is first launched, -## the format is: +## Is used to add administrative dashboard users when EMQX is launched for the first time. +## This config will not take any effect once EMQX database is populated with the provided users. +## The file content format is as below: ## ``` ##username1:password1 ##username2:password2 diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl index a0f62e7bd..b2589bd06 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -23,7 +23,7 @@ -include("emqx_dashboard.hrl"). -include_lib("emqx/include/logger.hrl"). -define(DEFAULT_PASSWORD, <<"public">>). --define(BOOTSTRAP_USER_TAG, <<"bootstrap user">>). +-define(BOOTSTRAP_USER_TAG, <<"bootstrapped">>). -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). @@ -242,7 +242,7 @@ add_bootstrap_user(File, Dev, MP, Line) -> case add_user(Username, Password, ?BOOTSTRAP_USER_TAG) of ok -> add_bootstrap_user(File, Dev, MP, Line + 1); - Reason -> + {error, Reason} -> throw(#{file => File, line => Line, content => Bin, reason => Reason}) end; _ -> @@ -254,7 +254,7 @@ add_bootstrap_user(File, Dev, MP, Line) -> end; eof -> ok; - Error -> + {error, Error} -> throw(#{file => File, line => Line, reason => Error}) end. From 9167055f561280f50e19dc55a21ee00be3847dad Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 28 Oct 2022 21:49:16 +0800 Subject: [PATCH 09/15] chore: refactor init_default_admin_user/0 function --- .../src/emqx_dashboard_admin.erl | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl index b2589bd06..223d99fa5 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -44,7 +44,7 @@ , change_password/3 , all_users/0 , check/2 - , add_bootstrap_users/0 + , init_bootstrap_users/0 ]). %% gen_server Function Exports @@ -197,18 +197,18 @@ check(Username, Password) -> {error, <<"Username/Password error">>} end. -add_bootstrap_users() -> +init_bootstrap_users() -> Bootstrap = application:get_env(emqx_dashboard, bootstrap_users_file, undefined), Size = mnesia:table_info(mqtt_admin, size), - add_bootstrap_users(Bootstrap, Size). + init_bootstrap_users(Bootstrap, Size). -add_bootstrap_users(undefined, _) -> ok; -add_bootstrap_users(_File, Size)when Size > 0 -> ok; -add_bootstrap_users(File, 0) -> +init_bootstrap_users(undefined, _) -> ok; +init_bootstrap_users(_File, Size)when Size > 0 -> ok; +init_bootstrap_users(File, 0) -> case file:open(File, [read, binary]) of {ok, Dev} -> {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), - case add_bootstrap_users(File, Dev, MP) of + case init_bootstrap_users(File, Dev, MP) of ok -> ok; Error -> %% if failed add bootstrap users, we should clear all bootstrap users @@ -223,7 +223,7 @@ add_bootstrap_users(File, 0) -> Error end. -add_bootstrap_users(File, Dev, MP) -> +init_bootstrap_users(File, Dev, MP) -> try add_bootstrap_user(File, Dev, MP, 1) catch @@ -270,21 +270,23 @@ is_valid_pwd(<>, Password) -> %%-------------------------------------------------------------------- init([]) -> - case add_bootstrap_users() of - ok -> - case binenv(default_user_username) of - <<>> -> ok; - UserName -> - %% Add default admin user - {ok, _} = mnesia:subscribe({table, mqtt_admin, simple}), - PasswordHash = ensure_default_user_in_db(UserName), - ok = ensure_default_user_passwd_hashed_in_pt(PasswordHash), - ok = maybe_warn_default_pwd() - end, - {ok, state}; - Error -> {stop, Error} + case init_bootstrap_users() of + ok -> init_default_admin_user(); + {error, Error} -> {stop, Error} end. +init_default_admin_user() -> + case binenv(default_user_username) of + <<>> -> ok; + UserName -> + %% Add default admin user + {ok, _} = mnesia:subscribe({table, mqtt_admin, simple}), + PasswordHash = ensure_default_user_in_db(UserName), + ok = ensure_default_user_passwd_hashed_in_pt(PasswordHash), + ok = maybe_warn_default_pwd() + end, + {ok, state}. + handle_call(_Req, _From, State) -> {reply, error, State}. From 3f17119e366bada5ad0e319b176202efa68fbc95 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 28 Oct 2022 22:23:52 +0800 Subject: [PATCH 10/15] fix: run the 'message.acked' hook when ACK received from CoAP devices --- apps/emqx_coap/src/emqx_coap.app.src | 2 +- apps/emqx_coap/src/emqx_coap.appup.src | 14 ++++-- apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl | 45 +++++++++++++------ .../src/emqx_coap_pubsub_resource.erl | 20 ++++++++- apps/emqx_coap/src/emqx_coap_resource.erl | 16 ++++++- apps/emqx_coap/test/emqx_coap_SUITE.erl | 9 +++- changes/v4.3.22-en.md | 5 +++ changes/v4.3.22-zh.md | 3 ++ 8 files changed, 91 insertions(+), 23 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src index dce5ef4f4..b4ea07898 100644 --- a/apps/emqx_coap/src/emqx_coap.app.src +++ b/apps/emqx_coap/src/emqx_coap.app.src @@ -1,6 +1,6 @@ {application, emqx_coap, [{description, "EMQ X CoAP Gateway"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gen_coap]}, diff --git a/apps/emqx_coap/src/emqx_coap.appup.src b/apps/emqx_coap/src/emqx_coap.appup.src index b73e26be6..1e766c45b 100644 --- a/apps/emqx_coap/src/emqx_coap.appup.src +++ b/apps/emqx_coap/src/emqx_coap.appup.src @@ -1,9 +1,15 @@ %% -*-: erlang -*- {VSN, - [{"4.3.0",[ - {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []}]}, + [{<<"4\\.3\\.[0-1]">>,[ + {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []}, + {load_module, emqx_coap_pubsub_resource, brutal_purge, soft_purge, []}, + {load_module, emqx_coap_resource, brutal_purge, soft_purge, []} + ]}, {<<".*">>, []}], - [{"4.3.0",[ - {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []}]}, + [{<<"4\\.3\\.[0-1]">>,[ + {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []}, + {load_module, emqx_coap_pubsub_resource, brutal_purge, soft_purge, []}, + {load_module, emqx_coap_resource, brutal_purge, soft_purge, []} + ]}, {<<".*">>, []}] }. diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl index 63647de5d..c10efd655 100644 --- a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl @@ -31,6 +31,9 @@ -export([ subscribe/2 , unsubscribe/2 , publish/3 + , received_puback/2 + , message_payload/1 + , message_topic/1 ]). -export([ client_pid/4 @@ -95,6 +98,15 @@ unsubscribe(Pid, Topic) -> publish(Pid, Topic, Payload) -> gen_server:call(Pid, {publish, Topic, Payload}). +received_puback(Pid, Msg) -> + gen_server:cast(Pid, {received_puback, Msg}). + +message_payload(#message{payload = Payload}) -> + Payload. + +message_topic(#message{topic = Topic}) -> + Topic. + %% For emqx_management plugin call(Pid, Msg) -> call(Pid, Msg, infinity). @@ -172,13 +184,19 @@ handle_call(Request, _From, State) -> ?LOG(error, "adapter unexpected call ~p", [Request]), {reply, ignored, State, hibernate}. +handle_cast({received_puback, Msg}, State) -> + %% NOTE: the counter named 'messages.acked', but the hook named 'message.acked'! + ok = emqx_metrics:inc('messages.acked'), + _ = emqx_hooks:run('message.acked', [conninfo(State), Msg]), + {noreply, State, hibernate}; + handle_cast(Msg, State) -> ?LOG(error, "broker_api unexpected cast ~p", [Msg]), {noreply, State, hibernate}. -handle_info({deliver, _Topic, #message{topic = Topic, payload = Payload}}, +handle_info({deliver, _Topic, #message{} = Msg}, State = #state{sub_topics = Subscribers}) -> - deliver([{Topic, Payload}], Subscribers), + deliver([Msg], Subscribers), {noreply, State, hibernate}; handle_info(check_alive, State = #state{sub_topics = []}) -> @@ -271,27 +289,25 @@ packet_to_message(Topic, Payload, %% Deliver deliver([], _) -> ok; -deliver([Pub | More], Subscribers) -> - ok = do_deliver(Pub, Subscribers), +deliver([Msg | More], Subscribers) -> + ok = do_deliver(Msg, Subscribers), deliver(More, Subscribers). -do_deliver({Topic, Payload}, Subscribers) -> +do_deliver(Msg, Subscribers) -> %% handle PUBLISH packet from broker - ?LOG(debug, "deliver message from broker Topic=~p, Payload=~p", [Topic, Payload]), - deliver_to_coap(Topic, Payload, Subscribers), + ?LOG(debug, "deliver message from broker, msg: ~p", [Msg]), + deliver_to_coap(Msg, Subscribers), ok. -deliver_to_coap(_TopicName, _Payload, []) -> +deliver_to_coap(_Msg, []) -> ok; -deliver_to_coap(TopicName, Payload, [{TopicFilter, {IsWild, CoapPid}} | T]) -> +deliver_to_coap(#message{topic = TopicName} = Msg, [{TopicFilter, {IsWild, CoapPid}} | T]) -> Matched = case IsWild of true -> emqx_topic:match(TopicName, TopicFilter); false -> TopicName =:= TopicFilter end, - %?LOG(debug, "deliver_to_coap Matched=~p, CoapPid=~p, TopicName=~p, Payload=~p, T=~p", - % [Matched, CoapPid, TopicName, Payload, T]), - Matched andalso (CoapPid ! {dispatch, TopicName, Payload}), - deliver_to_coap(TopicName, Payload, T). + Matched andalso (CoapPid ! {dispatch, Msg}), + deliver_to_coap(Msg, T). %%-------------------------------------------------------------------- %% Helper funcs @@ -328,12 +344,13 @@ chann_info(State) -> will_msg => undefined }. -conninfo(#state{peername = Peername, +conninfo(#state{peername = {PeerHost, _} = Peername, clientid = ClientId, connected_at = ConnectedAt}) -> #{socktype => udp, sockname => {{127, 0, 0, 1}, 5683}, peername => Peername, + peerhost => PeerHost, peercert => nossl, %% TODO: dtls conn_mod => ?MODULE, proto_name => <<"CoAP">>, diff --git a/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl b/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl index 8ab33628b..445c8b708 100644 --- a/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl +++ b/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl @@ -138,16 +138,32 @@ coap_unobserve({state, ChId, Prefix, TopicPath}) -> ok. handle_info({dispatch, Topic, Payload}, State) -> + %% This clause should never be matched any more. We keep it here to handle + %% the old format messages during the release upgrade. + %% In this case the second function clause of `coap_ack/2` will be called, + %% and the ACKs is discarded. ?LOG(debug, "dispatch Topic=~p, Payload=~p", [Topic, Payload]), {ok, Ret} = emqx_coap_pubsub_topics:reset_topic_info(Topic, Payload), ?LOG(debug, "Updated publish info of topic=~p, the Ret is ~p", [Topic, Ret]), {notify, [], #coap_content{format = <<"application/octet-stream">>, payload = Payload}, State}; +handle_info({dispatch, Msg}, State) -> + Payload = emqx_coap_mqtt_adapter:message_payload(Msg), + Topic = emqx_coap_mqtt_adapter:message_topic(Msg), + {ok, Ret} = emqx_coap_pubsub_topics:reset_topic_info(Topic, Payload), + ?LOG(debug, "Updated publish info of topic=~p, the Ret is ~p", [Topic, Ret]), + {notify, {pub, Msg}, #coap_content{format = <<"application/octet-stream">>, payload = Payload}, State}; handle_info(Message, State) -> ?LOG(error, "Unknown Message ~p", [Message]), {noreply, State}. -coap_ack(_Ref, State) -> {ok, State}. - +coap_ack({pub, Msg}, State) -> + ?LOG(debug, "received coap ack for publish msg: ~p", [Msg]), + Pid = get(mqtt_client_pid), + emqx_coap_mqtt_adapter:received_puback(Pid, Msg), + {ok, State}; +coap_ack(_Ref, State) -> + ?LOG(debug, "received coap ack: ~p", [_Ref]), + {ok, State}. %%-------------------------------------------------------------------- %% Internal Functions diff --git a/apps/emqx_coap/src/emqx_coap_resource.erl b/apps/emqx_coap/src/emqx_coap_resource.erl index ab5d31a63..daa536540 100644 --- a/apps/emqx_coap/src/emqx_coap_resource.erl +++ b/apps/emqx_coap/src/emqx_coap_resource.erl @@ -104,12 +104,26 @@ coap_unobserve({state, ChId, Prefix, Topic}) -> ok. handle_info({dispatch, Topic, Payload}, State) -> + %% This clause should never be matched any more. We keep it here to handle + %% the old format messages during the release upgrade. + %% In this case the second function clause of `coap_ack/2` will be called, + %% and the ACKs is discarded. ?LOG(debug, "dispatch Topic=~p, Payload=~p", [Topic, Payload]), {notify, [], #coap_content{format = <<"application/octet-stream">>, payload = Payload}, State}; +handle_info({dispatch, Msg}, State) -> + Payload = emqx_coap_mqtt_adapter:message_payload(Msg), + {notify, {pub, Msg}, #coap_content{format = <<"application/octet-stream">>, payload = Payload}, State}; handle_info(Message, State) -> emqx_coap_mqtt_adapter:handle_info(Message, State). -coap_ack(_Ref, State) -> {ok, State}. +coap_ack({pub, Msg}, State) -> + ?LOG(debug, "received coap ack for publish msg: ~p", [Msg]), + Pid = get(mqtt_client_pid), + emqx_coap_mqtt_adapter:received_puback(Pid, Msg), + {ok, State}; +coap_ack(_Ref, State) -> + ?LOG(debug, "received coap ack: ~p", [_Ref]), + {ok, State}. get_auth(Query) -> get_auth(Query, #coap_mqtt_auth{}). diff --git a/apps/emqx_coap/test/emqx_coap_SUITE.erl b/apps/emqx_coap/test/emqx_coap_SUITE.erl index 6d8294ef5..0973382c6 100644 --- a/apps/emqx_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_SUITE.erl @@ -91,7 +91,7 @@ t_observe(_Config) -> Topic = <<"abc">>, TopicStr = binary_to_list(Topic), Payload = <<"123">>, Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", - {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), + {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), ?LOGT("observer Pid=~p, N=~p, Code=~p, Content=~p", [Pid, N, Code, Content]), [SubPid] = emqx:subscribers(Topic), @@ -195,12 +195,16 @@ t_one_clientid_sub_2_topics(_Config) -> [SubPid] = emqx:subscribers(Topic2), ?assert(is_pid(SubPid)), + CntrAcked1 = emqx_metrics:val('messages.acked'), emqx:publish(emqx_message:make(Topic1, Payload1)), Notif1 = receive_notification(), ?LOGT("observer 1 get Notif=~p", [Notif1]), {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv1}} = Notif1, ?assertEqual(Payload1, PayloadRecv1), + timer:sleep(100), + CntrAcked2 = emqx_metrics:val('messages.acked'), + ?assertEqual(CntrAcked2, CntrAcked1 + 1), emqx:publish(emqx_message:make(Topic2, Payload2)), @@ -208,6 +212,9 @@ t_one_clientid_sub_2_topics(_Config) -> ?LOGT("observer 2 get Notif=~p", [Notif2]), {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv2}} = Notif2, ?assertEqual(Payload2, PayloadRecv2), + timer:sleep(100), + CntrAcked3 = emqx_metrics:val('messages.acked'), + ?assertEqual(CntrAcked3, CntrAcked2 + 1), er_coap_observer:stop(Pid1), er_coap_observer:stop(Pid2). diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index bac959ca9..98b0073d0 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -2,6 +2,11 @@ ## Enhancements +- We now trigger the `'message.acked'` hook after the CoAP gateway sends a message to the device and receives the ACK from the device [#9264](https://github.com/emqx/emqx/pull/9264). + With this change, the CoAP gateway can be combined with the offline message caching function (in the + emqx enterprise), so that CoAP devices are able to read the missed messages from the database when + it is online again. + - Support to use placeholders like `${var}` in the HTTP `Headers` of rule-engine's Webhook actions [#9239](https://github.com/emqx/emqx/pull/9239). - Asynchronously refresh the resources and rules during emqx boot-up [#9199](https://github.com/emqx/emqx/pull/9199). diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 286b2a2f0..f253fe4b5 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -2,6 +2,9 @@ ## 增强 +- 当 CoAP 网关给设备投递消息并收到设备发来的确认之后，回调 `'message.acked'` 钩子 [#9264](https://github.com/emqx/emqx/pull/9264)。 + 有了这个改动，CoAP 网关可以配合 EMQX (企业版)的离线消息缓存功能，让 CoAP 设备重新上线之后，从数据库读取其离线状态下错过的消息。 + - 支持在规则引擎的 Webhook 动作的 HTTP Headers 里使用 `${var}` 格式的占位符 [#9239](https://github.com/emqx/emqx/pull/9239)。 - 在 emqx 启动时，异步地刷新资源和规则 [#9199](https://github.com/emqx/emqx/pull/9199)。 From 3aeb21cc3792e9221ad47724b8ec87dcef4594ab Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 31 Oct 2022 19:54:34 +0800 Subject: [PATCH 11/15] Revert "feat: bootstrap dashboard users from dashboard.bootstrap_users_file" --- .../src/emqx_acl_mnesia_cli.erl | 4 +- .../src/emqx_auth_mnesia.app.src | 2 +- .../src/emqx_auth_mnesia.appup.src | 8 +- changes/v4.3.22-en.md | 3 - changes/v4.3.22-zh.md | 3 - lib-ce/emqx_dashboard/etc/emqx_dashboard.conf | 10 --- .../emqx_dashboard/priv/emqx_dashboard.schema | 5 -- .../src/emqx_dashboard_admin.erl | 69 -------------- .../emqx_dashboard_admin_bootstrap_user.erl | 89 ------------------- 9 files changed, 5 insertions(+), 188 deletions(-) delete mode 100644 lib-ce/emqx_dashboard/test/emqx_dashboard_admin_bootstrap_user.erl diff --git a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl index c4b11dbe9..34f9777b9 100644 --- a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl @@ -122,8 +122,8 @@ cli(_) -> , {"acl list ", "List all acls"} , {"acl show clientid ", "Lookup clientid acl detail"} , {"acl show username ", "Lookup username acl detail"} - , {"acl add clientid ", "Add clientid acl"} - , {"acl add username ", "Add username acl"} + , {"acl aad clientid ", "Add clientid acl"} + , {"acl add Username ", "Add username acl"} , {"acl add _all ", "Add $all acl"} , {"acl delete clientid ", "Delete clientid acl"} , {"acl delete username ", "Delete username acl"} diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src index 3bce055f6..6dd1dcdfc 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mnesia, [{description, "EMQ X Authentication with Mnesia"}, - {vsn, "4.3.10"}, % strict semver, bump manually + {vsn, "4.3.9"}, % strict semver, bump manually {modules, []}, {registered, []}, {applications, [kernel,stdlib,mnesia]}, diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index 5cf05d34d..7906449db 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -1,9 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[8-9]">>, - [{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, - {"4.3.7", + [{"4.3.7", [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[5-6]">>, @@ -35,9 +33,7 @@ {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[8-9]">>, - [{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, - {"4.3.7", + [{"4.3.7", [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[5-6]">>, diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 2b6cb1805..98b0073d0 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -22,9 +22,6 @@ - Enhanced log security in ACL modules, sensitive data will be obscured. [#9242](https://github.com/emqx/emqx/pull/9242). -- Add `dashboard.bootstrap_users_file` configuration to bulk import default administrative username and password when EMQX initializes the database [#9256](https://github.com/emqx/emqx/pull/9256). - - ## Bug fixes - Fix that after uploading a backup file with an UTF8 filename, HTTP API `GET /data/export` fails with status code 500 [#9224](https://github.com/emqx/emqx/pull/9224). diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 584300755..f253fe4b5 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -20,9 +20,6 @@ - 增强 ACL 模块中的日志安全性，敏感数据将被模糊化。[#9242](https://github.com/emqx/emqx/pull/9242)。 -- 增加 `dashboard.bootstrap_users_file` 配置，可以让 EMQX 初始化数据库时，从该文件批量导入一些控制台用户的用户名 / 密码 [#9256](https://github.com/emqx/emqx/pull/9256)。 - - ## 修复 - 修复若上传的备份文件名中包含 UTF8 字符，`GET /data/export` HTTP 接口返回 500 错误 [#9224](https://github.com/emqx/emqx/pull/9224)。 diff --git a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf index 18756f06a..f59f27a47 100644 --- a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf +++ b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf @@ -17,16 +17,6 @@ dashboard.default_user.login = admin ## Value: String dashboard.default_user.password = public -## Initialize users file -## Is used to add administrative dashboard users when EMQX is launched for the first time. -## This config will not take any effect once EMQX database is populated with the provided users. -## The file content format is as below: -## ``` -##username1:password1 -##username2:password2 -## ``` -# dashboard.bootstrap_users_file = {{ platform_etc_dir }}/bootstrap_users.txt - ##-------------------------------------------------------------------- ## HTTP Listener diff --git a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema index 93607b61b..7ef39ac8d 100644 --- a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema +++ b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema @@ -10,11 +10,6 @@ {override_env, "ADMIN_PASSWORD"} ]}. -{mapping, "dashboard.bootstrap_users_file", "emqx_dashboard.bootstrap_users_file", [ - {datatype, string}, - hidden -]}. - {mapping, "dashboard.listener.http", "emqx_dashboard.listeners", [ {datatype, [integer, ip]} ]}. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl index 223d99fa5..a76ed9cff 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -23,7 +23,6 @@ -include("emqx_dashboard.hrl"). -include_lib("emqx/include/logger.hrl"). -define(DEFAULT_PASSWORD, <<"public">>). --define(BOOTSTRAP_USER_TAG, <<"bootstrapped">>). -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). @@ -44,7 +43,6 @@ , change_password/3 , all_users/0 , check/2 - , init_bootstrap_users/0 ]). %% gen_server Function Exports @@ -197,67 +195,6 @@ check(Username, Password) -> {error, <<"Username/Password error">>} end. -init_bootstrap_users() -> - Bootstrap = application:get_env(emqx_dashboard, bootstrap_users_file, undefined), - Size = mnesia:table_info(mqtt_admin, size), - init_bootstrap_users(Bootstrap, Size). - -init_bootstrap_users(undefined, _) -> ok; -init_bootstrap_users(_File, Size)when Size > 0 -> ok; -init_bootstrap_users(File, 0) -> - case file:open(File, [read, binary]) of - {ok, Dev} -> - {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), - case init_bootstrap_users(File, Dev, MP) of - ok -> ok; - Error -> - %% if failed add bootstrap users, we should clear all bootstrap users - {atomic, ok} = mnesia:clear_table(mqtt_admin), - Error - end; - {error, Reason} = Error -> - ?LOG(error, - "failed to open the dashboard bootstrap users file(~s) for ~p", - [File, Reason] - ), - Error - end. - -init_bootstrap_users(File, Dev, MP) -> - try - add_bootstrap_user(File, Dev, MP, 1) - catch - throw:Error -> {error, Error}; - Type:Reason:Stacktrace -> - {error, {Type, Reason, Stacktrace}} - after - file:close(Dev) - end. - -add_bootstrap_user(File, Dev, MP, Line) -> - case file:read_line(Dev) of - {ok, Bin} -> - case re:run(Bin, MP, [global, {capture, all_but_first, binary}]) of - {match, [[Username, Password]]} -> - case add_user(Username, Password, ?BOOTSTRAP_USER_TAG) of - ok -> - add_bootstrap_user(File, Dev, MP, Line + 1); - {error, Reason} -> - throw(#{file => File, line => Line, content => Bin, reason => Reason}) - end; - _ -> - ?LOG(error, - "failed to bootstrap users file(~s) for Line(~w): ~ts", - [File, Line, Bin] - ), - throw(#{file => File, line => Line, content => Bin, reason => "invalid format"}) - end; - eof -> - ok; - {error, Error} -> - throw(#{file => File, line => Line, reason => Error}) - end. - bad_login_penalty() -> timer:sleep(2000), ok. @@ -270,12 +207,6 @@ is_valid_pwd(<>, Password) -> %%-------------------------------------------------------------------- init([]) -> - case init_bootstrap_users() of - ok -> init_default_admin_user(); - {error, Error} -> {stop, Error} - end. - -init_default_admin_user() -> case binenv(default_user_username) of <<>> -> ok; UserName -> diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_admin_bootstrap_user.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_admin_bootstrap_user.erl deleted file mode 100644 index 452cfc25a..000000000 --- a/lib-ce/emqx_dashboard/test/emqx_dashboard_admin_bootstrap_user.erl +++ /dev/null @@ -1,89 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_dashboard_admin_bootstrap_user). - --compile(export_all). --compile(nowarn_export_all). --import(emqx_dashboard_SUITE, [http_post/2]). - --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). - -%%-------------------------------------------------------------------- -%% Setups -%%-------------------------------------------------------------------- - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - Config. - -end_per_suite(_) -> - ok. - -%%-------------------------------------------------------------------- -%% Test cases -%%-------------------------------------------------------------------- - -t_load_ok(_) -> - Bin = <<"test-1:password-1\ntest-2:password-2">>, - File = "./bootstrap_users.txt", - ok = file:write_file(File, Bin), - _ = mnesia:clear_table(emqx_admin), - application:set_env(emqx_dashboard, bootstrap_users_file, File), - emqx_ct_helpers:start_apps([emqx_dashboard]), - ?assertEqual(#{<<"code">> => 0}, check_auth(<<"test-1">>, <<"password-1">>)), - ?assertEqual(#{<<"code">> => 0}, check_auth(<<"test-2">>, <<"password-2">>)), - ?assertEqual(#{<<"message">> => <<"Username/Password error">>}, - check_auth(<<"test-2">>, <<"password-1">>)), - emqx_ct_helpers:stop_apps([emqx_dashboard]). - -t_bootstrap_user_file_not_found(_) -> - File = "./bootstrap_users_not_exist.txt", - check_load_failed(File), - ok. - -t_load_invalid_username_failed(_) -> - Bin = <<"test-1:password-1\ntest&2:password-2">>, - File = "./bootstrap_users.txt", - ok = file:write_file(File, Bin), - check_load_failed(File), - ok. - -t_load_invalid_format_failed(_) -> - Bin = <<"test-1:password-1\ntest-2password-2">>, - File = "./bootstrap_users.txt", - ok = file:write_file(File, Bin), - check_load_failed(File), - ok. - -check_load_failed(File) -> - _ = mnesia:clear_table(emqx_admin), - application:set_env(emqx_dashboard, bootstrap_users_file, File), - ?assertError(_, emqx_ct_helpers:start_apps([emqx_dashboard])), - ?assertNot(lists:member(emqx_dashboard, application:which_applications())), - ?assertEqual(0, mnesia:table_info(mqtt_admin, size)). - - -check_auth(Username, Password) -> - {ok, Res} = http_post("auth", #{<<"username">> => Username, <<"password">> => Password}), - json(Res). - -json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. From e4473d34c04e3f0701e55f7881699bc1218a79ee Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 31 Oct 2022 19:50:55 +0800 Subject: [PATCH 12/15] feat: bootstrap app/secret from management.bootstrap_apps_file --- apps/emqx_management/etc/emqx_management.conf | 10 +++ .../priv/emqx_management.schema | 5 ++ apps/emqx_management/src/emqx_mgmt_app.erl | 15 ++-- apps/emqx_management/src/emqx_mgmt_auth.erl | 65 ++++++++++++++ .../test/emqx_mgmt_bootstrap_app_SUITE.erl | 89 +++++++++++++++++++ changes/v4.3.22-en.md | 2 + changes/v4.3.22-zh.md | 2 + 7 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl diff --git a/apps/emqx_management/etc/emqx_management.conf b/apps/emqx_management/etc/emqx_management.conf index f9e6a518c..1dbfc1583 100644 --- a/apps/emqx_management/etc/emqx_management.conf +++ b/apps/emqx_management/etc/emqx_management.conf @@ -20,6 +20,16 @@ management.default_application.id = admin ## Value: String management.default_application.secret = public +## Initialize apps file +## Is used to add administrative app/secrets when EMQX is launched for the first time. +## This config will not take any effect once EMQX database is populated with the provided apps. +## The file content format is as below: +## ``` +##819e5db182cf:l9C5suZClIF3FvdzWqmINrVU61WNfIjcglxw9CVM7y1VI +##bb5a6cf1c06a:WuNRRgcRTGiNcuyrE49Bpwz4PGPrRnP4hUMi647kNSbN +## ``` +# management.bootstrap_apps_file = {{ platform_etc_dir }}/bootstrap_apps.txt + ##-------------------------------------------------------------------- ## HTTP Listener diff --git a/apps/emqx_management/priv/emqx_management.schema b/apps/emqx_management/priv/emqx_management.schema index e0cc47d2f..0b91509f1 100644 --- a/apps/emqx_management/priv/emqx_management.schema +++ b/apps/emqx_management/priv/emqx_management.schema @@ -6,6 +6,11 @@ {datatype, integer} ]}. +{mapping, "management.bootstrap_apps_file", "emqx_management.bootstrap_apps_file", [ + {datatype, string}, + hidden +]}. + {mapping, "management.default_application.id", "emqx_management.default_application_id", [ {default, undefined}, {datatype, string} diff --git a/apps/emqx_management/src/emqx_mgmt_app.erl b/apps/emqx_management/src/emqx_mgmt_app.erl index f1708c906..60e552bd2 100644 --- a/apps/emqx_management/src/emqx_mgmt_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_app.erl @@ -25,11 +25,16 @@ ]). start(_Type, _Args) -> - {ok, Sup} = emqx_mgmt_sup:start_link(), - _ = emqx_mgmt_auth:add_default_app(), - emqx_mgmt_http:start_listeners(), - emqx_mgmt_cli:load(), - {ok, Sup}. + case emqx_mgmt_auth:init_bootstrap_apps() of + ok -> + {ok, Sup} = emqx_mgmt_sup:start_link(), + _ = emqx_mgmt_auth:add_default_app(), + emqx_mgmt_http:start_listeners(), + emqx_mgmt_cli:load(), + {ok, Sup}; + {error, _Reason} = Error -> + Error + end. stop(_State) -> emqx_mgmt_http:stop_listeners(). diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 6eca989cf..3f796177b 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -35,6 +35,7 @@ , update_app/5 , del_app/1 , list_apps/0 + , init_bootstrap_apps/0 ]). %% APP Auth/ACL API @@ -44,6 +45,8 @@ -record(mqtt_app, {id, secret, name, desc, status, expired}). +-define(BOOTSTRAP_TAG, <<"Bootstrapped From File">>). + -type(appid() :: binary()). -type(appsecret() :: binary()). @@ -77,6 +80,68 @@ add_default_app() -> add_app(AppId1, <<"Default">>, AppSecret1, <<"Application user">>, true, undefined) end. +init_bootstrap_apps() -> + Bootstrap = application:get_env(emqx_management, bootstrap_apps_file, undefined), + Size = mnesia:table_info(mqtt_app, size), + init_bootstrap_apps(Bootstrap, Size). + +init_bootstrap_apps(undefined, _) -> ok; +init_bootstrap_apps(_File, Size)when Size > 0 -> ok; +init_bootstrap_apps(File, 0) -> + case file:open(File, [read, binary]) of + {ok, Dev} -> + {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), + case init_bootstrap_apps(File, Dev, MP) of + ok -> ok; + Error -> + %% if failed add bootstrap users, we should clear all bootstrap apps + {atomic, ok} = mnesia:clear_table(mqtt_app), + Error + end; + {error, Reason} = Error -> + ?LOG(error, + "failed to open the mgmt bootstrap apps file(~s) for ~p", + [File, Reason] + ), + Error + end. + +init_bootstrap_apps(File, Dev, MP) -> + try + add_bootstrap_app(File, Dev, MP, 1) + catch + throw:Error -> {error, Error}; + Type:Reason:Stacktrace -> + {error, {Type, Reason, Stacktrace}} + after + file:close(Dev) + end. + +add_bootstrap_app(File, Dev, MP, Line) -> + case file:read_line(Dev) of + {ok, Bin} -> + case re:run(Bin, MP, [global, {capture, all_but_first, binary}]) of + {match, [[AppId, AppSecret]]} -> + Name = <<"bootstraped">>, + case add_app(AppId, Name, AppSecret, ?BOOTSTRAP_TAG, true, undefined) of + {ok, _} -> + add_bootstrap_app(File, Dev, MP, Line + 1); + {error, Reason} -> + throw(#{file => File, line => Line, content => Bin, reason => Reason}) + end; + _ -> + ?LOG(error, + "failed to bootstrap apps file(~s) for Line(~w): ~ts", + [File, Line, Bin] + ), + throw(#{file => File, line => Line, content => Bin, reason => "invalid format"}) + end; + eof -> + ok; + {error, Error} -> + throw(#{file => File, line => Line, reason => Error}) + end. + -spec(add_app(appid(), binary()) -> {ok, appsecret()} | {error, term()}). add_app(AppId, Name) when is_binary(AppId) -> add_app(AppId, Name, <<"Application user">>, true, undefined). diff --git a/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl new file mode 100644 index 000000000..7748c0224 --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl @@ -0,0 +1,89 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_bootstrap_app_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:boot_modules(all), + application:load(emqx_modules), + application:load(emqx_modules_spec), + application:load(emqx_management), + application:stop(emqx_rule_engine), + ekka_mnesia:start(), + emqx_ct_helpers:start_apps([]), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([]), + ok. + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_load_ok(_) -> + application:stop(emqx_management), + Bin = <<"test-1:secret-1\ntest-2:secret-2">>, + File = "./bootstrap_apps.txt", + ok = file:write_file(File, Bin), + _ = mnesia:clear_table(mqtt_app), + application:set_env(emqx_management, bootstrap_apps_file, File), + {ok, _} = application:ensure_all_started(emqx_management), + ?assert(emqx_mgmt_auth:is_authorized(<<"test-1">>, <<"secret-1">>)), + ?assert(emqx_mgmt_auth:is_authorized(<<"test-2">>, <<"secret-2">>)), + ?assertNot(emqx_mgmt_auth:is_authorized(<<"test-2">>, <<"secret-1">>)), + application:stop(emqx_management). + +t_bootstrap_user_file_not_found(_) -> + File = "./bootstrap_apps_not_exist.txt", + check_load_failed(File), + ok. + +t_load_invalid_username_failed(_) -> + Bin = <<"test-1:password-1\ntest&2:password-2">>, + File = "./bootstrap_apps.txt", + ok = file:write_file(File, Bin), + check_load_failed(File), + ok. + +t_load_invalid_format_failed(_) -> + Bin = <<"test-1:password-1\ntest-2password-2">>, + File = "./bootstrap_apps.txt", + ok = file:write_file(File, Bin), + check_load_failed(File), + ok. + +check_load_failed(File) -> + _ = mnesia:clear_table(mqtt_app), + application:stop(emqx_management), + application:set_env(emqx_management, bootstrap_apps_file, File), + ?assertMatch({error, _}, application:ensure_all_started(emqx_management)), + ?assertNot(lists:member(emqx_management, application:which_applications())), + ?assertEqual(0, mnesia:table_info(mqtt_app, size)). diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 2b6cb1805..8e0fc4701 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -24,6 +24,8 @@ - Add `dashboard.bootstrap_users_file` configuration to bulk import default administrative username and password when EMQX initializes the database [#9256](https://github.com/emqx/emqx/pull/9256). +- Add `management.bootstrap_apps_file` configuration to bulk import default app/secret when EMQX initializes the database [#9273](https://github.com/emqx/emqx/pull/9273). + ## Bug fixes diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 584300755..5cec12db3 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -22,6 +22,8 @@ - 增加 `dashboard.bootstrap_users_file` 配置，可以让 EMQX 初始化数据库时，从该文件批量导入一些控制台用户的用户名 / 密码 [#9256](https://github.com/emqx/emqx/pull/9256)。 +- 增加 `management.bootstrap_apps_file` 配置，可以让 EMQX 初始化数据库时，从该文件批量导入一些 APP / Secret [#9273](https://github.com/emqx/emqx/pull/9273)。 + ## 修复 From 7e322bd392ea2d66ca167e51b068ec2ff2569cd1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 31 Oct 2022 16:35:25 +0100 Subject: [PATCH 13/15] chore: fix typos in command line examples --- apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl index 34f9777b9..c4b11dbe9 100644 --- a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl @@ -122,8 +122,8 @@ cli(_) -> , {"acl list ", "List all acls"} , {"acl show clientid ", "Lookup clientid acl detail"} , {"acl show username ", "Lookup username acl detail"} - , {"acl aad clientid ", "Add clientid acl"} - , {"acl add Username ", "Add username acl"} + , {"acl add clientid ", "Add clientid acl"} + , {"acl add username ", "Add username acl"} , {"acl add _all ", "Add $all acl"} , {"acl delete clientid ", "Delete clientid acl"} , {"acl delete username ", "Delete username acl"} From 809f98502ac77c0e77bd4327f8bce736fcefe017 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 31 Oct 2022 17:33:51 +0100 Subject: [PATCH 14/15] chore(emqx_auth_mnesia): re-generate appup --- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src | 2 +- apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src index 6dd1dcdfc..3bce055f6 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mnesia, [{description, "EMQ X Authentication with Mnesia"}, - {vsn, "4.3.9"}, % strict semver, bump manually + {vsn, "4.3.10"}, % strict semver, bump manually {modules, []}, {registered, []}, {applications, [kernel,stdlib,mnesia]}, diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index 7906449db..82bff332d 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.7", + [{"4.3.9",[{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, + {"4.3.8",[{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, + {"4.3.7", [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[5-6]">>, @@ -33,7 +35,9 @@ {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.7", + [{"4.3.9",[{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, + {"4.3.8",[{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, + {"4.3.7", [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[5-6]">>, From 406428c8ca66dadd8c5f4bc6606cbf11230632b3 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 1 Nov 2022 07:45:58 +0800 Subject: [PATCH 15/15] chore: update lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl Co-authored-by: Zaiming (Stone) Shi --- lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 113a315ab..b05615e03 100644 --- a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -248,7 +248,7 @@ t_start_listener_failed_log(_Config) -> ?assertError(_, emqx_dashboard:start_listener({http, {"1.1.1.1", 8080}, Options})), ct:capture_stop(), I0 = ct:capture_get(), - ?assertNotMatch(nomatch, re:run(iolist_to_binary(I0), "eaddrnotavail", [])), + ?assertMatch({match, _}, re:run(iolist_to_binary(I0), "eaddrnotavail", [])), ok. %%------------------------------------------------------------------------------