Merge branch 'release-v43' into revert-9256-bootstrap-users

apps/emqx_management/etc/emqx_management.conf

@@ -20,6 +20,16 @@ management.default_application.id = admin
 ## Value: String
 management.default_application.secret = public
 
+## Initialize apps file
+## Is used to add administrative app/secrets when EMQX is launched for the first time.
+## This config will not take any effect once EMQX database is populated with the provided apps.
+## The file content format is as below:
+##  ```
+##819e5db182cf:l9C5suZClIF3FvdzWqmINrVU61WNfIjcglxw9CVM7y1VI
+##bb5a6cf1c06a:WuNRRgcRTGiNcuyrE49Bpwz4PGPrRnP4hUMi647kNSbN
+## ```
+# management.bootstrap_apps_file = {{ platform_etc_dir }}/bootstrap_apps.txt
+
 ##--------------------------------------------------------------------
 ## HTTP Listener
 

apps/emqx_management/priv/emqx_management.schema

@@ -6,6 +6,11 @@
   {datatype, integer}
 ]}.
 
+{mapping, "management.bootstrap_apps_file", "emqx_management.bootstrap_apps_file", [
+  {datatype, string},
+  hidden
+]}.
+
 {mapping, "management.default_application.id", "emqx_management.default_application_id", [
   {default, undefined},
   {datatype, string}

apps/emqx_management/src/emqx_mgmt_app.erl

@@ -25,11 +25,16 @@
         ]).
 
 start(_Type, _Args) ->
-    {ok, Sup} = emqx_mgmt_sup:start_link(),
+    case emqx_mgmt_auth:init_bootstrap_apps() of
-    _ = emqx_mgmt_auth:add_default_app(),
+        ok ->
-    emqx_mgmt_http:start_listeners(),
+            {ok, Sup} = emqx_mgmt_sup:start_link(),
-    emqx_mgmt_cli:load(),
+            _ = emqx_mgmt_auth:add_default_app(),
-    {ok, Sup}.
+            emqx_mgmt_http:start_listeners(),
+            emqx_mgmt_cli:load(),
+            {ok, Sup};
+        {error, _Reason} = Error ->
+            Error
+    end.
 
 stop(_State) ->
     emqx_mgmt_http:stop_listeners().

apps/emqx_management/src/emqx_mgmt_auth.erl

@@ -35,6 +35,7 @@
         , update_app/5
         , del_app/1
         , list_apps/0
+        , init_bootstrap_apps/0
         ]).
 
 %% APP Auth/ACL API
@@ -44,6 +45,8 @@
 
 -record(mqtt_app, {id, secret, name, desc, status, expired}).
 
+-define(BOOTSTRAP_TAG, <<"Bootstrapped From File">>).
+
 -type(appid() :: binary()).
 
 -type(appsecret() :: binary()).
@@ -77,6 +80,68 @@ add_default_app() ->
             add_app(AppId1, <<"Default">>, AppSecret1, <<"Application user">>, true, undefined)
     end.
 
+init_bootstrap_apps() ->
+    Bootstrap = application:get_env(emqx_management, bootstrap_apps_file, undefined),
+    Size = mnesia:table_info(mqtt_app, size),
+    init_bootstrap_apps(Bootstrap, Size).
+
+init_bootstrap_apps(undefined, _) -> ok;
+init_bootstrap_apps(_File, Size)when Size > 0 -> ok;
+init_bootstrap_apps(File, 0) ->
+    case file:open(File, [read, binary]) of
+        {ok, Dev} ->
+            {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]),
+            case init_bootstrap_apps(File, Dev, MP) of
+                ok -> ok;
+                Error ->
+                    %% if failed add bootstrap users, we should clear all bootstrap apps
+                    {atomic, ok} = mnesia:clear_table(mqtt_app),
+                    Error
+            end;
+        {error, Reason} = Error ->
+            ?LOG(error,
+                "failed to open the mgmt bootstrap apps file(~s) for ~p",
+                [File, Reason]
+            ),
+            Error
+    end.
+
+init_bootstrap_apps(File, Dev, MP) ->
+    try
+        add_bootstrap_app(File, Dev, MP, 1)
+    catch
+        throw:Error -> {error, Error};
+        Type:Reason:Stacktrace ->
+            {error, {Type, Reason, Stacktrace}}
+    after
+        file:close(Dev)
+    end.
+
+add_bootstrap_app(File, Dev, MP, Line) ->
+    case file:read_line(Dev) of
+        {ok, Bin} ->
+            case re:run(Bin, MP, [global, {capture, all_but_first, binary}]) of
+                {match, [[AppId, AppSecret]]} ->
+                    Name = <<"bootstraped">>,
+                    case add_app(AppId, Name, AppSecret, ?BOOTSTRAP_TAG, true, undefined) of
+                        {ok, _} ->
+                            add_bootstrap_app(File, Dev, MP, Line + 1);
+                        {error, Reason} ->
+                            throw(#{file => File, line => Line, content => Bin, reason => Reason})
+                    end;
+                _ ->
+                    ?LOG(error,
+                        "failed to bootstrap apps file(~s) for Line(~w): ~ts",
+                        [File, Line, Bin]
+                    ),
+                    throw(#{file => File, line => Line, content => Bin, reason => "invalid format"})
+            end;
+        eof ->
+            ok;
+        {error, Error} ->
+            throw(#{file => File, line => Line, reason => Error})
+    end.
+
 -spec(add_app(appid(), binary()) -> {ok, appsecret()} | {error, term()}).
 add_app(AppId, Name) when is_binary(AppId) ->
     add_app(AppId, Name, <<"Application user">>, true, undefined).

apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl

@@ -0,0 +1,89 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_mgmt_bootstrap_app_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx/include/emqx.hrl").
+
+%%--------------------------------------------------------------------
+%% Setups
+%%--------------------------------------------------------------------
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:boot_modules(all),
+    application:load(emqx_modules),
+    application:load(emqx_modules_spec),
+    application:load(emqx_management),
+    application:stop(emqx_rule_engine),
+    ekka_mnesia:start(),
+    emqx_ct_helpers:start_apps([]),
+    Config.
+
+end_per_suite(_) ->
+    emqx_ct_helpers:stop_apps([]),
+    ok.
+
+%%--------------------------------------------------------------------
+%% Test cases
+%%--------------------------------------------------------------------
+
+t_load_ok(_) ->
+    application:stop(emqx_management),
+    Bin = <<"test-1:secret-1\ntest-2:secret-2">>,
+    File = "./bootstrap_apps.txt",
+    ok = file:write_file(File, Bin),
+    _ = mnesia:clear_table(mqtt_app),
+    application:set_env(emqx_management, bootstrap_apps_file, File),
+    {ok, _} = application:ensure_all_started(emqx_management),
+    ?assert(emqx_mgmt_auth:is_authorized(<<"test-1">>, <<"secret-1">>)),
+    ?assert(emqx_mgmt_auth:is_authorized(<<"test-2">>, <<"secret-2">>)),
+    ?assertNot(emqx_mgmt_auth:is_authorized(<<"test-2">>, <<"secret-1">>)),
+    application:stop(emqx_management).
+
+t_bootstrap_user_file_not_found(_) ->
+    File = "./bootstrap_apps_not_exist.txt",
+    check_load_failed(File),
+    ok.
+
+t_load_invalid_username_failed(_) ->
+    Bin = <<"test-1:password-1\ntest&2:password-2">>,
+    File = "./bootstrap_apps.txt",
+    ok = file:write_file(File, Bin),
+    check_load_failed(File),
+    ok.
+
+t_load_invalid_format_failed(_) ->
+    Bin = <<"test-1:password-1\ntest-2password-2">>,
+    File = "./bootstrap_apps.txt",
+    ok = file:write_file(File, Bin),
+    check_load_failed(File),
+    ok.
+
+check_load_failed(File) ->
+    _ = mnesia:clear_table(mqtt_app),
+    application:stop(emqx_management),
+    application:set_env(emqx_management, bootstrap_apps_file, File),
+    ?assertMatch({error, _}, application:ensure_all_started(emqx_management)),
+    ?assertNot(lists:member(emqx_management, application:which_applications())),
+    ?assertEqual(0, mnesia:table_info(mqtt_app, size)).

changes/v4.3.22-en.md

@@ -22,6 +22,8 @@
 
 - Enhanced log security in ACL modules, sensitive data will be obscured. [#9242](https://github.com/emqx/emqx/pull/9242).
 
+- Add `management.bootstrap_apps_file` configuration to bulk import default app/secret when EMQX initializes the database [#9273](https://github.com/emqx/emqx/pull/9273).
+
 ## Bug fixes
 
 - Fix that after uploading a backup file with an UTF8 filename, HTTP API `GET /data/export` fails with status code 500 [#9224](https://github.com/emqx/emqx/pull/9224).

changes/v4.3.22-zh.md

@@ -20,6 +20,8 @@
 
 - 增强 ACL 模块中的日志安全性，敏感数据将被模糊化。[#9242](https://github.com/emqx/emqx/pull/9242)。
 
+- 增加 `management.bootstrap_apps_file` 配置，可以让 EMQX 初始化数据库时，从该文件批量导入一些 APP / Secret [#9273](https://github.com/emqx/emqx/pull/9273)。
+
 ## 修复
 
 - 修复若上传的备份文件名中包含 UTF8 字符，`GET /data/export` HTTP 接口返回 500 错误 [#9224](https://github.com/emqx/emqx/pull/9224)。