Merge pull request #9713 from zhongwencool/api-keys-bootstrap-file

feat: introduce API keys bootstrap_file config

apps/emqx/src/emqx_packet.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_packet).
 
+-elvis([{elvis_style, no_spec_with_records, disable}]).
+
 -include("emqx.hrl").
 -include("emqx_mqtt.hrl").
 
@@ -492,7 +494,7 @@ format_variable(undefined, _, _) ->
 format_variable(Variable, undefined, PayloadEncode) ->
     format_variable(Variable, PayloadEncode);
 format_variable(Variable, Payload, PayloadEncode) ->
-    [format_variable(Variable, PayloadEncode), format_payload(Payload, PayloadEncode)].
+    [format_variable(Variable, PayloadEncode), ",", format_payload(Payload, PayloadEncode)].
 
 format_variable(
     #mqtt_packet_connect{

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -60,7 +60,8 @@
     emqx_exhook_schema,
     emqx_psk_schema,
     emqx_limiter_schema,
-    emqx_slow_subs_schema
+    emqx_slow_subs_schema,
+    emqx_mgmt_api_key_schema
 ]).
 
 %% root config should not have a namespace

apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf

@@ -199,23 +199,12 @@ its own from which a browser should permit loading resources."""
   }
   bootstrap_users_file {
     desc {
-      en: "Initialize users file."
+      en: "Deprecated, use api_key.bootstrap_file"
-      zh: "初始化用户文件"
+      zh: "已废弃，请使用 api_key.bootstrap_file"
     }
     label {
-      en: """Is used to add an administrative user to Dashboard when emqx is first launched,
+      en: """Deprecated"""
-      the format is:
+      zh: """已废弃"""
-       ```
-       username1:password1
-       username2:password2
-       ```
-"""
-      zh: """用于在首次启动 emqx 时，为 Dashboard 添加管理用户，其格式为：
-      ```
-      username1:password1
-      username2:password2
-      ```
-"""
     }
   }
 }

apps/emqx_dashboard/src/emqx_dashboard_admin.erl

@@ -51,8 +51,7 @@
 
 -export([
     add_default_user/0,
-    default_username/0,
+    default_username/0
-    add_bootstrap_users/0
 ]).
 
 -type emqx_admin() :: #?ADMIN{}.
@@ -85,21 +84,6 @@ mnesia(boot) ->
 add_default_user() ->
     add_default_user(binenv(default_username), binenv(default_password)).
 
--spec add_bootstrap_users() -> ok | {error, _}.
-add_bootstrap_users() ->
-    case emqx:get_config([dashboard, bootstrap_users_file], undefined) of
-        undefined ->
-            ok;
-        File ->
-            case mnesia:table_info(?ADMIN, size) of
-                0 ->
-                    ?SLOG(debug, #{msg => "Add dashboard bootstrap users", file => File}),
-                    add_bootstrap_users(File);
-                _ ->
-                    ok
-            end
-    end.
-
 %%--------------------------------------------------------------------
 %% API
 %%--------------------------------------------------------------------
@@ -311,44 +295,3 @@ add_default_user(Username, Password) ->
         [] -> add_user(Username, Password, <<"administrator">>);
         _ -> {ok, default_user_exists}
     end.
-
-add_bootstrap_users(File) ->
-    case file:open(File, [read]) of
-        {ok, Dev} ->
-            {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]),
-            try
-                load_bootstrap_user(Dev, MP)
-            catch
-                Type:Reason ->
-                    {error, {Type, Reason}}
-            after
-                file:close(Dev)
-            end;
-        {error, Reason} = Error ->
-            ?SLOG(error, #{
-                msg => "failed to open the dashboard bootstrap users file",
-                file => File,
-                reason => Reason
-            }),
-            Error
-    end.
-
-load_bootstrap_user(Dev, MP) ->
-    case file:read_line(Dev) of
-        {ok, Line} ->
-            case re:run(Line, MP, [global, {capture, all_but_first, binary}]) of
-                {match, [[Username, Password]]} ->
-                    case add_user(Username, Password, ?BOOTSTRAP_USER_TAG) of
-                        {ok, _} ->
-                            load_bootstrap_user(Dev, MP);
-                        Error ->
-                            Error
-                    end;
-                _ ->
-                    load_bootstrap_user(Dev, MP)
-            end;
-        eof ->
-            ok;
-        Error ->
-            Error
-    end.

apps/emqx_dashboard/src/emqx_dashboard_app.erl

@@ -31,13 +31,8 @@ start(_StartType, _StartArgs) ->
     case emqx_dashboard:start_listeners() of
         ok ->
             emqx_dashboard_cli:load(),
-            case emqx_dashboard_admin:add_bootstrap_users() of
+            {ok, _} = emqx_dashboard_admin:add_default_user(),
-                ok ->
+            {ok, Sup};
-                    {ok, _} = emqx_dashboard_admin:add_default_user(),
-                    {ok, Sup};
-                Error ->
-                    Error
-            end;
         {error, Reason} ->
             {error, Reason}
     end.

apps/emqx_dashboard/src/emqx_dashboard_schema.erl

@@ -56,7 +56,15 @@ fields("dashboard") ->
         {cors, fun cors/1},
         {i18n_lang, fun i18n_lang/1},
         {bootstrap_users_file,
-            ?HOCON(binary(), #{desc => ?DESC(bootstrap_users_file), required => false})}
+            ?HOCON(
+                binary(),
+                #{
+                    desc => ?DESC(bootstrap_users_file),
+                    required => false,
+                    default => <<>>
+                    %% deprecated => {since, "5.1.0"}
+                }
+            )}
     ];
 fields("listeners") ->
     [

apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf

@@ -0,0 +1,33 @@
+emqx_mgmt_api_key_schema {
+  api_key {
+    desc {
+      en: """API Key, can be used to request API other than the management API key and the Dashboard user management API"""
+      zh: """API 密钥， 可用于请求除管理 API 密钥及 Dashboard 用户管理 API 的其它接口"""
+    }
+    label {
+      en: "API Key"
+      zh: "API 密钥"
+    }
+  }
+  bootstrap_file {
+    desc {
+      en: """Bootstrap file is used to add an api_key when emqx is launched,
+      the format is:
+       ```
+       7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK
+       ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL
+       ```
+"""
+      zh: """用于在启动 emqx 时，添加 API 密钥，其格式为：
+      ```
+      7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK
+      ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL
+      ```
+"""
+    }
+    label {
+      en: "Initialize api_key file."
+      zh: "API 密钥初始化文件"
+    }
+  }
+}

apps/emqx_management/src/emqx_mgmt_api_api_keys.erl

@@ -13,7 +13,7 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--module(emqx_mgmt_api_app).
+-module(emqx_mgmt_api_api_keys).
 
 -behaviour(minirest_api).
 

apps/emqx_management/src/emqx_mgmt_api_configs.erl

@@ -63,7 +63,8 @@
         <<"prometheus">>,
         <<"telemetry">>,
         <<"listeners">>,
-        <<"license">>
+        <<"license">>,
+        <<"api_key">>
     ] ++ global_zone_roots()
 ).
 

apps/emqx_management/src/emqx_mgmt_api_key_schema.erl

@@ -0,0 +1,44 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_mgmt_api_key_schema).
+
+-include_lib("hocon/include/hoconsc.hrl").
+
+-export([
+    roots/0,
+    fields/1,
+    namespace/0,
+    desc/1
+]).
+
+namespace() -> api_key.
+roots() -> ["api_key"].
+
+fields("api_key") ->
+    [
+        {bootstrap_file,
+            ?HOCON(
+                binary(),
+                #{
+                    desc => ?DESC(bootstrap_file),
+                    required => false,
+                    default => <<>>
+                }
+            )}
+    ].
+
+desc("api_key") ->
+    ?DESC(api_key).

apps/emqx_management/src/emqx_mgmt_app.erl

@@ -28,10 +28,15 @@
 -include("emqx_mgmt.hrl").
 
 start(_Type, _Args) ->
-    {ok, Sup} = emqx_mgmt_sup:start_link(),
     ok = mria_rlog:wait_for_shards([?MANAGEMENT_SHARD], infinity),
-    emqx_mgmt_cli:load(),
+    case emqx_mgmt_auth:init_bootstrap_file() of
-    {ok, Sup}.
+        ok ->
+            {ok, Sup} = emqx_mgmt_sup:start_link(),
+            ok = emqx_mgmt_cli:load(),
+            {ok, Sup};
+        {error, Reason} ->
+            {error, Reason}
+    end.
 
 stop(_State) ->
     ok.

apps/emqx_management/src/emqx_mgmt_auth.erl

@@ -15,6 +15,7 @@
 %%--------------------------------------------------------------------
 -module(emqx_mgmt_auth).
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 %% API
 -export([mnesia/1]).
@@ -25,7 +26,8 @@
     read/1,
     update/4,
     delete/1,
-    list/0
+    list/0,
+    init_bootstrap_file/0
 ]).
 
 -export([authorize/3]).
@@ -34,7 +36,8 @@
 -export([
     do_update/4,
     do_delete/1,
-    do_create_app/3
+    do_create_app/3,
+    do_force_create_app/3
 ]).
 
 -define(APP, emqx_app).
@@ -45,7 +48,7 @@
     api_secret_hash = <<>> :: binary() | '_',
     enable = true :: boolean() | '_',
     desc = <<>> :: binary() | '_',
-    expired_at = 0 :: integer() | undefined | '_',
+    expired_at = 0 :: integer() | undefined | infinity | '_',
     created_at = 0 :: integer() | '_'
 }).
 
@@ -58,8 +61,14 @@ mnesia(boot) ->
         {attributes, record_info(fields, ?APP)}
     ]).
 
+-spec init_bootstrap_file() -> ok | {error, _}.
+init_bootstrap_file() ->
+    File = bootstrap_file(),
+    ?SLOG(debug, #{msg => "init_bootstrap_api_keys_from_file", file => File}),
+    init_bootstrap_file(File).
+
 create(Name, Enable, ExpiredAt, Desc) ->
-    case mnesia:table_info(?APP, size) < 30 of
+    case mnesia:table_info(?APP, size) < 100 of
         true -> create_app(Name, Enable, ExpiredAt, Desc);
         false -> {error, "Maximum ApiKey"}
     end.
@@ -169,6 +178,9 @@ create_app(Name, Enable, ExpiredAt, Desc) ->
 create_app(App = #?APP{api_key = ApiKey, name = Name}) ->
     trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]).
 
+force_create_app(NamePrefix, App = #?APP{api_key = ApiKey}) ->
+    trans(fun ?MODULE:do_force_create_app/3, [App, ApiKey, NamePrefix]).
+
 do_create_app(App, ApiKey, Name) ->
     case mnesia:read(?APP, Name) of
         [_] ->
@@ -183,6 +195,22 @@ do_create_app(App, ApiKey, Name) ->
             end
     end.
 
+do_force_create_app(App, ApiKey, NamePrefix) ->
+    case mnesia:match_object(?APP, #?APP{api_key = ApiKey, _ = '_'}, read) of
+        [] ->
+            NewName = generate_unique_name(NamePrefix),
+            ok = mnesia:write(App#?APP{name = NewName});
+        [#?APP{name = Name}] ->
+            ok = mnesia:write(App#?APP{name = Name})
+    end.
+
+generate_unique_name(NamePrefix) ->
+    New = list_to_binary(NamePrefix ++ emqx_misc:gen_id(16)),
+    case mnesia:read(?APP, New) of
+        [] -> New;
+        _ -> generate_unique_name(NamePrefix)
+    end.
+
 trans(Fun, Args) ->
     case mria:transaction(?COMMON_SHARD, Fun, Args) of
         {atomic, Res} -> {ok, Res};
@@ -192,3 +220,84 @@ trans(Fun, Args) ->
 generate_api_secret() ->
     Random = crypto:strong_rand_bytes(32),
     emqx_base62:encode(Random).
+
+bootstrap_file() ->
+    case emqx:get_config([api_key, bootstrap_file], <<>>) of
+        %% For compatible remove until 5.1.0
+        <<>> ->
+            emqx:get_config([dashboard, bootstrap_users_file], <<>>);
+        File ->
+            File
+    end.
+
+init_bootstrap_file(<<>>) ->
+    ok;
+init_bootstrap_file(File) ->
+    case file:open(File, [read, binary]) of
+        {ok, Dev} ->
+            {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]),
+            init_bootstrap_file(File, Dev, MP);
+        {error, Reason0} ->
+            Reason = emqx_misc:explain_posix(Reason0),
+            ?SLOG(
+                error,
+                #{
+                    msg => "failed_to_open_the_bootstrap_file",
+                    file => File,
+                    reason => Reason
+                }
+            ),
+            {error, Reason}
+    end.
+
+init_bootstrap_file(File, Dev, MP) ->
+    try
+        add_bootstrap_file(File, Dev, MP, 1)
+    catch
+        throw:Error -> {error, Error};
+        Type:Reason:Stacktrace -> {error, {Type, Reason, Stacktrace}}
+    after
+        file:close(Dev)
+    end.
+
+-define(BOOTSTRAP_TAG, <<"Bootstrapped From File">>).
+
+add_bootstrap_file(File, Dev, MP, Line) ->
+    case file:read_line(Dev) of
+        {ok, Bin} ->
+            case re:run(Bin, MP, [global, {capture, all_but_first, binary}]) of
+                {match, [[AppKey, ApiSecret]]} ->
+                    App =
+                        #?APP{
+                            enable = true,
+                            expired_at = infinity,
+                            desc = ?BOOTSTRAP_TAG,
+                            created_at = erlang:system_time(second),
+                            api_secret_hash = emqx_dashboard_admin:hash(ApiSecret),
+                            api_key = AppKey
+                        },
+                    case force_create_app("from_bootstrap_file_", App) of
+                        {ok, ok} ->
+                            add_bootstrap_file(File, Dev, MP, Line + 1);
+                        {error, Reason} ->
+                            throw(#{file => File, line => Line, content => Bin, reason => Reason})
+                    end;
+                _ ->
+                    Reason = "invalid_format",
+                    ?SLOG(
+                        error,
+                        #{
+                            msg => "failed_to_load_bootstrap_file",
+                            file => File,
+                            line => Line,
+                            content => Bin,
+                            reason => Reason
+                        }
+                    ),
+                    throw(#{file => File, line => Line, content => Bin, reason => Reason})
+            end;
+        eof ->
+            ok;
+        {error, Reason} ->
+            throw(#{file => File, line => Line, reason => Reason})
+    end.

apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl

@@ -13,7 +13,7 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--module(emqx_mgmt_api_app_SUITE).
+-module(emqx_mgmt_api_api_keys_SUITE).
 
 -compile(export_all).
 -compile(nowarn_export_all).
@@ -25,15 +25,62 @@ suite() -> [{timetrap, {minutes, 1}}].
 groups() ->
     [
         {parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]},
-        {sequence, [], [t_create_failed]}
+        {sequence, [], [t_bootstrap_file, t_create_failed]}
     ].
 
 init_per_suite(Config) ->
-    emqx_mgmt_api_test_util:init_suite(),
+    emqx_mgmt_api_test_util:init_suite([emqx_conf]),
     Config.
 
 end_per_suite(_) ->
-    emqx_mgmt_api_test_util:end_suite().
+    emqx_mgmt_api_test_util:end_suite([emqx_conf]).
+
+t_bootstrap_file(_) ->
+    TestPath = <<"/api/v5/status">>,
+    Bin = <<"test-1:secret-1\ntest-2:secret-2">>,
+    File = "./bootstrap_api_keys.txt",
+    ok = file:write_file(File, Bin),
+    emqx:update_config([api_key, bootstrap_file], File),
+    ok = emqx_mgmt_auth:init_bootstrap_file(),
+    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
+    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
+    ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-1">>)),
+
+    %% relaunch to check if the table is changed.
+    Bin1 = <<"test-1:new-secret-1\ntest-2:new-secret-2">>,
+    ok = file:write_file(File, Bin1),
+    ok = emqx_mgmt_auth:init_bootstrap_file(),
+    ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)),
+    ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)),
+    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)),
+    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)),
+
+    %% Compatibility
+    Bin2 = <<"test-3:new-secret-3\ntest-4:new-secret-4">>,
+    ok = file:write_file(File, Bin2),
+    emqx:update_config([api_key, bootstrap_file], <<>>),
+    emqx:update_config([dashboard, bootstrap_users_file], File),
+    ok = emqx_mgmt_auth:init_bootstrap_file(),
+    ?assertMatch(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)),
+    ?assertMatch(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)),
+    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-3">>, <<"new-secret-3">>)),
+    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-4">>, <<"new-secret-4">>)),
+
+    %% not found
+    NotFoundFile = "./bootstrap_apps_not_exist.txt",
+    emqx:update_config([api_key, bootstrap_file], NotFoundFile),
+    ?assertMatch({error, "No such file or directory"}, emqx_mgmt_auth:init_bootstrap_file()),
+
+    %% bad format
+    BadBin = <<"test-1:secret-11\ntest-2 secret-12">>,
+    ok = file:write_file(File, BadBin),
+    emqx:update_config([api_key, bootstrap_file], File),
+    ?assertMatch({error, #{reason := "invalid_format"}}, emqx_mgmt_auth:init_bootstrap_file()),
+    ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-11">>)),
+    ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-12">>)),
+    emqx:update_config([api_key, bootstrap_file], <<>>),
+    emqx:update_config([dashboard, bootstrap_users_file], <<>>),
+    ok.
 
 t_create(_Config) ->
     Name = <<"EMQX-API-KEY-1">>,
@@ -69,7 +116,7 @@ t_create_failed(_Config) ->
     ?assertEqual(BadRequest, create_app(LongName)),
 
     {ok, List} = list_app(),
-    CreateNum = 30 - erlang:length(List),
+    CreateNum = 100 - erlang:length(List),
     Names = lists:map(
         fun(Seq) ->
             <<"EMQX-API-FAILED-KEY-", (integer_to_binary(Seq))/binary>>

changes/v5.0.14/feat-9713.en.md

@@ -0,0 +1,3 @@
+Introduce `api_key.bootstrap_file` to initialize the api key at boot time.
+Deprecate `dashboard.bootstrap_users_file`.
+Limit the maximum number of api keys to 100 instead of 30.

changes/v5.0.14/feat-9713.zh.md

@@ -0,0 +1,3 @@
+引入 `api_key.bootstrap_file`，用于启动时初始化api密钥。
+废弃 `dashboard.boostrap_users_file`。
+将 API 密钥的最大数量限制提升为 100（原来为30）。