From f6a47e5cf61270580bbee7c0c6c7f369bc99a0f8 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 10 Jan 2023 09:43:04 +0800 Subject: [PATCH] chore: support api_key.bootstrap_file config --- apps/emqx/src/emqx_packet.erl | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 3 +- .../i18n/emqx_dashboard_i18n.conf | 19 +- .../src/emqx_dashboard_admin.erl | 59 +---- .../emqx_dashboard/src/emqx_dashboard_app.erl | 9 +- .../src/emqx_dashboard_schema.erl | 10 +- .../i18n/emqx_mgmt_api_key.conf | 221 ++++++++++++++++++ .../i18n/emqx_mgmt_api_key_i18n.conf | 33 +++ ...api_app.erl => emqx_mgmt_api_api_keys.erl} | 2 +- .../src/emqx_mgmt_api_configs.erl | 3 +- .../src/emqx_mgmt_api_key_schema.erl | 44 ++++ apps/emqx_management/src/emqx_mgmt_app.erl | 11 +- apps/emqx_management/src/emqx_mgmt_auth.erl | 112 ++++++++- 13 files changed, 438 insertions(+), 90 deletions(-) create mode 100644 apps/emqx_management/i18n/emqx_mgmt_api_key.conf create mode 100644 apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf rename apps/emqx_management/src/{emqx_mgmt_api_app.erl => emqx_mgmt_api_api_keys.erl} (99%) create mode 100644 apps/emqx_management/src/emqx_mgmt_api_key_schema.erl diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index 8f539563e..d0de1f018 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -492,7 +492,7 @@ format_variable(undefined, _, _) -> format_variable(Variable, undefined, PayloadEncode) -> format_variable(Variable, PayloadEncode); format_variable(Variable, Payload, PayloadEncode) -> - [format_variable(Variable, PayloadEncode), format_payload(Payload, PayloadEncode)]. + [format_variable(Variable, PayloadEncode), ",", format_payload(Payload, PayloadEncode)]. format_variable( #mqtt_packet_connect{ diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 2a46e95a5..a7b388964 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -60,7 +60,8 @@ emqx_exhook_schema, emqx_psk_schema, emqx_limiter_schema, - emqx_slow_subs_schema + emqx_slow_subs_schema, + emqx_mgmt_api_key_schema ]). %% root config should not have a namespace diff --git a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf index e6758d0de..872cfdf26 100644 --- a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf +++ b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf @@ -199,23 +199,12 @@ its own from which a browser should permit loading resources.""" } bootstrap_users_file { desc { - en: "Initialize users file." - zh: "初始化用户文件" + en: "Deprecated, use api_key.bootstrap_file" + zh: "已废弃,请使用 api_key.bootstrap_file" } label { - en: """Is used to add an administrative user to Dashboard when emqx is first launched, - the format is: - ``` - username1:password1 - username2:password2 - ``` -""" - zh: """用于在首次启动 emqx 时,为 Dashboard 添加管理用户,其格式为: - ``` - username1:password1 - username2:password2 - ``` -""" + en: """Deprecated""" + zh: """已废弃""" } } } diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 77c77d5b9..e36c2628b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -51,8 +51,7 @@ -export([ add_default_user/0, - default_username/0, - add_bootstrap_users/0 + default_username/0 ]). -type emqx_admin() :: #?ADMIN{}. @@ -85,21 +84,6 @@ mnesia(boot) -> add_default_user() -> add_default_user(binenv(default_username), binenv(default_password)). --spec add_bootstrap_users() -> ok | {error, _}. -add_bootstrap_users() -> - case emqx:get_config([dashboard, bootstrap_users_file], undefined) of - undefined -> - ok; - File -> - case mnesia:table_info(?ADMIN, size) of - 0 -> - ?SLOG(debug, #{msg => "Add dashboard bootstrap users", file => File}), - add_bootstrap_users(File); - _ -> - ok - end - end. - %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -311,44 +295,3 @@ add_default_user(Username, Password) -> [] -> add_user(Username, Password, <<"administrator">>); _ -> {ok, default_user_exists} end. - -add_bootstrap_users(File) -> - case file:open(File, [read]) of - {ok, Dev} -> - {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), - try - load_bootstrap_user(Dev, MP) - catch - Type:Reason -> - {error, {Type, Reason}} - after - file:close(Dev) - end; - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "failed to open the dashboard bootstrap users file", - file => File, - reason => Reason - }), - Error - end. - -load_bootstrap_user(Dev, MP) -> - case file:read_line(Dev) of - {ok, Line} -> - case re:run(Line, MP, [global, {capture, all_but_first, binary}]) of - {match, [[Username, Password]]} -> - case add_user(Username, Password, ?BOOTSTRAP_USER_TAG) of - {ok, _} -> - load_bootstrap_user(Dev, MP); - Error -> - Error - end; - _ -> - load_bootstrap_user(Dev, MP) - end; - eof -> - ok; - Error -> - Error - end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_app.erl b/apps/emqx_dashboard/src/emqx_dashboard_app.erl index 6956f3fc8..2c3f9b8bc 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_app.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_app.erl @@ -31,13 +31,8 @@ start(_StartType, _StartArgs) -> case emqx_dashboard:start_listeners() of ok -> emqx_dashboard_cli:load(), - case emqx_dashboard_admin:add_bootstrap_users() of - ok -> - {ok, _} = emqx_dashboard_admin:add_default_user(), - {ok, Sup}; - Error -> - Error - end; + {ok, _} = emqx_dashboard_admin:add_default_user(), + {ok, Sup}; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 4605d911d..6742032d5 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -56,7 +56,15 @@ fields("dashboard") -> {cors, fun cors/1}, {i18n_lang, fun i18n_lang/1}, {bootstrap_users_file, - ?HOCON(binary(), #{desc => ?DESC(bootstrap_users_file), required => false})} + ?HOCON( + binary(), + #{ + desc => ?DESC(bootstrap_users_file), + required => false, + default => <<>> + %% deprecated => {since, "5.1.0"} + } + )} ]; fields("listeners") -> [ diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_key.conf b/apps/emqx_management/i18n/emqx_mgmt_api_key.conf new file mode 100644 index 000000000..e6758d0de --- /dev/null +++ b/apps/emqx_management/i18n/emqx_mgmt_api_key.conf @@ -0,0 +1,221 @@ +emqx_dashboard_schema { + listeners { + desc { + en: """HTTP(s) listeners are identified by their protocol type and are +used to serve dashboard UI and restful HTTP API. +Listeners must have a unique combination of port number and IP address. +For example, an HTTP listener can listen on all configured IP addresses +on a given port for a machine by specifying the IP address 0.0.0.0. +Alternatively, the HTTP listener can specify a unique IP address for each listener, +but use the same port.""" + zh: """仪表盘监听器设置。""" + } + label { + en: "Listeners" + zh: "监听器" + } + } + sample_interval { + desc { + en: """How often to update metrics displayed in the dashboard. +Note: `sample_interval` should be a divisor of 60.""" + zh: """更新仪表板中显示的指标的时间间隔。必须小于60,且被60的整除。""" + } + } + token_expired_time { + desc { + en: "JWT token expiration time." + zh: "JWT token 过期时间" + } + label { + en: "Token expired time" + zh: "JWT 过期时间" + } + } + num_acceptors { + desc { + en: "Socket acceptor pool size for TCP protocols." + zh: "TCP协议的Socket acceptor池大小" + } + label { + en: "Number of acceptors" + zh: "Acceptor 数量" + } + } + max_connections { + desc { + en: "Maximum number of simultaneous connections." + zh: "同时处理的最大连接数" + } + label { + en: "Maximum connections" + zh: "最大连接数" + } + } + backlog { + desc { + en: "Defines the maximum length that the queue of pending connections can grow to." + zh: "排队等待连接的队列的最大长度" + } + label { + en: "Backlog" + zh: "排队长度" + } + } + send_timeout { + desc { + en: "Send timeout for the socket." + zh: "Socket发送超时时间" + } + label { + en: "Send timeout" + zh: "发送超时时间" + } + } + inet6 { + desc { + en: "Enable IPv6 support, default is false, which means IPv4 only." + zh: "启用IPv6, 如果机器不支持IPv6,请关闭此选项,否则会导致仪表盘无法使用。" + } + label { + en: "IPv6" + zh: "IPv6" + } + } + ipv6_v6only { + desc { + en: "Disable IPv4-to-IPv6 mapping for the listener." + zh: "当开启 inet6 功能的同时禁用 IPv4-to-IPv6 映射。该配置仅在 inet6 功能开启时有效。" + } + label { + en: "IPv6 only" + zh: "IPv6 only" + } + } + desc_dashboard { + desc { + en: "Configuration for EMQX dashboard." + zh: "EMQX仪表板配置" + } + label { + en: "Dashboard" + zh: "仪表板" + } + } + desc_listeners { + desc { + en: "Configuration for the dashboard listener." + zh: "仪表板监听器配置" + } + label { + en: "Listeners" + zh: "监听器" + } + } + desc_http { + desc { + en: "Configuration for the dashboard listener (plaintext)." + zh: "仪表板监听器(HTTP)配置" + } + label { + en: "HTTP" + zh: "HTTP" + } + } + desc_https { + desc { + en: "Configuration for the dashboard listener (TLS)." + zh: "仪表板监听器(HTTPS)配置" + } + label { + en: "HTTPS" + zh: "HTTPS" + } + } + listener_enable { + desc { + en: "Ignore or enable this listener" + zh: "忽略或启用该监听器配置" + } + label { + en: "Enable" + zh: "启用" + } + } + bind { + desc { + en: "Port without IP(18083) or port with specified IP(127.0.0.1:18083)." + zh: "监听的地址与端口,在dashboard更新此配置时,会重启dashboard服务。" + } + label { + en: "Bind" + zh: "绑定端口" + } + } + default_username { + desc { + en: "The default username of the automatically created dashboard user." + zh: "默认的仪表板用户名" + } + label { + en: "Default username" + zh: "默认用户名" + } + } + default_password { + desc { + en: """The initial default password for dashboard 'admin' user. +For safety, it should be changed as soon as possible.""" + zh: """默认的仪表板用户密码 +为了安全,应该尽快修改密码。""" + } + label { + en: "Default password" + zh: "默认密码" + } + } + cors { + desc { + en: """Support Cross-Origin Resource Sharing (CORS). +Allows a server to indicate any origins (domain, scheme, or port) other than +its own from which a browser should permit loading resources.""" + zh: """支持跨域资源共享(CORS) +允许服务器指示任何来源(域名、协议或端口),除了本服务器之外的任何浏览器应允许加载资源。""" + } + label { + en: "CORS" + zh: "跨域资源共享" + } + } + i18n_lang { + desc { + en: "Internationalization language support." + zh: "swagger多语言支持" + } + label { + en: "I18n language" + zh: "多语言支持" + } + } + bootstrap_users_file { + desc { + en: "Initialize users file." + zh: "初始化用户文件" + } + label { + en: """Is used to add an administrative user to Dashboard when emqx is first launched, + the format is: + ``` + username1:password1 + username2:password2 + ``` +""" + zh: """用于在首次启动 emqx 时,为 Dashboard 添加管理用户,其格式为: + ``` + username1:password1 + username2:password2 + ``` +""" + } + } +} diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf new file mode 100644 index 000000000..1651f85ea --- /dev/null +++ b/apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf @@ -0,0 +1,33 @@ +emqx_mgmt_api_key_schema { + api_key { + desc { + en: """API Key, can be used to request API other than the management API key and the Dashboard user management API""" + zh: """API 密钥, 可用于请求除管理 API 密钥及 Dashboard 用户管理 API 的其它接口""" + } + label { + en: "API Key" + zh: "API 密钥" + } + } + bootstrap_file { + desc { + en: """Is used to add an api_key when emqx is launched, + the format is: + ``` + 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK + ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL + ``` +""" + zh: """用于在启动 emqx 时,添加 API 密钥,其格式为: + ``` + 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK + ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL + ``` +""" + } + label { + en: "Initialize api_key file." + zh: "API 密钥初始化文件" + } + } +} diff --git a/apps/emqx_management/src/emqx_mgmt_api_app.erl b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl similarity index 99% rename from apps/emqx_management/src/emqx_mgmt_api_app.erl rename to apps/emqx_management/src/emqx_mgmt_api_api_keys.erl index d317bea70..c39b11273 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_mgmt_api_app). +-module(emqx_mgmt_api_api_keys). -behaviour(minirest_api). diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 976dd29f2..d9cdf6477 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -63,7 +63,8 @@ <<"prometheus">>, <<"telemetry">>, <<"listeners">>, - <<"license">> + <<"license">>, + <<"api_key">> ] ++ global_zone_roots() ). diff --git a/apps/emqx_management/src/emqx_mgmt_api_key_schema.erl b/apps/emqx_management/src/emqx_mgmt_api_key_schema.erl new file mode 100644 index 000000000..556e4308f --- /dev/null +++ b/apps/emqx_management/src/emqx_mgmt_api_key_schema.erl @@ -0,0 +1,44 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_api_key_schema). + +-include_lib("hocon/include/hoconsc.hrl"). + +-export([ + roots/0, + fields/1, + namespace/0, + desc/1 +]). + +namespace() -> api_key. +roots() -> ["api_key"]. + +fields("api_key") -> + [ + {bootstrap_file, + ?HOCON( + binary(), + #{ + desc => ?DESC(bootstrap_file), + required => false, + default => <<>> + } + )} + ]. + +desc("api_key") -> + ?DESC(api_key). diff --git a/apps/emqx_management/src/emqx_mgmt_app.erl b/apps/emqx_management/src/emqx_mgmt_app.erl index 164ac1b36..137f4502c 100644 --- a/apps/emqx_management/src/emqx_mgmt_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_app.erl @@ -28,10 +28,15 @@ -include("emqx_mgmt.hrl"). start(_Type, _Args) -> - {ok, Sup} = emqx_mgmt_sup:start_link(), ok = mria_rlog:wait_for_shards([?MANAGEMENT_SHARD], infinity), - emqx_mgmt_cli:load(), - {ok, Sup}. + case emqx_mgmt_auth:init_bootstrap_file() of + ok -> + {ok, Sup} = emqx_mgmt_sup:start_link(), + ok = emqx_mgmt_cli:load(), + {ok, Sup}; + {error, Reason} -> + {error, Reason} + end. stop(_State) -> ok. diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index e92a9310d..19317da42 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -15,6 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_mgmt_auth). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). %% API -export([mnesia/1]). @@ -25,7 +26,8 @@ read/1, update/4, delete/1, - list/0 + list/0, + init_bootstrap_file/0 ]). -export([authorize/3]). @@ -34,7 +36,8 @@ -export([ do_update/4, do_delete/1, - do_create_app/3 + do_create_app/3, + do_force_create_app/3 ]). -define(APP, emqx_app). @@ -58,6 +61,12 @@ mnesia(boot) -> {attributes, record_info(fields, ?APP)} ]). +-spec init_bootstrap_file() -> ok | {error, _}. +init_bootstrap_file() -> + File = bootstrap_file(), + ?SLOG(debug, #{msg => "init_bootstrap_api_keys_from_file", file => File}), + init_bootstrap_file(File). + create(Name, Enable, ExpiredAt, Desc) -> case mnesia:table_info(?APP, size) < 1024 of true -> create_app(Name, Enable, ExpiredAt, Desc); @@ -169,6 +178,9 @@ create_app(Name, Enable, ExpiredAt, Desc) -> create_app(App = #?APP{api_key = ApiKey, name = Name}) -> trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]). +force_create_app(NamePrefix, App = #?APP{api_key = ApiKey}) -> + trans(fun ?MODULE:do_force_create_app/3, [App, ApiKey, NamePrefix]). + do_create_app(App, ApiKey, Name) -> case mnesia:read(?APP, Name) of [_] -> @@ -183,6 +195,22 @@ do_create_app(App, ApiKey, Name) -> end end. +do_force_create_app(App, ApiKey, NamePrefix) -> + case mnesia:match_object(?APP, #?APP{api_key = ApiKey, _ = '_'}, read) of + [] -> + NewName = generate_unique_name(NamePrefix), + ok = mnesia:write(App#?APP{name = NewName}); + [#?APP{name = Name}] -> + ok = mnesia:write(App#?APP{name = Name}) + end. + +generate_unique_name(NamePrefix) -> + New = list_to_binary(NamePrefix ++ emqx_misc:gen_id(16)), + case mnesia:read(?APP, New) of + [] -> New; + _ -> generate_unique_name(NamePrefix) + end. + trans(Fun, Args) -> case mria:transaction(?COMMON_SHARD, Fun, Args) of {atomic, Res} -> {ok, Res}; @@ -192,3 +220,83 @@ trans(Fun, Args) -> generate_api_secret() -> Random = crypto:strong_rand_bytes(32), emqx_base62:encode(Random). + +bootstrap_file() -> + case emqx:get_config([api_key, bootstrap_file], <<>>) of + %% For compatible remove until 5.1.0 + <<>> -> + emqx:get_config([dashboard, bootstrap_users_file], <<>>); + File -> + File + end. + +init_bootstrap_file(<<>>) -> + ok; +init_bootstrap_file(File) -> + case file:open(File, [read, binary]) of + {ok, Dev} -> + {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), + init_bootstrap_file(File, Dev, MP); + {error, Reason} = Error -> + ?SLOG( + error, + #{ + msg => "failed_to_open_the_bootstrap_file", + file => File, + reason => emqx_misc:explain_posix(Reason) + } + ), + Error + end. + +init_bootstrap_file(File, Dev, MP) -> + try + add_bootstrap_file(File, Dev, MP, 1) + catch + throw:Error -> {error, Error}; + Type:Reason:Stacktrace -> {error, {Type, Reason, Stacktrace}} + after + file:close(Dev) + end. + +-define(BOOTSTRAP_TAG, <<"Bootstrapped From File">>). + +add_bootstrap_file(File, Dev, MP, Line) -> + case file:read_line(Dev) of + {ok, Bin} -> + case re:run(Bin, MP, [global, {capture, all_but_first, binary}]) of + {match, [[AppKey, ApiSecret]]} -> + App = + #?APP{ + enable = true, + expired_at = infinity, + desc = ?BOOTSTRAP_TAG, + created_at = erlang:system_time(second), + api_secret_hash = emqx_dashboard_admin:hash(ApiSecret), + api_key = AppKey + }, + case force_create_app("from_bootstrap_file_", App) of + {ok, ok} -> + add_bootstrap_file(File, Dev, MP, Line + 1); + {error, Reason} -> + throw(#{file => File, line => Line, content => Bin, reason => Reason}) + end; + _ -> + Reason = "invalid_format", + ?SLOG( + error, + #{ + msg => "failed_to_load_bootstrap_file", + file => File, + line => Line, + content => Bin, + reason => Reason + } + ), + throw(#{file => File, line => Line, content => Bin, reason => Reason}) + end; + eof -> + ok; + {error, Reason} -> + throw(#{file => File, line => Line, reason => Reason}) + end.