From 2eb7a71e499ed7cbafe95e429fadd09cead61b3f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 1 Dec 2021 08:52:40 -0300 Subject: [PATCH 01/31] chore(appup): make update_appup.escript output only differences for external dependencies [5.0] Same as #6307 . Currently, the update_appup.escript outputs as an error the full appup file for external dependencies, even if all the changes are already contained in the depency. Here, we make it only output the missing actions to be inserted, to aid in seeing what are the differences. --- scripts/update_appup.escript | 85 ++++++++++++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 8 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 8c420c1bd..8dc624346 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -99,8 +99,26 @@ main(Options, Baseline) -> [] -> ok; _ -> - set_invalid(), - log("ERROR: The appup files are incomplete. Missing changes:~n ~p", [AppupChanges]) + Diffs = + lists:filtermap( + fun({App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}) -> + case parse_appup_diffs(Upgrade, OldUpgrade, + Downgrade, OldDowngrade) of + ok -> + false; + {diffs, Diffs} -> + {true, {App, Diffs}} + end + end, + AppupChanges), + case Diffs =:= [] of + true -> + ok; + false -> + set_invalid(), + log("ERROR: The appup files are incomplete. Missing changes:~n ~p", + [Diffs]) + end end; false -> update_appups(AppupChanges) @@ -189,9 +207,52 @@ find_appup_actions(App, CurrAppIdx, PrevAppIdx = #app{version = PrevVersion}) -> %% The appup file has been already updated: []; true -> - [{App, {Upgrade, Downgrade}}] + [{App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}] end. +%% For external dependencies, show only the changes that are missing +%% in their current appup. +diff_appup_instructions(ComputedChanges, PresentChanges) -> + lists:foldr( + fun({Vsn, ComputedActions}, Acc) -> + case find_matching_version(Vsn, PresentChanges) of + undefined -> + [{Vsn, ComputedActions} | Acc]; + PresentActions -> + DiffActions = ComputedActions -- PresentActions, + case DiffActions of + [] -> + %% no diff + Acc; + _ -> + [{Vsn, DiffActions} | Acc] + end + end + end, + [], + ComputedChanges). + +%% For external dependencies, checks if any missing diffs are present +%% and groups them by `up' and `down' types. +parse_appup_diffs(Upgrade, OldUpgrade, Downgrade, OldDowngrade) -> + DiffUp = diff_appup_instructions(Upgrade, OldUpgrade), + DiffDown = diff_appup_instructions(Downgrade, OldDowngrade), + case {DiffUp, DiffDown} of + {[], []} -> + %% no diff for external dependency; ignore + ok; + _ -> + set_invalid(), + Diffs = #{ up => DiffUp + , down => DiffDown + }, + {diffs, Diffs} + end. + +%% TODO: handle regexes +find_matching_version(Vsn, PresentChanges) -> + proplists:get_value(Vsn, PresentChanges). + find_old_appup_actions(App, PrevVersion) -> {Upgrade0, Downgrade0} = case locate(ebin_current, App, ".appup") of @@ -270,12 +331,12 @@ check_appup_files() -> update_appups(Changes) -> lists:foreach( - fun({App, {Upgrade, Downgrade}}) -> - do_update_appup(App, Upgrade, Downgrade) + fun({App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}) -> + do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) end, Changes). -do_update_appup(App, Upgrade, Downgrade) -> +do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) -> case locate(src, App, ".appup.src") of {ok, AppupFile} -> render_appfile(AppupFile, Upgrade, Downgrade); @@ -284,8 +345,16 @@ do_update_appup(App, Upgrade, Downgrade) -> {ok, AppupFile} -> render_appfile(AppupFile, Upgrade, Downgrade); false -> - set_invalid(), - log("ERROR: Appup file for the external dependency '~p' is not complete.~n Missing changes: ~p~n", [App, Upgrade]) + case parse_appup_diffs(Upgrade, OldUpgrade, + Downgrade, OldDowngrade) of + ok -> + %% no diff for external dependency; ignore + ok; + {diffs, Diffs} -> + set_invalid(), + log("ERROR: Appup file for the external dependency '~p' is not complete.~n Missing changes: ~100p~n", [App, Diffs]), + log("NOTE: Some changes above might be already covered by regexes.~n") + end end end. From 2b0a3e8ba3665dc9455aae7c4c31d01f5c14faf5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 6 Dec 2021 22:50:42 +0300 Subject: [PATCH 02/31] refactor(authn): unify password hashing --- apps/emqx/src/emqx_channel.erl | 2 +- apps/emqx/src/emqx_passwd.erl | 116 ++++++++------- apps/emqx/test/emqx_passwd_SUITE.erl | 76 +++++++++- .../src/emqx_authn_password_hashing.erl | 139 ++++++++++++++++++ apps/emqx_authn/src/emqx_authn_utils.erl | 47 ++---- .../src/simple_authn/emqx_authn_mnesia.erl | 87 +++-------- .../src/simple_authn/emqx_authn_mongodb.erl | 40 +---- .../src/simple_authn/emqx_authn_mysql.erl | 20 +-- .../src/simple_authn/emqx_authn_pgsql.erl | 22 +-- .../src/simple_authn/emqx_authn_redis.erl | 18 +-- .../test/emqx_authn_mongo_SUITE.erl | 48 +++--- .../test/emqx_authn_mysql_SUITE.erl | 45 +++--- .../emqx_authn_password_hashing_SUITE.erl | 135 +++++++++++++++++ .../test/emqx_authn_pgsql_SUITE.erl | 42 +++--- .../test/emqx_authn_redis_SUITE.erl | 40 +++-- 15 files changed, 543 insertions(+), 334 deletions(-) create mode 100644 apps/emqx_authn/src/emqx_authn_password_hashing.erl create mode 100644 apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 50f9fcf70..a304f6d7a 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -255,7 +255,7 @@ set_peercert_infos(Peercert, ClientInfo, Zone) -> dn -> DN; crt -> Peercert; pem when is_binary(Peercert) -> base64:encode(Peercert); - md5 when is_binary(Peercert) -> emqx_passwd:hash(md5, Peercert); + md5 when is_binary(Peercert) -> emqx_passwd:hash_data(md5, Peercert); _ -> undefined end end, diff --git a/apps/emqx/src/emqx_passwd.erl b/apps/emqx/src/emqx_passwd.erl index 2104f1850..f729ce49e 100644 --- a/apps/emqx/src/emqx_passwd.erl +++ b/apps/emqx/src/emqx_passwd.erl @@ -17,76 +17,92 @@ -module(emqx_passwd). -export([ hash/2 - , check_pass/2 + , hash_data/2 + , check_pass/3 ]). +-export_type([ password/0 + , password_hash/0 + , hash_type_simple/0 + , hash_type/0 + , salt_position/0 + , salt/0]). + -include("logger.hrl"). --type(hash_type() :: plain | md5 | sha | sha256 | sha512 | pbkdf2 | bcrypt). +-type(password() :: binary()). +-type(password_hash() :: binary()). --export_type([hash_type/0]). +-type(hash_type_simple() :: plain | md5 | sha | sha256 | sha512). +-type(hash_type() :: hash_type_simple() | bcrypt). + +-type(salt_position() :: prefix | suffix). +-type(salt() :: binary()). + +-type(hash_params() :: {bcrypt, salt()} | {hash_type_simple(), salt(), salt_position()}). %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- --spec(check_pass(binary() | tuple(), binary() | tuple()) - -> ok | {error, term()}). -check_pass({PassHash, Password}, bcrypt) -> - try - Salt = binary:part(PassHash, {0, 29}), - check_pass(PassHash, emqx_passwd:hash(bcrypt, {Salt, Password})) - catch - error:badarg -> {error, incorrect_hash} +-spec(check_pass(hash_params(), password_hash(), password()) -> boolean()). +check_pass({bcrypt, Salt}, PasswordHash, Password) -> + case bcrypt:hashpw(Password, Salt) of + {ok, HashPasswd} -> + compare_secure(list_to_binary(HashPasswd), PasswordHash); + {error, _Reason}-> + false end; -check_pass({PassHash, Password}, HashType) -> - check_pass(PassHash, emqx_passwd:hash(HashType, Password)); -check_pass({PassHash, Salt, Password}, {pbkdf2, Macfun, Iterations, Dklen}) -> - check_pass(PassHash, emqx_passwd:hash(pbkdf2, {Salt, Password, Macfun, Iterations, Dklen})); -check_pass({PassHash, Salt, Password}, {salt, bcrypt}) -> - check_pass(PassHash, emqx_passwd:hash(bcrypt, {Salt, Password})); -check_pass({PassHash, Salt, Password}, {bcrypt, salt}) -> - check_pass(PassHash, emqx_passwd:hash(bcrypt, {Salt, Password})); -check_pass({PassHash, Salt, Password}, {salt, HashType}) -> - check_pass(PassHash, emqx_passwd:hash(HashType, <>)); -check_pass({PassHash, Salt, Password}, {HashType, salt}) -> - check_pass(PassHash, emqx_passwd:hash(HashType, <>)); -check_pass(PassHash, PassHash) -> ok; -check_pass(_Hash1, _Hash2) -> {error, password_error}. +check_pass({_SimpleHash, _Salt, _SaltPosition} = HashParams, PasswordHash, Password) -> + Hash = hash(HashParams, Password), + compare_secure(Hash, PasswordHash). --spec(hash(hash_type(), binary() | tuple()) -> binary()). -hash(plain, Password) -> - Password; -hash(md5, Password) -> - hexstring(crypto:hash(md5, Password)); -hash(sha, Password) -> - hexstring(crypto:hash(sha, Password)); -hash(sha256, Password) -> - hexstring(crypto:hash(sha256, Password)); -hash(sha512, Password) -> - hexstring(crypto:hash(sha512, Password)); -hash(pbkdf2, {Salt, Password, Macfun, Iterations, Dklen}) -> - case pbkdf2:pbkdf2(Macfun, Password, Salt, Iterations, Dklen) of - {ok, Hexstring} -> - pbkdf2:to_hex(Hexstring); - {error, Reason} -> - ?SLOG(error, #{msg => "pbkdf2_hash_error", reason => Reason}), - <<>> - end; -hash(bcrypt, {Salt, Password}) -> - {ok, _} = application:ensure_all_started(bcrypt), +-spec(hash(hash_params(), password()) -> password_hash()). +hash({bcrypt, Salt}, Password) -> case bcrypt:hashpw(Password, Salt) of {ok, HashPasswd} -> list_to_binary(HashPasswd); {error, Reason}-> - ?SLOG(error, #{msg => "bcrypt_hash_error", reason => Reason}), - <<>> - end. + error(Reason) + end; +hash({SimpleHash, Salt, prefix}, Password) when is_binary(Password), is_binary(Salt) -> + hash_data(SimpleHash, <>); +hash({SimpleHash, Salt, suffix}, Password) when is_binary(Password), is_binary(Salt) -> + hash_data(SimpleHash, <>). + + +-spec(hash_data(hash_type(), binary()) -> binary()). +hash_data(plain, Data) when is_binary(Data) -> + Data; +hash_data(md5, Data) when is_binary(Data) -> + hexstring(crypto:hash(md5, Data)); +hash_data(sha, Data) when is_binary(Data) -> + hexstring(crypto:hash(sha, Data)); +hash_data(sha256, Data) when is_binary(Data) -> + hexstring(crypto:hash(sha256, Data)); +hash_data(sha512, Data) when is_binary(Data) -> + hexstring(crypto:hash(sha512, Data)). %%-------------------------------------------------------------------- -%% Internal funcs +%% Internal functions %%-------------------------------------------------------------------- +compare_secure(X, Y) when is_binary(X), is_binary(Y) -> + compare_secure(binary_to_list(X), binary_to_list(Y)); +compare_secure(X, Y) when is_list(X), is_list(Y) -> + case length(X) == length(Y) of + true -> + compare_secure(X, Y, 0); + false -> + false + end. + +compare_secure([X | RestX], [Y | RestY], Result) -> + compare_secure(RestX, RestY, (X bxor Y) bor Result); +compare_secure([], [], Result) -> + Result == 0. + + hexstring(<>) -> iolist_to_binary(io_lib:format("~32.16.0b", [X])); hexstring(<>) -> diff --git a/apps/emqx/test/emqx_passwd_SUITE.erl b/apps/emqx/test/emqx_passwd_SUITE.erl index fe4694294..e3c904a2e 100644 --- a/apps/emqx/test/emqx_passwd_SUITE.erl +++ b/apps/emqx/test/emqx_passwd_SUITE.erl @@ -19,13 +19,73 @@ -compile(nowarn_export_all). -compile(export_all). -all() -> [t_hash]. +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(bcrypt), + Config. + +end_per_suite(_Config) -> + ok. + +t_hash_data(_) -> + Password = <<"password">>, + Password = emqx_passwd:hash_data(plain, Password), + + <<"5f4dcc3b5aa765d61d8327deb882cf99">> + = emqx_passwd:hash_data(md5, Password), + + <<"5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8">> + = emqx_passwd:hash_data(sha, Password), + + <<"5e884898da28047151d0e56f8dc6292773603d0d6aabbdd62a11ef721d1542d8">> + = emqx_passwd:hash_data(sha256, Password), + + Sha512 = iolist_to_binary( + [<<"b109f3bbbc244eb82441917ed06d618b9008dd09b3befd1b5e07394c706a8bb9">>, + <<"80b1d7785e5976ec049b46df5f1326af5a2ea6d103fd07c95385ffab0cacbc86">>]), + + Sha512 = emqx_passwd:hash_data(sha512, Password). t_hash(_) -> - Password = <<"password">>, Salt = <<"salt">>, - _ = emqx_passwd:hash(plain, Password), - _ = emqx_passwd:hash(md5, Password), - _ = emqx_passwd:hash(sha, Password), - _ = emqx_passwd:hash(sha256, Password), - _ = emqx_passwd:hash(bcrypt, {Salt, Password}), - _ = emqx_passwd:hash(pbkdf2, {Salt, Password, sha256, 1000, 20}). + Password = <<"password">>, + Salt = <<"salt">>, + WrongPassword = <<"wrongpass">>, + + Md5 = <<"67a1e09bb1f83f5007dc119c14d663aa">>, + Md5 = emqx_passwd:hash({md5, Salt, prefix}, Password), + true = emqx_passwd:check_pass({md5, Salt, prefix}, Md5, Password), + false = emqx_passwd:check_pass({md5, Salt, prefix}, Md5, WrongPassword), + + Sha = <<"59b3e8d637cf97edbe2384cf59cb7453dfe30789">>, + Sha = emqx_passwd:hash({sha, Salt, prefix}, Password), + true = emqx_passwd:check_pass({sha, Salt, prefix}, Sha, Password), + false = emqx_passwd:check_pass({sha, Salt, prefix}, Sha, WrongPassword), + + Sha256 = <<"7a37b85c8918eac19a9089c0fa5a2ab4dce3f90528dcdeec108b23ddf3607b99">>, + Sha256 = emqx_passwd:hash({sha256, Salt, suffix}, Password), + true = emqx_passwd:check_pass({sha256, Salt, suffix}, Sha256, Password), + false = emqx_passwd:check_pass({sha256, Salt, suffix}, Sha256, WrongPassword), + + Sha512 = iolist_to_binary( + [<<"fa6a2185b3e0a9a85ef41ffb67ef3c1fb6f74980f8ebf970e4e72e353ed9537d">>, + <<"593083c201dfd6e43e1c8a7aac2bc8dbb119c7dfb7d4b8f131111395bd70e97f">>]), + Sha512 = emqx_passwd:hash({sha512, Salt, suffix}, Password), + true = emqx_passwd:check_pass({sha512, Salt, suffix}, Sha512, Password), + false = emqx_passwd:check_pass({sha512, Salt, suffix}, Sha512, WrongPassword), + + BcryptSalt = <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + Bcrypt = <<"$2b$12$wtY3h20mUjjmeaClpqZVvehyw7F.V78F3rbK2xDkCzRTMi6pmfUB6">>, + Bcrypt = emqx_passwd:hash({bcrypt, BcryptSalt}, Password), + true = emqx_passwd:check_pass({bcrypt, Bcrypt}, Bcrypt, Password), + false = emqx_passwd:check_pass({bcrypt, Bcrypt}, Bcrypt, WrongPassword), + false = emqx_passwd:check_pass({bcrypt, <<>>}, <<>>, WrongPassword), + + %% Invalid salt, bcrypt fails + ?assertException(error, _, emqx_passwd:hash({bcrypt, Salt}, Password)). diff --git a/apps/emqx_authn/src/emqx_authn_password_hashing.erl b/apps/emqx_authn/src/emqx_authn_password_hashing.erl new file mode 100644 index 000000000..4cb1ad401 --- /dev/null +++ b/apps/emqx_authn/src/emqx_authn_password_hashing.erl @@ -0,0 +1,139 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_password_hashing). + +-include_lib("typerefl/include/types.hrl"). + +-type(simple_algorithm_name() :: plain | md5 | sha | sha256 | sha512). +-type(salt_position() :: prefix | suffix). + +-type(simple_algorithm() :: #{name := simple_algorithm_name(), + salt_position := salt_position()}). + +-type(bcrypt_algorithm() :: #{name := bcrypt}). +-type(bcrypt_algorithm_rw() :: #{name := bcrypt, salt_rounds := integer()}). + +-type(algorithm() :: simple_algorithm() | bcrypt_algorithm()). +-type(algorithm_rw() :: simple_algorithm() | bcrypt_algorithm_rw()). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +-behaviour(hocon_schema). + +-export([roots/0, + fields/1]). + +-export([type_ro/1, + type_rw/1]). + +-export([init/1, + gen_salt/1, + hash/2, + check_password/4]). + +roots() -> [bcrypt, bcrypt_rw, other_algorithms]. + +fields(bcrypt_rw) -> + fields(bcrypt) ++ + [{salt_rounds, fun salt_rounds/1}]; + +fields(bcrypt) -> + [{name, {enum, [bcrypt]}}]; + +fields(other_algorithms) -> + [{name, {enum, [plain, md5, sha, sha256, sha512]}}, + {salt_position, fun salt_position/1}]. + +salt_position(type) -> {enum, [prefix, suffix]}; +salt_position(default) -> prefix; +salt_position(_) -> undefined. + +salt_rounds(type) -> integer(); +salt_rounds(default) -> 10; +salt_rounds(_) -> undefined. + +type_rw(type) -> + hoconsc:union(rw_refs()); +type_rw(default) -> #{<<"name">> => sha256, <<"salt_position">> => prefix}; +type_rw(_) -> undefined. + +type_ro(type) -> + hoconsc:union(ro_refs()); +type_ro(default) -> #{<<"name">> => sha256, <<"salt_position">> => prefix}; +type_ro(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +-spec(init(algorithm()) -> ok). +init(#{name := bcrypt}) -> + {ok, _} = application:ensure_all_started(bcrypt), + ok; +init(#{name := _Other}) -> + ok. + + +-spec(gen_salt(algorithm_rw()) -> emqx_passwd:salt()). +gen_salt(#{name := plain}) -> + <<>>; +gen_salt(#{name := bcrypt, + salt_rounds := Rounds}) -> + {ok, Salt} = bcrypt:gen_salt(Rounds), + list_to_binary(Salt); +gen_salt(#{name := Other}) when Other =/= plain, Other =/= bcrypt -> + <> = crypto:strong_rand_bytes(16), + iolist_to_binary(io_lib:format("~32.16.0b", [X])). + + +-spec(hash(algorithm_rw(), emqx_passwd:password()) -> {emqx_passwd:hash(), emqx_passwd:salt()}). +hash(#{name := bcrypt, salt_rounds := _} = Algorithm, Password) -> + Salt0 = gen_salt(Algorithm), + Hash = emqx_passwd:hash({bcrypt, Salt0}, Password), + Salt = Hash, + {Hash, Salt}; + +hash(#{name := Other, salt_position := SaltPosition} = Algorithm, Password) -> + Salt = gen_salt(Algorithm), + Hash = emqx_passwd:hash({Other, Salt, SaltPosition}, Password), + {Hash, Salt}. + + +-spec(check_password( + algorithm(), + emqx_passwd:salt(), + emqx_passwd:hash(), + emqx_passwd:password()) -> boolean()). +check_password(#{name := bcrypt}, _Salt, PasswordHash, Password) -> + emqx_passwd:check_pass({bcrypt, PasswordHash}, PasswordHash, Password); + +check_password(#{name := Other, salt_position := SaltPosition}, Salt, PasswordHash, Password) -> + emqx_passwd:check_pass({Other, Salt, SaltPosition}, PasswordHash, Password). + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +rw_refs() -> + [hoconsc:ref(?MODULE, bcrypt_rw), + hoconsc:ref(?MODULE, other_algorithms)]. + +ro_refs() -> + [hoconsc:ref(?MODULE, bcrypt), + hoconsc:ref(?MODULE, other_algorithms)]. diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 2205d237d..b211cc1bd 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -18,12 +18,10 @@ -include_lib("emqx/include/emqx_placeholder.hrl"). --export([ replace_placeholders/2 +-export([ check_password_from_selected_map/3 + , replace_placeholders/2 , replace_placeholder/2 - , check_password/3 , is_superuser/1 - , hash/4 - , gen_salt/0 , bin/1 , ensure_apps_started/1 , cleanup_resources/0 @@ -36,6 +34,17 @@ %% APIs %%------------------------------------------------------------------------------ +check_password_from_selected_map(_Algorithm, _Selected, undefined) -> + {error, bad_username_or_password}; +check_password_from_selected_map( + Algorithm, #{<<"password_hash">> := Hash} = Selected, Password) -> + Salt = maps:get(<<"salt">>, Selected, <<>>), + case emqx_authn_password_hashing:check_password(Algorithm, Salt, Hash, Password) of + true -> ok; + false -> + {error, bad_username_or_password} + end. + replace_placeholders(PlaceHolders, Data) -> replace_placeholders(PlaceHolders, Data, []). @@ -64,27 +73,6 @@ replace_placeholder(?PH_CERT_CN_NAME, Credential) -> replace_placeholder(Constant, _) -> Constant. -check_password(undefined, _Selected, _State) -> - {error, bad_username_or_password}; -check_password(Password, - #{<<"password_hash">> := Hash}, - #{password_hash_algorithm := bcrypt}) -> - case emqx_passwd:hash(bcrypt, {Hash, Password}) of - Hash -> ok; - _ -> - {error, bad_username_or_password} - end; -check_password(Password, - #{<<"password_hash">> := Hash} = Selected, - #{password_hash_algorithm := Algorithm, - salt_position := SaltPosition}) -> - Salt = maps:get(<<"salt">>, Selected, <<>>), - case hash(Algorithm, Password, Salt, SaltPosition) of - Hash -> ok; - _ -> - {error, bad_username_or_password} - end. - is_superuser(#{<<"is_superuser">> := <<"">>}) -> #{is_superuser => false}; is_superuser(#{<<"is_superuser">> := <<"0">>}) -> @@ -108,15 +96,6 @@ ensure_apps_started(bcrypt) -> ensure_apps_started(_) -> ok. -hash(Algorithm, Password, Salt, prefix) -> - emqx_passwd:hash(Algorithm, <>); -hash(Algorithm, Password, Salt, suffix) -> - emqx_passwd:hash(Algorithm, <>). - -gen_salt() -> - <> = crypto:strong_rand_bytes(16), - iolist_to_binary(io_lib:format("~32.16.0b", [X])). - bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(L) when is_list(L) -> list_to_binary(L); bin(X) -> X. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index f609d8cac..2c68c034d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -91,31 +91,13 @@ fields(?CONF_NS) -> [ {mechanism, emqx_authn_schema:mechanism('password-based')} , {backend, emqx_authn_schema:backend('built-in-database')} , {user_id_type, fun user_id_type/1} - , {password_hash_algorithm, fun password_hash_algorithm/1} - ] ++ emqx_authn_schema:common_fields(); - -fields(bcrypt) -> - [ {name, {enum, [bcrypt]}} - , {salt_rounds, fun salt_rounds/1} - ]; - -fields(other_algorithms) -> - [ {name, {enum, [plain, md5, sha, sha256, sha512]}} - ]. + , {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1} + ] ++ emqx_authn_schema:common_fields(). user_id_type(type) -> user_id_type(); user_id_type(default) -> <<"username">>; user_id_type(_) -> undefined. -password_hash_algorithm(type) -> hoconsc:union([hoconsc:ref(?MODULE, bcrypt), - hoconsc:ref(?MODULE, other_algorithms)]); -password_hash_algorithm(default) -> #{<<"name">> => sha256}; -password_hash_algorithm(_) -> undefined. - -salt_rounds(type) -> integer(); -salt_rounds(default) -> 10; -salt_rounds(_) -> undefined. - %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ @@ -125,22 +107,11 @@ refs() -> create(AuthenticatorID, #{user_id_type := Type, - password_hash_algorithm := #{name := bcrypt, - salt_rounds := SaltRounds}}) -> - ok = emqx_authn_utils:ensure_apps_started(bcrypt), + password_hash_algorithm := Algorithm}) -> + ok = emqx_authn_password_hashing:init(Algorithm), State = #{user_group => AuthenticatorID, user_id_type => Type, - password_hash_algorithm => bcrypt, - salt_rounds => SaltRounds}, - {ok, State}; - -create(AuthenticatorID, - #{user_id_type := Type, - password_hash_algorithm := #{name := Name}}) -> - ok = emqx_authn_utils:ensure_apps_started(Name), - State = #{user_group => AuthenticatorID, - user_id_type => Type, - password_hash_algorithm => Name}, + password_hash_algorithm => Algorithm}, {ok, State}. update(Config, #{user_group := ID}) -> @@ -156,12 +127,9 @@ authenticate(#{password := Password} = Credential, case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of [] -> ignore; - [#user_info{password_hash = PasswordHash, salt = Salt0, is_superuser = IsSuperuser}] -> - Salt = case Algorithm of - bcrypt -> PasswordHash; - _ -> Salt0 - end, - case PasswordHash =:= hash(Algorithm, Password, Salt) of + [#user_info{password_hash = PasswordHash, salt = Salt, is_superuser = IsSuperuser}] -> + case emqx_authn_password_hashing:check_password( + Algorithm, Salt, PasswordHash, Password) of true -> {ok, #{is_superuser => IsSuperuser}}; false -> {error, bad_username_or_password} end @@ -193,12 +161,13 @@ import_users(Filename0, State) -> add_user(#{user_id := UserID, password := Password} = UserInfo, - #{user_group := UserGroup} = State) -> + #{user_group := UserGroup, + password_hash_algorithm := Algorithm}) -> trans( fun() -> case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> - {PasswordHash, Salt} = hash(Password, State), + {PasswordHash, Salt} = emqx_authn_password_hashing:hash(Algorithm, Password), IsSuperuser = maps:get(is_superuser, UserInfo, false), insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), {ok, #{user_id => UserID, is_superuser => IsSuperuser}}; @@ -219,7 +188,8 @@ delete_user(UserID, #{user_group := UserGroup}) -> end). update_user(UserID, UserInfo, - #{user_group := UserGroup} = State) -> + #{user_group := UserGroup, + password_hash_algorithm := Algorithm}) -> trans( fun() -> case mnesia:read(?TAB, {UserGroup, UserID}, write) of @@ -229,11 +199,12 @@ update_user(UserID, UserInfo, , salt = Salt , is_superuser = IsSuperuser}] -> NSuperuser = maps:get(is_superuser, UserInfo, IsSuperuser), - {NPasswordHash, NSalt} = case maps:get(password, UserInfo, undefined) of - undefined -> - {PasswordHash, Salt}; - Password -> - hash(Password, State) + {NPasswordHash, NSalt} = case UserInfo of + #{password := Password} -> + emqx_authn_password_hashing:hash( + Algorithm, Password); + #{} -> + {PasswordHash, Salt} end, insert_user(UserGroup, UserID, NPasswordHash, NSalt, NSuperuser), {ok, #{user_id => UserID, is_superuser => NSuperuser}} @@ -349,26 +320,6 @@ get_user_info_by_seq([<<"false">> | More1], [<<"is_superuser">> | More2], Acc) - get_user_info_by_seq(_, _, _) -> {error, bad_format}. -gen_salt(#{password_hash_algorithm := plain}) -> - <<>>; -gen_salt(#{password_hash_algorithm := bcrypt, - salt_rounds := Rounds}) -> - {ok, Salt} = bcrypt:gen_salt(Rounds), - Salt; -gen_salt(_) -> - emqx_authn_utils:gen_salt(). - -hash(bcrypt, Password, Salt) -> - {ok, Hash} = bcrypt:hashpw(Password, Salt), - list_to_binary(Hash); -hash(Algorithm, Password, Salt) -> - emqx_passwd:hash(Algorithm, <>). - -hash(Password, #{password_hash_algorithm := Algorithm} = State) -> - Salt = gen_salt(State), - PasswordHash = hash(Algorithm, Password, Salt), - {PasswordHash, Salt}. - insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) -> UserInfo = #user_info{user_id = {UserGroup, UserID}, password_hash = PasswordHash, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 3b47bcd7b..8f8b53f14 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -63,8 +63,7 @@ common_fields() -> , {password_hash_field, fun password_hash_field/1} , {salt_field, fun salt_field/1} , {is_superuser_field, fun is_superuser_field/1} - , {password_hash_algorithm, fun password_hash_algorithm/1} - , {salt_position, fun salt_position/1} + , {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1} ] ++ emqx_authn_schema:common_fields(). collection(type) -> binary(); @@ -84,14 +83,6 @@ is_superuser_field(type) -> binary(); is_superuser_field(nullable) -> true; is_superuser_field(_) -> undefined. -password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; -password_hash_algorithm(default) -> sha256; -password_hash_algorithm(_) -> undefined. - -salt_position(type) -> {enum, [prefix, suffix]}; -salt_position(default) -> prefix; -salt_position(_) -> undefined. - %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ @@ -116,7 +107,7 @@ create(#{selector := Selector} = Config) -> salt_position], Config), #{password_hash_algorithm := Algorithm} = State, - ok = emqx_authn_utils:ensure_apps_started(Algorithm), + ok = emqx_authn_password_hashing:init(Algorithm), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), NState = State#{ selector => NSelector, @@ -203,24 +194,10 @@ normalize_selector(Selector) -> check_password(undefined, _Selected, _State) -> {error, bad_username_or_password}; -check_password(Password, - Doc, - #{password_hash_algorithm := bcrypt, - password_hash_field := PasswordHashField}) -> - case maps:get(PasswordHashField, Doc, undefined) of - undefined -> - {error, {cannot_find_password_hash_field, PasswordHashField}}; - Hash -> - case {ok, to_list(Hash)} =:= bcrypt:hashpw(Password, Hash) of - true -> ok; - false -> {error, bad_username_or_password} - end - end; check_password(Password, Doc, #{password_hash_algorithm := Algorithm, - password_hash_field := PasswordHashField, - salt_position := SaltPosition} = State) -> + password_hash_field := PasswordHashField} = State) -> case maps:get(PasswordHashField, Doc, undefined) of undefined -> {error, {cannot_find_password_hash_field, PasswordHashField}}; @@ -229,7 +206,7 @@ check_password(Password, undefined -> <<>>; SaltField -> maps:get(SaltField, Doc, <<>>) end, - case Hash =:= hash(Algorithm, Password, Salt, SaltPosition) of + case emqx_authn_password_hashing:check_password(Algorithm, Salt, Hash, Password) of true -> ok; false -> {error, bad_username_or_password} end @@ -240,12 +217,3 @@ is_superuser(Doc, #{is_superuser_field := IsSuperuserField}) -> emqx_authn_utils:is_superuser(#{<<"is_superuser">> => IsSuperuser}); is_superuser(_, _) -> emqx_authn_utils:is_superuser(#{<<"is_superuser">> => false}). - -hash(Algorithm, Password, Salt, prefix) -> - emqx_passwd:hash(Algorithm, <>); -hash(Algorithm, Password, Salt, suffix) -> - emqx_passwd:hash(Algorithm, <>). - -to_list(L) when is_list(L) -> L; -to_list(L) when is_binary(L) -> binary_to_list(L); -to_list(X) -> X. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index fd0d09f57..852789363 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -46,22 +46,13 @@ roots() -> [?CONF_NS]. fields(?CONF_NS) -> [ {mechanism, emqx_authn_schema:mechanism('password-based')} , {backend, emqx_authn_schema:backend(mysql)} - , {password_hash_algorithm, fun password_hash_algorithm/1} - , {salt_position, fun salt_position/1} + , {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1} , {query, fun query/1} , {query_timeout, fun query_timeout/1} ] ++ emqx_authn_schema:common_fields() ++ emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). -password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; -password_hash_algorithm(default) -> sha256; -password_hash_algorithm(_) -> undefined. - -salt_position(type) -> {enum, [prefix, suffix]}; -salt_position(default) -> prefix; -salt_position(_) -> undefined. - query(type) -> string(); query(_) -> undefined. @@ -80,14 +71,13 @@ create(_AuthenticatorID, Config) -> create(Config). create(#{password_hash_algorithm := Algorithm, - salt_position := SaltPosition, query := Query0, query_timeout := QueryTimeout } = Config) -> + ok = emqx_authn_password_hashing:init(Algorithm), {Query, PlaceHolders} = parse_query(Query0), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), State = #{password_hash_algorithm => Algorithm, - salt_position => SaltPosition, query => Query, placeholders => PlaceHolders, query_timeout => QueryTimeout, @@ -116,13 +106,15 @@ authenticate(#{password := Password} = Credential, #{placeholders := PlaceHolders, query := Query, query_timeout := Timeout, - resource_id := ResourceId} = State) -> + resource_id := ResourceId, + password_hash_algorithm := Algorithm}) -> Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), case emqx_resource:query(ResourceId, {sql, Query, Params, Timeout}) of {ok, _Columns, []} -> ignore; {ok, Columns, [Row | _]} -> Selected = maps:from_list(lists:zip(Columns, Row)), - case emqx_authn_utils:check_password(Password, Selected, State) of + case emqx_authn_utils:check_password_from_selected_map( + Algorithm, Selected, Password) of ok -> {ok, emqx_authn_utils:is_superuser(Selected)}; {error, Reason} -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index fdd30b618..0ed7d282a 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -52,21 +52,12 @@ roots() -> [?CONF_NS]. fields(?CONF_NS) -> [ {mechanism, emqx_authn_schema:mechanism('password-based')} , {backend, emqx_authn_schema:backend(postgresql)} - , {password_hash_algorithm, fun password_hash_algorithm/1} - , {salt_position, fun salt_position/1} + , {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1} , {query, fun query/1} ] ++ emqx_authn_schema:common_fields() ++ emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). -password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; -password_hash_algorithm(default) -> sha256; -password_hash_algorithm(_) -> undefined. - -salt_position(type) -> {enum, [prefix, suffix]}; -salt_position(default) -> prefix; -salt_position(_) -> undefined. - query(type) -> string(); query(_) -> undefined. @@ -81,14 +72,13 @@ create(_AuthenticatorID, Config) -> create(Config). create(#{query := Query0, - password_hash_algorithm := Algorithm, - salt_position := SaltPosition} = Config) -> + password_hash_algorithm := Algorithm} = Config) -> + ok = emqx_authn_password_hashing:init(Algorithm), {Query, PlaceHolders} = parse_query(Query0), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), State = #{query => Query, placeholders => PlaceHolders, password_hash_algorithm => Algorithm, - salt_position => SaltPosition, resource_id => ResourceId}, case emqx_resource:create_local(ResourceId, emqx_connector_pgsql, Config) of {ok, already_created} -> @@ -113,14 +103,16 @@ authenticate(#{auth_method := _}, _) -> authenticate(#{password := Password} = Credential, #{query := Query, placeholders := PlaceHolders, - resource_id := ResourceId} = State) -> + resource_id := ResourceId, + password_hash_algorithm := Algorithm}) -> Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), case emqx_resource:query(ResourceId, {sql, Query, Params}) of {ok, _Columns, []} -> ignore; {ok, Columns, [Row | _]} -> NColumns = [Name || #column{name = Name} <- Columns], Selected = maps:from_list(lists:zip(NColumns, erlang:tuple_to_list(Row))), - case emqx_authn_utils:check_password(Password, Selected, State) of + case emqx_authn_utils:check_password_from_selected_map( + Algorithm, Selected, Password) of ok -> {ok, emqx_authn_utils:is_superuser(Selected)}; {error, Reason} -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index e17d0ad8f..1927ab822 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -59,21 +59,12 @@ common_fields() -> [ {mechanism, emqx_authn_schema:mechanism('password-based')} , {backend, emqx_authn_schema:backend(redis)} , {cmd, fun cmd/1} - , {password_hash_algorithm, fun password_hash_algorithm/1} - , {salt_position, fun salt_position/1} + , {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1} ] ++ emqx_authn_schema:common_fields(). cmd(type) -> string(); cmd(_) -> undefined. -password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; -password_hash_algorithm(default) -> sha256; -password_hash_algorithm(_) -> undefined. - -salt_position(type) -> {enum, [prefix, suffix]}; -salt_position(default) -> prefix; -salt_position(_) -> undefined. - %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ @@ -89,6 +80,7 @@ create(_AuthenticatorID, Config) -> create(#{cmd := Cmd, password_hash_algorithm := Algorithm} = Config) -> + ok = emqx_authn_password_hashing:init(Algorithm), try NCmd = parse_cmd(Cmd), ok = emqx_authn_utils:ensure_apps_started(Algorithm), @@ -129,13 +121,15 @@ authenticate(#{auth_method := _}, _) -> ignore; authenticate(#{password := Password} = Credential, #{cmd := {Command, Key, Fields}, - resource_id := ResourceId} = State) -> + resource_id := ResourceId, + password_hash_algorithm := Algorithm}) -> NKey = binary_to_list(iolist_to_binary(replace_placeholders(Key, Credential))), case emqx_resource:query(ResourceId, {cmd, [Command, NKey | Fields]}) of {ok, Values} -> case merge(Fields, Values) of #{<<"password_hash">> := _} = Selected -> - case emqx_authn_utils:check_password(Password, Selected, State) of + case emqx_authn_utils:check_password_from_selected_map( + Algorithm, Selected, Password) of ok -> {ok, emqx_authn_utils:is_superuser(Selected)}; {error, Reason} -> diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl index 562c5aa1b..edd91be55 100644 --- a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl @@ -238,22 +238,22 @@ test_is_superuser({Value, ExpectedValue}) -> raw_mongo_auth_config() -> #{ - mechanism => <<"password-based">>, - password_hash_algorithm => <<"plain">>, - salt_position => <<"suffix">>, - enable => <<"true">>, + mechanism => <<"password-based">>, + password_hash_algorithm => #{name => <<"plain">>, + salt_position => <<"suffix">>}, + enable => <<"true">>, - backend => <<"mongodb">>, - mongo_type => <<"single">>, - database => <<"mqtt">>, - collection => <<"users">>, - server => mongo_server(), + backend => <<"mongodb">>, + mongo_type => <<"single">>, + database => <<"mqtt">>, + collection => <<"users">>, + server => mongo_server(), - selector => #{<<"username">> => <<"${username}">>}, - password_hash_field => <<"password_hash">>, - salt_field => <<"salt">>, - is_superuser_field => <<"is_superuser">> - }. + selector => #{<<"username">> => <<"${username}">>}, + password_hash_field => <<"password_hash">>, + salt_field => <<"salt">>, + is_superuser_field => <<"is_superuser">> + }. user_seeds() -> [#{data => #{ @@ -282,8 +282,8 @@ user_seeds() -> password => <<"md5">> }, config_params => #{ - password_hash_algorithm => <<"md5">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"md5">>, + salt_position => <<"suffix">> } }, result => {ok,#{is_superuser => false}} }, @@ -300,8 +300,8 @@ user_seeds() -> }, config_params => #{ selector => #{<<"username">> => <<"${clientid}">>}, - password_hash_algorithm => <<"sha256">>, - salt_position => <<"prefix">> + password_hash_algorithm => #{name => <<"sha256">>, + salt_position => <<"prefix">>} }, result => {ok,#{is_superuser => true}} }, @@ -317,8 +317,7 @@ user_seeds() -> password => <<"bcrypt">> }, config_params => #{ - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> % should be ignored + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {ok,#{is_superuser => false}} }, @@ -336,8 +335,7 @@ user_seeds() -> config_params => #{ % clientid variable & username credentials selector => #{<<"username">> => <<"${clientid}">>}, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,not_authorized} }, @@ -354,8 +352,7 @@ user_seeds() -> }, config_params => #{ selector => #{<<"userid">> => <<"${clientid}">>}, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,not_authorized} }, @@ -372,8 +369,7 @@ user_seeds() -> password => <<"wrongpass">> }, config_params => #{ - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,bad_username_or_password} } diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index bf66b034a..95eecdead 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -204,20 +204,20 @@ t_update(_Config) -> raw_mysql_auth_config() -> #{ - mechanism => <<"password-based">>, - password_hash_algorithm => <<"plain">>, - salt_position => <<"suffix">>, - enable => <<"true">>, + mechanism => <<"password-based">>, + password_hash_algorithm => #{name => <<"plain">>, + salt_position => <<"suffix">>}, + enable => <<"true">>, - backend => <<"mysql">>, - database => <<"mqtt">>, - username => <<"root">>, - password => <<"public">>, + backend => <<"mysql">>, + database => <<"mqtt">>, + username => <<"root">>, + password => <<"public">>, - query => <<"SELECT password_hash, salt, is_superuser_str as is_superuser + query => <<"SELECT password_hash, salt, is_superuser_str as is_superuser FROM users where username = ${username} LIMIT 1">>, - server => mysql_server() - }. + server => mysql_server() + }. user_seeds() -> [#{data => #{ @@ -244,8 +244,8 @@ user_seeds() -> password => <<"md5">> }, config_params => #{ - password_hash_algorithm => <<"md5">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"md5">>, + salt_position => <<"suffix">>} }, result => {ok,#{is_superuser => false}} }, @@ -263,8 +263,8 @@ user_seeds() -> config_params => #{ query => <<"SELECT password_hash, salt, is_superuser_int as is_superuser FROM users where username = ${clientid} LIMIT 1">>, - password_hash_algorithm => <<"sha256">>, - salt_position => <<"prefix">> + password_hash_algorithm => #{name => <<"sha256">>, + salt_position => <<"prefix">>} }, result => {ok,#{is_superuser => true}} }, @@ -282,8 +282,7 @@ user_seeds() -> config_params => #{ query => <<"SELECT password_hash, salt, is_superuser_int as is_superuser FROM users where username = ${username} LIMIT 1">>, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> % should be ignored + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {ok,#{is_superuser => false}} }, @@ -300,8 +299,7 @@ user_seeds() -> config_params => #{ query => <<"SELECT password_hash, salt, is_superuser_int as is_superuser FROM users where username = ${username} LIMIT 1">>, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> % should be ignored + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {ok,#{is_superuser => false}} }, @@ -320,8 +318,7 @@ user_seeds() -> % clientid variable & username credentials query => <<"SELECT password_hash, salt, is_superuser_int as is_superuser FROM users where username = ${clientid} LIMIT 1">>, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,not_authorized} }, @@ -340,8 +337,7 @@ user_seeds() -> % Bad keys in query query => <<"SELECT 1 AS unknown_field FROM users where username = ${username} LIMIT 1">>, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,not_authorized} }, @@ -358,8 +354,7 @@ user_seeds() -> password => <<"wrongpass">> }, config_params => #{ - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,bad_username_or_password} } diff --git a/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl b/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl new file mode 100644 index 000000000..e0273e24f --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl @@ -0,0 +1,135 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_password_hashing_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(SIMPLE_HASHES, [plain, md5, sha, sha256, sha512]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(bcrypt), + Config. + +end_per_suite(_Config) -> + ok. + +t_gen_salt(_Config) -> + Algorithms = [#{name => Type, salt_position => suffix} || Type <- ?SIMPLE_HASHES] + ++ [#{name => bcrypt, salt_rounds => 10}], + + lists:foreach( + fun(Algorithm) -> + Salt = emqx_authn_password_hashing:gen_salt(Algorithm), + ct:pal("gen_salt(~p): ~p", [Algorithm, Salt]), + ?assert(is_binary(Salt)) + end, + Algorithms). + +t_init(_Config) -> + Algorithms = [#{name => Type, salt_position => suffix} || Type <- ?SIMPLE_HASHES] + ++ [#{name => bcrypt, salt_rounds => 10}], + + lists:foreach( + fun(Algorithm) -> + ok = emqx_authn_password_hashing:init(Algorithm) + end, + Algorithms). + +t_check_password(_Config) -> + lists:foreach( + fun test_check_password/1, + hash_examples()). + +test_check_password(#{ + password_hash := Hash, + salt := Salt, + password := Password, + password_hash_algorithm := Algorithm + } = Sample) -> + ct:pal("t_check_password sample: ~p", [Sample]), + true = emqx_authn_password_hashing:check_password(Algorithm, Salt, Hash, Password), + false = emqx_authn_password_hashing:check_password(Algorithm, Salt, Hash, <<"wrongpass">>). + +t_hash(_Config) -> + lists:foreach( + fun test_hash/1, + hash_examples()). + +test_hash(#{password := Password, + password_hash_algorithm := Algorithm + } = Sample) -> + ct:pal("t_hash sample: ~p", [Sample]), + {Hash, Salt} = emqx_authn_password_hashing:hash(Algorithm, Password), + true = emqx_authn_password_hashing:check_password(Algorithm, Salt, Hash, Password). + +hash_examples() -> + [#{ + password_hash => <<"plainsalt">>, + salt => <<"salt">>, + password => <<"plain">>, + password_hash_algorithm => #{name => plain, + salt_position => suffix} + }, + #{ + password_hash => <<"9b4d0c43d206d48279e69b9ad7132e22">>, + salt => <<"salt">>, + password => <<"md5">>, + password_hash_algorithm => #{name => md5, + salt_position => suffix} + }, + #{ + password_hash => <<"c665d4c0a9e5498806b7d9fd0b417d272853660e">>, + salt => <<"salt">>, + password => <<"sha">>, + password_hash_algorithm => #{name => sha, + salt_position => prefix} + }, + #{ + password_hash => <<"ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf">>, + salt => <<"salt">>, + password => <<"sha256">>, + password_hash_algorithm => #{name => sha256, + salt_position => prefix} + }, + #{ + password_hash => iolist_to_binary( + [<<"a1509ab67bfacbad020927b5ac9d91e9100a82e33a0ebb01459367ce921c0aa8">>, + <<"157aa5652f94bc84fa3babc08283e44887d61c48bcf8ad7bcb3259ee7d0eafcd">>]), + salt => <<"salt">>, + password => <<"sha512">>, + password_hash_algorithm => #{name => sha512, + salt_position => prefix} + }, + #{ + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + password => <<"bcrypt">>, + + password_hash_algorithm => #{name => bcrypt, + salt_rounds => 10} + } + ]. diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index 2a79179e1..8f1f12690 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -272,20 +272,20 @@ t_parse_query(_) -> raw_pgsql_auth_config() -> #{ - mechanism => <<"password-based">>, - password_hash_algorithm => <<"plain">>, - salt_position => <<"suffix">>, - enable => <<"true">>, + mechanism => <<"password-based">>, + password_hash_algorithm => #{name => <<"plain">>, + salt_position => <<"suffix">>}, + enable => <<"true">>, - backend => <<"postgresql">>, - database => <<"mqtt">>, - username => <<"root">>, - password => <<"public">>, + backend => <<"postgresql">>, + database => <<"mqtt">>, + username => <<"root">>, + password => <<"public">>, - query => <<"SELECT password_hash, salt, is_superuser_str as is_superuser + query => <<"SELECT password_hash, salt, is_superuser_str as is_superuser FROM users where username = ${username} LIMIT 1">>, - server => pgsql_server() - }. + server => pgsql_server() + }. user_seeds() -> [#{data => #{ @@ -312,8 +312,8 @@ user_seeds() -> password => <<"md5">> }, config_params => #{ - password_hash_algorithm => <<"md5">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"md5">>, + salt_position => <<"suffix">>} }, result => {ok,#{is_superuser => false}} }, @@ -331,8 +331,8 @@ user_seeds() -> config_params => #{ query => <<"SELECT password_hash, salt, is_superuser_int as is_superuser FROM users where username = ${clientid} LIMIT 1">>, - password_hash_algorithm => <<"sha256">>, - salt_position => <<"prefix">> + password_hash_algorithm => #{name => <<"sha256">>, + salt_position => <<"prefix">>} }, result => {ok,#{is_superuser => true}} }, @@ -350,8 +350,7 @@ user_seeds() -> config_params => #{ query => <<"SELECT password_hash, salt, is_superuser_int as is_superuser FROM users where username = ${username} LIMIT 1">>, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> % should be ignored + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {ok,#{is_superuser => false}} }, @@ -370,8 +369,7 @@ user_seeds() -> % clientid variable & username credentials query => <<"SELECT password_hash, salt, is_superuser_int as is_superuser FROM users where username = ${clientid} LIMIT 1">>, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,not_authorized} }, @@ -390,8 +388,7 @@ user_seeds() -> % Bad keys in query query => <<"SELECT 1 AS unknown_field FROM users where username = ${username} LIMIT 1">>, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,not_authorized} }, @@ -408,8 +405,7 @@ user_seeds() -> password => <<"wrongpass">> }, config_params => #{ - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,bad_username_or_password} } diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 2e941e72f..938ca8714 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -208,17 +208,17 @@ t_update(_Config) -> raw_redis_auth_config() -> #{ - mechanism => <<"password-based">>, - password_hash_algorithm => <<"plain">>, - salt_position => <<"suffix">>, - enable => <<"true">>, + mechanism => <<"password-based">>, + password_hash_algorithm => #{name => <<"plain">>, + salt_position => <<"suffix">>}, + enable => <<"true">>, - backend => <<"redis">>, - cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, - database => <<"1">>, - password => <<"public">>, - server => redis_server() - }. + backend => <<"redis">>, + cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, + database => <<"1">>, + password => <<"public">>, + server => redis_server() + }. user_seeds() -> [#{data => #{ @@ -245,8 +245,8 @@ user_seeds() -> }, key => "mqtt_user:md5", config_params => #{ - password_hash_algorithm => <<"md5">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"md5">>, + salt_position => <<"suffix">>} }, result => {ok,#{is_superuser => false}} }, @@ -263,8 +263,8 @@ user_seeds() -> key => "mqtt_user:sha256", config_params => #{ cmd => <<"HMGET mqtt_user:${clientid} password_hash salt is_superuser">>, - password_hash_algorithm => <<"sha256">>, - salt_position => <<"prefix">> + password_hash_algorithm => #{name => <<"sha256">>, + salt_position => <<"prefix">>} }, result => {ok,#{is_superuser => true}} }, @@ -280,8 +280,7 @@ user_seeds() -> }, key => "mqtt_user:bcrypt", config_params => #{ - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> % should be ignored + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {ok,#{is_superuser => false}} }, @@ -299,8 +298,7 @@ user_seeds() -> config_params => #{ % clientid variable & username credentials cmd => <<"HMGET mqtt_client:${clientid} password_hash salt is_superuser">>, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,not_authorized} }, @@ -318,8 +316,7 @@ user_seeds() -> config_params => #{ % Bad key in cmd cmd => <<"HMGET badkey:${username} password_hash salt is_superuser">>, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,not_authorized} }, @@ -337,8 +334,7 @@ user_seeds() -> key => "mqtt_user:bcrypt2", config_params => #{ cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, - password_hash_algorithm => <<"bcrypt">>, - salt_position => <<"suffix">> + password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {error,bad_username_or_password} } From 0eff5358caaeb8967526b08374294386ead14117 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 8 Dec 2021 10:52:27 +0800 Subject: [PATCH 03/31] chore(gw): add examples for gw&client api --- apps/emqx_gateway/src/emqx_gateway_api.erl | 129 +++++++++- .../src/emqx_gateway_api_clients.erl | 223 ++++++++++++++---- apps/emqx_gateway/src/emqx_gateway_conf.erl | 26 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 2 +- .../test/emqx_gateway_api_SUITE.erl | 32 +-- .../test/emqx_gateway_conf_SUITE.erl | 16 +- 6 files changed, 343 insertions(+), 85 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 596d42c9a..7c961223d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -79,9 +79,9 @@ gateway(post, Request) -> undefined -> error(badarg); _ -> GwConf = maps:without([<<"name">>], Body), - case emqx_gateway_conf:load_gateway(GwName, GwConf) of - ok -> - {204}; + case emqx_gateway_conf:load_gateway(GwName, GwConf) of + {ok, NGwConf} -> + {201, NGwConf}; {error, Reason} -> return_http_error(500, Reason) end @@ -131,8 +131,8 @@ gateway_insta(put, #{body := GwConf, }) -> with_gateway(Name0, fun(GwName, _) -> case emqx_gateway_conf:update_gateway(GwName, GwConf) of - ok -> - {204}; + {ok, Gateway} -> + {200, Gateway}; {error, Reason} -> return_http_error(500, Reason) end @@ -201,7 +201,7 @@ schema("/gateway/:name/stats") -> params_gateway_name_in_path() -> [{name, - mk(binary(), + mk(string(), #{ in => path , desc => <<"Gateway Name">> })} @@ -209,7 +209,7 @@ params_gateway_name_in_path() -> params_gateway_status_in_qs() -> [{status, - mk(binary(), + mk(string(), #{ in => query , nullable => true , desc => <<"Gateway Status">> @@ -270,7 +270,7 @@ fields(Gw) when Gw == stomp; Gw == mqttsn; Gw == coap; Gw == lwm2m; Gw == exproto -> [{name, - mk(string(), #{ desc => <<"Gateway Name">>})} + mk(hoconsc:union([Gw]), #{ desc => <<"Gateway Name">>})} ] ++ convert_listener_struct(emqx_gateway_schema:fields(Gw)); fields(Listener) when Listener == tcp_listener; Listener == ssl_listener; @@ -330,21 +330,130 @@ examples_gateway_confs() -> #{ summary => <<"A simple STOMP gateway configs">> , value => #{ enable => true + , name => <<"stomp">> , enable_stats => true , idle_timeout => <<"30s">> , mountpoint => <<"stomp/">> , frame => - #{ max_header => 10 - , make_header_length => 1024 + #{ max_headers => 10 + , max_headers_length => 1024 , max_body_length => 65535 } + , listeners => + [ #{ type => <<"tcp">> + , name => <<"default">> + , bind => <<"61613">> + , max_connections => 1024000 + , max_conn_rate => 1000 + } + ] } } , mqttsn_gateway => #{ summary => <<"A simple MQTT-SN gateway configs">> , value => #{ enable => true + , name => <<"mqttsn">> , enable_stats => true + , idle_timeout => <<"30s">> + , mountpoint => <<"mqttsn/">> + , gateway_id => 1 + , broadcast => true + , enable_qos3 => true + , predefined => + [ #{ id => <<"1001">> + , topic => <<"pred/1001">> + } + , #{ id => <<"1002">> + , topic => <<"pred/1002">> + } + ] + , listeners => + [ #{ type => <<"udp">> + , name => <<"default">> + , bind => <<"1884">> + , max_connections => 1024000 + , max_conn_rate => 1000 + } + ] + } + } + , coap_gateway => + #{ summary => <<"A simple CoAP gateway configs">> + , value => + #{ enable => true + , name => <<"coap">> + , enable_stats => true + , idle_timeout => <<"30s">> + , mountpoint => <<"coap/">> + , heartbeat => <<"30s">> + , connection_required => false + , notify_type => <<"qos">> + , subscribe_qos => <<"coap">> + , publish_qos => <<"coap">> + , listeners => + [ #{ type => <<"udp">> + , name => <<"coap">> + , bind => <<"5683">> + , max_connections => 1024000 + , max_conn_rate => 1000 + } + ] + } + } + , lwm2m_gateway => + #{ summary => <<"A simple LwM2M gateway configs">> + , value => + #{ enable => true + , name => <<"lwm2m">> + , enable_stats => true + , idle_timeout => <<"30s">> + , mountpoint => <<"lwm2m/">> + , xml_dir => <<"etc/lwm2m_xml">> + , lifetime_min => <<"1s">> + , lifetime_max => <<"86400s">> + , qmode_time_window => <<"22s">> + , auto_observe => false + , update_msg_publish_condition => <<"always">> + , translators => + #{ command => #{topic => <<"/dn/#">>} + , response => #{topic => <<"/up/resp">>} + , notify => #{topic => <<"/up/notify">>} + , register => #{topic => <<"/up/resp">>} + , update => #{topic => <<"/up/resp">>} + } + , listeners => + [ #{ type => <<"udp">> + , name => <<"lwm2m">> + , bind => <<"5783">> + , max_connections => 1024000 + , max_conn_rate => 1000 + } + ] + } + } + , exproto_gateway => + #{ summary => <<"A simple ExProto gateway configs">> + , value => + #{ enable => true + , name => <<"exproto">> + , enable_stats => true + , idle_timeout => <<"30s">> + , mountpoint => <<"exproto/">> + , server => + #{ bind => <<"9100">> + } + , handler => + #{ address => <<"http://127.0.0.1:9001">> + } + , listeners => + [ #{ type => <<"tcp">> + , name => <<"default">> + , bind => <<"7993">> + , max_connections => 1024000 + , max_conn_rate => 1000 + } + ] } } }. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index c7a77eb02..7d6feacf8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -412,10 +412,7 @@ schema("/gateway/:name/clients") -> #{ description => <<"Get the gateway client list">> , parameters => params_client_query() , responses => - ?STANDARD_RESP( - #{ 200 => emqx_dashboard_swagger:schema_with_examples( - hoconsc:array(ref(client)), - examples_client_list())}) + ?STANDARD_RESP(#{200 => schema_client_list()}) } }; schema("/gateway/:name/clients/:clientid") -> @@ -424,10 +421,7 @@ schema("/gateway/:name/clients/:clientid") -> #{ description => <<"Get the gateway client infomation">> , parameters => params_client_insta() , responses => - ?STANDARD_RESP( - #{ 200 => emqx_dashboard_swagger:schema_with_examples( - ref(client), - examples_client())}) + ?STANDARD_RESP(#{200 => schema_client()}) } , delete => #{ description => <<"Kick out the gateway client">> @@ -443,9 +437,9 @@ schema("/gateway/:name/clients/:clientid/subscriptions") -> , parameters => params_client_insta() , responses => ?STANDARD_RESP( - #{ 200 => emqx_dashboard_swagger:schema_with_examples( - hoconsc:array(ref(subscription)), - examples_subsctiption_list())}) + #{200 => emqx_dashboard_swagger:schema_with_examples( + hoconsc:array(ref(subscription)), + examples_subsctiption_list())}) } , post => #{ description => <<"Create a subscription membership">> @@ -567,13 +561,85 @@ params_topic_name_in_path() -> %%-------------------------------------------------------------------- %% schemas +schema_client_list() -> + emqx_dashboard_swagger:schema_with_examples( + hoconsc:union([hoconsc:array(ref(?MODULE, stomp_client)), + hoconsc:array(ref(?MODULE, mqttsn_client)), + hoconsc:array(ref(?MODULE, coap_client)), + hoconsc:array(ref(?MODULE, lwm2m_client)), + hoconsc:array(ref(?MODULE, exproto_client)) + ]), + examples_client_list() + ). + +schema_client() -> + emqx_dashboard_swagger:schema_with_examples( + hoconsc:union([ref(?MODULE, stomp_client), + ref(?MODULE, mqttsn_client), + ref(?MODULE, coap_client), + ref(?MODULE, lwm2m_client), + ref(?MODULE, exproto_client) + ]), + examples_client() + ). + roots() -> - [ client + [ stomp_client + , mqttsn_client + , coap_client + , lwm2m_client + , exproto_client , subscription ]. -fields(client) -> - %% XXX: enum for every protocol's client +fields(test) -> + [{key, mk(string(), #{ desc => <<"Desc">>})}]; + +fields(stomp_client) -> + common_client_props(); +fields(mqttsn_client) -> + common_client_props(); +fields(coap_client) -> + common_client_props(); +fields(lwm2m_client) -> + [ {endpoint_name, + mk(string(), + #{ desc => <<"The LwM2M client endpoint name">>})} + , {lifetime, + mk(integer(), + #{ desc => <<"Life time">>})} + ] ++ common_client_props(); +fields(exproto_client) -> + common_client_props(); + +fields(subscription) -> + [ {topic, + mk(string(), + #{ desc => <<"Topic Fillter">>})} + , {qos, + mk(integer(), + #{ desc => <<"QoS level, enum: 0, 1, 2">>})} + , {nl, + mk(integer(), %% FIXME: why not boolean? + #{ desc => <<"No Local option, enum: 0, 1">>})} + , {rap, + mk(integer(), + #{ desc => <<"Retain as Published option, enum: 0, 1">>})} + , {rh, + mk(integer(), + #{ desc => <<"Retain Handling option, enum: 0, 1, 2">>})} + , {sub_props, + mk(ref(extra_sub_props), + #{desc => <<"Subscription properties">>})} + ]; +fields(extra_sub_props) -> + [ {subid, + mk(string(), + #{ desc => <<"Only stomp protocol, an uniquely identity for " + "the subscription. range: 1-65535.">>})} + ]. + +common_client_props() -> [ {node, mk(string(), #{ desc => <<"Name of the node to which the client is " @@ -699,45 +765,114 @@ fields(client) -> , {reductions, mk(integer(), #{ desc => <<"Erlang reduction">>})} - ]; -fields(subscription) -> - [ {topic, - mk(string(), - #{ desc => <<"Topic Fillter">>})} - , {qos, - mk(integer(), - #{ desc => <<"QoS level, enum: 0, 1, 2">>})} - , {nl, - mk(integer(), %% FIXME: why not boolean? - #{ desc => <<"No Local option, enum: 0, 1">>})} - , {rap, - mk(integer(), - #{ desc => <<"Retain as Published option, enum: 0, 1">>})} - , {rh, - mk(integer(), - #{ desc => <<"Retain Handling option, enum: 0, 1, 2">>})} - , {sub_props, - mk(ref(extra_sub_props), - #{desc => <<"Subscription properties">>})} - ]; -fields(extra_sub_props) -> - [ {subid, - mk(string(), - #{ desc => <<"Only stomp protocol, an uniquely identity for " - "the subscription. range: 1-65535.">>})} ]. %%-------------------------------------------------------------------- %% examples examples_client_list() -> - #{}. + #{ general_client_list => + #{ summary => <<"General Client List">> + , value => [example_general_client()] + } + , lwm2m_client_list => + #{ summary => <<"LwM2M Client List">> + , value => [example_lwm2m_client()] + } + }. examples_client() -> - #{}. + #{ general_client => + #{ summary => <<"General Client Info">> + , value => example_general_client() + } + , lwm2m_client => + #{ summary => <<"LwM2M Client Info">> + , value => example_lwm2m_client() + } + }. examples_subsctiption_list() -> - #{}. + #{ general_subscription_list => + #{ summary => <<"A General Subscription List">> + , value => [example_general_subscription()] + } + , stomp_subscription_list => + #{ summary => <<"The Stomp Subscription List">> + , value => [example_stomp_subscription] + } + }. examples_subsctiption() -> - #{}. + #{ general_subscription => + #{ summary => <<"A General Subscription">> + , value => example_general_subscription() + } + , stomp_subscription => + #{ summary => <<"A Stomp Subscription">> + , value => example_stomp_subscription() + } + }. + +example_lwm2m_client() -> + maps:merge( + example_general_client(), + #{ proto_name => <<"LwM2M">> + , proto_ver => <<"1.0">> + , endpoint_name => <<"urn:imei:154928475237123">> + , lifetime => 86400 + }). + +example_general_client() -> + #{ clientid => <<"MzAyMzEzNTUwNzk1NDA1MzYyMzIwNzUxNjQwMTY1NzQ0NjE">> + , username => <<"guest">> + , node => <<"emqx@127.0.0.1">> + , proto_name => "STOMP" + , proto_ver => <<"1.0">> + , ip_address => <<"127.0.0.1">> + , port => 50675 + , clean_start => true + , connected => true + , is_bridge => false + , keepalive => 0 + , expiry_interval => 0 + , subscriptions_cnt => 0 + , subscriptions_max => <<"infinity">> + , awaiting_rel_cnt => 0 + , awaiting_rel_max => <<"infinity">> + , mqueue_len => 0 + , mqueue_max => <<"infinity">> + , mqueue_dropped => 0 + , inflight_cnt => 0 + , inflight_max => <<"infinity">> + , heap_size => 4185 + , recv_oct => 56 + , recv_cnt => 1 + , recv_pkt => 1 + , recv_msg => 0 + , send_oct => 61 + , send_cnt => 1 + , send_pkt => 1 + , send_msg => 0 + , reductions => 72022 + , mailbox_len => 0 + , created_at => <<"2021-12-07T10:44:02.721+08:00">> + , connected_at => <<"2021-12-07T10:44:02.721+08:00">> + , disconnected_at => null + }. + +example_stomp_subscription() -> + maps:merge( + example_general_subscription(), + #{ topic => <<"stomp/topic">> + , sub_props => #{subid => <<"10">>} + }). + +example_general_subscription() -> + #{ topic => <<"test/topic">> + , qos => 1 + , nl => 0 + , rap => 0 + , rh => 0 + , sub_props => #{} + }. diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index a799b7fb7..da06e3a6d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -79,15 +79,14 @@ unload() -> %%-------------------------------------------------------------------- %% APIs --spec load_gateway(atom_or_bin(), map()) -> ok_or_err(). +-spec load_gateway(atom_or_bin(), map()) -> map_or_err(). load_gateway(GwName, Conf) -> NConf = case maps:take(<<"listeners">>, Conf) of error -> Conf; {Ls, Conf1} -> Conf1#{<<"listeners">> => unconvert_listeners(Ls)} end, - %% TODO: - ret_ok_err(update({?FUNCTION_NAME, bin(GwName), NConf})). + ret_gw(GwName, update({?FUNCTION_NAME, bin(GwName), NConf})). %% @doc convert listener array to map unconvert_listeners(Ls) when is_list(Ls) -> @@ -108,13 +107,12 @@ maps_key_take([K | Ks], M, Acc) -> maps_key_take(Ks, M1, [V | Acc]) end. --spec update_gateway(atom_or_bin(), map()) -> ok_or_err(). +-spec update_gateway(atom_or_bin(), map()) -> map_or_err(). update_gateway(GwName, Conf0) -> Exclude0 = [listeners, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM], Exclude1 = [atom_to_binary(K, utf8) || K <- Exclude0], Conf = maps:without(Exclude0 ++ Exclude1, Conf0), - - ret_ok_err(update({?FUNCTION_NAME, bin(GwName), Conf})). + ret_gw(GwName, update({?FUNCTION_NAME, bin(GwName), Conf})). %% FIXME: delete cert files ?? @@ -261,6 +259,22 @@ bin(B) when is_binary(B) -> ret_ok_err({ok, _}) -> ok; ret_ok_err(Err) -> Err. +ret_gw(GwName, {ok, #{raw_config := GwConf}}) -> + GwConf1 = emqx_map_lib:deep_get([bin(GwName)], GwConf), + LsConf = emqx_map_lib:deep_get( + [bin(GwName), <<"listeners">>], + GwConf, #{}), + NLsConf = + lists:foldl(fun({LType, SubConf}, Acc) -> + NLConfs = + lists:map(fun({LName, LConf}) -> + do_convert_listener2(GwName, LType, LName, LConf) + end, proplists:from_map(SubConf)), + [NLConfs|Acc] + end, [], proplists:from_map(LsConf)), + {ok, maps:merge(GwConf1, #{<<"listeners">> => NLsConf})}; +ret_gw(_GwName, Err) -> Err. + ret_authn(GwName, {ok, #{raw_config := GwConf}}) -> Authn = emqx_map_lib:deep_get( [bin(GwName), <<"authentication">>], diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 2215c3a96..18b195c5b 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -221,7 +221,7 @@ fields(lwm2m) -> })} , {lifetime_min, sc(duration(), - #{ default => "1s" + #{ default => "15s" , desc => "Minimum value of lifetime allowed to be set by the LwM2M client" })} , {lifetime_max, diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 3ae9bcc12..18a380984 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -59,7 +59,7 @@ t_gateway(_) -> lists:foreach(fun assert_gw_unloaded/1, Gateways), {400, BadReq} = request(get, "/gateway/uname_gateway"), assert_bad_request(BadReq), - {204, _} = request(post, "/gateway", #{name => <<"stomp">>}), + {201, _} = request(post, "/gateway", #{name => <<"stomp">>}), {200, StompGw1} = request(get, "/gateway/stomp"), assert_feilds_apperence([name, status, enable, created_at, started_at], StompGw1), @@ -81,12 +81,12 @@ t_gateway_stomp(_) -> #{name => <<"def">>, type => <<"tcp">>, bind => <<"61613">>} ] }, - {204, _} = request(post, "/gateway", GwConf), + {201, _} = request(post, "/gateway", GwConf), {200, ConfResp} = request(get, "/gateway/stomp"), assert_confs(GwConf, ConfResp), %% put GwConf2 = emqx_map_lib:deep_merge(GwConf, #{frame => #{max_headers => 10}}), - {204, _} = request(put, "/gateway/stomp", maps:without([name], GwConf2)), + {200, _} = request(put, "/gateway/stomp", maps:without([name], GwConf2)), {200, ConfResp2} = request(get, "/gateway/stomp"), assert_confs(GwConf2, ConfResp2), {204, _} = request(delete, "/gateway/stomp"). @@ -104,12 +104,12 @@ t_gateway_mqttsn(_) -> #{name => <<"def">>, type => <<"udp">>, bind => <<"1884">>} ] }, - {204, _} = request(post, "/gateway", GwConf), + {201, _} = request(post, "/gateway", GwConf), {200, ConfResp} = request(get, "/gateway/mqttsn"), assert_confs(GwConf, ConfResp), %% put GwConf2 = emqx_map_lib:deep_merge(GwConf, #{predefined => []}), - {204, _} = request(put, "/gateway/mqttsn", maps:without([name], GwConf2)), + {200, _} = request(put, "/gateway/mqttsn", maps:without([name], GwConf2)), {200, ConfResp2} = request(get, "/gateway/mqttsn"), assert_confs(GwConf2, ConfResp2), {204, _} = request(delete, "/gateway/mqttsn"). @@ -125,12 +125,12 @@ t_gateway_coap(_) -> #{name => <<"def">>, type => <<"udp">>, bind => <<"5683">>} ] }, - {204, _} = request(post, "/gateway", GwConf), + {201, _} = request(post, "/gateway", GwConf), {200, ConfResp} = request(get, "/gateway/coap"), assert_confs(GwConf, ConfResp), %% put GwConf2 = emqx_map_lib:deep_merge(GwConf, #{heartbeat => <<"10s">>}), - {204, _} = request(put, "/gateway/coap", maps:without([name], GwConf2)), + {200, _} = request(put, "/gateway/coap", maps:without([name], GwConf2)), {200, ConfResp2} = request(get, "/gateway/coap"), assert_confs(GwConf2, ConfResp2), {204, _} = request(delete, "/gateway/coap"). @@ -156,12 +156,12 @@ t_gateway_lwm2m(_) -> #{name => <<"def">>, type => <<"udp">>, bind => <<"5783">>} ] }, - {204, _} = request(post, "/gateway", GwConf), + {201, _} = request(post, "/gateway", GwConf), {200, ConfResp} = request(get, "/gateway/lwm2m"), assert_confs(GwConf, ConfResp), %% put GwConf2 = emqx_map_lib:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}), - {204, _} = request(put, "/gateway/lwm2m", maps:without([name], GwConf2)), + {200, _} = request(put, "/gateway/lwm2m", maps:without([name], GwConf2)), {200, ConfResp2} = request(get, "/gateway/lwm2m"), assert_confs(GwConf2, ConfResp2), {204, _} = request(delete, "/gateway/lwm2m"). @@ -177,19 +177,19 @@ t_gateway_exproto(_) -> #{name => <<"def">>, type => <<"tcp">>, bind => <<"7993">>} ] }, - {204, _} = request(post, "/gateway", GwConf), + {201, _} = request(post, "/gateway", GwConf), {200, ConfResp} = request(get, "/gateway/exproto"), assert_confs(GwConf, ConfResp), %% put GwConf2 = emqx_map_lib:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}), - {204, _} = request(put, "/gateway/exproto", maps:without([name], GwConf2)), + {200, _} = request(put, "/gateway/exproto", maps:without([name], GwConf2)), {200, ConfResp2} = request(get, "/gateway/exproto"), assert_confs(GwConf2, ConfResp2), {204, _} = request(delete, "/gateway/exproto"). t_authn(_) -> GwConf = #{name => <<"stomp">>}, - {204, _} = request(post, "/gateway", GwConf), + {201, _} = request(post, "/gateway", GwConf), {204, _} = request(get, "/gateway/stomp/authentication"), AuthConf = #{mechanism => <<"password-based">>, @@ -212,7 +212,7 @@ t_authn(_) -> t_authn_data_mgmt(_) -> GwConf = #{name => <<"stomp">>}, - {204, _} = request(post, "/gateway", GwConf), + {201, _} = request(post, "/gateway", GwConf), {204, _} = request(get, "/gateway/stomp/authentication"), AuthConf = #{mechanism => <<"password-based">>, @@ -256,7 +256,7 @@ t_authn_data_mgmt(_) -> t_listeners(_) -> GwConf = #{name => <<"stomp">>}, - {204, _} = request(post, "/gateway", GwConf), + {201, _} = request(post, "/gateway", GwConf), {404, _} = request(get, "/gateway/stomp/listeners"), LisConf = #{name => <<"def">>, type => <<"tcp">>, @@ -289,7 +289,7 @@ t_listeners_authn(_) -> type => <<"tcp">>, bind => <<"61613">> }]}, - {204, _} = request(post, "/gateway", GwConf), + {201, _} = request(post, "/gateway", GwConf), {200, ConfResp} = request(get, "/gateway/stomp"), assert_confs(GwConf, ConfResp), @@ -316,7 +316,7 @@ t_listeners_authn_data_mgmt(_) -> type => <<"tcp">>, bind => <<"61613">> }]}, - {204, _} = request(post, "/gateway", GwConf), + {201, _} = request(post, "/gateway", GwConf), {200, ConfResp} = request(get, "/gateway/stomp"), assert_confs(GwConf, ConfResp), diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index 916913856..f3859532e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -244,12 +244,12 @@ t_load_unload_gateway(_) -> StompConf2 = compose(?CONF_STOMP_BAISC_2, ?CONF_STOMP_AUTHN_1, ?CONF_STOMP_LISTENER_1), - ok = emqx_gateway_conf:load_gateway(stomp, StompConf1), + {ok, _} = emqx_gateway_conf:load_gateway(stomp, StompConf1), {error, already_exist} = emqx_gateway_conf:load_gateway(stomp, StompConf1), assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])), - ok = emqx_gateway_conf:update_gateway(stomp, StompConf2), + {ok, _} = emqx_gateway_conf:update_gateway(stomp, StompConf2), assert_confs(StompConf2, emqx:get_raw_config([gateway, stomp])), ok = emqx_gateway_conf:unload_gateway(stomp), @@ -265,7 +265,7 @@ t_load_unload_gateway(_) -> t_load_remove_authn(_) -> StompConf = compose_listener(?CONF_STOMP_BAISC_1, ?CONF_STOMP_LISTENER_1), - ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), + {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), {ok, _} = emqx_gateway_conf:add_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_1), @@ -292,7 +292,7 @@ t_load_remove_authn(_) -> t_load_remove_listeners(_) -> StompConf = compose_authn(?CONF_STOMP_BAISC_1, ?CONF_STOMP_AUTHN_1), - ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), + {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), {ok, _} = emqx_gateway_conf:add_listener( @@ -338,7 +338,7 @@ t_load_remove_listener_authn(_) -> ?CONF_STOMP_AUTHN_2 ), - ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), + {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), {ok, _} = emqx_gateway_conf:add_authn( @@ -368,7 +368,7 @@ t_load_gateway_with_certs_content(_) -> ?CONF_STOMP_BAISC_1, ?CONF_STOMP_LISTENER_SSL ), - ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), + {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), SslConf = emqx_map_lib:deep_get( [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl">>], @@ -388,7 +388,7 @@ t_load_gateway_with_certs_content(_) -> % ?CONF_STOMP_BAISC_1, % ?CONF_STOMP_LISTENER_SSL_PATH % ), -% ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), +% {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), % assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), % SslConf = emqx_map_lib:deep_get( % [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl">>], @@ -402,7 +402,7 @@ t_load_gateway_with_certs_content(_) -> t_add_listener_with_certs_content(_) -> StompConf = ?CONF_STOMP_BAISC_1, - ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), + {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), {ok, _} = emqx_gateway_conf:add_listener( From a937a3d4dcb3dd17cb83ae03a74282db7874aae3 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 8 Dec 2021 11:13:00 +0800 Subject: [PATCH 04/31] chore(gw): fix bad argument type --- apps/emqx_gateway/src/emqx_gateway_api.erl | 20 ++++++------- .../src/emqx_gateway_api_clients.erl | 30 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 7c961223d..34165d492 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -201,7 +201,7 @@ schema("/gateway/:name/stats") -> params_gateway_name_in_path() -> [{name, - mk(string(), + mk(binary(), #{ in => path , desc => <<"Gateway Name">> })} @@ -209,7 +209,7 @@ params_gateway_name_in_path() -> params_gateway_status_in_qs() -> [{status, - mk(string(), + mk(binary(), #{ in => query , nullable => true , desc => <<"Gateway Status">> @@ -226,20 +226,20 @@ roots() -> fields(gateway_overview) -> [ {name, - mk(string(), + mk(binary(), #{ desc => <<"Gateway Name">>})} , {status, mk(hoconsc:enum([running, stopped, unloaded]), #{ desc => <<"The Gateway status">>})} , {created_at, - mk(string(), + mk(binary(), #{desc => <<"The Gateway created datetime">>})} , {started_at, - mk(string(), + mk(binary(), #{ nullable => true , desc => <<"The Gateway started datetime">>})} , {stopped_at, - mk(string(), + mk(binary(), #{ nullable => true , desc => <<"The Gateway stopped datetime">>})} , {max_connections, @@ -256,7 +256,7 @@ fields(gateway_overview) -> ]; fields(gateway_listener_overview) -> [ {id, - mk(string(), + mk(binary(), #{ desc => <<"Listener ID">>})} , {running, mk(boolean(), @@ -277,14 +277,14 @@ fields(Listener) when Listener == tcp_listener; Listener == udp_listener; Listener == dtls_listener -> [ {id, - mk(string(), + mk(binary(), #{ nullable => true , desc => <<"Listener ID">>})} , {type, mk(hoconsc:union([tcp, ssl, udp, dtls]), #{ desc => <<"Listener type">>})} , {name, - mk(string(), + mk(binary(), #{ desc => <<"Listener Name">>})} , {running, mk(boolean(), @@ -293,7 +293,7 @@ fields(Listener) when Listener == tcp_listener; ] ++ emqx_gateway_schema:fields(Listener); fields(gateway_stats) -> - [{key, mk(string(), #{})}]. + [{key, mk(binary(), #{})}]. schema_gateways_conf() -> %% XXX: We need convert the emqx_gateway_schema's listener map diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index 7d6feacf8..b67961e12 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -593,7 +593,7 @@ roots() -> ]. fields(test) -> - [{key, mk(string(), #{ desc => <<"Desc">>})}]; + [{key, mk(binary(), #{ desc => <<"Desc">>})}]; fields(stomp_client) -> common_client_props(); @@ -603,7 +603,7 @@ fields(coap_client) -> common_client_props(); fields(lwm2m_client) -> [ {endpoint_name, - mk(string(), + mk(binary(), #{ desc => <<"The LwM2M client endpoint name">>})} , {lifetime, mk(integer(), @@ -614,7 +614,7 @@ fields(exproto_client) -> fields(subscription) -> [ {topic, - mk(string(), + mk(binary(), #{ desc => <<"Topic Fillter">>})} , {qos, mk(integer(), @@ -634,30 +634,30 @@ fields(subscription) -> ]; fields(extra_sub_props) -> [ {subid, - mk(string(), + mk(binary(), #{ desc => <<"Only stomp protocol, an uniquely identity for " "the subscription. range: 1-65535.">>})} ]. common_client_props() -> [ {node, - mk(string(), + mk(binary(), #{ desc => <<"Name of the node to which the client is " "connected">>})} , {clientid, - mk(string(), + mk(binary(), #{ desc => <<"Client identifier">>})} , {username, - mk(string(), + mk(binary(), #{ desc => <<"Username of client when connecting">>})} , {proto_name, - mk(string(), + mk(binary(), #{ desc => <<"Client protocol name">>})} , {proto_ver, - mk(string(), + mk(binary(), #{ desc => <<"Protocol version used by the client">>})} , {ip_address, - mk(string(), + mk(binary(), #{ desc => <<"Client's IP address">>})} , {port, mk(integer(), @@ -667,10 +667,10 @@ common_client_props() -> #{ desc => <<"Indicates whether the client is connected via " "bridge">>})} , {connected_at, - mk(string(), + mk(binary(), #{ desc => <<"Client connection time">>})} , {disconnected_at, - mk(string(), + mk(binary(), #{ desc => <<"Client offline time, This field is only valid and " "returned when connected is false">>})} , {connected, @@ -681,10 +681,10 @@ common_client_props() -> %% want it %% %, {will_msg, - % mk(string(), + % mk(binary(), % #{ desc => <<"Client will message">>})} %, {zone, - % mk(string(), + % mk(binary(), % #{ desc => <<"Indicate the configuration group used by the " % "client">>})} , {keepalive, @@ -699,7 +699,7 @@ common_client_props() -> #{ desc => <<"Session expiration interval, with the unit of " "second">>})} , {created_at, - mk(string(), + mk(binary(), #{ desc => <<"Session creation time">>})} , {subscriptions_cnt, mk(integer(), From e48f10d2d65a531af8ff99cb0b7e47d7db1482e5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 8 Dec 2021 11:33:11 +0800 Subject: [PATCH 05/31] chore(gw): ignore needless args --- apps/emqx_gateway/src/emqx_gateway_api.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 34165d492..8f9cb99f7 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -126,10 +126,12 @@ gateway_insta(get, #{bindings := #{name := Name0}}) -> error : badarg -> return_http_error(400, "Bad gateway name") end; -gateway_insta(put, #{body := GwConf, +gateway_insta(put, #{body := GwConf0, bindings := #{name := Name0} }) -> with_gateway(Name0, fun(GwName, _) -> + %% XXX: Clear the unused fields + GwConf = maps:without([<<"name">>], GwConf0), case emqx_gateway_conf:update_gateway(GwName, GwConf) of {ok, Gateway} -> {200, Gateway}; From 7c261b53562e1d2b1db73dad04597c17a30f407c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 8 Dec 2021 17:18:39 +0800 Subject: [PATCH 06/31] chore(gw): improve examples for _gateway_api --- apps/emqx_gateway/src/emqx_gateway_api.erl | 174 +++++++++++++++++- .../src/exproto/emqx_exproto_impl.erl | 2 +- 2 files changed, 169 insertions(+), 7 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 8f9cb99f7..5036286b4 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -153,10 +153,14 @@ schema("/gateway") -> #{ description => <<"Get gateway list">> , parameters => params_gateway_status_in_qs() , responses => - ?STANDARD_RESP(#{200 => ref(gateway_overview)}) + ?STANDARD_RESP( + #{200 => emqx_dashboard_swagger:schema_with_example( + hoconsc:array(ref(gateway_overview)), + examples_gateway_overview())}) }, post => #{ description => <<"Load a gateway">> + %% TODO: distinguish create & response swagger schema , 'requestBody' => schema_gateways_conf() , responses => ?STANDARD_RESP(#{201 => schema_gateways_conf()}) @@ -179,7 +183,7 @@ schema("/gateway/:name") -> put => #{ description => <<"Update the gateway configurations/status">> , parameters => params_gateway_name_in_path() - , 'requestBody' => schema_gateways_conf() + , 'requestBody' => schema_update_gateways_conf() , responses => ?STANDARD_RESP(#{200 => schema_gateways_conf()}) } @@ -210,6 +214,7 @@ params_gateway_name_in_path() -> ]. params_gateway_status_in_qs() -> + %% FIXME: enum in swagger ?? [{status, mk(binary(), #{ in => query @@ -274,6 +279,14 @@ fields(Gw) when Gw == stomp; Gw == mqttsn; [{name, mk(hoconsc:union([Gw]), #{ desc => <<"Gateway Name">>})} ] ++ convert_listener_struct(emqx_gateway_schema:fields(Gw)); + +fields(Gw) when Gw == update_stomp; Gw == update_mqttsn; + Gw == update_coap; Gw == update_lwm2m; + Gw == update_exproto -> + "update_" ++ GwStr = atom_to_list(Gw), + Gw1 = list_to_existing_atom(GwStr), + remove_listener_and_authn(emqx_gateway_schema:fields(Gw1)); + fields(Listener) when Listener == tcp_listener; Listener == ssl_listener; Listener == udp_listener; @@ -297,9 +310,17 @@ fields(Listener) when Listener == tcp_listener; fields(gateway_stats) -> [{key, mk(binary(), #{})}]. +schema_update_gateways_conf() -> + emqx_dashboard_swagger:schema_with_examples( + hoconsc:union([ref(?MODULE, update_stomp), + ref(?MODULE, update_mqttsn), + ref(?MODULE, update_coap), + ref(?MODULE, update_lwm2m), + ref(?MODULE, update_exproto)]), + examples_update_gateway_confs() + ). + schema_gateways_conf() -> - %% XXX: We need convert the emqx_gateway_schema's listener map - %% structure to array emqx_dashboard_swagger:schema_with_examples( hoconsc:union([ref(?MODULE, stomp), ref(?MODULE, mqttsn), ref(?MODULE, coap), ref(?MODULE, lwm2m), @@ -316,6 +337,11 @@ convert_listener_struct(Schema) -> }), lists:keystore(listeners, 1, Schema1, {listeners, ListenerSchema}). +remove_listener_and_authn(Schmea) -> + lists:keydelete( + authentication, 1, + lists:keydelete(listeners, 1, Schmea)). + listeners_schema(?R_REF(_Mod, tcp_listeners)) -> hoconsc:array(hoconsc:union([ref(tcp_listener), ref(ssl_listener)])); listeners_schema(?R_REF(_Mod, udp_listeners)) -> @@ -327,6 +353,57 @@ listeners_schema(?R_REF(_Mod, udp_tcp_listeners)) -> %%-------------------------------------------------------------------- %% examples +examples_gateway_overview() -> + [ #{ name => <<"coap">> + , status => <<"unloaded">> + } + , #{ name => <<"exproto">> + , status => <<"unloaded">> + } + , #{ name => <<"lwm2m">> + , status => <<"running">> + , current_connections => 0 + , max_connections => 1024000 + , listeners => + [ #{ id => <<"lwm2m:udp:default">> + , type => <<"udp">> + , name => <<"default">> + , running => true + } + ] + , created_at => <<"2021-12-08T14:41:26.171+08:00">> + , started_at => <<"2021-12-08T14:41:26.202+08:00">> + } + , #{ name => <<"mqttsn">> + , status => <<"stopped">> + , current_connections => 0 + , max_connections => 1024000 + , listeners => + [ #{ id => <<"mqttsn:udp:default">> + , name => <<"default">> + , running => false + , type => <<"udp">> + } + ] + , created_at => <<"2021-12-08T14:41:45.071+08:00">> + , stopped_at => <<"2021-12-08T14:56:35.576+08:00">> + } + , #{ name => <<"stomp">> + , status => <<"running">> + , current_connections => 0 + , max_connections => 1024000 + , listeners => + [ #{ id => <<"stomp:tcp:default">> + , name => <<"default">> + , running => true + , type => <<"tcp">> + } + ] + , created_at => <<"2021-12-08T14:42:15.272+08:00">> + , started_at => <<"2021-12-08T14:42:15.274+08:00">> + } + ]. + examples_gateway_confs() -> #{ stomp_gateway => #{ summary => <<"A simple STOMP gateway configs">> @@ -395,7 +472,7 @@ examples_gateway_confs() -> , publish_qos => <<"coap">> , listeners => [ #{ type => <<"udp">> - , name => <<"coap">> + , name => <<"default">> , bind => <<"5683">> , max_connections => 1024000 , max_conn_rate => 1000 @@ -426,7 +503,7 @@ examples_gateway_confs() -> } , listeners => [ #{ type => <<"udp">> - , name => <<"lwm2m">> + , name => <<"default">> , bind => <<"5783">> , max_connections => 1024000 , max_conn_rate => 1000 @@ -460,5 +537,90 @@ examples_gateway_confs() -> } }. +examples_update_gateway_confs() -> + #{ stomp_gateway => + #{ summary => <<"A simple STOMP gateway configs">> + , value => + #{ enable => true + , enable_stats => true + , idle_timeout => <<"30s">> + , mountpoint => <<"stomp2/">> + , frame => + #{ max_headers => 100 + , max_headers_length => 10240 + , max_body_length => 655350 + } + } + } + , mqttsn_gateway => + #{ summary => <<"A simple MQTT-SN gateway configs">> + , value => + #{ enable => true + , enable_stats => true + , idle_timeout => <<"30s">> + , mountpoint => <<"mqttsn2/">> + , gateway_id => 1 + , broadcast => true + , enable_qos3 => false + , predefined => + [ #{ id => <<"1003">> + , topic => <<"pred/1003">> + } + ] + } + } + , coap_gateway => + #{ summary => <<"A simple CoAP gateway configs">> + , value => + #{ enable => true + , enable_stats => true + , idle_timeout => <<"30s">> + , mountpoint => <<"coap2/">> + , heartbeat => <<"30s">> + , connection_required => false + , notify_type => <<"qos">> + , subscribe_qos => <<"coap">> + , publish_qos => <<"coap">> + } + } + , lwm2m_gateway => + #{ summary => <<"A simple LwM2M gateway configs">> + , value => + #{ enable => true + , enable_stats => true + , idle_timeout => <<"30s">> + , mountpoint => <<"lwm2m2/">> + , xml_dir => <<"etc/lwm2m_xml">> + , lifetime_min => <<"1s">> + , lifetime_max => <<"86400s">> + , qmode_time_window => <<"22s">> + , auto_observe => false + , update_msg_publish_condition => <<"always">> + , translators => + #{ command => #{topic => <<"/dn/#">>} + , response => #{topic => <<"/up/resp">>} + , notify => #{topic => <<"/up/notify">>} + , register => #{topic => <<"/up/resp">>} + , update => #{topic => <<"/up/resp">>} + } + } + } + , exproto_gateway => + #{ summary => <<"A simple ExProto gateway configs">> + , value => + #{ enable => true + , enable_stats => true + , idle_timeout => <<"30s">> + , mountpoint => <<"exproto2/">> + , server => + #{ bind => <<"9100">> + } + , handler => + #{ address => <<"http://127.0.0.1:9001">> + } + } + } + }. + examples_gateway_stats() -> #{}. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl index 82eb0b52c..d0ac84322 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl @@ -61,7 +61,7 @@ start_grpc_server(GwName, Options = #{bind := ListenOn}) -> end, case grpc:start_server(GwName, ListenOn, Services, SvrOptions) of {ok, _SvrPid} -> - console_print("Start ~ts gRPC server on ~p successfully.", + console_print("Start ~ts gRPC server on ~p successfully.~n", [GwName, ListenOn]); {error, Reason} -> ?ELOG("Falied to start ~ts gRPC server on ~p, reason: ~p", From 9fbc247e29395c157f4685721eb660c51b713fa4 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Mon, 6 Dec 2021 16:47:15 +0100 Subject: [PATCH 07/31] feat(system_monitor): Add system_monitor application --- apps/emqx_conf/src/emqx_conf_schema.erl | 62 +++++++++++++++++++++++++ apps/emqx_machine/src/emqx_machine.erl | 13 ++++++ rebar.config | 1 + rebar.config.erl | 1 + 4 files changed, 77 insertions(+) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 617703cd2..0a4841e64 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -105,6 +105,10 @@ natively in the EMQ X node;
sc(ref("db"), #{ desc => "Settings of the embedded database." })} + , {"system_monitor", + sc(ref("system_monitor"), + #{ desc => "Erlang process and application monitoring." + })} ] ++ emqx_schema:roots(medium) ++ emqx_schema:roots(low) ++ @@ -327,6 +331,64 @@ a crash dump )} ]; +fields("system_monitor") -> + [ {"top_num_items", + sc(non_neg_integer(), + #{ mapping => "system_monitor.top_num_items" + , default => 10 + , desc => "The number of top processes per monitoring group" + }) + } + , {"top_sample_interval", + sc(emqx_schema:duration(), + #{ mapping => "system_monitor.top_sample_interval" + , default => "2s" + , desc => "Specifies how often process top should be collected" + }) + } + , {"top_max_procs", + sc(non_neg_integer(), + #{ mapping => "system_monitor.top_max_procs" + , default => 200000 + , desc => "Stop collecting data when the number of processes exceeds this value" + }) + } + , {"db_hostname", + sc(string(), + #{ mapping => "system_monitor.db_hostname" + , desc => "Hostname of the postgres database that collects the data points" + }) + } + , {"db_port", + sc(integer(), + #{ mapping => "system_monitor.db_port" + , default => 5432 + , desc => "Port of the postgres database that collects the data points" + }) + } + , {"db_username", + sc(string(), + #{ mapping => "system_monitor.db_username" + , default => "system_monitor" + , desc => "EMQX user name in the postgres database" + }) + } + , {"db_password", + sc(binary(), + #{ mapping => "system_monitor.db_password" + , default => "system_monitor_password" + , desc => "EMQX user password in the postgres database" + }) + } + , {"db_name", + sc(string(), + #{ mapping => "system_monitor.db_name" + , default => "postgres" + , desc => "Postgres database name" + }) + } + ]; + fields("db") -> [ {"backend", sc(hoconsc:enum([mnesia, rlog]), diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index fae21eece..178cf8fc7 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -32,6 +32,7 @@ start() -> os:set_signal(sigterm, handle) %% default is handle end, ok = set_backtrace_depth(), + start_sysmon(), ekka:start(), ok = print_otp_version_warning(). @@ -54,3 +55,15 @@ print_otp_version_warning() -> ?ULOG("WARNING: Running on Erlang/OTP version ~p. Recommended: 23~n", [?OTP_RELEASE]). -endif. % OTP_RELEASE > 22 + +start_sysmon() -> + case application:get_env(system_monitor, db_hostname) of + undefined -> + %% If there is no sink for the events, there is no reason + %% to run system_monitor_top, ignore it: + ok; + _ -> + application:set_env(system_monitor, callback_mod, system_monitor_pg), + _ = application:ensure_all_started(system_monitor, temporary), + ok + end. diff --git a/rebar.config b/rebar.config index 4fbf1b136..c7d6d70bd 100644 --- a/rebar.config +++ b/rebar.config @@ -62,6 +62,7 @@ , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x + , {system_monitor, {git, "https://github.com/klarna-incubator/system_monitor", {tag, "2.2.0"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 22949f233..5bdcd2eb5 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -281,6 +281,7 @@ relx_apps(ReleaseType, Edition) -> , {emqx_plugin_libs, load} , {esasl, load} , observer_cli + , system_monitor , emqx_http_lib , emqx_resource , emqx_connector From 056558e445e54d7328822bd3ba6d9a8e0bf003a4 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Sat, 4 Dec 2021 01:42:10 +0300 Subject: [PATCH 08/31] chore(authz): encapsulate authz backend logic --- apps/emqx_authz/include/emqx_authz.hrl | 2 - apps/emqx_authz/src/emqx_authz.erl | 150 ++++++------------ apps/emqx_authz/src/emqx_authz_app.erl | 2 +- apps/emqx_authz/src/emqx_authz_http.erl | 24 ++- apps/emqx_authz/src/emqx_authz_mnesia.erl | 13 +- apps/emqx_authz/src/emqx_authz_mongodb.erl | 21 ++- apps/emqx_authz/src/emqx_authz_mysql.erl | 20 ++- apps/emqx_authz/src/emqx_authz_postgresql.erl | 22 ++- apps/emqx_authz/src/emqx_authz_redis.erl | 23 ++- apps/emqx_authz/src/emqx_authz_rule.erl | 36 ++++- apps/emqx_authz/src/emqx_authz_utils.erl | 54 +++++++ 11 files changed, 249 insertions(+), 118 deletions(-) create mode 100644 apps/emqx_authz/src/emqx_authz_utils.erl diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index 4e0baa8fd..b11915d14 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -17,8 +17,6 @@ -type(rule() :: {permission(), who(), action(), list(emqx_types:topic())}). -type(rules() :: [rule()]). --type(sources() :: [map()]). - -define(APP, emqx_authz). -define(ALLOW_DENY(A), ((A =:= allow) orelse (A =:= <<"allow">>) orelse diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 0b5534608..e2bf18a70 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -27,6 +27,7 @@ -export([ register_metrics/0 , init/0 + , deinit/0 , lookup/0 , lookup/1 , move/2 @@ -42,6 +43,31 @@ -export([ph_to_re/1]). +-type(source() :: map()). + +-type(match_result() :: {matched, allow} | {matched, deny} | nomatch). + +-type(default_result() :: allow | deny). + +-type(authz_result() :: {stop, allow} | {ok, deny}). + +-type(sources() :: [source()]). + + +-callback(init(source()) -> source()). + +-callback(description() -> string()). + +-callback(destroy(source()) -> ok). + +-callback(dry_run(source()) -> ok | {error, term()}). + +-callback(authorize( + emqx_types:clientinfo(), + emqx_types:pubsub(), + emqx_types:topic(), + source()) -> match_result()). + -spec(register_metrics() -> ok). register_metrics() -> lists:foreach(fun emqx_metrics:ensure/1, ?AUTHZ_METRICS). @@ -54,6 +80,11 @@ init() -> NSources = init_sources(Sources), ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NSources]}, -1). +deinit() -> + ok = emqx_hooks:del('client.authorize', {?MODULE, authorize}), + emqx_conf:remove_handler(?CONF_KEY_PATH), + emqx_authz_utils:cleanup_resources(). + lookup() -> {_M, _F, [A]}= find_action_in_hooks(), A. @@ -115,7 +146,7 @@ do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := true} = Source}, Conf) when i NConf = Front ++ [Source | Rear], ok = check_dup_types(NConf), NConf; - Error -> Error + {error, _} = Error -> Error end; do_update({{?CMD_REPLACE, Type}, Source}, Conf) when is_map(Source), is_list(Conf) -> {_Old, Front, Rear} = take(Type, Conf), @@ -178,9 +209,9 @@ do_post_update(_, NewSources) -> ok = emqx_authz_cache:drain_cache(). ensure_resource_deleted(#{enable := false}) -> ok; -ensure_resource_deleted(#{type := file}) -> ok; -ensure_resource_deleted(#{type := 'built-in-database'}) -> ok; -ensure_resource_deleted(#{annotations := #{id := Id}}) -> ok = emqx_resource:remove(Id). +ensure_resource_deleted(#{type := Type} = Source) -> + Module = authz_module(Type), + Module:destroy(Source). check_dup_types(Sources) -> check_dup_types(Sources, []). @@ -204,26 +235,10 @@ check_dup_types([Source | Sources], Checked) -> check_dup_types(Sources, [Type | Checked]) end. -create_dry_run(T, Source) -> - case is_connector_source(T) of - true -> - [CheckedSource] = check_sources([Source]), - case T of - http -> - URIMap = maps:get(url, CheckedSource), - NSource = maps:put(base_url, maps:remove(query, URIMap), CheckedSource) - end, - emqx_resource:create_dry_run(connector_module(T), NSource); - false -> - ok -end. - -is_connector_source(http) -> true; -is_connector_source(mongodb) -> true; -is_connector_source(mysql) -> true; -is_connector_source(postgresql) -> true; -is_connector_source(redis) -> true; -is_connector_source(_) -> false. +create_dry_run(Type, Source) -> + [CheckedSource] = check_sources([Source]), + Module = authz_module(Type), + Module:dry_run(CheckedSource). init_sources(Sources) -> {_Enabled, Disabled} = lists:partition(fun(#{enable := Enable}) -> Enable end, Sources), @@ -234,54 +249,9 @@ init_sources(Sources) -> lists:map(fun init_source/1, Sources). init_source(#{enable := false} = Source) -> Source; -init_source(#{type := file, - path := Path - } = Source) -> - Rules = case file:consult(Path) of - {ok, Terms} -> - [emqx_authz_rule:compile(Term) || Term <- Terms]; - {error, eacces} -> - ?SLOG(alert, #{msg => "insufficient_permissions_to_read_file", path => Path}), - error(eaccess); - {error, enoent} -> - ?SLOG(alert, #{msg => "file_does_not_exist", path => Path}), - error(enoent); - {error, Reason} -> - ?SLOG(alert, #{msg => "failed_to_read_file", path => Path, reason => Reason}), - error(Reason) - end, - Source#{annotations => #{rules => Rules}}; -init_source(#{type := http, - url := Url - } = Source) -> - NSource= maps:put(base_url, maps:remove(query, Url), Source), - case create_resource(NSource) of - {error, Reason} -> error({load_config_error, Reason}); - Id -> Source#{annotations => #{id => Id}} - end; -init_source(#{type := 'built-in-database' - } = Source) -> - Source; -init_source(#{type := DB - } = Source) when DB =:= redis; - DB =:= mongodb -> - case create_resource(Source) of - {error, Reason} -> error({load_config_error, Reason}); - Id -> Source#{annotations => #{id => Id}} - end; -init_source(#{type := DB, - query := SQL - } = Source) when DB =:= mysql; - DB =:= postgresql -> - Mod = authz_module(DB), - case create_resource(Source) of - {error, Reason} -> error({load_config_error, Reason}); - Id -> Source#{annotations => - #{id => Id, - query => erlang:apply(Mod, parse_query, [SQL]) - } - } - end. +init_source(#{type := Type} = Source) -> + Module = authz_module(Type), + Module:init(Source). %%-------------------------------------------------------------------- %% AuthZ callbacks @@ -289,11 +259,11 @@ init_source(#{type := DB, %% @doc Check AuthZ -spec(authorize( emqx_types:clientinfo() - , emqx_types:all() + , emqx_types:pubsub() , emqx_types:topic() - , allow | deny + , default_result() , sources()) - -> {stop, allow} | {ok, deny}). + -> authz_result()). authorize(#{username := Username, peerhost := IpAddress } = Client, PubSub, Topic, DefaultResult, Sources) -> @@ -325,16 +295,10 @@ do_authorize(_Client, _PubSub, _Topic, []) -> nomatch; do_authorize(Client, PubSub, Topic, [#{enable := false} | Rest]) -> do_authorize(Client, PubSub, Topic, Rest); -do_authorize(Client, PubSub, Topic, [#{type := file} = F | Tail]) -> - #{annotations := #{rules := Rules}} = F, - case emqx_authz_rule:matches(Client, PubSub, Topic, Rules) of - nomatch -> do_authorize(Client, PubSub, Topic, Tail); - Matched -> Matched - end; do_authorize(Client, PubSub, Topic, [Connector = #{type := Type} | Tail] ) -> - Mod = authz_module(Type), - case erlang:apply(Mod, authorize, [Client, PubSub, Topic, Connector]) of + Module = authz_module(Type), + case Module:authorize(Client, PubSub, Topic, Connector) of nomatch -> do_authorize(Client, PubSub, Topic, Tail); Matched -> Matched end. @@ -367,29 +331,13 @@ find_action_in_hooks() -> [Action] = [Action || {callback,{?MODULE, authorize, _} = Action, _, _} <- Callbacks ], Action. -gen_id(Type) -> - iolist_to_binary([io_lib:format("~ts_~ts",[?APP, Type])]). - -create_resource(#{type := DB} = Source) -> - ResourceID = gen_id(DB), - case emqx_resource:create(ResourceID, connector_module(DB), Source) of - {ok, already_created} -> ResourceID; - {ok, _} -> ResourceID; - {error, Reason} -> {error, Reason} - end. - authz_module('built-in-database') -> emqx_authz_mnesia; +authz_module(file) -> + emqx_authz_rule; authz_module(Type) -> list_to_existing_atom("emqx_authz_" ++ atom_to_list(Type)). -connector_module(mongodb) -> - emqx_connector_mongo; -connector_module(postgresql) -> - emqx_connector_pgsql; -connector_module(Type) -> - list_to_existing_atom("emqx_connector_" ++ atom_to_list(Type)). - type(#{type := Type}) -> type(Type); type(#{<<"type">> := Type}) -> type(Type); type(file) -> file; diff --git a/apps/emqx_authz/src/emqx_authz_app.erl b/apps/emqx_authz/src/emqx_authz_app.erl index cf9685650..0fb5c4e02 100644 --- a/apps/emqx_authz/src/emqx_authz_app.erl +++ b/apps/emqx_authz/src/emqx_authz_app.erl @@ -34,7 +34,7 @@ start(_StartType, _StartArgs) -> {ok, Sup}. stop(_State) -> - emqx_conf:remove_handler(?CONF_KEY_PATH), + ok = emqx_authz:deinit(), ok. %% internal functions diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 6d1324c47..62719b9ed 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -21,9 +21,14 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-behaviour(emqx_authz). + %% AuthZ Callbacks --export([ authorize/4 - , description/0 +-export([ description/0 + , init/1 + , destroy/1 + , dry_run/1 + , authorize/4 , parse_url/1 ]). @@ -35,6 +40,21 @@ description() -> "AuthZ with http". +init(#{url := Url} = Source) -> + NSource= maps:put(base_url, maps:remove(query, Url), Source), + case emqx_authz_utils:create_resource(emqx_connector_http, NSource) of + {error, Reason} -> error({load_config_error, Reason}); + {ok, Id} -> Source#{annotations => #{id => Id}} + end. + +destroy(#{annotations := #{id := Id}}) -> + ok = emqx_resource:remove(Id). + +dry_run(Source) -> + URIMap = maps:get(url, Source), + NSource = maps:put(base_url, maps:remove(query, URIMap), Source), + emqx_resource:create_dry_run(emqx_connector_http, NSource). + authorize(Client, PubSub, Topic, #{type := http, url := #{path := Path} = URL, diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl index 3851affed..d652c6731 100644 --- a/apps/emqx_authz/src/emqx_authz_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl @@ -20,10 +20,15 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-behaviour(emqx_authz). + %% AuthZ Callbacks -export([ mnesia/1 - , authorize/4 , description/0 + , init/1 + , destroy/1 + , dry_run/1 + , authorize/4 ]). -ifdef(TEST). @@ -45,6 +50,12 @@ mnesia(boot) -> description() -> "AuthZ with Mnesia". +init(Source) -> Source. + +destroy(_Source) -> ok. + +dry_run(_Source) -> ok. + authorize(#{username := Username, clientid := Clientid } = Client, PubSub, Topic, #{type := 'built-in-database'}) -> diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index 5b55c23b7..439c2c853 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -21,9 +21,14 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-behaviour(emqx_authz). + %% AuthZ Callbacks --export([ authorize/4 - , description/0 +-export([ description/0 + , init/1 + , destroy/1 + , dry_run/1 + , authorize/4 ]). -ifdef(TEST). @@ -34,6 +39,18 @@ description() -> "AuthZ with MongoDB". +init(Source) -> + case emqx_authz_utils:create_resource(emqx_connector_mongo, Source) of + {error, Reason} -> error({load_config_error, Reason}); + {ok, Id} -> Source#{annotations => #{id => Id}} + end. + +dry_run(Source) -> + emqx_resource:create_dry_run(emqx_connector_mongo, Source). + +destroy(#{annotations := #{id := Id}}) -> + ok = emqx_resource:remove(Id). + authorize(Client, PubSub, Topic, #{collection := Collection, selector := Selector, diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index a3a5e1ed9..118b00a4f 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -21,9 +21,13 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-behaviour(emqx_authz). + %% AuthZ Callbacks -export([ description/0 - , parse_query/1 + , init/1 + , destroy/1 + , dry_run/1 , authorize/4 ]). @@ -35,6 +39,20 @@ description() -> "AuthZ with Mysql". +init(#{query := SQL} = Source) -> + case emqx_authz_utils:create_resource(emqx_connector_mysql, Source) of + {error, Reason} -> error({load_config_error, Reason}); + {ok, Id} -> Source#{annotations => + #{id => Id, + query => parse_query(SQL)}} + end. + +dry_run(Source) -> + emqx_resource:create_dry_run(emqx_connector_mysql, Source). + +destroy(#{annotations := #{id := Id}}) -> + ok = emqx_resource:remove(Id). + parse_query(undefined) -> undefined; parse_query(Sql) -> diff --git a/apps/emqx_authz/src/emqx_authz_postgresql.erl b/apps/emqx_authz/src/emqx_authz_postgresql.erl index 5bae5f674..6034bfd15 100644 --- a/apps/emqx_authz/src/emqx_authz_postgresql.erl +++ b/apps/emqx_authz/src/emqx_authz_postgresql.erl @@ -21,9 +21,13 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-behaviour(emqx_authz). + %% AuthZ Callbacks -export([ description/0 - , parse_query/1 + , init/1 + , destroy/1 + , dry_run/1 , authorize/4 ]). @@ -33,7 +37,21 @@ -endif. description() -> - "AuthZ with postgresql". + "AuthZ with Postgresql". + +init(#{query := SQL} = Source) -> + case emqx_authz_utils:create_resource(emqx_connector_pgsql, Source) of + {error, Reason} -> error({load_config_error, Reason}); + {ok, Id} -> Source#{annotations => + #{id => Id, + query => parse_query(SQL)}} + end. + +destroy(#{annotations := #{id := Id}}) -> + ok = emqx_resource:remove(Id). + +dry_run(Source) -> + emqx_resource:create_dry_run(emqx_connector_pgsql, Source). parse_query(undefined) -> undefined; diff --git a/apps/emqx_authz/src/emqx_authz_redis.erl b/apps/emqx_authz/src/emqx_authz_redis.erl index 8fa1e94c3..fc60d57ad 100644 --- a/apps/emqx_authz/src/emqx_authz_redis.erl +++ b/apps/emqx_authz/src/emqx_authz_redis.erl @@ -21,9 +21,14 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-behaviour(emqx_authz). + %% AuthZ Callbacks --export([ authorize/4 - , description/0 +-export([ description/0 + , init/1 + , destroy/1 + , dry_run/1 + , authorize/4 ]). -ifdef(TEST). @@ -32,7 +37,19 @@ -endif. description() -> - "AuthZ with redis". + "AuthZ with Redis". + +init(Source) -> + case emqx_authz_utils:create_resource(emqx_connector_redis, Source) of + {error, Reason} -> error({load_config_error, Reason}); + {ok, Id} -> Source#{annotations => #{id => Id}} + end. + +destroy(#{annotations := #{id := Id}}) -> + ok = emqx_resource:remove(Id). + +dry_run(Source) -> + emqx_resource:create_dry_run(emqx_connector_redis, Source). authorize(Client, PubSub, Topic, #{cmd := CMD, diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index 5b6885e22..ade364788 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -20,19 +20,49 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-behaviour(emqx_authz). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). -endif. %% APIs --export([ match/4 - , matches/4 - , compile/1 +-export([ description/0 + , init/1 + , destroy/1 + , dry_run/1 + , authorize/4 ]). -export_type([rule/0]). +description() -> + "AuthZ with static rules". + +init(#{path := Path} = Source) -> + Rules = case file:consult(Path) of + {ok, Terms} -> + [compile(Term) || Term <- Terms]; + {error, eacces} -> + ?SLOG(alert, #{msg => "insufficient_permissions_to_read_file", path => Path}), + error(eaccess); + {error, enoent} -> + ?SLOG(alert, #{msg => "file_does_not_exist", path => Path}), + error(enoent); + {error, Reason} -> + ?SLOG(alert, #{msg => "failed_to_read_file", path => Path, reason => Reason}), + error(Reason) + end, + Source#{annotations => #{rules => Rules}}. + +destroy(_Source) -> ok. + +dry_run(_Source) -> ok. + +authorize(Client, PubSub, Topic, #{annotations := #{rules := Rules}}) -> + matches(Client, PubSub, Topic, Rules). + compile({Permission, all}) when ?ALLOW_DENY(Permission) -> {Permission, all, all, [compile_topic(<<"#">>)]}; compile({Permission, Who, Action, TopicFilters}) diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl new file mode 100644 index 000000000..73132aacb --- /dev/null +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -0,0 +1,54 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_utils). + +-include_lib("emqx/include/emqx_placeholder.hrl"). + +-export([cleanup_resources/0, + make_resource_id/1, + create_resource/2]). + +-define(RESOURCE_GROUP, <<"emqx_authz">>). + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create_resource(Module, Config) -> + ResourceID = make_resource_id(Module), + case emqx_resource:create(ResourceID, Module, Config) of + {ok, already_created} -> {ok, ResourceID}; + {ok, _} -> {ok, ResourceID}; + {error, Reason} -> {error, Reason} + end. + +cleanup_resources() -> + lists:foreach( + fun emqx_resource:remove/1, + emqx_resource:list_group_instances(?RESOURCE_GROUP)). + +make_resource_id(Name) -> + NameBin = bin(Name), + emqx_resource:generate_id(?RESOURCE_GROUP, NameBin). + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(L) when is_list(L) -> list_to_binary(L); +bin(X) -> X. From b7d227d2ef9b34b7b8b80fedf390913c59d3abf5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Sat, 4 Dec 2021 15:40:26 +0300 Subject: [PATCH 09/31] chore(authz): move file backend to a separate module --- apps/emqx_authz/include/emqx_authz.hrl | 22 ++------- apps/emqx_authz/src/emqx_authz.erl | 2 - apps/emqx_authz/src/emqx_authz_file.erl | 61 +++++++++++++++++++++++++ apps/emqx_authz/src/emqx_authz_rule.erl | 50 +++++++------------- 4 files changed, 81 insertions(+), 54 deletions(-) create mode 100644 apps/emqx_authz/src/emqx_authz_file.erl diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index b11915d14..de5dca73e 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -1,22 +1,3 @@ --type(ipaddress() :: {ipaddr, esockd_cidr:cidr_string()} | - {ipaddrs, list(esockd_cidr:cidr_string())}). - --type(username() :: {username, binary()}). - --type(clientid() :: {clientid, binary()}). - --type(who() :: ipaddress() | username() | clientid() | - {'and', [ipaddress() | username() | clientid()]} | - {'or', [ipaddress() | username() | clientid()]} | - all). - --type(action() :: subscribe | publish | all). - --type(permission() :: allow | deny). - --type(rule() :: {permission(), who(), action(), list(emqx_types:topic())}). --type(rules() :: [rule()]). - -define(APP, emqx_authz). -define(ALLOW_DENY(A), ((A =:= allow) orelse (A =:= <<"allow">>) orelse @@ -36,6 +17,9 @@ -define(ACL_TABLE_USERNAME, 1). -define(ACL_TABLE_CLIENTID, 2). +-type(action() :: subscribe | publish | all). +-type(permission() :: allow | deny). + -record(emqx_acl, { who :: ?ACL_TABLE_ALL| {?ACL_TABLE_USERNAME, binary()} | {?ACL_TABLE_CLIENTID, binary()}, rules :: [ {permission(), action(), emqx_topic:topic()} ] diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index e2bf18a70..af16c3892 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -333,8 +333,6 @@ find_action_in_hooks() -> authz_module('built-in-database') -> emqx_authz_mnesia; -authz_module(file) -> - emqx_authz_rule; authz_module(Type) -> list_to_existing_atom("emqx_authz_" ++ atom_to_list(Type)). diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl new file mode 100644 index 000000000..ba4f9c2b7 --- /dev/null +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -0,0 +1,61 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_file). + +-include("emqx_authz.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-behaviour(emqx_authz). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + +%% APIs +-export([ description/0 + , init/1 + , destroy/1 + , dry_run/1 + , authorize/4 + ]). + +description() -> + "AuthZ with static rules". + +init(#{path := Path} = Source) -> + Rules = case file:consult(Path) of + {ok, Terms} -> + [emqx_authz_rule:compile(Term) || Term <- Terms]; + {error, eacces} -> + ?SLOG(alert, #{msg => "insufficient_permissions_to_read_file", path => Path}), + error(eaccess); + {error, enoent} -> + ?SLOG(alert, #{msg => "file_does_not_exist", path => Path}), + error(enoent); + {error, Reason} -> + ?SLOG(alert, #{msg => "failed_to_read_file", path => Path, reason => Reason}), + error(Reason) + end, + Source#{annotations => #{rules => Rules}}. + +destroy(_Source) -> ok. + +dry_run(_Source) -> ok. + +authorize(Client, PubSub, Topic, #{annotations := #{rules := Rules}}) -> + emqx_authz_rule:matches(Client, PubSub, Topic, Rules). diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index ade364788..952d6b7a5 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -20,49 +20,33 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). --behaviour(emqx_authz). - -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). -endif. %% APIs --export([ description/0 - , init/1 - , destroy/1 - , dry_run/1 - , authorize/4 +-export([ match/4 + , matches/4 + , compile/1 ]). +-type(ipaddress() :: {ipaddr, esockd_cidr:cidr_string()} | + {ipaddrs, list(esockd_cidr:cidr_string())}). + +-type(username() :: {username, binary()}). + +-type(clientid() :: {clientid, binary()}). + +-type(who() :: ipaddress() | username() | clientid() | + {'and', [ipaddress() | username() | clientid()]} | + {'or', [ipaddress() | username() | clientid()]} | + all). + +-type(rule() :: {permission(), who(), action(), list(emqx_types:topic())}). + -export_type([rule/0]). -description() -> - "AuthZ with static rules". - -init(#{path := Path} = Source) -> - Rules = case file:consult(Path) of - {ok, Terms} -> - [compile(Term) || Term <- Terms]; - {error, eacces} -> - ?SLOG(alert, #{msg => "insufficient_permissions_to_read_file", path => Path}), - error(eaccess); - {error, enoent} -> - ?SLOG(alert, #{msg => "file_does_not_exist", path => Path}), - error(enoent); - {error, Reason} -> - ?SLOG(alert, #{msg => "failed_to_read_file", path => Path, reason => Reason}), - error(Reason) - end, - Source#{annotations => #{rules => Rules}}. - -destroy(_Source) -> ok. - -dry_run(_Source) -> ok. - -authorize(Client, PubSub, Topic, #{annotations := #{rules := Rules}}) -> - matches(Client, PubSub, Topic, Rules). - compile({Permission, all}) when ?ALLOW_DENY(Permission) -> {Permission, all, all, [compile_topic(<<"#">>)]}; compile({Permission, Who, Action, TopicFilters}) From 1c9d867c5ec8b5a9f396882f55105154248f8e00 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 6 Dec 2021 12:11:45 +0300 Subject: [PATCH 10/31] chore(authz): add test for authz backend creation dry run --- apps/emqx_authz/test/emqx_authz_SUITE.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index d965affee..e18901fc5 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -33,7 +33,8 @@ init_per_suite(Config) -> meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end), - meck:expect(emqx_resource, remove, fun(_) -> ok end ), + meck:expect(emqx_resource, remove, fun(_) -> ok end), + meck:expect(emqx_resource, create_dry_run, fun(_, _) -> ok end), ok = emqx_common_test_helpers:start_apps( [emqx_connector, emqx_conf, emqx_authz], @@ -137,6 +138,13 @@ t_update_source(_) -> , #{type := file, enable := true} ], emqx_conf:get([authorization, sources], [])), + {ok, _} = emqx_authz:update({?CMD_REPLACE, http}, ?SOURCE1#{<<"enable">> := true}), + {ok, _} = emqx_authz:update({?CMD_REPLACE, mongodb}, ?SOURCE2#{<<"enable">> := true}), + {ok, _} = emqx_authz:update({?CMD_REPLACE, mysql}, ?SOURCE3#{<<"enable">> := true}), + {ok, _} = emqx_authz:update({?CMD_REPLACE, postgresql}, ?SOURCE4#{<<"enable">> := true}), + {ok, _} = emqx_authz:update({?CMD_REPLACE, redis}, ?SOURCE5#{<<"enable">> := true}), + {ok, _} = emqx_authz:update({?CMD_REPLACE, file}, ?SOURCE6#{<<"enable">> := true}), + {ok, _} = emqx_authz:update({?CMD_REPLACE, http}, ?SOURCE1#{<<"enable">> := false}), {ok, _} = emqx_authz:update({?CMD_REPLACE, mongodb}, ?SOURCE2#{<<"enable">> := false}), {ok, _} = emqx_authz:update({?CMD_REPLACE, mysql}, ?SOURCE3#{<<"enable">> := false}), From 028f1fa71c63e661e70c1295d9d363fa94f3b230 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 9 Dec 2021 11:33:09 +0800 Subject: [PATCH 11/31] chore(gw): fix listener examples --- .../src/emqx_gateway_api_listeners.erl | 69 +++++++++++++++---- apps/emqx_gateway/src/emqx_gateway_http.erl | 4 ++ 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index e73dd707e..142469313 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -233,6 +233,8 @@ schema("/gateway/:name/listeners") -> post => #{ description => <<"Create the gateway listener">> , parameters => params_gateway_name_in_path() + %% XXX: How to distinguish the different listener supported by + %% different types of gateways? , 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(listener), examples_listener()) @@ -288,7 +290,7 @@ schema("/gateway/:name/listeners/:id/authentication") -> , responses => ?STANDARD_RESP( #{ 200 => schema_authn() - , 204 => <<"Authentication does not initiated">> + , 204 => <<"Authentication or listener does not existed">> }) }, post => @@ -487,7 +489,6 @@ fields(ssl_listener_opts) -> , {keyfile, binary()} , {verify, binary()} , {fail_if_no_peer_cert, boolean()} - , {server_name_indication, boolean()} , {depth, integer()} , {password, binary()} , {handshake_timeout, binary()} @@ -586,7 +587,9 @@ examples_listener() -> #{ tcp_listener=> #{ summary => <<"A simple tcp listener example">> , value => - #{ bind => <<"61613">> + #{ name => <<"tcp-def">> + , type => <<"tcp">> + , bind => <<"22210">> , acceptors => 16 , max_connections => 1024000 , max_conn_rate => 1000 @@ -607,7 +610,9 @@ examples_listener() -> , ssl_listener => #{ summary => <<"A simple ssl listener example">> , value => - #{ bind => <<"61614">> + #{ name => <<"ssl-def">> + , type => <<"ssl">> + , bind => <<"22211">> , acceptors => 16 , max_connections => 1024000 , max_conn_rate => 1000 @@ -620,7 +625,6 @@ examples_listener() -> , keyfile => <<"etc/certs/key.pem">> , verify => <<"verify_none">> , fail_if_no_peer_cert => false - , server_name_indication => disable } , tcp => #{ active_n => 100 @@ -631,7 +635,9 @@ examples_listener() -> , udp_listener => #{ summary => <<"A simple udp listener example">> , value => - #{ bind => <<"0.0.0.0:1884">> + #{ name => <<"udp-def">> + , type => udp + , bind => <<"22212">> , udp => #{ active_n => 100 , recbuf => <<"10KB">> @@ -644,32 +650,67 @@ examples_listener() -> , dtls_listener => #{ summary => <<"A simple dtls listener example">> , value => - #{ bind => <<"5684">> + #{ name => <<"dtls-def">> + , type => <<"dtls">> + , bind => <<"22213">> , acceptors => 16 , max_connections => 1024000 , max_conn_rate => 1000 , access_rules => [<<"allow all">>] - , ssl => + , dtls => #{ versions => [<<"dtlsv1.2">>, <<"dtlsv1">>] , cacertfile => <<"etc/certs/cacert.pem">> , certfile => <<"etc/certs/cert.pem">> , keyfile => <<"etc/certs/key.pem">> , verify => <<"verify_none">> , fail_if_no_peer_cert => false - , server_name_indication => disable } - , tcp => + , udp => #{ active_n => 100 , backlog => 1024 } } } , dtls_listener_with_psk_ciphers => - #{ summary => <<"todo">> + #{ summary => <<"A dtls listener with PSK example">> , value => - #{} + #{ name => <<"dtls-psk">> + , type => <<"dtls">> + , bind => <<"22214">> + , acceptors => 16 + , max_connections => 1024000 + , max_conn_rate => 1000 + , dtls => + #{ versions => [<<"dtlsv1.2">>, <<"dtlsv1">>] + , cacertfile => <<"etc/certs/cacert.pem">> + , certfile => <<"etc/certs/cert.pem">> + , keyfile => <<"etc/certs/key.pem">> + , verify => <<"verify_none">> + , user_lookup_fun => <<"emqx_tls_psk:lookup">> + , ciphers => +<<"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384,RSA-PSK-AES128-GCM-SHA256," + "RSA-PSK-AES128-CBC-SHA256,RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA">> + , fail_if_no_peer_cert => false + } + } } , lisetner_with_authn => - #{ summary => <<"todo">> - , value => #{}} + #{ summary => <<"A tcp listener with authentication example">> + , value => + #{ name => <<"tcp-with-authn">> + , type => <<"tcp">> + , bind => <<"22215">> + , acceptors => 16 + , max_connections => 1024000 + , max_conn_rate => 1000 + , authentication => + #{ backend => <<"built-in-database">> + , mechanism => <<"password-based">> + , password_hash_algorithm => + #{ name => <<"sha256">> + } + , user_id_type => <<"username">> + } + } + } }. diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 2a9840a9c..434a0bc49 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -223,6 +223,8 @@ remove_authn(GwName, ListenerId) -> confexp(ok) -> ok; confexp({ok, Res}) -> {ok, Res}; +confexp({error, badarg}) -> + error({update_conf_error, badarg}); confexp({error, not_found}) -> error({update_conf_error, not_found}); confexp({error, already_exist}) -> @@ -372,6 +374,8 @@ with_gateway(GwName0, Fun) -> lists:join(".", lists:map(fun to_list/1, Path0))), return_http_error(404, "Resource not found. path: " ++ Path); %% Exceptions from: confexp/1 + error : {update_conf_error, badarg} -> + return_http_error(400, "Bad arguments"); error : {update_conf_error, not_found} -> return_http_error(404, "Resource not found"); error : {update_conf_error, already_exist} -> From e66262858130c5a6d4007f9c789ed2efc0318fe7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 9 Dec 2021 14:50:28 +0800 Subject: [PATCH 12/31] chore(gw): fix bad default value in query-string --- apps/emqx_gateway/src/emqx_gateway_api.erl | 2 ++ .../src/emqx_gateway_api_authn.erl | 4 ++++ .../src/emqx_gateway_api_clients.erl | 18 +++++++++++------- .../src/emqx_gateway_api_listeners.erl | 5 +++++ 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 5036286b4..7bf52b4ec 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -210,6 +210,7 @@ params_gateway_name_in_path() -> mk(binary(), #{ in => path , desc => <<"Gateway Name">> + , example => <<"">> })} ]. @@ -220,6 +221,7 @@ params_gateway_status_in_qs() -> #{ in => query , nullable => true , desc => <<"Gateway Status">> + , example => <<"">> })} ]. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index d9bdf4d7c..701890633 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -272,6 +272,7 @@ params_gateway_name_in_path() -> mk(binary(), #{ in => path , desc => <<"Gateway Name">> + , example => <<"">> })} ]. @@ -279,6 +280,7 @@ params_userid_in_path() -> [{uid, mk(binary(), #{ in => path , desc => <<"User ID">> + , example => <<"">> })} ]. @@ -287,11 +289,13 @@ params_paging_in_qs() -> #{ in => query , nullable => true , desc => <<"Page Index">> + , example => 1 })}, {limit, mk(integer(), #{ in => query , nullable => true , desc => <<"Page Limit">> + , example => 100 })} ]. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index b67961e12..9fe36d25e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -475,7 +475,7 @@ params_client_insta() -> ++ params_gateway_name_in_path(). params_client_searching_in_qs() -> - M = #{in => query, nullable => true}, + M = #{in => query, nullable => true, example => <<"">>}, [ {node, mk(binary(), M#{desc => <<"Match the client's node name">>})} @@ -526,12 +526,16 @@ params_paging() -> mk(integer(), #{ in => query , nullable => true - , desc => <<"Page Index">>})} - , {limit, - mk(integer(), - #{ in => query - , desc => <<"Page Limit">> - , nullable => true})} + , desc => <<"Page Index">> + , example => 1 + })} + , {limit, + mk(integer(), + #{ in => query + , desc => <<"Page Limit">> + , nullable => true + , example => 100 + })} ]. params_gateway_name_in_path() -> diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 142469313..fbf923700 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -410,6 +410,7 @@ params_gateway_name_in_path() -> mk(binary(), #{ in => path , desc => <<"Gateway Name">> + , example => <<"">> })} ]. @@ -418,6 +419,7 @@ params_listener_id_in_path() -> mk(binary(), #{ in => path , desc => <<"Listener ID">> + , example => <<"">> })} ]. @@ -425,6 +427,7 @@ params_userid_in_path() -> [{uid, mk(binary(), #{ in => path , desc => <<"User ID">> + , example => <<"">> })} ]. @@ -433,11 +436,13 @@ params_paging_in_qs() -> #{ in => query , nullable => true , desc => <<"Page Index">> + , example => 1 })}, {limit, mk(integer(), #{ in => query , nullable => true , desc => <<"Page Limit">> + , example => 100 })} ]. From a40c50f2e89cd10c0c45a3961cdf4993477fb320 Mon Sep 17 00:00:00 2001 From: Spycsh <757407490@qq.com> Date: Fri, 3 Dec 2021 16:46:24 +0100 Subject: [PATCH 13/31] fix: fix apiVersion for networking.k8s.io/v1 --- deploy/charts/emqx/templates/ingress.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/charts/emqx/templates/ingress.yaml b/deploy/charts/emqx/templates/ingress.yaml index 926023f61..a69ee06f8 100644 --- a/deploy/charts/emqx/templates/ingress.yaml +++ b/deploy/charts/emqx/templates/ingress.yaml @@ -49,7 +49,9 @@ spec: --- {{- end }} {{- if .Values.ingress.mgmt.enabled -}} -{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1beta1 {{- else -}} apiVersion: extensions/v1beta1 From fa11c6df9cc7c236fba99c88fa54eb3736ff2c9f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 9 Dec 2021 16:37:19 +0800 Subject: [PATCH 14/31] test(gw): fix bad test cases --- apps/emqx_gateway/src/emqx_gateway_cli.erl | 6 +++--- apps/emqx_gateway/src/emqx_gateway_conf.erl | 4 ++-- apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index a441b384e..03d55e27e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -69,7 +69,7 @@ gateway(["load", Name, Conf]) -> bin(Name), emqx_json:decode(Conf, [return_maps]) ) of - ok -> + {ok, _} -> print("ok~n"); {error, Reason} -> print("Error: ~p~n", [Reason]) @@ -88,7 +88,7 @@ gateway(["stop", Name]) -> bin(Name), #{<<"enable">> => <<"false">>} ) of - ok -> + {ok, _} -> print("ok~n"); {error, Reason} -> print("Error: ~p~n", [Reason]) @@ -99,7 +99,7 @@ gateway(["start", Name]) -> bin(Name), #{<<"enable">> => <<"true">>} ) of - ok -> + {ok, _} -> print("ok~n"); {error, Reason} -> print("Error: ~p~n", [Reason]) diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index da06e3a6d..351093e0f 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -269,9 +269,9 @@ ret_gw(GwName, {ok, #{raw_config := GwConf}}) -> NLConfs = lists:map(fun({LName, LConf}) -> do_convert_listener2(GwName, LType, LName, LConf) - end, proplists:from_map(SubConf)), + end, maps:to_list(SubConf)), [NLConfs|Acc] - end, [], proplists:from_map(LsConf)), + end, [], maps:to_list(LsConf)), {ok, maps:merge(GwConf1, #{<<"listeners">> => NLsConf})}; ret_gw(_GwName, Err) -> Err. diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 18a380984..f91347a6e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -86,7 +86,7 @@ t_gateway_stomp(_) -> assert_confs(GwConf, ConfResp), %% put GwConf2 = emqx_map_lib:deep_merge(GwConf, #{frame => #{max_headers => 10}}), - {200, _} = request(put, "/gateway/stomp", maps:without([name], GwConf2)), + {200, _} = request(put, "/gateway/stomp", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateway/stomp"), assert_confs(GwConf2, ConfResp2), {204, _} = request(delete, "/gateway/stomp"). @@ -109,7 +109,7 @@ t_gateway_mqttsn(_) -> assert_confs(GwConf, ConfResp), %% put GwConf2 = emqx_map_lib:deep_merge(GwConf, #{predefined => []}), - {200, _} = request(put, "/gateway/mqttsn", maps:without([name], GwConf2)), + {200, _} = request(put, "/gateway/mqttsn", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateway/mqttsn"), assert_confs(GwConf2, ConfResp2), {204, _} = request(delete, "/gateway/mqttsn"). @@ -130,7 +130,7 @@ t_gateway_coap(_) -> assert_confs(GwConf, ConfResp), %% put GwConf2 = emqx_map_lib:deep_merge(GwConf, #{heartbeat => <<"10s">>}), - {200, _} = request(put, "/gateway/coap", maps:without([name], GwConf2)), + {200, _} = request(put, "/gateway/coap", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateway/coap"), assert_confs(GwConf2, ConfResp2), {204, _} = request(delete, "/gateway/coap"). @@ -161,7 +161,7 @@ t_gateway_lwm2m(_) -> assert_confs(GwConf, ConfResp), %% put GwConf2 = emqx_map_lib:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}), - {200, _} = request(put, "/gateway/lwm2m", maps:without([name], GwConf2)), + {200, _} = request(put, "/gateway/lwm2m", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateway/lwm2m"), assert_confs(GwConf2, ConfResp2), {204, _} = request(delete, "/gateway/lwm2m"). @@ -182,7 +182,7 @@ t_gateway_exproto(_) -> assert_confs(GwConf, ConfResp), %% put GwConf2 = emqx_map_lib:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}), - {200, _} = request(put, "/gateway/exproto", maps:without([name], GwConf2)), + {200, _} = request(put, "/gateway/exproto", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateway/exproto"), assert_confs(GwConf2, ConfResp2), {204, _} = request(delete, "/gateway/exproto"). From 8493b61cb5510a8057427014d4bff6b7db150025 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 10 Dec 2021 00:46:11 +0800 Subject: [PATCH 15/31] refactor(eqmx_limiter): use the new hierarchical token bucket to replace the old ratelimit code --- apps/emqx/src/emqx_channel.erl | 35 +- apps/emqx/src/emqx_connection.erl | 229 ++++-- .../src/emqx_limiter/etc/emqx_limiter.conf | 52 ++ .../src/emqx_limiter/src/emqx_htb_limiter.erl | 358 ++++++++++ .../emqx_limiter/src/emqx_limiter.app.src | 4 +- .../emqx_limiter/src/emqx_limiter_app.erl | 2 +- .../src/emqx_limiter_bucket_ref.erl | 102 +++ .../src/emqx_limiter_container.erl | 157 +++++ .../emqx_limiter/src/emqx_limiter_decimal.erl | 9 +- .../emqx_limiter/src/emqx_limiter_manager.erl | 73 +- .../emqx_limiter/src/emqx_limiter_schema.erl | 176 +++++ .../emqx_limiter/src/emqx_limiter_server.erl | 582 ++++++++++++++++ .../src/emqx_limiter_server_sup.erl | 10 +- .../emqx_limiter/src/emqx_limiter_sup.erl | 2 +- apps/emqx/src/emqx_listeners.erl | 11 +- apps/emqx/src/emqx_misc.erl | 9 + apps/emqx/src/emqx_schema.erl | 2 + apps/emqx/src/emqx_sup.erl | 3 +- apps/emqx/src/emqx_ws_connection.erl | 183 +++-- apps/emqx/test/emqx_channel_SUITE.erl | 51 +- apps/emqx/test/emqx_common_test_helpers.erl | 1 + apps/emqx/test/emqx_connection_SUITE.erl | 70 +- apps/emqx/test/emqx_listeners_SUITE.erl | 1 + apps/emqx/test/emqx_ratelimiter_SUITE.erl | 659 ++++++++++++++++++ apps/emqx/test/emqx_ws_connection_SUITE.erl | 62 +- .../src/emqx_dashboard_swagger.erl | 11 +- .../test/emqx_lwm2m_api_SUITE.erl | 5 +- apps/emqx_limiter/etc/emqx_limiter.conf | 50 -- apps/emqx_limiter/src/emqx_limiter_client.erl | 144 ---- apps/emqx_limiter/src/emqx_limiter_schema.erl | 140 ---- apps/emqx_limiter/src/emqx_limiter_server.erl | 426 ----------- apps/emqx_limiter/test/emqx_limiter_SUITE.erl | 272 -------- .../test/emqx_mgmt_clients_api_SUITE.erl | 2 +- rebar.config.erl | 1 - scripts/merge-config.escript | 58 +- 35 files changed, 2672 insertions(+), 1280 deletions(-) create mode 100644 apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf create mode 100644 apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl rename apps/{ => emqx/src}/emqx_limiter/src/emqx_limiter.app.src (75%) rename apps/{ => emqx/src}/emqx_limiter/src/emqx_limiter_app.erl (96%) create mode 100644 apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl create mode 100644 apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl rename apps/{ => emqx/src}/emqx_limiter/src/emqx_limiter_decimal.erl (92%) rename apps/{ => emqx/src}/emqx_limiter/src/emqx_limiter_manager.erl (80%) create mode 100644 apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl create mode 100644 apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl rename apps/{ => emqx/src}/emqx_limiter/src/emqx_limiter_server_sup.erl (89%) rename apps/{ => emqx/src}/emqx_limiter/src/emqx_limiter_sup.erl (97%) create mode 100644 apps/emqx/test/emqx_ratelimiter_SUITE.erl delete mode 100644 apps/emqx_limiter/etc/emqx_limiter.conf delete mode 100644 apps/emqx_limiter/src/emqx_limiter_client.erl delete mode 100644 apps/emqx_limiter/src/emqx_limiter_schema.erl delete mode 100644 apps/emqx_limiter/src/emqx_limiter_server.erl delete mode 100644 apps/emqx_limiter/test/emqx_limiter_SUITE.erl diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index cf22fa0ae..4e48be2cb 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -82,7 +82,7 @@ %% Authentication Data Cache auth_cache :: maybe(map()), %% Quota checkers - quota :: maybe(emqx_limiter:limiter()), + quota :: maybe(emqx_limiter_container:limiter()), %% Timers timers :: #{atom() => disabled | maybe(reference())}, %% Conn State @@ -120,6 +120,7 @@ }). -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). +-define(LIMITER_ROUTING, message_routing). -dialyzer({no_match, [shutdown/4, ensure_timer/2, interval/2]}). @@ -200,14 +201,13 @@ caps(#channel{clientinfo = #{zone := Zone}}) -> -spec(init(emqx_types:conninfo(), opts()) -> channel()). init(ConnInfo = #{peername := {PeerHost, _Port}, sockname := {_Host, SockPort}}, - #{zone := Zone, listener := {Type, Listener}}) -> + #{zone := Zone, limiter := LimiterCfg, listener := {Type, Listener}}) -> Peercert = maps:get(peercert, ConnInfo, undefined), Protocol = maps:get(protocol, ConnInfo, mqtt), MountPoint = case emqx_config:get_listener_conf(Type, Listener, [mountpoint]) of <<>> -> undefined; MP -> MP end, - QuotaPolicy = emqx_config:get_zone_conf(Zone, [quota], #{}), ClientInfo = set_peercert_infos( Peercert, #{zone => Zone, @@ -228,7 +228,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, outbound => #{} }, auth_cache = #{}, - quota = emqx_limiter:init(Zone, quota_policy(QuotaPolicy)), + quota = emqx_limiter_container:get_limiter_by_names([?LIMITER_ROUTING], LimiterCfg), timers = #{}, conn_state = idle, takeover = false, @@ -236,11 +236,6 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, pendings = [] }. -quota_policy(RawPolicy) -> - [{Name, {list_to_integer(StrCount), - erlang:trunc(hocon_postprocess:duration(StrWind) / 1000)}} - || {Name, [StrCount, StrWind]} <- maps:to_list(RawPolicy)]. - set_peercert_infos(NoSSL, ClientInfo, _) when NoSSL =:= nossl; NoSSL =:= undefined -> @@ -653,10 +648,10 @@ ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> ({_, _, {ok, I}}, N) -> N + I; (_, N) -> N end, 1, PubRes), - case emqx_limiter:check(#{cnt => Cnt, oct => 0}, Limiter) of + case emqx_limiter_container:check(Cnt, ?LIMITER_ROUTING, Limiter) of {ok, NLimiter} -> Channel#channel{quota = NLimiter}; - {pause, Intv, NLimiter} -> + {_, Intv, NLimiter} -> ensure_timer(quota_timer, Intv, Channel#channel{quota = NLimiter}) end. @@ -1005,10 +1000,9 @@ handle_call({takeover, 'end'}, Channel = #channel{session = Session, handle_call(list_authz_cache, Channel) -> {reply, emqx_authz_cache:list_authz_cache(), Channel}; -handle_call({quota, Policy}, Channel) -> - Zone = info(zone, Channel), - Quota = emqx_limiter:init(Zone, Policy), - reply(ok, Channel#channel{quota = Quota}); +handle_call({quota, Bucket}, #channel{quota = Quota} = Channel) -> + Quota2 = emqx_limiter_container:update_by_name(message_routing, Bucket, Quota), + reply(ok, Channel#channel{quota = Quota2}); handle_call({keepalive, Interval}, Channel = #channel{keepalive = KeepAlive, conninfo = ConnInfo}) -> @@ -1147,8 +1141,15 @@ handle_timeout(_TRef, will_message, Channel = #channel{will_msg = WillMsg}) -> (WillMsg =/= undefined) andalso publish_will_msg(WillMsg), {ok, clean_timer(will_timer, Channel#channel{will_msg = undefined})}; -handle_timeout(_TRef, expire_quota_limit, Channel) -> - {ok, clean_timer(quota_timer, Channel)}; +handle_timeout(_TRef, expire_quota_limit, + #channel{quota = Quota} = Channel) -> + case emqx_limiter_container:retry(?LIMITER_ROUTING, Quota) of + {_, Intv, Quota2} -> + Channel2 = ensure_timer(quota_timer, Intv, Channel#channel{quota = Quota2}), + {ok, Channel2}; + {_, Quota2} -> + {ok, clean_timer(quota_timer, Channel#channel{quota = Quota2})} + end; handle_timeout(_TRef, Msg, Channel) -> ?SLOG(error, #{msg => "unexpected_timeout", timeout_msg => Msg}), diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 9ba126fc9..6919c6ff8 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -67,8 +67,7 @@ -export([set_field/3]). -import(emqx_misc, - [ maybe_apply/2 - , start_timer/2 + [ start_timer/2 ]). -record(state, { @@ -82,11 +81,6 @@ sockname :: emqx_types:peername(), %% Sock State sockstate :: emqx_types:sockstate(), - %% Limiter - limiter :: maybe(emqx_limiter:limiter()), - %% Limit Timer - limit_timer :: maybe(reference()), - %% Parse State parse_state :: emqx_frame:parse_state(), %% Serialize options serialize :: emqx_frame:serialize_opts(), @@ -103,10 +97,30 @@ %% Zone name zone :: atom(), %% Listener Type and Name - listener :: {Type::atom(), Name::atom()} - }). + listener :: {Type::atom(), Name::atom()}, + + %% Limiter + limiter :: maybe(limiter()), + + %% cache operation when overload + limiter_cache :: queue:queue(cache()), + + %% limiter timers + limiter_timer :: undefined | reference() + }). + +-record(retry, { types :: list(limiter_type()) + , data :: any() + , next :: check_succ_handler() + }). + +-record(cache, { need :: list({pos_integer(), limiter_type()}) + , data :: any() + , next :: check_succ_handler() + }). -type(state() :: #state{}). +-type cache() :: #cache{}. -define(ACTIVE_N, 100). -define(INFO_KEYS, [socktype, peername, sockname, sockstate]). @@ -127,6 +141,11 @@ -define(ALARM_SOCK_STATS_KEYS, [send_pend, recv_cnt, recv_oct, send_cnt, send_oct]). -define(ALARM_SOCK_OPTS_KEYS, [high_watermark, high_msgq_watermark, sndbuf, recbuf, buffer]). +%% use macro to do compile time limiter's type check +-define(LIMITER_BYTES_IN, bytes_in). +-define(LIMITER_MESSAGE_IN, message_in). +-define(EMPTY_QUEUE, {[], []}). + -dialyzer({no_match, [info/2]}). -dialyzer({nowarn_function, [ init/4 , init_state/3 @@ -170,10 +189,10 @@ info(sockstate, #state{sockstate = SockSt}) -> SockSt; info(stats_timer, #state{stats_timer = StatsTimer}) -> StatsTimer; -info(limit_timer, #state{limit_timer = LimitTimer}) -> - LimitTimer; info(limiter, #state{limiter = Limiter}) -> - maybe_apply(fun emqx_limiter:info/1, Limiter). + Limiter; +info(limiter_timer, #state{limiter_timer = Timer}) -> + Timer. %% @doc Get stats of the connection/channel. -spec(stats(pid() | state()) -> emqx_types:stats()). @@ -244,7 +263,8 @@ init(Parent, Transport, RawSocket, Options) -> exit_on_sock_error(Reason) end. -init_state(Transport, Socket, #{zone := Zone, listener := Listener} = Opts) -> +init_state(Transport, Socket, + #{zone := Zone, limiter := LimiterCfg, listener := Listener} = Opts) -> {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]), {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]), Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]), @@ -254,7 +274,10 @@ init_state(Transport, Socket, #{zone := Zone, listener := Listener} = Opts) -> peercert => Peercert, conn_mod => ?MODULE }, - Limiter = emqx_limiter:init(Zone, undefined, undefined, []), + + LimiterTypes = [?LIMITER_BYTES_IN, ?LIMITER_MESSAGE_IN], + Limiter = emqx_limiter_container:get_limiter_by_names(LimiterTypes, LimiterCfg), + FrameOpts = #{ strict_mode => emqx_config:get_zone_conf(Zone, [mqtt, strict_mode]), max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size]) @@ -286,7 +309,9 @@ init_state(Transport, Socket, #{zone := Zone, listener := Listener} = Opts) -> idle_timeout = IdleTimeout, idle_timer = IdleTimer, zone = Zone, - listener = Listener + listener = Listener, + limiter_cache = queue:new(), + limiter_timer = undefined }. run_loop(Parent, State = #state{transport = Transport, @@ -428,14 +453,23 @@ handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> Oct = iolist_size(Data), inc_counter(incoming_bytes, Oct), ok = emqx_metrics:inc('bytes.received', Oct), - parse_incoming(Data, State); + when_bytes_in(Oct, Data, State); handle_msg({quic, Data, _Sock, _, _, _}, State) -> ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => quic}), Oct = iolist_size(Data), inc_counter(incoming_bytes, Oct), ok = emqx_metrics:inc('bytes.received', Oct), - parse_incoming(Data, State); + when_bytes_in(Oct, Data, State); + +handle_msg(check_cache, #state{limiter_cache = Cache} = State) -> + case queue:peek(Cache) of + empty -> + activate_socket(State); + {value, #cache{need = Needs, data = Data, next = Next}} -> + State2 = State#state{limiter_cache = queue:drop(Cache)}, + check_limiter(Needs, Data, Next, [check_cache], State2) + end; handle_msg({incoming, Packet = ?CONNECT_PACKET(ConnPkt)}, State = #state{idle_timer = IdleTimer}) -> @@ -466,14 +500,12 @@ handle_msg({Passive, _Sock}, State) Pubs = emqx_pd:reset_counter(incoming_pubs), Bytes = emqx_pd:reset_counter(incoming_bytes), InStats = #{cnt => Pubs, oct => Bytes}, - %% Ensure Rate Limit - NState = ensure_rate_limit(InStats, State), %% Run GC and Check OOM - NState1 = check_oom(run_gc(InStats, NState)), + NState1 = check_oom(run_gc(InStats, State)), handle_info(activate_socket, NState1); -handle_msg(Deliver = {deliver, _Topic, _Msg}, #state{ - listener = {Type, Listener}} = State) -> +handle_msg(Deliver = {deliver, _Topic, _Msg}, + #state{listener = {Type, Listener}} = State) -> ActiveN = get_active_n(Type, Listener), Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); @@ -579,10 +611,12 @@ handle_call(_From, info, State) -> handle_call(_From, stats, State) -> {reply, stats(State), State}; -handle_call(_From, {ratelimit, Policy}, State = #state{channel = Channel}) -> - Zone = emqx_channel:info(zone, Channel), - Limiter = emqx_limiter:init(Zone, Policy), - {reply, ok, State#state{limiter = Limiter}}; +handle_call(_From, {ratelimit, Changes}, State = #state{limiter = Limiter}) -> + Fun = fun({Type, Bucket}, Acc) -> + emqx_limiter_container:update_by_name(Type, Bucket, Acc) + end, + Limiter2 = lists:foldl(Fun, Limiter, Changes), + {reply, ok, State#state{limiter = Limiter2}}; handle_call(_From, Req, State = #state{channel = Channel}) -> case emqx_channel:handle_call(Req, Channel) of @@ -603,10 +637,7 @@ handle_timeout(_TRef, idle_timeout, State) -> shutdown(idle_timeout, State); handle_timeout(_TRef, limit_timeout, State) -> - NState = State#state{sockstate = idle, - limit_timer = undefined - }, - handle_info(activate_socket, NState); + retry_limiter(State); handle_timeout(_TRef, emit_stats, State = #state{channel = Channel, transport = Transport, socket = Socket}) -> @@ -634,11 +665,23 @@ handle_timeout(TRef, Msg, State) -> %%-------------------------------------------------------------------- %% Parse incoming data - --compile({inline, [parse_incoming/2]}). -parse_incoming(Data, State) -> +-compile({inline, [when_bytes_in/3]}). +when_bytes_in(Oct, Data, State) -> {Packets, NState} = parse_incoming(Data, [], State), - {ok, next_incoming_msgs(Packets), NState}. + Len = erlang:length(Packets), + check_limiter([{Oct, ?LIMITER_BYTES_IN}, {Len, ?LIMITER_MESSAGE_IN}], + Packets, + fun next_incoming_msgs/3, + [], + NState). + +-compile({inline, [next_incoming_msgs/3]}). +next_incoming_msgs([Packet], Msgs, State) -> + {ok, [{incoming, Packet} | Msgs], State}; +next_incoming_msgs(Packets, Msgs, State) -> + Fun = fun(Packet, Acc) -> [{incoming, Packet} | Acc] end, + Msgs2 = lists:foldl(Fun, Msgs, Packets), + {ok, Msgs2, State}. parse_incoming(<<>>, Packets, State) -> {Packets, State}; @@ -668,12 +711,6 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> {[{frame_error, Reason} | Packets], State} end. --compile({inline, [next_incoming_msgs/1]}). -next_incoming_msgs([Packet]) -> - {incoming, Packet}; -next_incoming_msgs(Packets) -> - [{incoming, Packet} || Packet <- lists:reverse(Packets)]. - %%-------------------------------------------------------------------- %% Handle incoming packet @@ -810,20 +847,82 @@ handle_cast(Req, State) -> State. %%-------------------------------------------------------------------- -%% Ensure rate limit +%% rate limit -ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> - case ?ENABLED(Limiter) andalso emqx_limiter:check(Stats, Limiter) of - false -> State; - {ok, Limiter1} -> - State#state{limiter = Limiter1}; - {pause, Time, Limiter1} -> - ?SLOG(warning, #{msg => "pause_time_due_to_rate_limit", time_in_ms => Time}), - TRef = start_timer(Time, limit_timeout), - State#state{sockstate = blocked, - limiter = Limiter1, - limit_timer = TRef - } +-type limiter_type() :: emqx_limiter_container:limiter_type(). +-type limiter() :: emqx_limiter_container:limiter(). +-type check_succ_handler() :: + fun((any(), list(any()), state()) -> _). + +%% check limiters, if successed call WhenOk with Data and Msgs +%% Data is the data to be processed +%% Msgs include the next msg which after Data processed +-spec check_limiter(list({pos_integer(), limiter_type()}), + any(), + check_succ_handler(), + list(any()), + state()) -> _. +check_limiter(Needs, + Data, + WhenOk, + Msgs, + #state{limiter = Limiter, + limiter_timer = LimiterTimer, + limiter_cache = Cache} = State) when Limiter =/= undefined -> + case LimiterTimer of + undefined -> + case emqx_limiter_container:check_list(Needs, Limiter) of + {ok, Limiter2} -> + WhenOk(Data, Msgs, State#state{limiter = Limiter2}); + {pause, Time, Limiter2} -> + ?SLOG(warning, #{msg => "pause time dueto rate limit", + needs => Needs, + time_in_ms => Time}), + + Retry = #retry{types = [Type || {_, Type} <- Needs], + data = Data, + next = WhenOk}, + + Limiter3 = emqx_limiter_container:set_retry_context(Retry, Limiter2), + + TRef = start_timer(Time, limit_timeout), + + {ok, State#state{limiter = Limiter3, + limiter_timer = TRef}}; + {drop, Limiter2} -> + {ok, State#state{limiter = Limiter2}} + end; + _ -> + %% if there has a retry timer, cache the operation and execute it after the retry is over + %% TODO: maybe we need to set socket to passive if size of queue is very large + %% because we queue up lots of ops that checks with the limiters. + New = #cache{need = Needs, data = Data, next = WhenOk}, + {ok, State#state{limiter_cache = queue:in(New, Cache)}} + end; + +check_limiter(_, Data, WhenOk, Msgs, State) -> + WhenOk(Data, Msgs, State). + +%% try to perform a retry +-spec retry_limiter(state()) -> _. +retry_limiter(#state{limiter = Limiter} = State) -> + #retry{types = Types, data = Data, next = Next} = emqx_limiter_container:get_retry_context(Limiter), + case emqx_limiter_container:retry_list(Types, Limiter) of + {ok, Limiter2} -> + Next(Data, + [check_cache], + State#state{ limiter = Limiter2 + , limiter_timer = undefined + }); + {pause, Time, Limiter2} -> + ?SLOG(warning, #{msg => "pause time dueto rate limit", + types => Types, + time_in_ms => Time}), + + TRef = start_timer(Time, limit_timeout), + + {ok, State#state{limiter = Limiter2, + limiter_timer = TRef}} end. %%-------------------------------------------------------------------- @@ -852,19 +951,25 @@ check_oom(State = #state{channel = Channel}) -> %%-------------------------------------------------------------------- %% Activate Socket - +%% TODO: maybe we could keep socket passive for receiving socket closed event. -compile({inline, [activate_socket/1]}). -activate_socket(State = #state{sockstate = closed}) -> - {ok, State}; -activate_socket(State = #state{sockstate = blocked}) -> - {ok, State}; -activate_socket(State = #state{transport = Transport, socket = Socket, - listener = {Type, Listener}}) -> +activate_socket(#state{limiter_timer = Timer} = State) + when Timer =/= undefined -> + {ok, State#state{sockstate = blocked}}; + +activate_socket(#state{transport = Transport, + sockstate = SockState, + socket = Socket, + listener = {Type, Listener}} = State) + when SockState =/= closed -> ActiveN = get_active_n(Type, Listener), case Transport:setopts(Socket, [{active, ActiveN}]) of ok -> {ok, State#state{sockstate = running}}; Error -> Error - end. + end; + +activate_socket(State) -> + {ok, State}. %%-------------------------------------------------------------------- %% Close Socket @@ -943,6 +1048,6 @@ get_state(Pid) -> maps:from_list(lists:zip(record_info(fields, state), tl(tuple_to_list(State)))). -get_active_n(quic, _Listener) -> 100; +get_active_n(quic, _Listener) -> ?ACTIVE_N; get_active_n(Type, Listener) -> emqx_config:get_listener_conf(Type, Listener, [tcp, active_n]). diff --git a/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf b/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf new file mode 100644 index 000000000..5c199e63f --- /dev/null +++ b/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf @@ -0,0 +1,52 @@ +##-------------------------------------------------------------------- +## Emq X Rate Limiter +##-------------------------------------------------------------------- +emqx_limiter { + bytes_in { + global.rate = infinity # token generation rate + zone.default.rate = infinity + bucket.default { + zone = default + aggregated.rate = infinity + aggregated.capacity = infinity + per_client.rate = infinity + per_client.capacity = infinity + } + } + + message_in { + global.rate = infinity + zone.default.rate = infinity + bucket.default { + zone = default + aggregated.rate = infinity + aggregated.capacity = infinity + per_client.rate = infinity + per_client.capacity = infinity + } + } + + connection { + global.rate = infinity + zone.default.rate = infinity + bucket.default { + zone = default + aggregated.rate = infinity + aggregated.capacity = infinity + per_client.rate = infinity + per_client.capacity = infinity + } + } + + message_routing { + global.rate = infinity + zone.default.rate = infinity + bucket.default { + zone = default + aggregated.rate = infinity + aggregated.capacity = infinity + per_client.rate = infinity + per_client.capacity = infinity + } + } +} diff --git a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl new file mode 100644 index 000000000..a183d6a81 --- /dev/null +++ b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl @@ -0,0 +1,358 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_htb_limiter). + +%% @doc the limiter of the hierarchical token limiter system +%% this module provides api for creating limiters, consume tokens, check tokens and retry +%% @end + +%% API +-export([ make_token_bucket_limiter/2, make_ref_limiter/2, check/2 + , consume/2, set_retry/2, retry/1, make_infinity_limiter/1 + , make_future/1, available/1 + ]). +-export_type([token_bucket_limiter/0]). + +%% a token bucket limiter with a limiter server's bucket reference +-type token_bucket_limiter() :: #{ tokens := non_neg_integer() %% the number of tokens currently available + , rate := decimal() + , capacity := decimal() + , lasttime := millisecond() + , max_retry_time := non_neg_integer() %% @see emqx_limiter_schema + , failure_strategy := failure_strategy() %% @see emqx_limiter_schema + , divisible := boolean() %% @see emqx_limiter_schema + , low_water_mark := non_neg_integer() %% @see emqx_limiter_schema + , bucket := bucket() %% the limiter server's bucket + + %% retry contenxt + , retry_ctx => undefined %% undefined meaning there is no retry context or no need to retry + | retry_context(token_bucket_limiter()) %% the retry context + , atom => any() %% allow to add other keys + }. + +%% a limiter server's bucket reference +-type ref_limiter() :: #{ max_retry_time := non_neg_integer() + , failure_strategy := failure_strategy() + , divisible := boolean() + , low_water_mark := non_neg_integer() + , bucket := bucket() + + , retry_ctx => undefined | retry_context(ref_limiter()) + , atom => any() %% allow to add other keys + }. + +-type retry_fun(Limiter) :: fun((pos_integer(), Limiter) -> inner_check_result(Limiter)). +-type acquire_type(Limiter) :: integer() | retry_context(Limiter). +-type retry_context(Limiter) :: #{ continuation := undefined | retry_fun(Limiter) + , diff := non_neg_integer() %% how many tokens are left to obtain + + , need => pos_integer() + , start => millisecond() + }. + +-type bucket() :: emqx_limiter_bucket_ref:bucket_ref(). +-type limiter() :: token_bucket_limiter() | ref_limiter() | infinity. +-type millisecond() :: non_neg_integer(). + +-type pause_type() :: pause | partial. +-type check_result_ok(Limiter) :: {ok, Limiter}. +-type check_result_pause(Limiter) :: {pause_type(), millisecond(), retry_context(Limiter), Limiter}. +-type result_drop(Limiter) :: {drop, Limiter}. + +-type check_result(Limiter) :: check_result_ok(Limiter) + | check_result_pause(Limiter) + | result_drop(Limiter). + +-type inner_check_result(Limiter) :: check_result_ok(Limiter) + | check_result_pause(Limiter). + +-type consume_result(Limiter) :: check_result_ok(Limiter) + | result_drop(Limiter). + +-type decimal() :: emqx_limiter_decimal:decimal(). +-type failure_strategy() :: emqx_limiter_schema:failure_strategy(). + +-type limiter_bucket_cfg() :: #{ rate := decimal() + , initial := non_neg_integer() + , low_water_mark := non_neg_integer() + , capacity := decimal() + , divisible := boolean() + , max_retry_time := non_neg_integer() + , failure_strategy := failure_strategy() + }. + +-type future() :: pos_integer(). + +-define(NOW, erlang:monotonic_time(millisecond)). +-define(MINIMUM_PAUSE, 50). +-define(MAXIMUM_PAUSE, 5000). + +-import(emqx_limiter_decimal, [sub/2, mul/2, floor_div/2, add/2]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +%%@doc create a limiter +-spec make_token_bucket_limiter(limiter_bucket_cfg(), bucket()) -> _. +make_token_bucket_limiter(Cfg, Bucket) -> + Cfg#{ tokens => emqx_limiter_server:get_initial_val(Cfg) + , lasttime => ?NOW + , bucket => Bucket + }. + +%%@doc create a limiter server's reference +-spec make_ref_limiter(limiter_bucket_cfg(), bucket()) -> ref_limiter(). +make_ref_limiter(Cfg, Bucket) when Bucket =/= infinity -> + Cfg#{bucket => Bucket}. + +-spec make_infinity_limiter(limiter_bucket_cfg()) -> infinity. +make_infinity_limiter(_) -> + infinity. + +%% @doc request some tokens +%% it will automatically retry when failed until the maximum retry time is reached +%% @end +-spec consume(integer(), Limiter) -> consume_result(Limiter) + when Limiter :: limiter(). +consume(Need, #{max_retry_time := RetryTime} = Limiter) when Need > 0 -> + try_consume(RetryTime, Need, Limiter); + +consume(_, Limiter) -> + {ok, Limiter}. + +%% @doc try to request the token and return the result without automatically retrying +-spec check(acquire_type(Limiter), Limiter) -> check_result(Limiter) + when Limiter :: limiter(). +check(_, infinity) -> + {ok, infinity}; + +check(Need, Limiter) when is_integer(Need), Need > 0 -> + case do_check(Need, Limiter) of + {ok, _} = Done -> + Done; + {PauseType, Pause, Ctx, Limiter2} -> + {PauseType, + Pause, + Ctx#{start => ?NOW, need => Need}, Limiter2} + end; + +%% check with retry context. +%% when continuation = undefined, the diff will be 0 +%% so there is no need to check continuation here +check(#{continuation := Cont, + diff := Diff, + start := Start} = Retry, + #{failure_strategy := Failure, + max_retry_time := RetryTime} = Limiter) when Diff > 0 -> + case Cont(Diff, Limiter) of + {ok, _} = Done -> + Done; + {PauseType, Pause, Ctx, Limiter2} -> + IsFailed = ?NOW - Start >= RetryTime, + Retry2 = maps:merge(Retry, Ctx), + case IsFailed of + false -> + {PauseType, Pause, Retry2, Limiter2}; + _ -> + on_failure(Failure, try_restore(Retry2, Limiter2)) + end + end; + +check(_, Limiter) -> + {ok, Limiter}. + +%% @doc pack the retry context into the limiter data +-spec set_retry(retry_context(Limiter), Limiter) -> Limiter + when Limiter :: limiter(). +set_retry(Retry, Limiter) -> + Limiter#{retry_ctx => Retry}. + +%% @doc check if there is a retry context, and try again if there is +-spec retry(Limiter) -> check_result(Limiter) when Limiter :: limiter(). +retry(#{retry_ctx := Retry} = Limiter) when is_map(Retry) -> + check(Retry, Limiter#{retry_ctx := undefined}); + +retry(Limiter) -> + {ok, Limiter}. + +%% @doc make a future value +%% this similar to retry context, but represents a value that will be checked in the future +%% @end +-spec make_future(pos_integer()) -> future(). +make_future(Need) -> + Need. + +%% @doc get the number of tokens currently available +-spec available(limiter()) -> decimal(). +available(#{tokens := Tokens, + rate := Rate, + lasttime := LastTime, + capacity := Capacity, + bucket := Bucket}) -> + Tokens2 = apply_elapsed_time(Rate, ?NOW - LastTime, Tokens, Capacity), + erlang:min(Tokens2, emqx_limiter_bucket_ref:available(Bucket)); + +available(#{bucket := Bucket}) -> + emqx_limiter_bucket_ref:available(Bucket); + +available(infinity) -> + infinity. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +-spec try_consume(millisecond(), + acquire_type(Limiter), + Limiter) -> consume_result(Limiter) when Limiter :: limiter(). +try_consume(LeftTime, Retry, #{failure_strategy := Failure} = Limiter) + when LeftTime =< 0, is_map(Retry) -> + on_failure(Failure, try_restore(Retry, Limiter)); + +try_consume(LeftTime, Need, Limiter) when is_integer(Need) -> + case do_check(Need, Limiter) of + {ok, _} = Done -> + Done; + {_, Pause, Ctx, Limiter2} -> + timer:sleep(erlang:min(LeftTime, Pause)), + try_consume(LeftTime - Pause, Ctx#{need => Need}, Limiter2) + end; + +try_consume(LeftTime, + #{continuation := Cont, + diff := Diff} = Retry, Limiter) when Diff > 0 -> + case Cont(Diff, Limiter) of + {ok, _} = Done -> + Done; + {_, Pause, Ctx, Limiter2} -> + timer:sleep(erlang:min(LeftTime, Pause)), + try_consume(LeftTime - Pause, maps:merge(Retry, Ctx), Limiter2) + end; + +try_consume(_, _, Limiter) -> + {ok, Limiter}. + +-spec do_check(acquire_type(Limiter), Limiter) -> inner_check_result(Limiter) + when Limiter :: limiter(). +do_check(Need, #{tokens := Tokens} = Limiter) -> + if Need =< Tokens -> + do_check_with_parent_limiter(Need, Limiter); + true -> + do_reset(Need, Limiter) + end; + +do_check(Need, #{divisible := Divisible, + bucket := Bucket} = Ref) -> + case emqx_limiter_bucket_ref:check(Need, Bucket, Divisible) of + {ok, Tokens} -> + may_return_or_pause(Tokens, Ref); + {PauseType, Rate, Obtained} -> + return_pause(Rate, + PauseType, + fun ?FUNCTION_NAME/2, Need - Obtained, Ref) + end. + +on_failure(force, Limiter) -> + {ok, Limiter}; + +on_failure(drop, Limiter) -> + {drop, Limiter}; + +on_failure(throw, Limiter) -> + Message = io_lib:format("limiter consume failed, limiter:~p~n", [Limiter]), + erlang:throw({rate_check_fail, Message}). + +-spec do_check_with_parent_limiter(pos_integer(), token_bucket_limiter()) -> inner_check_result(token_bucket_limiter()). +do_check_with_parent_limiter(Need, + #{tokens := Tokens, + divisible := Divisible, + bucket := Bucket} = Limiter) -> + case emqx_limiter_bucket_ref:check(Need, Bucket, Divisible) of + {ok, RefLeft} -> + Left = sub(Tokens, Need), + may_return_or_pause(erlang:min(RefLeft, Left), Limiter#{tokens := Left}); + {PauseType, Rate, Obtained} -> + return_pause(Rate, + PauseType, + fun ?FUNCTION_NAME/2, + Need - Obtained, + Limiter#{tokens := sub(Tokens, Obtained)}) + end. + +-spec do_reset(pos_integer(), token_bucket_limiter()) -> inner_check_result(token_bucket_limiter()). +do_reset(Need, + #{tokens := Tokens, + rate := Rate, + lasttime := LastTime, + divisible := Divisible, + capacity := Capacity} = Limiter) -> + Now = ?NOW, + Tokens2 = apply_elapsed_time(Rate, Now - LastTime, Tokens, Capacity), + if Tokens2 >= Need -> + Limiter2 = Limiter#{tokens := Tokens2, lasttime := Now}, + do_check_with_parent_limiter(Need, Limiter2); + Divisible andalso Tokens2 > 0 -> + %% must be allocated here, because may be Need > Capacity + return_pause(Rate, + partial, + fun do_reset/2, + Need - Tokens2, + Limiter#{tokens := 0, lasttime := Now}); + true -> + return_pause(Rate, pause, fun do_reset/2, Need, Limiter) + end. + +-spec return_pause(decimal(), pause_type(), retry_fun(Limiter), pos_integer(), Limiter) + -> check_result_pause(Limiter) when Limiter :: limiter(). +return_pause(infinity, PauseType, Fun, Diff, Limiter) -> + %% workaround when emqx_limiter_server's rate is infinity + {PauseType, ?MINIMUM_PAUSE, make_retry_context(Fun, Diff), Limiter}; + +return_pause(Rate, PauseType, Fun, Diff, Limiter) -> + Val = erlang:round(Diff * emqx_limiter_schema:minimum_period() / Rate), + Pause = emqx_misc:clamp(Val, ?MINIMUM_PAUSE, ?MAXIMUM_PAUSE), + {PauseType, Pause, make_retry_context(Fun, Diff), Limiter}. + +-spec make_retry_context(undefined | retry_fun(Limiter), non_neg_integer()) -> retry_context(Limiter) + when Limiter :: limiter(). +make_retry_context(Fun, Diff) -> + #{continuation => Fun, diff => Diff}. + +-spec try_restore(retry_context(Limiter), Limiter) -> Limiter + when Limiter :: limiter(). +try_restore(#{need := Need, diff := Diff}, + #{tokens := Tokens, capcacity := Capacity, bucket := Bucket} = Limiter) -> + Back = Need - Diff, + Tokens2 = erlang:min(Capacity, Back + Tokens), + emqx_limiter_bucket_ref:try_restore(Back, Bucket), + Limiter#{tokens := Tokens2}; + +try_restore(#{need := Need, diff := Diff}, #{bucket := Bucket} = Limiter) -> + emqx_limiter_bucket_ref:try_restore(Need - Diff, Bucket), + Limiter. + +-spec may_return_or_pause(non_neg_integer(), Limiter) -> check_result(Limiter) + when Limiter :: limiter(). +may_return_or_pause(Left, #{low_water_mark := Mark} = Limiter) when Left >= Mark -> + {ok, Limiter}; + +may_return_or_pause(_, Limiter) -> + {pause, ?MINIMUM_PAUSE, make_retry_context(undefined, 0), Limiter}. + +%% @doc apply the elapsed time to the limiter +apply_elapsed_time(Rate, Elapsed, Tokens, Capacity) -> + Inc = floor_div(mul(Elapsed, Rate), emqx_limiter_schema:minimum_period()), + erlang:min(add(Tokens, Inc), Capacity). diff --git a/apps/emqx_limiter/src/emqx_limiter.app.src b/apps/emqx/src/emqx_limiter/src/emqx_limiter.app.src similarity index 75% rename from apps/emqx_limiter/src/emqx_limiter.app.src rename to apps/emqx/src/emqx_limiter/src/emqx_limiter.app.src index 70fe89e97..3ad320849 100644 --- a/apps/emqx_limiter/src/emqx_limiter.app.src +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter.app.src @@ -9,7 +9,5 @@ {env, []}, {licenses, ["Apache-2.0"]}, {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-retainer"} - ]} + {links, []} ]}. diff --git a/apps/emqx_limiter/src/emqx_limiter_app.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_app.erl similarity index 96% rename from apps/emqx_limiter/src/emqx_limiter_app.erl rename to apps/emqx/src/emqx_limiter/src/emqx_limiter_app.erl index 2244a0e91..0581e3e1a 100644 --- a/apps/emqx_limiter/src/emqx_limiter_app.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl new file mode 100644 index 000000000..4e84e40b5 --- /dev/null +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl @@ -0,0 +1,102 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_limiter_bucket_ref). + +%% @doc limiter bucket reference +%% this module is used to manage the bucket reference of the limiter server +%% @end + +%% API +-export([ new/3, check/3, try_restore/2 + , available/1]). + +-export_type([bucket_ref/0]). + +-type infinity_bucket_ref() :: infinity. +-type finite_bucket_ref() :: #{ counter := counters:counters_ref() + , index := index() + , rate := rate()}. + +-type bucket_ref() :: infinity_bucket_ref() + | finite_bucket_ref(). + +-type index() :: emqx_limiter_server:index(). +-type rate() :: emqx_limiter_decimal:decimal(). +-type check_failure_type() :: partial | pause. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new(undefined | counters:countres_ref(), + undefined | index(), + rate()) -> bucket_ref(). +new(undefined, _, _) -> + infinity; + +new(Counter, Index, Rate) -> + #{counter => Counter, + index => Index, + rate => Rate}. + +%% @doc check tokens +-spec check(pos_integer(), bucket_ref(), Disivisble :: boolean()) -> + HasToken :: {ok, emqx_limiter_decimal:decimal()} + | {check_failure_type(), rate(), pos_integer()}. +check(_, infinity, _) -> + {ok, infinity}; + +check(Need, + #{counter := Counter, + index := Index, + rate := Rate}, + Divisible)-> + RefToken = counters:get(Counter, Index), + if RefToken >= Need -> + counters:sub(Counter, Index, Need), + {ok, RefToken - Need}; + Divisible andalso RefToken > 0 -> + counters:sub(Counter, Index, RefToken), + {partial, Rate, RefToken}; + true -> + {pause, Rate, 0} + end. + +%% @doc try to restore token when consume failed +-spec try_restore(non_neg_integer(), bucket_ref()) -> ok. +try_restore(0, _) -> + ok; +try_restore(_, infinity) -> + ok; +try_restore(Inc, #{counter := Counter, index := Index}) -> + case counters:get(Counter, Index) of + Tokens when Tokens < 0 -> + counters:add(Counter, Index, Inc); + _ -> + ok + end. + +%% @doc get the number of tokens currently available +-spec available(bucket_ref()) -> emqx_limiter_decimal:decimal(). +available(#{counter := Counter, index := Index}) -> + counters:get(Counter, Index); + +available(infinity) -> + infinity. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl new file mode 100644 index 000000000..f927ee823 --- /dev/null +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl @@ -0,0 +1,157 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_limiter_container). + +%% @doc the container of emqx_htb_limiter +%% used to merge limiters of different type of limiters to simplify operations +%% @end + +%% API +-export([ new/0, new/1, get_limiter_by_names/2 + , add_new/3, update_by_name/3, set_retry_context/2 + , check/3, retry/2, get_retry_context/1 + , check_list/2, retry_list/2 + ]). + +-export_type([container/0, check_result/0]). + +-type container() :: #{ limiter_type() => undefined | limiter() + , retry_key() => undefined | retry_context() | future() %% the retry context of the limiter + , retry_ctx := undefined | any() %% the retry context of the container + }. + +-type future() :: pos_integer(). +-type limiter_type() :: emqx_limiter_schema:limiter_type(). +-type limiter() :: emqx_htb_limiter:limiter(). +-type retry_context() :: emqx_htb_limiter:retry_context(). +-type bucket_name() :: emqx_limiter_schema:bucket_name(). +-type millisecond() :: non_neg_integer(). +-type check_result() :: {ok, container()} + | {drop, container()} + | {pause, millisecond(), container()}. + +-define(RETRY_KEY(Type), {retry, Type}). +-type retry_key() :: ?RETRY_KEY(limiter_type()). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new() -> container(). +new() -> + new([]). + +%% @doc generate default data according to the type of limiter +-spec new(list(limiter_type())) -> container(). +new(Types) -> + get_limiter_by_names(Types, #{}). + +%% @doc generate a container +%% according to the type of limiter and the bucket name configuration of the limiter +%% @end +-spec get_limiter_by_names(list(limiter_type()), #{limiter_type() => emqx_limiter_schema:bucket_name()}) -> container(). +get_limiter_by_names(Types, BucketNames) -> + Init = fun(Type, Acc) -> + Limiter = emqx_limiter_server:connect(Type, BucketNames), + add_new(Type, Limiter, Acc) + end, + lists:foldl(Init, #{retry_ctx => undefined}, Types). + +%% @doc add the specified type of limiter to the container +-spec update_by_name(limiter_type(), + bucket_name() | #{limiter_type() => bucket_name()}, + container()) -> container(). +update_by_name(Type, Buckets, Container) -> + Limiter = emqx_limiter_server:connect(Type, Buckets), + add_new(Type, Limiter, Container). + +-spec add_new(limiter_type(), limiter(), container()) -> container(). +add_new(Type, Limiter, Container) -> + Container#{ Type => Limiter + , ?RETRY_KEY(Type) => undefined + }. + +%% @doc check the specified limiter +-spec check(pos_integer(), limiter_type(), container()) -> check_result(). +check(Need, Type, Container) -> + check_list([{Need, Type}], Container). + +%% @doc check multiple limiters +-spec check_list(list({pos_integer(), limiter_type()}), container()) -> check_result(). +check_list([{Need, Type} | T], Container) -> + Limiter = maps:get(Type, Container), + case emqx_htb_limiter:check(Need, Limiter) of + {ok, Limiter2} -> + check_list(T, Container#{Type := Limiter2}); + {_, PauseMs, Ctx, Limiter2} -> + Fun = fun({FN, FT}, Acc) -> + Future = emqx_htb_limiter:make_future(FN), + Acc#{?RETRY_KEY(FT) := Future} + end, + C2 = lists:foldl(Fun, + Container#{Type := Limiter2, + ?RETRY_KEY(Type) := Ctx}, + T), + {pause, PauseMs, C2}; + {drop, Limiter2} -> + {drop, Container#{Type := Limiter2}} + end; + +check_list([], Container) -> + {ok, Container}. + +%% @doc retry the specified limiter +-spec retry(limiter_type(), container()) -> check_result(). +retry(Type, Container) -> + retry_list([Type], Container). + +%% @doc retry multiple limiters +-spec retry_list(list(limiter_type()), container()) -> check_result(). +retry_list([Type | T], Container) -> + Key = ?RETRY_KEY(Type), + case Container of + #{Type := Limiter, + Key := Retry} when Retry =/= undefined -> + case emqx_htb_limiter:check(Retry, Limiter) of + {ok, Limiter2} -> + %% undefined meaning there is no retry context or there is no need to retry + %% when a limiter has a undefined retry context, the check will always success + retry_list(T, Container#{Type := Limiter2, Key := undefined}); + {_, PauseMs, Ctx, Limiter2} -> + {pause, + PauseMs, + Container#{Type := Limiter2, Key := Ctx}}; + {drop, Limiter2} -> + {drop, Container#{Type := Limiter2}} + end; + _ -> + retry_list(T, Container) + end; + +retry_list([], Container) -> + {ok, Container}. + +-spec set_retry_context(any(), container()) -> container(). +set_retry_context(Data, Container) -> + Container#{retry_ctx := Data}. + +-spec get_retry_context(container()) -> any(). +get_retry_context(#{retry_ctx := Data}) -> + Data. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- diff --git a/apps/emqx_limiter/src/emqx_limiter_decimal.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_decimal.erl similarity index 92% rename from apps/emqx_limiter/src/emqx_limiter_decimal.erl rename to apps/emqx/src/emqx_limiter/src/emqx_limiter_decimal.erl index 26ae611e8..28b6f3385 100644 --- a/apps/emqx_limiter/src/emqx_limiter_decimal.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_decimal.erl @@ -20,7 +20,7 @@ %% API -export([ add/2, sub/2, mul/2 - , add_to_counter/3, put_to_counter/3]). + , add_to_counter/3, put_to_counter/3, floor_div/2]). -export_type([decimal/0, zero_or_float/0]). -type decimal() :: infinity | number(). @@ -53,6 +53,13 @@ mul(A, B) when A =:= infinity mul(A, B) -> A * B. +-spec floor_div(decimal(), number()) -> decimal(). +floor_div(infinity, _) -> + infinity; + +floor_div(A, B) -> + erlang:floor(A / B). + -spec add_to_counter(counters:counters_ref(), pos_integer(), decimal()) -> {zero_or_float(), zero_or_float()}. add_to_counter(_, _, infinity) -> diff --git a/apps/emqx_limiter/src/emqx_limiter_manager.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl similarity index 80% rename from apps/emqx_limiter/src/emqx_limiter_manager.erl rename to apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl index 471098242..3d46590c5 100644 --- a/apps/emqx_limiter/src/emqx_limiter_manager.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl @@ -22,29 +22,27 @@ -include_lib("stdlib/include/ms_transform.hrl"). %% API --export([ start_link/0, start_server/1, find_counter/1 - , find_counter/3, insert_counter/4, insert_counter/6 +-export([ start_link/0, start_server/1, find_bucket/1 + , find_bucket/3, insert_bucket/2, insert_bucket/4 , make_path/3, restart_server/1]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3, format_status/2]). +-export_type([path/0]). + -type path() :: list(atom()). -type limiter_type() :: emqx_limiter_schema:limiter_type(). -type zone_name() :: emqx_limiter_schema:zone_name(). -type bucket_name() :: emqx_limiter_schema:bucket_name(). %% counter record in ets table --record(element, {path :: path(), - counter :: counters:counters_ref(), - index :: index(), - rate :: rate() - }). +-record(bucket, { path :: path() + , bucket :: bucket_ref() + }). - --type index() :: emqx_limiter_server:index(). --type rate() :: emqx_limiter_decimal:decimal(). +-type bucket_ref() :: emqx_limiter_bucket_ref:bucket_ref(). -define(TAB, emqx_limiter_counters). @@ -59,43 +57,32 @@ start_server(Type) -> restart_server(Type) -> emqx_limiter_server_sup:restart(Type). --spec find_counter(limiter_type(), zone_name(), bucket_name()) -> - {ok, counters:counters_ref(), index(), rate()} | undefined. -find_counter(Type, Zone, BucketId) -> - find_counter(make_path(Type, Zone, BucketId)). +-spec find_bucket(limiter_type(), zone_name(), bucket_name()) -> + {ok, bucket_ref()} | undefined. +find_bucket(Type, Zone, BucketId) -> + find_bucket(make_path(Type, Zone, BucketId)). --spec find_counter(path()) -> - {ok, counters:counters_ref(), index(), rate()} | undefined. -find_counter(Path) -> +-spec find_bucket(path()) -> {ok, bucket_ref()} | undefined. +find_bucket(Path) -> case ets:lookup(?TAB, Path) of - [#element{counter = Counter, index = Index, rate = Rate}] -> - {ok, Counter, Index, Rate}; + [#bucket{bucket = Bucket}] -> + {ok, Bucket}; _ -> undefined end. --spec insert_counter(limiter_type(), - zone_name(), - bucket_name(), - counters:counters_ref(), - index(), - rate()) -> boolean(). -insert_counter(Type, Zone, BucketId, Counter, Index, Rate) -> - insert_counter(make_path(Type, Zone, BucketId), - Counter, - Index, - Rate). +-spec insert_bucket(limiter_type(), + zone_name(), + bucket_name(), + bucket_ref()) -> boolean(). +insert_bucket(Type, Zone, BucketId, Bucket) -> + inner_insert_bucket(make_path(Type, Zone, BucketId), + Bucket). --spec insert_counter(path(), - counters:counters_ref(), - index(), - rate()) -> boolean(). -insert_counter(Path, Counter, Index, Rate) -> - ets:insert(?TAB, - #element{path = Path, - counter = Counter, - index = Index, - rate = Rate}). + +-spec insert_bucket(path(), bucket_ref()) -> true. +insert_bucket(Path, Bucket) -> + inner_insert_bucket(Path, Bucket). -spec make_path(limiter_type(), zone_name(), bucket_name()) -> path(). make_path(Type, Name, BucketId) -> @@ -129,7 +116,7 @@ start_link() -> {stop, Reason :: term()} | ignore. init([]) -> - _ = ets:new(?TAB, [ set, public, named_table, {keypos, #element.path} + _ = ets:new(?TAB, [ set, public, named_table, {keypos, #bucket.path} , {write_concurrency, true}, {read_concurrency, true} , {heir, erlang:whereis(emqx_limiter_sup), none} ]), @@ -227,3 +214,7 @@ format_status(_Opt, Status) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +-spec inner_insert_bucket(path(), bucket_ref()) -> true. +inner_insert_bucket(Path, Bucket) -> + ets:insert(?TAB, + #bucket{path = Path, bucket = Bucket}). diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl new file mode 100644 index 000000000..20844a3df --- /dev/null +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -0,0 +1,176 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_limiter_schema). + +-include_lib("typerefl/include/types.hrl"). + +-export([ roots/0, fields/1, to_rate/1, to_capacity/1 + , minimum_period/0, to_burst_rate/1, to_initial/1]). + +-define(KILOBYTE, 1024). + +-type limiter_type() :: bytes_in + | message_in + | connection + | message_routing. + +-type bucket_name() :: atom(). +-type zone_name() :: atom(). +-type rate() :: infinity | float(). +-type burst_rate() :: 0 | float(). +-type capacity() :: infinity | number(). %% the capacity of the token bucket +-type initial() :: non_neg_integer(). %% initial capacity of the token bucket + +%% the processing strategy after the failure of the token request +-type failure_strategy() :: force %% Forced to pass + | drop %% discard the current request + | throw. %% throw an exception + +-typerefl_from_string({rate/0, ?MODULE, to_rate}). +-typerefl_from_string({burst_rate/0, ?MODULE, to_burst_rate}). +-typerefl_from_string({capacity/0, ?MODULE, to_capacity}). +-typerefl_from_string({initial/0, ?MODULE, to_initial}). + +-reflect_type([ rate/0 + , burst_rate/0 + , capacity/0 + , initial/0 + , failure_strategy/0 + ]). + +-export_type([limiter_type/0, bucket_name/0, zone_name/0]). + +-import(emqx_schema, [sc/2, map/2]). + +roots() -> [emqx_limiter]. + +fields(emqx_limiter) -> + [ {bytes_in, sc(ref(limiter), #{})} + , {message_in, sc(ref(limiter), #{})} + , {connection, sc(ref(limiter), #{})} + , {message_routing, sc(ref(limiter), #{})} + ]; + +fields(limiter) -> + [ {global, sc(ref(rate_burst), #{})} + , {zone, sc(map("zone name", ref(rate_burst)), #{})} + , {bucket, sc(map("bucket id", ref(bucket)), + #{desc => "token bucket"})} + ]; + +fields(rate_burst) -> + [ {rate, sc(rate(), #{})} + , {burst, sc(burst_rate(), #{default => "0/0s"})} + ]; + +fields(bucket) -> + [ {zone, sc(atom(), #{desc => "the zone which the bucket in"})} + , {aggregated, sc(ref(bucket_aggregated), #{})} + , {per_client, sc(ref(client_bucket), #{})} + ]; + +fields(bucket_aggregated) -> + [ {rate, sc(rate(), #{})} + , {initial, sc(initial(), #{default => "0"})} + , {capacity, sc(capacity(), #{})} + ]; + +fields(client_bucket) -> + [ {rate, sc(rate(), #{})} + , {initial, sc(initial(), #{default => "0"})} + %% low_water_mark add for emqx_channel and emqx_session + %% both modules consume first and then check + %% so we need to use this value to prevent excessive consumption (e.g, consumption from an empty bucket) + , {low_water_mark, sc(initial(), + #{desc => "if the remaining tokens are lower than this value, +the check/consume will succeed, but it will be forced to hang for a short period of time", + default => "0"})} + , {capacity, sc(capacity(), #{desc => "the capacity of the token bucket"})} + , {divisible, sc(boolean(), + #{desc => "is it possible to split the number of tokens requested", + default => false})} + , {max_retry_time, sc(emqx_schema:duration(), + #{ desc => "the maximum retry time when acquire failed" + , default => "5s"})} + , {failure_strategy, sc(failure_strategy(), + #{ desc => "the strategy when all retry failed" + , default => force})} + ]. + +%% minimum period is 100ms +minimum_period() -> + 100. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +ref(Field) -> hoconsc:ref(?MODULE, Field). + +to_rate(Str) -> + to_rate(Str, true, false). + +to_burst_rate(Str) -> + to_rate(Str, false, true). + +to_rate(Str, CanInfinity, CanZero) -> + Tokens = [string:trim(T) || T <- string:tokens(Str, "/")], + case Tokens of + ["infinity"] when CanInfinity -> + {ok, infinity}; + ["0", _] when CanZero -> + {ok, 0}; %% for burst + [Quota, Interval] -> + {ok, Val} = to_capacity(Quota), + case emqx_schema:to_duration_ms(Interval) of + {ok, Ms} when Ms > 0 -> + {ok, Val * minimum_period() / Ms}; + _ -> + {error, Str} + end; + _ -> + {error, Str} + end. + +to_capacity(Str) -> + Regex = "^\s*(?:(?:([1-9][0-9]*)([a-zA-z]*))|infinity)\s*$", + to_quota(Str, Regex). + +to_initial(Str) -> + Regex = "^\s*([0-9]+)([a-zA-z]*)\s*$", + to_quota(Str, Regex). + +to_quota(Str, Regex) -> + {ok, MP} = re:compile(Regex), + Result = re:run(Str, MP, [{capture, all_but_first, list}]), + case Result of + {match, [Quota, Unit]} -> + Val = erlang:list_to_integer(Quota), + Unit2 = string:to_lower(Unit), + {ok, apply_unit(Unit2, Val)}; + {match, [Quota]} -> + {ok, erlang:list_to_integer(Quota)}; + {match, []} -> + {ok, infinity}; + _ -> + {error, Str} + end. + +apply_unit("", Val) -> Val; +apply_unit("kb", Val) -> Val * ?KILOBYTE; +apply_unit("mb", Val) -> Val * ?KILOBYTE * ?KILOBYTE; +apply_unit("gb", Val) -> Val * ?KILOBYTE * ?KILOBYTE * ?KILOBYTE; +apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl new file mode 100644 index 000000000..799d623bf --- /dev/null +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -0,0 +1,582 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% A hierarchical token bucket algorithm +%% Note: this is not the linux HTB algorithm(http://luxik.cdi.cz/~devik/qos/htb/manual/theory.htm) +%% Algorithm: +%% 1. the root node periodically generates tokens and then distributes them +%% just like the oscillation of water waves +%% 2. the leaf node has a counter, which is the place where the token is actually held. +%% 3. other nodes only play the role of transmission, and the rate of the node is like a valve, +%% limiting the oscillation transmitted from the parent node + +-module(emqx_limiter_server). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3, format_status/2]). + +-export([ start_link/1, connect/2, info/2 + , name/1, get_initial_val/1]). + +-type root() :: #{ rate := rate() %% number of tokens generated per period + , burst := rate() + , period := pos_integer() %% token generation interval(second) + , childs := list(node_id()) %% node children + , consumed := non_neg_integer() + }. + +-type zone() :: #{ id := node_id() + , name := zone_name() + , rate := rate() + , burst := rate() + , obtained := non_neg_integer() %% number of tokens obtained + , childs := list(node_id()) + }. + +-type bucket() :: #{ id := node_id() + , name := bucket_name() + , zone := zone_name() %% pointer to zone node, use for burst + , rate := rate() + , obtained := non_neg_integer() + , correction := emqx_limiter_decimal:zero_or_float() %% token correction value + , capacity := capacity() + , counter := undefined | counters:counters_ref() + , index := undefined | index() + }. + +-type state() :: #{ root := undefined | root() + , counter := undefined | counters:counters_ref() %% current counter to alloc + , index := index() + , zones := #{zone_name() => node_id()} + , buckets := list(node_id()) + , nodes := nodes() + , type := limiter_type() + }. + +-type node_id() :: pos_integer(). +-type node_data() :: zone() | bucket(). +-type nodes() :: #{node_id() => node_data()}. +-type zone_name() :: emqx_limiter_schema:zone_name(). +-type limiter_type() :: emqx_limiter_schema:limiter_type(). +-type bucket_name() :: emqx_limiter_schema:bucket_name(). +-type rate() :: decimal(). +-type flow() :: decimal(). +-type capacity() :: decimal(). +-type decimal() :: emqx_limiter_decimal:decimal(). +-type index() :: pos_integer(). + +-define(CALL(Type, Msg), gen_server:call(name(Type), {?FUNCTION_NAME, Msg})). +-define(OVERLOAD_MIN_ALLOC, 0.3). %% minimum coefficient for overloaded limiter + +-export_type([index/0]). +-import(emqx_limiter_decimal, [add/2, sub/2, mul/2, add_to_counter/3, put_to_counter/3]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec connect(limiter_type(), + bucket_name() | #{limiter_type() => bucket_name()}) -> emqx_htb_limiter:limiter(). +connect(Type, BucketName) when is_atom(BucketName) -> + Path = [emqx_limiter, Type, bucket, BucketName], + case emqx:get_config(Path, undefined) of + undefined -> + ?LOG(error, "can't find the config of this bucket: ~p~n", [Path]), + throw("bucket's config not found"); + #{zone := Zone, + aggregated := #{rate := AggrRate, capacity := AggrSize}, + per_client := #{rate := CliRate, capacity := CliSize} = Cfg} -> + case emqx_limiter_manager:find_bucket(Type, Zone, BucketName) of + {ok, Bucket} -> + if CliRate < AggrRate orelse CliSize < AggrSize -> + emqx_htb_limiter:make_token_bucket_limiter(Cfg, Bucket); + Bucket =:= infinity -> + emqx_htb_limiter:make_infinity_limiter(Cfg); + true -> + emqx_htb_limiter:make_ref_limiter(Cfg, Bucket) + end; + undefined -> + ?LOG(error, "can't find the bucket:~p~n", [Path]), + throw("invalid bucket") + end + end; + +connect(Type, Names) -> + connect(Type, maps:get(Type, Names, default)). + +-spec info(limiter_type(), atom()) -> term(). +info(Type, Info) -> + ?CALL(Type, Info). + +-spec name(limiter_type()) -> atom(). +name(Type) -> + erlang:list_to_atom(io_lib:format("~s_~s", [?MODULE, Type])). + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% @end +%%-------------------------------------------------------------------- +-spec start_link(limiter_type()) -> _. +start_link(Type) -> + gen_server:start_link({local, name(Type)}, ?MODULE, [Type], []). + +%%-------------------------------------------------------------------- +%%% gen_server callbacks +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the server +%% @end +%%-------------------------------------------------------------------- +-spec init(Args :: term()) -> {ok, State :: term()} | + {ok, State :: term(), Timeout :: timeout()} | + {ok, State :: term(), hibernate} | + {stop, Reason :: term()} | + ignore. +init([Type]) -> + State = #{root => undefined, + counter => undefined, + index => 1, + zones => #{}, + nodes => #{}, + buckets => [], + type => Type}, + State2 = init_tree(Type, State), + #{root := #{period := Perido}} = State2, + oscillate(Perido), + {ok, State2}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_call(Request :: term(), From :: {pid(), term()}, State :: term()) -> + {reply, Reply :: term(), NewState :: term()} | + {reply, Reply :: term(), NewState :: term(), Timeout :: timeout()} | + {reply, Reply :: term(), NewState :: term(), hibernate} | + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: term(), Reply :: term(), NewState :: term()} | + {stop, Reason :: term(), NewState :: term()}. +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_cast(Request :: term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: term(), NewState :: term()}. +handle_cast(Req, State) -> + ?LOG(error, "Unexpected cast: ~p", [Req]), + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_info(Info :: timeout() | term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: normal | term(), NewState :: term()}. +handle_info(oscillate, State) -> + {noreply, oscillation(State)}; + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% @end +%%-------------------------------------------------------------------- +-spec terminate(Reason :: normal | shutdown | {shutdown, term()} | term(), + State :: term()) -> any(). +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% @end +%%-------------------------------------------------------------------- +-spec code_change(OldVsn :: term() | {down, term()}, + State :: term(), + Extra :: term()) -> {ok, NewState :: term()} | + {error, Reason :: term()}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called for changing the form and appearance +%% of gen_server status when it is returned from sys:get_status/1,2 +%% or when it appears in termination error logs. +%% @end +%%-------------------------------------------------------------------- +-spec format_status(Opt :: normal | terminate, + Status :: list()) -> Status :: term(). +format_status(_Opt, Status) -> + Status. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +oscillate(Interval) -> + erlang:send_after(Interval, self(), ?FUNCTION_NAME). + +%% @doc generate tokens, and then spread to leaf nodes +-spec oscillation(state()) -> state(). +oscillation(#{root := #{rate := Flow, + period := Interval, + childs := ChildIds, + consumed := Consumed} = Root, + nodes := Nodes} = State) -> + oscillate(Interval), + Childs = get_ordered_childs(ChildIds, Nodes), + {Alloced, Nodes2} = transverse(Childs, Flow, 0, Nodes), + maybe_burst(State#{nodes := Nodes2, + root := Root#{consumed := Consumed + Alloced}}). + +%% @doc horizontal spread +-spec transverse(list(node_data()), + flow(), + non_neg_integer(), + nodes()) -> {non_neg_integer(), nodes()}. +transverse([H | T], InFlow, Alloced, Nodes) when InFlow > 0 -> + {NodeAlloced, Nodes2} = longitudinal(H, InFlow, Nodes), + InFlow2 = sub(InFlow, NodeAlloced), + Alloced2 = Alloced + NodeAlloced, + transverse(T, InFlow2, Alloced2, Nodes2); + +transverse(_, _, Alloced, Nodes) -> + {Alloced, Nodes}. + +%% @doc vertical spread +-spec longitudinal(node_data(), flow(), nodes()) -> + {non_neg_integer(), nodes()}. +longitudinal(#{id := Id, + rate := Rate, + obtained := Obtained, + childs := ChildIds} = Node, InFlow, Nodes) -> + Flow = erlang:min(InFlow, Rate), + + if Flow > 0 -> + Childs = get_ordered_childs(ChildIds, Nodes), + {Alloced, Nodes2} = transverse(Childs, Flow, 0, Nodes), + if Alloced > 0 -> + {Alloced, + Nodes2#{Id => Node#{obtained := Obtained + Alloced}}}; + true -> + %% childs are empty or all counter childs are full + {0, Nodes2} + end; + true -> + {0, Nodes} + end; + +longitudinal(#{id := Id, + rate := Rate, + capacity := Capacity, + correction := Correction, + counter := Counter, + index := Index, + obtained := Obtained} = Node, + InFlow, Nodes) when Counter =/= undefined -> + Flow = add(erlang:min(InFlow, Rate), Correction), + + ShouldAlloc = + case counters:get(Counter, Index) of + Tokens when Tokens < 0 -> + %% toknes's value mayb be a negative value(stolen from the future) + %% because ∃ x. add(Capacity, x) < 0, so here we must compare with minimum value + erlang:max(add(Capacity, Tokens), + mul(Capacity, ?OVERLOAD_MIN_ALLOC)); + Tokens -> + %% is it possible that Tokens > Capacity ??? + erlang:max(sub(Capacity, Tokens), 0) + end, + + case lists:min([ShouldAlloc, Flow, Capacity]) of + Avaiable when Avaiable > 0 -> + %% XXX if capacity is infinity, and flow always > 0, the value in counter + %% will be overflow at some point in the future, do we need to deal with this situation??? + {Alloced, Decimal} = add_to_counter(Counter, Index, Avaiable), + + {Alloced, + Nodes#{Id := Node#{obtained := Obtained + Alloced, + correction := Decimal}}}; + _ -> + {0, Nodes} + end; + +longitudinal(_, _, Nodes) -> + {0, Nodes}. + +-spec get_ordered_childs(list(node_id()), nodes()) -> list(node_data()). +get_ordered_childs(Ids, Nodes) -> + Childs = [maps:get(Id, Nodes) || Id <- Ids], + + %% sort by obtained, avoid node goes hungry + lists:sort(fun(#{obtained := A}, #{obtained := B}) -> + A < B + end, + Childs). + +-spec maybe_burst(state()) -> state(). +maybe_burst(#{buckets := Buckets, + zones := Zones, + root := #{burst := Burst}, + nodes := Nodes} = State) when Burst > 0 -> + %% find empty buckets and group by zone name + GroupFun = fun(Id, Groups) -> + #{counter := Counter, + index := Index, + zone := Zone} = maps:get(Id, Nodes), + case counters:get(Counter, Index) of + Any when Any =< 0 -> + Group = maps:get(Zone, Groups, []), + maps:put(Zone, [Id | Group], Groups); + _ -> + Groups + end + end, + + case lists:foldl(GroupFun, #{}, Buckets) of + Groups when map_size(Groups) > 0 -> + %% remove the zone which don't support burst + Filter = fun({Name, Childs}, Acc) -> + ZoneId = maps:get(Name, Zones), + #{burst := ZoneBurst} = Zone = maps:get(ZoneId, Nodes), + case ZoneBurst > 0 of + true -> + [{Zone, Childs} | Acc]; + _ -> + Acc + end + end, + + FilterL = lists:foldl(Filter, [], maps:to_list(Groups)), + dispatch_burst(FilterL, State); + _ -> + State + end; + +maybe_burst(State) -> + State. + +-spec dispatch_burst(list({zone(), list(node_id())}), state()) -> state(). +dispatch_burst([], State) -> + State; + +dispatch_burst(GroupL, + #{root := #{burst := Burst}, + nodes := Nodes} = State) -> + InFlow = erlang:floor(Burst / erlang:length(GroupL)), + Dispatch = fun({Zone, Childs}, NodeAcc) -> + #{id := ZoneId, + burst := ZoneBurst, + obtained := Obtained} = Zone, + + ZoneFlow = erlang:min(InFlow, ZoneBurst), + EachFlow = ZoneFlow div erlang:length(Childs), + Zone2 = Zone#{obtained := Obtained + ZoneFlow}, + NodeAcc2 = NodeAcc#{ZoneId := Zone2}, + dispatch_burst_to_buckets(Childs, EachFlow, NodeAcc2) + end, + State#{nodes := lists:foldl(Dispatch, Nodes, GroupL)}. + +-spec dispatch_burst_to_buckets(list(node_id()), + non_neg_integer(), nodes()) -> nodes(). +dispatch_burst_to_buckets(Childs, InFlow, Nodes) -> + Each = fun(ChildId, NodeAcc) -> + #{counter := Counter, + index := Index, + obtained := Obtained} = Bucket = maps:get(ChildId, NodeAcc), + counters:add(Counter, Index, InFlow), + NodeAcc#{ChildId := Bucket#{obtained := Obtained + InFlow}} + end, + lists:foldl(Each, Nodes, Childs). + +-spec init_tree(emqx_limiter_schema:limiter_type(), state()) -> state(). +init_tree(Type, State) -> + #{global := Global, + zone := Zone, + bucket := Bucket} = emqx:get_config([emqx_limiter, Type]), + {Factor, Root} = make_root(Global, Zone), + State2 = State#{root := Root}, + {NodeId, State3} = make_zone(maps:to_list(Zone), Factor, 1, State2), + State4 = State3#{counter := counters:new(maps:size(Bucket), + [write_concurrency])}, + make_bucket(maps:to_list(Bucket), Global, Zone, Factor, NodeId, [], State4). + +-spec make_root(hocons:confg(), hocon:config()) -> {number(), root()}. +make_root(#{rate := Rate, burst := Burst}, Zone) -> + ZoneNum = maps:size(Zone), + Childs = lists:seq(1, ZoneNum), + MiniPeriod = emqx_limiter_schema:minimum_period(), + if Rate >= 1 -> + {1, #{rate => Rate, + burst => Burst, + period => MiniPeriod, + childs => Childs, + consumed => 0}}; + true -> + Factor = 1 / Rate, + {Factor, #{rate => 1, + burst => Burst * Factor, + period => erlang:floor(Factor * MiniPeriod), + childs => Childs, + consumed => 0}} + end. + +make_zone([{Name, ZoneCfg} | T], Factor, NodeId, State) -> + #{rate := Rate, burst := Burst} = ZoneCfg, + #{zones := Zones, nodes := Nodes} = State, + Zone = #{id => NodeId, + name => Name, + rate => mul(Rate, Factor), + burst => Burst, + obtained => 0, + childs => []}, + State2 = State#{zones := Zones#{Name => NodeId}, + nodes := Nodes#{NodeId => Zone}}, + make_zone(T, Factor, NodeId + 1, State2); + +make_zone([], _, NodeId, State2) -> + {NodeId, State2}. + +make_bucket([{Name, Conf} | T], Global, Zone, Factor, Id, Buckets, #{type := Type} = State) -> + #{zone := ZoneName, + aggregated := Aggregated} = Conf, + Path = emqx_limiter_manager:make_path(Type, ZoneName, Name), + case get_counter_rate(Conf, Zone, Global) of + infinity -> + State2 = State, + Rate = infinity, + Capacity = infinity, + Counter = undefined, + Index = undefined, + Ref = emqx_limiter_bucket_ref:new(Counter, Index, Rate), + emqx_limiter_manager:insert_bucket(Path, Ref); + RawRate -> + #{capacity := Capacity} = Aggregated, + Initial = get_initial_val(Aggregated), + {Counter, Index, State2} = alloc_counter(Path, RawRate, Initial, State), + Rate = mul(RawRate, Factor) + end, + + Node = #{ id => Id + , name => Name + , zone => ZoneName + , rate => Rate + , obtained => 0 + , correction => 0 + , capacity => Capacity + , counter => Counter + , index => Index}, + + State3 = add_zone_child(Id, Node, ZoneName, State2), + make_bucket(T, Global, Zone, Factor, Id + 1, [Id | Buckets], State3); + +make_bucket([], _, _, _, _, Buckets, State) -> + State#{buckets := Buckets}. + +-spec alloc_counter(emqx_limiter_manager:path(), rate(), capacity(), state()) -> + {counters:counters_ref(), pos_integer(), state()}. +alloc_counter(Path, Rate, Initial, + #{counter := Counter, index := Index} = State) -> + case emqx_limiter_manager:find_bucket(Path) of + {ok, #{counter := ECounter, + index := EIndex}} when ECounter =/= undefined -> + init_counter(Path, ECounter, EIndex, Rate, Initial, State); + _ -> + init_counter(Path, Counter, Index, + Rate, Initial, State#{index := Index + 1}) + end. + +init_counter(Path, Counter, Index, Rate, Initial, State) -> + _ = put_to_counter(Counter, Index, Initial), + Ref = emqx_limiter_bucket_ref:new(Counter, Index, Rate), + emqx_limiter_manager:insert_bucket(Path, Ref), + {Counter, Index, State}. + +-spec add_zone_child(node_id(), bucket(), zone_name(), state()) -> state(). +add_zone_child(NodeId, Bucket, Name, #{zones := Zones, nodes := Nodes} = State) -> + ZoneId = maps:get(Name, Zones), + #{childs := Childs} = Zone = maps:get(ZoneId, Nodes), + Nodes2 = Nodes#{ZoneId => Zone#{childs := [NodeId | Childs]}, + NodeId => Bucket}, + State#{nodes := Nodes2}. + +%% @doc find first limited node +get_counter_rate(#{zone := ZoneName, + aggregated := Cfg}, ZoneCfg, Global) -> + Zone = maps:get(ZoneName, ZoneCfg), + Search = lists:search(fun(E) -> is_limited(E) end, + [Cfg, Zone, Global]), + case Search of + {value, #{rate := Rate}} -> + Rate; + false -> + infinity + end. + +is_limited(#{rate := Rate, capacity := Capacity}) -> + Rate =/= infinity orelse Capacity =/= infinity; + +is_limited(#{rate := Rate}) -> + Rate =/= infinity. + +get_initial_val(#{initial := Initial, + rate := Rate, + capacity := Capacity}) -> + %% initial will nevner be infinity(see the emqx_limiter_schema) + if Initial > 0 -> + Initial; + Rate =/= infinity -> + erlang:min(Rate, Capacity); + Capacity =/= infinity -> + Capacity; + true -> + 0 + end. diff --git a/apps/emqx_limiter/src/emqx_limiter_server_sup.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl similarity index 89% rename from apps/emqx_limiter/src/emqx_limiter_server_sup.erl rename to apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl index 56a2dd2dc..ce8c08913 100644 --- a/apps/emqx_limiter/src/emqx_limiter_server_sup.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -24,9 +24,9 @@ %% Supervisor callbacks -export([init/1]). -%%--================================================================== +%%-------------------------------------------------------------------- %% API functions -%%--================================================================== +%%-------------------------------------------------------------------- %%-------------------------------------------------------------------- %% @doc @@ -52,9 +52,9 @@ restart(Type) -> _ = supervisor:terminate_child(?MODULE, Id), supervisor:restart_child(?MODULE, Id). -%%--================================================================== +%%-------------------------------------------------------------------- %% Supervisor callbacks -%%--================================================================== +%%-------------------------------------------------------------------- %%-------------------------------------------------------------------- %% @private diff --git a/apps/emqx_limiter/src/emqx_limiter_sup.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_sup.erl similarity index 97% rename from apps/emqx_limiter/src/emqx_limiter_sup.erl rename to apps/emqx/src/emqx_limiter/src/emqx_limiter_sup.erl index 957f053af..e5cd7a0b5 100644 --- a/apps/emqx_limiter/src/emqx_limiter_sup.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 2869dfc75..2af2673d1 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -228,7 +228,8 @@ do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) esockd:open(listener_id(Type, ListenerName), ListenOn, merge_default(esockd_opts(Type, Opts)), {emqx_connection, start_link, [#{listener => {Type, ListenerName}, - zone => zone(Opts)}]}); + zone => zone(Opts), + limiter => limiter(Opts)}]}); %% Start MQTT/WS listener do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) @@ -260,6 +261,7 @@ do_start_listener(quic, ListenerName, #{bind := ListenOn} = Opts) -> , peer_bidi_stream_count => 10 , zone => zone(Opts) , listener => {quic, ListenerName} + , limiter => limiter(Opts) }, StreamOpts = [{stream_callback, emqx_quic_stream}], quicer:start_listener(listener_id(quic, ListenerName), @@ -315,7 +317,9 @@ esockd_opts(Type, Opts0) -> ws_opts(Type, ListenerName, Opts) -> WsPaths = [{maps:get(mqtt_path, Opts, "/mqtt"), emqx_ws_connection, - #{zone => zone(Opts), listener => {Type, ListenerName}}}], + #{zone => zone(Opts), + listener => {Type, ListenerName}, + limiter => limiter(Opts)}}], Dispatch = cowboy_router:compile([{'_', WsPaths}]), ProxyProto = maps:get(proxy_protocol, Opts, false), #{env => #{dispatch => Dispatch}, proxy_header => ProxyProto}. @@ -380,6 +384,9 @@ parse_listener_id(Id) -> zone(Opts) -> maps:get(zone, Opts, undefined). +limiter(Opts) -> + maps:get(limiter, Opts). + ssl_opts(Opts) -> maps:to_list( emqx_tls_lib:drop_tls13_for_old_otp( diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index 0d3edd551..9c3e94dac 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -55,6 +55,8 @@ , hexstr2bin/1 ]). +-export([clamp/3]). + -define(SHORT, 8). %% @doc Parse v4 or v6 string format address to tuple. @@ -305,6 +307,13 @@ gen_id(Len) -> <> = crypto:strong_rand_bytes(Len div 2), int_to_hex(R, Len). +-spec clamp(number(), number(), number()) -> number(). +clamp(Val, Min, Max) -> + if Val < Min -> Min; + Val > Max -> Max; + true -> Val + end. + %%------------------------------------------------------------------------------ %% Internal Functions %%------------------------------------------------------------------------------ diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 1c471da90..c4f77b9a0 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1017,6 +1017,8 @@ base_listener() -> sc(atom(), #{ default => 'default' })} + , {"limiter", + sc(map("ratelimit bucket's name", atom()), #{default => #{}})} ]. %% utils diff --git a/apps/emqx/src/emqx_sup.erl b/apps/emqx/src/emqx_sup.erl index 9c8af33d6..446564d1f 100644 --- a/apps/emqx/src/emqx_sup.erl +++ b/apps/emqx/src/emqx_sup.erl @@ -68,12 +68,13 @@ init([]) -> SessionSup = child_spec(emqx_persistent_session_sup, supervisor), CMSup = child_spec(emqx_cm_sup, supervisor), SysSup = child_spec(emqx_sys_sup, supervisor), + Limiter = child_spec(emqx_limiter_sup, supervisor), Children = [KernelSup] ++ [SessionSup || emqx_persistent_session:is_store_enabled()] ++ [RouterSup || emqx_boot:is_enabled(router)] ++ [BrokerSup || emqx_boot:is_enabled(broker)] ++ [CMSup || emqx_boot:is_enabled(broker)] ++ - [SysSup], + [SysSup, Limiter], SupFlags = #{strategy => one_for_all, intensity => 0, period => 1 diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 9ac8a03d0..375b1ae2f 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -63,10 +63,6 @@ sockstate :: emqx_types:sockstate(), %% MQTT Piggyback mqtt_piggyback :: single | multiple, - %% Limiter - limiter :: maybe(emqx_limiter:limiter()), - %% Limit Timer - limit_timer :: maybe(reference()), %% Parse State parse_state :: emqx_frame:parse_state(), %% Serialize options @@ -86,10 +82,30 @@ %% Zone name zone :: atom(), %% Listener Type and Name - listener :: {Type::atom(), Name::atom()} - }). + listener :: {Type::atom(), Name::atom()}, + + %% Limiter + limiter :: maybe(container()), + + %% cache operation when overload + limiter_cache :: queue:queue(cache()), + + %% limiter timers + limiter_timer :: undefined | reference() + }). + +-record(retry, { types :: list(limiter_type()) + , data :: any() + , next :: check_succ_handler() + }). + +-record(cache, { need :: list({pos_integer(), limiter_type()}) + , data :: any() + , next :: check_succ_handler() + }). -type(state() :: #state{}). +-type cache() :: #cache{}. -type(ws_cmd() :: {active, boolean()}|close). @@ -99,6 +115,8 @@ -define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). -define(ENABLED(X), (X =/= undefined)). +-define(LIMITER_BYTES_IN, bytes_in). +-define(LIMITER_MESSAGE_IN, message_in). -dialyzer({no_match, [info/2]}). -dialyzer({nowarn_function, [websocket_init/1]}). @@ -126,7 +144,7 @@ info(sockname, #state{sockname = Sockname}) -> info(sockstate, #state{sockstate = SockSt}) -> SockSt; info(limiter, #state{limiter = Limiter}) -> - maybe_apply(fun emqx_limiter:info/1, Limiter); + Limiter; info(channel, #state{channel = Channel}) -> emqx_channel:info(Channel); info(gc_state, #state{gc_state = GcSt}) -> @@ -242,7 +260,8 @@ check_origin_header(Req, #{listener := {Type, Listener}} = Opts) -> false -> ok end. -websocket_init([Req, #{zone := Zone, listener := {Type, Listener}} = Opts]) -> +websocket_init([Req, + #{zone := Zone, limiter := LimiterCfg, listener := {Type, Listener}} = Opts]) -> {Peername, Peercert} = case emqx_config:get_listener_conf(Type, Listener, [proxy_protocol]) andalso maps:get(proxy_header, Req) of @@ -279,7 +298,7 @@ websocket_init([Req, #{zone := Zone, listener := {Type, Listener}} = Opts]) -> ws_cookie => WsCookie, conn_mod => ?MODULE }, - Limiter = emqx_limiter:init(Zone, undefined, undefined, []), + Limiter = emqx_limiter_container:get_limiter_by_names([?LIMITER_BYTES_IN, ?LIMITER_MESSAGE_IN], LimiterCfg), MQTTPiggyback = get_ws_opts(Type, Listener, mqtt_piggyback), FrameOpts = #{ strict_mode => emqx_config:get_zone_conf(Zone, [mqtt, strict_mode]), @@ -319,7 +338,9 @@ websocket_init([Req, #{zone := Zone, listener := {Type, Listener}} = Opts]) -> idle_timeout = IdleTimeout, idle_timer = IdleTimer, zone = Zone, - listener = {Type, Listener} + listener = {Type, Listener}, + limiter_timer = undefined, + limiter_cache = queue:new() }, hibernate}. websocket_handle({binary, Data}, State) when is_list(Data) -> @@ -327,9 +348,17 @@ websocket_handle({binary, Data}, State) when is_list(Data) -> websocket_handle({binary, Data}, State) -> ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => websocket}), - ok = inc_recv_stats(1, iolist_size(Data)), - NState = ensure_stats_timer(State), - return(parse_incoming(Data, NState)); + State2 = ensure_stats_timer(State), + {Packets, State3} = parse_incoming(Data, [], State2), + LenMsg = erlang:length(Packets), + ByteSize = erlang:iolist_size(Data), + inc_recv_stats(LenMsg, ByteSize), + State4 = check_limiter([{ByteSize, ?LIMITER_BYTES_IN}, {LenMsg, ?LIMITER_MESSAGE_IN}], + Packets, + fun when_msg_in/3, + [], + State3), + return(State4); %% Pings should be replied with pongs, cowboy does it automatically %% Pongs can be safely ignored. Clause here simply prevents crash. @@ -343,7 +372,6 @@ websocket_handle({Frame, _}, State) -> %% TODO: should not close the ws connection ?SLOG(error, #{msg => "unexpected_frame", frame => Frame}), shutdown(unexpected_ws_frame, State). - websocket_info({call, From, Req}, State) -> handle_call(From, Req, State); @@ -351,8 +379,7 @@ websocket_info({cast, rate_limit}, State) -> Stats = #{cnt => emqx_pd:reset_counter(incoming_pubs), oct => emqx_pd:reset_counter(incoming_bytes) }, - NState = postpone({check_gc, Stats}, State), - return(ensure_rate_limit(Stats, NState)); + return(postpone({check_gc, Stats}, State)); websocket_info({cast, Msg}, State) -> handle_info(Msg, State); @@ -377,12 +404,18 @@ websocket_info(Deliver = {deliver, _Topic, _Msg}, Delivers = [Deliver|emqx_misc:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); -websocket_info({timeout, TRef, limit_timeout}, - State = #state{limit_timer = TRef}) -> - NState = State#state{sockstate = running, - limit_timer = undefined - }, - return(enqueue({active, true}, NState)); +websocket_info({timeout, _, limit_timeout}, + State) -> + return(retry_limiter(State)); + +websocket_info(check_cache, #state{limiter_cache = Cache} = State) -> + case queue:peek(Cache) of + empty -> + return(enqueue({active, true}, State#state{sockstate = running})); + {value, #cache{need = Needs, data = Data, next = Next}} -> + State2 = State#state{limiter_cache = queue:drop(Cache)}, + return(check_limiter(Needs, Data, Next, [check_cache], State2)) + end; websocket_info({timeout, TRef, Msg}, State) when is_reference(TRef) -> handle_timeout(TRef, Msg, State); @@ -421,10 +454,9 @@ handle_call(From, stats, State) -> gen_server:reply(From, stats(State)), return(State); -handle_call(_From, {ratelimit, Policy}, State = #state{channel = Channel}) -> - Zone = emqx_channel:info(zone, Channel), - Limiter = emqx_limiter:init(Zone, Policy), - {reply, ok, State#state{limiter = Limiter}}; +handle_call(_From, {ratelimit, Type, Bucket}, State = #state{limiter = Limiter}) -> + Limiter2 = emqx_limiter_container:update_by_name(Type, Bucket, Limiter), + {reply, ok, State#state{limiter = Limiter2}}; handle_call(From, Req, State = #state{channel = Channel}) -> case emqx_channel:handle_call(Req, Channel) of @@ -495,21 +527,80 @@ handle_timeout(TRef, TMsg, State) -> %% Ensure rate limit %%-------------------------------------------------------------------- -ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> - case ?ENABLED(Limiter) andalso emqx_limiter:check(Stats, Limiter) of - false -> State; - {ok, Limiter1} -> - State#state{limiter = Limiter1}; - {pause, Time, Limiter1} -> - ?SLOG(warning, #{msg => "pause_due_to_rate_limit", time => Time}), - TRef = start_timer(Time, limit_timeout), - NState = State#state{sockstate = blocked, - limiter = Limiter1, - limit_timer = TRef - }, - enqueue({active, false}, NState) +-type limiter_type() :: emqx_limiter_container:limiter_type(). +-type container() :: emqx_limiter_container:container(). +-type check_succ_handler() :: + fun((any(), list(any()), state()) -> state()). + +-spec check_limiter(list({pos_integer(), limiter_type()}), + any(), + check_succ_handler(), + list(any()), + state()) -> state(). +check_limiter(Needs, + Data, + WhenOk, + Msgs, + #state{limiter = Limiter, + limiter_timer = LimiterTimer, + limiter_cache = Cache} = State) -> + case LimiterTimer of + undefined -> + case emqx_limiter_container:check_list(Needs, Limiter) of + {ok, Limiter2} -> + WhenOk(Data, Msgs, State#state{limiter = Limiter2}); + {pause, Time, Limiter2} -> + ?SLOG(warning, #{msg => "pause time dueto rate limit", + needs => Needs, + time_in_ms => Time}), + + Retry = #retry{types = [Type || {_, Type} <- Needs], + data = Data, + next = WhenOk}, + + Limiter3 = emqx_limiter_container:set_retry_context(Retry, Limiter2), + + TRef = start_timer(Time, limit_timeout), + + enqueue({active, false}, + State#state{sockstate = blocked, + limiter = Limiter3, + limiter_timer = TRef}); + {drop, Limiter2} -> + {ok, State#state{limiter = Limiter2}} + end; + _ -> + New = #cache{need = Needs, data = Data, next = WhenOk}, + State#state{limiter_cache = queue:in(New, Cache)} end. + +-spec retry_limiter(state()) -> state(). +retry_limiter(#state{limiter = Limiter} = State) -> + #retry{types = Types, data = Data, next = Next} = emqx_limiter_container:get_retry_context(Limiter), + case emqx_limiter_container:retry_list(Types, Limiter) of + {ok, Limiter2} -> + Next(Data, + [check_cache], + State#state{ limiter = Limiter2 + , limiter_timer = undefined + }); + {pause, Time, Limiter2} -> + ?SLOG(warning, #{msg => "pause time dueto rate limit", + types => Types, + time_in_ms => Time}), + + TRef = start_timer(Time, limit_timeout), + + State#state{limiter = Limiter2, limiter_timer = TRef} + end. + +when_msg_in(Packets, [], State) -> + postpone(Packets, State); + +when_msg_in(Packets, Msgs, State) -> + postpone(Packets, enqueue(Msgs, State)). + %%-------------------------------------------------------------------- %% Run GC, Check OOM %%-------------------------------------------------------------------- @@ -538,16 +629,16 @@ check_oom(State = #state{channel = Channel}) -> %% Parse incoming data %%-------------------------------------------------------------------- -parse_incoming(<<>>, State) -> - State; +parse_incoming(<<>>, Packets, State) -> + {Packets, State}; -parse_incoming(Data, State = #state{parse_state = ParseState}) -> +parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> try emqx_frame:parse(Data, ParseState) of {more, NParseState} -> - State#state{parse_state = NParseState}; + {Packets, State#state{parse_state = NParseState}}; {ok, Packet, Rest, NParseState} -> NState = State#state{parse_state = NParseState}, - parse_incoming(Rest, postpone({incoming, Packet}, NState)) + parse_incoming(Rest, [{incoming, Packet} | Packets], NState) catch throw : ?FRAME_PARSE_ERROR(Reason) -> ?SLOG(info, #{ reason => Reason @@ -555,7 +646,7 @@ parse_incoming(Data, State = #state{parse_state = ParseState}) -> , input_bytes => Data }), FrameError = {frame_error, Reason}, - postpone({incoming, FrameError}, State); + {[{incoming, FrameError} | Packets], State}; error : Reason : Stacktrace -> ?SLOG(error, #{ at_state => emqx_frame:describe_state(ParseState) , input_bytes => Data @@ -563,7 +654,7 @@ parse_incoming(Data, State = #state{parse_state = ParseState}) -> , stacktrace => Stacktrace }), FrameError = {frame_error, Reason}, - postpone({incoming, FrameError}, State) + {[{incoming, FrameError} | Packets], State} end. %%-------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 40f0d1c91..45b00ff29 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -129,7 +129,8 @@ basic_conf() -> rpc => rpc_conf(), stats => stats_conf(), listeners => listeners_conf(), - zones => zone_conf() + zones => zone_conf(), + emqx_limiter => emqx:get_config([emqx_limiter]) }. set_test_listener_confs() -> @@ -178,14 +179,48 @@ end_per_suite(_Config) -> emqx_banned ]). -init_per_testcase(_TestCase, Config) -> +init_per_testcase(TestCase, Config) -> NewConf = set_test_listener_confs(), + emqx_common_test_helpers:start_apps([]), + modify_limiter(TestCase, NewConf), [{config, NewConf}|Config]. end_per_testcase(_TestCase, Config) -> emqx_config:put(?config(config, Config)), + emqx_common_test_helpers:stop_apps([]), Config. +modify_limiter(TestCase, NewConf) -> + Checks = [t_quota_qos0, t_quota_qos1, t_quota_qos2], + case lists:member(TestCase, Checks) of + true -> + modify_limiter(NewConf); + _ -> + ok + end. + +%% per_client 5/1s,5 +%% aggregated 10/1s,10 +modify_limiter(#{emqx_limiter := Limiter} = NewConf) -> + #{message_routing := #{bucket := Bucket} = Routing} = Limiter, + #{default := #{per_client := Client} = Default} = Bucket, + Client2 = Client#{rate := 5, + initial := 0, + capacity := 5, + low_water_mark := 1}, + Default2 = Default#{per_client := Client2, + aggregated := #{rate => 10, + initial => 0, + capacity => 10 + }}, + Bucket2 = Bucket#{default := Default2}, + Routing2 = Routing#{bucket := Bucket2}, + + NewConf2 = NewConf#{emqx_limiter := Limiter#{message_routing := Routing2}}, + emqx_config:put(NewConf2), + emqx_limiter_manager:restart_server(message_routing), + ok. + %%-------------------------------------------------------------------- %% Test cases for channel info/stats/caps %%-------------------------------------------------------------------- @@ -547,6 +582,7 @@ t_quota_qos0(_) -> {ok, Chann1} = emqx_channel:handle_in(Pub, Chann), {ok, Chann2} = emqx_channel:handle_in(Pub, Chann1), M1 = emqx_metrics:val('packets.publish.dropped') - 1, + timer:sleep(1000), {ok, Chann3} = emqx_channel:handle_timeout(ref, expire_quota_limit, Chann2), {ok, _} = emqx_channel:handle_in(Pub, Chann3), M1 = emqx_metrics:val('packets.publish.dropped') - 1, @@ -718,7 +754,7 @@ t_handle_call_takeover_end(_) -> t_handle_call_quota(_) -> {reply, ok, _Chan} = emqx_channel:handle_call( - {quota, [{conn_messages_routing, {100,1}}]}, + {quota, default}, channel() ). @@ -886,7 +922,7 @@ t_ws_cookie_init(_) -> conn_mod => emqx_ws_connection, ws_cookie => WsCookie }, - Channel = emqx_channel:init(ConnInfo, #{zone => default, listener => {tcp, default}}), + Channel = emqx_channel:init(ConnInfo, #{zone => default, limiter => limiter_cfg(), listener => {tcp, default}}), ?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)). %%-------------------------------------------------------------------- @@ -911,7 +947,7 @@ channel(InitFields) -> maps:fold(fun(Field, Value, Channel) -> emqx_channel:set_field(Field, Value, Channel) end, - emqx_channel:init(ConnInfo, #{zone => default, listener => {tcp, default}}), + emqx_channel:init(ConnInfo, #{zone => default, limiter => limiter_cfg(), listener => {tcp, default}}), maps:merge(#{clientinfo => clientinfo(), session => session(), conn_state => connected @@ -957,5 +993,6 @@ session(InitFields) when is_map(InitFields) -> %% conn: 5/s; overall: 10/s quota() -> - emqx_limiter:init(zone, [{conn_messages_routing, {5, 1}}, - {overall_messages_routing, {10, 1}}]). + emqx_limiter_container:get_limiter_by_names([message_routing], limiter_cfg()). + +limiter_cfg() -> #{}. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 127a0892c..a5d80bab5 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -134,6 +134,7 @@ start_apps(Apps, Handler) when is_function(Handler) -> %% Because, minirest, ekka etc.. application will scan these modules lists:foreach(fun load/1, [emqx | Apps]), ekka:start(), + ok = emqx_ratelimiter_SUITE:base_conf(), lists:foreach(fun(App) -> start_app(App, Handler) end, [emqx | Apps]). load(App) -> diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 90f1bdca1..073ec0ae3 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -39,7 +39,7 @@ init_per_suite(Config) -> ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Meck Limiter - ok = meck:new(emqx_limiter, [passthrough, no_history, no_link]), + ok = meck:new(emqx_htb_limiter, [passthrough, no_history, no_link]), %% Meck Pd ok = meck:new(emqx_pd, [passthrough, no_history, no_link]), %% Meck Metrics @@ -60,17 +60,19 @@ init_per_suite(Config) -> ok = meck:expect(emqx_alarm, deactivate, fun(_, _) -> ok end), emqx_channel_SUITE:set_test_listener_confs(), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> ok = meck:unload(emqx_transport), catch meck:unload(emqx_channel), ok = meck:unload(emqx_cm), - ok = meck:unload(emqx_limiter), + ok = meck:unload(emqx_htb_limiter), ok = meck:unload(emqx_pd), ok = meck:unload(emqx_metrics), ok = meck:unload(emqx_hooks), ok = meck:unload(emqx_alarm), + emqx_common_test_helpers:stop_apps([]), ok. init_per_testcase(TestCase, Config) when @@ -129,8 +131,9 @@ t_info(_) -> socktype := tcp}, SockInfo). t_info_limiter(_) -> - St = st(#{limiter => emqx_limiter:init(default, [])}), - ?assertEqual(undefined, emqx_connection:info(limiter, St)). + Limiter = init_limiter(), + St = st(#{limiter => Limiter}), + ?assertEqual(Limiter, emqx_connection:info(limiter, St)). t_stats(_) -> CPid = spawn(fun() -> @@ -250,24 +253,22 @@ t_handle_msg_shutdown(_) -> ?assertMatch({stop, {shutdown, for_testing}, _St}, handle_msg({shutdown, for_testing}, st())). t_handle_call(_) -> - St = st(), + St = st(#{limiter => init_limiter()}), ?assertMatch({ok, _St}, handle_msg({event, undefined}, St)), ?assertMatch({reply, _Info, _NSt}, handle_call(self(), info, St)), ?assertMatch({reply, _Stats, _NSt}, handle_call(self(), stats, St)), ?assertMatch({reply, ok, _NSt}, handle_call(self(), {ratelimit, []}, St)), ?assertMatch({reply, ok, _NSt}, - handle_call(self(), {ratelimit, [{conn_messages_in, {100, 1}}]}, St)), + handle_call(self(), {ratelimit, [{bytes_in, default}]}, St)), ?assertEqual({reply, ignored, St}, handle_call(self(), for_testing, St)), ?assertMatch({stop, {shutdown,kicked}, ok, _NSt}, handle_call(self(), kick, St)). t_handle_timeout(_) -> TRef = make_ref(), - State = st(#{idle_timer => TRef, limit_timer => TRef, stats_timer => TRef}), + State = st(#{idle_timer => TRef, stats_timer => TRef, limiter => init_limiter()}), ?assertMatch({stop, {shutdown,idle_timeout}, _NState}, emqx_connection:handle_timeout(TRef, idle_timeout, State)), - ?assertMatch({ok, {event,running}, _NState}, - emqx_connection:handle_timeout(TRef, limit_timeout, State)), ?assertMatch({ok, _NState}, emqx_connection:handle_timeout(TRef, emit_stats, State)), ?assertMatch({ok, _NState}, @@ -279,13 +280,15 @@ t_handle_timeout(_) -> ?assertMatch({ok, _NState}, emqx_connection:handle_timeout(TRef, undefined, State)). t_parse_incoming(_) -> - ?assertMatch({ok, [], _NState}, emqx_connection:parse_incoming(<<>>, st())), + ?assertMatch({[], _NState}, emqx_connection:parse_incoming(<<>>, [], st())), ?assertMatch({[], _NState}, emqx_connection:parse_incoming(<<"for_testing">>, [], st())). t_next_incoming_msgs(_) -> - ?assertEqual({incoming, packet}, emqx_connection:next_incoming_msgs([packet])), - ?assertEqual([{incoming, packet2}, {incoming, packet1}], - emqx_connection:next_incoming_msgs([packet1, packet2])). + State = st(#{}), + ?assertEqual({ok, [{incoming, packet}], State}, + emqx_connection:next_incoming_msgs([packet], [], State)), + ?assertEqual({ok, [{incoming, packet2}, {incoming, packet1}], State}, + emqx_connection:next_incoming_msgs([packet1, packet2], [], State)). t_handle_incoming(_) -> ?assertMatch({ok, _Out, _NState}, @@ -331,26 +334,28 @@ t_handle_info(_) -> ?assertMatch({ok, _NState}, emqx_connection:handle_info(for_testing, st())). t_ensure_rate_limit(_) -> - State = emqx_connection:ensure_rate_limit(#{}, st(#{limiter => undefined})), + WhenOk = fun emqx_connection:next_incoming_msgs/3, + {ok, [], State} = emqx_connection:check_limiter([], [], WhenOk, [], st(#{limiter => undefined})), ?assertEqual(undefined, emqx_connection:info(limiter, State)), - ok = meck:expect(emqx_limiter, check, - fun(_, _) -> {ok, emqx_limiter:init(default, [])} end), - State1 = emqx_connection:ensure_rate_limit(#{}, st(#{limiter => #{}})), - ?assertEqual(undefined, emqx_connection:info(limiter, State1)), + Limiter = init_limiter(), + {ok, [], State1} = emqx_connection:check_limiter([], [], WhenOk, [], st(#{limiter => Limiter})), + ?assertEqual(Limiter, emqx_connection:info(limiter, State1)), - ok = meck:expect(emqx_limiter, check, - fun(_, _) -> {pause, 3000, emqx_limiter:init(default, [])} end), - State2 = emqx_connection:ensure_rate_limit(#{}, st(#{limiter => #{}})), - ?assertEqual(undefined, emqx_connection:info(limiter, State2)), - ?assertEqual(blocked, emqx_connection:info(sockstate, State2)). + ok = meck:expect(emqx_htb_limiter, check, + fun(_, Client) -> {pause, 3000, undefined, Client} end), + {ok, State2} = emqx_connection:check_limiter([{1000, bytes_in}], [], WhenOk, [], st(#{limiter => Limiter})), + meck:unload(emqx_htb_limiter), + ok = meck:new(emqx_htb_limiter, [passthrough, no_history, no_link]), + ?assertNotEqual(undefined, emqx_connection:info(limiter_timer, State2)). t_activate_socket(_) -> - State = st(), + Limiter = init_limiter(), + State = st(#{limiter => Limiter}), {ok, NStats} = emqx_connection:activate_socket(State), ?assertEqual(running, emqx_connection:info(sockstate, NStats)), - State1 = st(#{sockstate => blocked}), + State1 = st(#{sockstate => blocked, limiter_timer => any_timer}), ?assertEqual({ok, State1}, emqx_connection:activate_socket(State1)), State2 = st(#{sockstate => closed}), @@ -458,7 +463,10 @@ with_conn(TestFun, Opts) when is_map(Opts) -> TrapExit = maps:get(trap_exit, Opts, false), process_flag(trap_exit, TrapExit), {ok, CPid} = emqx_connection:start_link(emqx_transport, sock, - maps:merge(Opts, #{zone => default, listener => {tcp, default}})), + maps:merge(Opts, + #{zone => default, + limiter => limiter_cfg(), + listener => {tcp, default}})), TestFun(CPid), TrapExit orelse emqx_connection:stop(CPid), ok. @@ -481,7 +489,8 @@ st(InitFields) when is_map(InitFields) -> st(InitFields, #{}). st(InitFields, ChannelFields) when is_map(InitFields) -> St = emqx_connection:init_state(emqx_transport, sock, #{zone => default, - listener => {tcp, default}}), + limiter => limiter_cfg(), + listener => {tcp, default}}), maps:fold(fun(N, V, S) -> emqx_connection:set_field(N, V, S) end, emqx_connection:set_field(channel, channel(ChannelFields), St), InitFields @@ -515,7 +524,7 @@ channel(InitFields) -> maps:fold(fun(Field, Value, Channel) -> emqx_channel:set_field(Field, Value, Channel) end, - emqx_channel:init(ConnInfo, #{zone => default, listener => {tcp, default}}), + emqx_channel:init(ConnInfo, #{zone => default, limiter => limiter_cfg(), listener => {tcp, default}}), maps:merge(#{clientinfo => ClientInfo, session => Session, conn_state => connected @@ -524,3 +533,8 @@ channel(InitFields) -> handle_msg(Msg, St) -> emqx_connection:handle_msg(Msg, St). handle_call(Pid, Call, St) -> emqx_connection:handle_call(Pid, Call, St). + +limiter_cfg() -> #{}. + +init_limiter() -> + emqx_limiter_container:get_limiter_by_names([bytes_in, message_in], limiter_cfg()). diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index e4e04fb6a..0733bad53 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -46,6 +46,7 @@ init_per_testcase(Case, Config) emqx_config:put([listeners, tcp], #{ listener_test => #{ bind => {"127.0.0.1", 9999} , max_connections => 4321 + , limiter => #{} } }), emqx_config:put([rate_limit], #{max_conn_rate => 1000}), diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl new file mode 100644 index 000000000..61eef166b --- /dev/null +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -0,0 +1,659 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ratelimiter_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(APP, emqx). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(BASE_CONF, <<""" +emqx_limiter { + bytes_in { + global.rate = infinity + zone.default.rate = infinity + bucket.default { + zone = default + aggregated.rate = infinity + aggregated.capacity = infinity + per_client.rate = \"100MB/1s\" + per_client.capacity = infinity + } + } + + message_in { + global.rate = infinity + zone.default.rate = infinity + bucket.default { + zone = default + aggregated.rate = infinity + aggregated.capacity = infinity + per_client.rate = infinity + per_client.capacity = infinity + } + } + + connection { + global.rate = infinity + zone.default.rate = infinity + bucket.default { + zone = default + aggregated.rate = infinity + aggregated.capacity = infinity + per_client.rate = infinity + per_client.capacity = infinity + } + } + + message_routing { + global.rate = infinity + zone.default.rate = infinity + bucket.default { + zone = default + aggregated.rate = infinity + aggregated.capacity = infinity + per_client.rate = infinity + per_client.capacity = infinity + } + } +} + +""">>). + +-record(client, { counter :: counters:counter_ref() + , start :: pos_integer() + , endtime :: pos_integer() + , obtained :: pos_integer() + , rate :: float() + , client :: emqx_htb_limiter:client() + }). + +-define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). +-define(RATE(Rate), to_rate(Rate)). +-define(NOW, erlang:system_time(millisecond)). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_config:init_load(emqx_limiter_schema, ?BASE_CONF), + emqx_common_test_helpers:start_apps([?APP]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([?APP]). + +init_per_testcase(_TestCase, Config) -> + Config. + +base_conf() -> + emqx_config:init_load(emqx_limiter_schema, ?BASE_CONF). + +%%-------------------------------------------------------------------- +%% Test Cases Bucket Level +%%-------------------------------------------------------------------- +t_max_retry_time(_) -> + Cfg = fun(Cfg) -> + Cfg#{rate := 1, + capacity := 1, + max_retry_time := 500, + failure_strategy := drop} + end, + Case = fun() -> + Client = connect(default), + Begin = ?NOW, + Result = emqx_htb_limiter:consume(101, Client), + ?assertMatch({drop, _}, Result), + Time = ?NOW - Begin, + ?assert(Time >= 500 andalso Time < 550) + end, + with_per_client(default, Cfg, Case). + +t_divisible(_) -> + Cfg = fun(Cfg) -> + Cfg#{divisible := true, + rate := ?RATE("1000/1s"), + initial := 600, + capacity := 600} + end, + Case = fun() -> + Client = connect(default), + Result = emqx_htb_limiter:check(1000, Client), + ?assertMatch({partial, + 400, + #{continuation := _, + diff := 400, + start := _, + need := 1000}, + _}, Result) + end, + with_per_client(default, Cfg, Case). + +t_low_water_mark(_) -> + Cfg = fun(Cfg) -> + Cfg#{low_water_mark := 400, + rate := ?RATE("1000/1s"), + initial := 1000, + capacity := 1000} + end, + Case = fun() -> + Client = connect(default), + Result = emqx_htb_limiter:check(500, Client), + ?assertMatch({ok, _}, Result), + {_, Client2} = Result, + Result2 = emqx_htb_limiter:check(101, Client2), + ?assertMatch({pause, + _, + #{continuation := undefined, + diff := 0}, + _}, Result2) + end, + with_per_client(default, Cfg, Case). + +t_infinity_client(_) -> + Fun = fun(#{aggregated := Aggr, per_client := Cli} = Bucket) -> + Aggr2 = Aggr#{rate := infinity, + capacity := infinity}, + Cli2 = Cli#{rate := infinity, capacity := infinity}, + Bucket#{aggregated := Aggr2, + per_client := Cli2} + end, + Case = fun() -> + Client = connect(default), + ?assertEqual(infinity, Client), + Result = emqx_htb_limiter:check(100000, Client), + ?assertEqual({ok, Client}, Result) + end, + with_bucket(default, Fun, Case). + +t_short_board(_) -> + Fun = fun(#{aggregated := Aggr, per_client := Cli} = Bucket) -> + Aggr2 = Aggr#{rate := ?RATE("100/1s"), + initial := 0, + capacity := 100}, + Cli2 = Cli#{rate := ?RATE("600/1s"), + capacity := 600, + initial := 600}, + Bucket#{aggregated := Aggr2, + per_client := Cli2} + end, + Case = fun() -> + Counter = counters:new(1, [write_concurrency]), + start_client(default, ?NOW + 2000, Counter, 20), + timer:sleep(2100), + check_average_rate(Counter, 2, 100, 20) + end, + with_bucket(default, Fun, Case). + +t_rate(_) -> + Fun = fun(#{aggregated := Aggr, per_client := Cli} = Bucket) -> + Aggr2 = Aggr#{rate := ?RATE("100/100ms"), + initial := 0, + capacity := infinity}, + Cli2 = Cli#{rate := infinity, + capacity := infinity, + initial := 0}, + Bucket#{aggregated := Aggr2, + per_client := Cli2} + end, + Case = fun() -> + Client = connect(default), + Ts1 = erlang:system_time(millisecond), + C1 = emqx_htb_limiter:available(Client), + timer:sleep(1000), + Ts2 = erlang:system_time(millisecond), + C2 = emqx_htb_limiter:available(Client), + ShouldInc = floor((Ts2 - Ts1) / 100) * 100, + Inc = C2 - C1, + ?assert(in_range(Inc, ShouldInc - 100, ShouldInc + 100), "test bucket rate") + end, + with_bucket(default, Fun, Case). + +t_capacity(_) -> + Capacity = 600, + Fun = fun(#{aggregated := Aggr, per_client := Cli} = Bucket) -> + Aggr2 = Aggr#{rate := ?RATE("100/100ms"), + initial := 0, + capacity := 600}, + Cli2 = Cli#{rate := infinity, + capacity := infinity, + initial := 0}, + Bucket#{aggregated := Aggr2, + per_client := Cli2} + end, + Case = fun() -> + Client = connect(default), + timer:sleep(1000), + C1 = emqx_htb_limiter:available(Client), + ?assertEqual(Capacity, C1, "test bucket capacity") + end, + with_bucket(default, Fun, Case). + +%%-------------------------------------------------------------------- +%% Test Cases Zone Level +%%-------------------------------------------------------------------- +t_limit_zone_with_unlimit_bucket(_) -> + ZoneMod = fun(Cfg) -> + Cfg#{rate := ?RATE("600/1s"), + burst := ?RATE("60/1s")} + end, + + Bucket = fun(#{aggregated := Aggr, per_client := Cli} = Bucket) -> + Aggr2 = Aggr#{rate := infinity, + initial := 0, + capacity := infinity}, + Cli2 = Cli#{rate := infinity, + initial := 0, + capacity := infinity, + divisible := true}, + Bucket#{aggregated := Aggr2, per_client := Cli2} + end, + + Case = fun() -> + C1 = counters:new(1, [write_concurrency]), + start_client(b1, ?NOW + 2000, C1, 20), + timer:sleep(2100), + check_average_rate(C1, 2, 600, 1000) + end, + + with_zone(default, ZoneMod, [{b1, Bucket}], Case). + + +%%-------------------------------------------------------------------- +%% Test Cases Global Level +%%-------------------------------------------------------------------- +t_burst_and_fairness(_) -> + GlobalMod = fun(Cfg) -> + Cfg#{burst := ?RATE("60/1s")} + end, + + ZoneMod = fun(Cfg) -> + Cfg#{rate := ?RATE("600/1s"), + burst := ?RATE("60/1s")} + end, + + Bucket = fun(#{aggregated := Aggr, per_client := Cli} = Bucket) -> + Aggr2 = Aggr#{rate := ?RATE("500/1s"), + initial := 0, + capacity := 500}, + Cli2 = Cli#{rate := ?RATE("600/1s"), + capacity := 600, + initial := 600}, + Bucket#{aggregated := Aggr2, + per_client := Cli2} + end, + + Case = fun() -> + C1 = counters:new(1, [write_concurrency]), + C2 = counters:new(1, [write_concurrency]), + start_client(b1, ?NOW + 2000, C1, 20), + start_client(b2, ?NOW + 2000, C2, 30), + timer:sleep(2100), + check_average_rate(C1, 2, 330, 25), + check_average_rate(C2, 2, 330, 25) + end, + + with_global(GlobalMod, + default, + ZoneMod, + [{b1, Bucket}, {b2, Bucket}], + Case). + +t_limit_global_with_unlimit_other(_) -> + GlobalMod = fun(Cfg) -> + Cfg#{rate := ?RATE("600/1s")} + end, + + ZoneMod = fun(Cfg) -> Cfg#{rate := infinity} end, + + Bucket = fun(#{aggregated := Aggr, per_client := Cli} = Bucket) -> + Aggr2 = Aggr#{rate := infinity, + initial := 0, + capacity := infinity}, + Cli2 = Cli#{rate := infinity, + capacity := infinity, + initial := 0}, + Bucket#{aggregated := Aggr2, + per_client := Cli2} + end, + + Case = fun() -> + C1 = counters:new(1, [write_concurrency]), + start_client(b1, ?NOW + 2000, C1, 20), + timer:sleep(2100), + check_average_rate(C1, 2, 600, 100) + end, + + with_global(GlobalMod, + default, + ZoneMod, + [{b1, Bucket}], + Case). + +t_multi_zones(_) -> + GlobalMod = fun(Cfg) -> + Cfg#{rate := ?RATE("600/1s")} + end, + + Zone1 = fun(Cfg) -> + Cfg#{rate := ?RATE("400/1s")} + end, + + Zone2 = fun(Cfg) -> + Cfg#{rate := ?RATE("500/1s")} + end, + + Bucket = fun(Zone, Rate) -> + fun(#{aggregated := Aggr, per_client := Cli} = Bucket) -> + Aggr2 = Aggr#{rate := infinity, + initial := 0, + capacity := infinity}, + Cli2 = Cli#{rate := Rate, + capacity := infinity, + initial := 0}, + Bucket#{aggregated := Aggr2, + per_client := Cli2, + zone := Zone} + end + end, + + Case = fun() -> + C1 = counters:new(1, [write_concurrency]), + C2 = counters:new(1, [write_concurrency]), + start_client(b1, ?NOW + 2000, C1, 25), + start_client(b2, ?NOW + 2000, C2, 20), + timer:sleep(2100), + check_average_rate(C1, 2, 300, 25), + check_average_rate(C2, 2, 300, 25) + end, + + with_global(GlobalMod, + [z1, z2], + [Zone1, Zone2], + [{b1, Bucket(z1, ?RATE("400/1s"))}, {b2, Bucket(z2, ?RATE("500/1s"))}], + Case). + +%% because the simulated client will try to reach the maximum rate +%% when divisiable = true, a large number of divided tokens will be generated +%% so this is not an accurate test +t_multi_zones_with_divisible(_) -> + GlobalMod = fun(Cfg) -> + Cfg#{rate := ?RATE("600/1s")} + end, + + Zone1 = fun(Cfg) -> + Cfg#{rate := ?RATE("400/1s")} + end, + + Zone2 = fun(Cfg) -> + Cfg#{rate := ?RATE("500/1s")} + end, + + Bucket = fun(Zone, Rate) -> + fun(#{aggregated := Aggr, per_client := Cli} = Bucket) -> + Aggr2 = Aggr#{rate := Rate, + initial := 0, + capacity := infinity}, + Cli2 = Cli#{rate := Rate, + divisible := true, + capacity := infinity, + initial := 0}, + Bucket#{aggregated := Aggr2, + per_client := Cli2, + zone := Zone} + end + end, + + Case = fun() -> + C1 = counters:new(1, [write_concurrency]), + C2 = counters:new(1, [write_concurrency]), + start_client(b1, ?NOW + 2000, C1, 25), + start_client(b2, ?NOW + 2000, C2, 20), + timer:sleep(2100), + check_average_rate(C1, 2, 300, 120), + check_average_rate(C2, 2, 300, 120) + end, + + with_global(GlobalMod, + [z1, z2], + [Zone1, Zone2], + [{b1, Bucket(z1, ?RATE("400/1s"))}, {b2, Bucket(z2, ?RATE("500/1s"))}], + Case). + +t_zone_hunger_and_fair(_) -> + GlobalMod = fun(Cfg) -> + Cfg#{rate := ?RATE("600/1s")} + end, + + Zone1 = fun(Cfg) -> + Cfg#{rate := ?RATE("600/1s")} + end, + + Zone2 = fun(Cfg) -> + Cfg#{rate := ?RATE("50/1s")} + end, + + Bucket = fun(Zone, Rate) -> + fun(#{aggregated := Aggr, per_client := Cli} = Bucket) -> + Aggr2 = Aggr#{rate := infinity, + initial := 0, + capacity := infinity}, + Cli2 = Cli#{rate := Rate, + capacity := infinity, + initial := 0}, + Bucket#{aggregated := Aggr2, + per_client := Cli2, + zone := Zone} + end + end, + + Case = fun() -> + C1 = counters:new(1, [write_concurrency]), + C2 = counters:new(1, [write_concurrency]), + start_client(b1, ?NOW + 2000, C1, 20), + start_client(b2, ?NOW + 2000, C2, 20), + timer:sleep(2100), + check_average_rate(C1, 2, 550, 25), + check_average_rate(C2, 2, 50, 25) + end, + + with_global(GlobalMod, + [z1, z2], + [Zone1, Zone2], + [{b1, Bucket(z1, ?RATE("600/1s"))}, {b2, Bucket(z2, ?RATE("50/1s"))}], + Case). + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +start_client(Name, EndTime, Counter, Number) -> + lists:foreach(fun(_) -> + spawn(fun() -> + start_client(Name, EndTime, Counter) + end) + end, + lists:seq(1, Number)). + +start_client(Name, EndTime, Counter) -> + #{per_client := PerClient} = + emqx_config:get([emqx_limiter, message_routing, bucket, Name]), + #{rate := Rate} = PerClient, + Client = #client{start = ?NOW, + endtime = EndTime, + counter = Counter, + obtained = 0, + rate = Rate, + client = connect(Name) + }, + client_loop(Client). + +%% the simulated client will try to reach the configured rate as much as possible +%% note this client will not considered the capacity, so must make sure rate < capacity +client_loop(#client{start = Start, + endtime = EndTime, + obtained = Obtained, + rate = Rate} = State) -> + Now = ?NOW, + Period = emqx_limiter_schema:minimum_period(), + MinPeriod = erlang:ceil(0.25 * Period), + if Now >= EndTime -> + stop; + Now - Start < MinPeriod -> + timer:sleep(client_random_val(MinPeriod)), + client_loop(State); + Obtained =< 0 -> + Rand = client_random_val(Rate), + client_try_check(Rand, State); + true -> + Span = Now - Start, + CurrRate = Obtained * Period / Span, + if CurrRate < Rate -> + Rand = client_random_val(Rate), + client_try_check(Rand, State); + true -> + LeftTime = EndTime - Now, + CanSleep = erlang:min(LeftTime, client_random_val(MinPeriod div 2)), + timer:sleep(CanSleep), + client_loop(State) + end + end. + +client_try_check(Need, #client{counter = Counter, + endtime = EndTime, + obtained = Obtained, + client = Client} = State) -> + case emqx_htb_limiter:check(Need, Client) of + {ok, Client2} -> + case Need of + #{need := Val} -> ok; + Val -> ok + end, + counters:add(Counter, 1, Val), + client_loop(State#client{obtained = Obtained + Val, client = Client2}); + {_, Pause, Retry, Client2} -> + LeftTime = EndTime - ?NOW, + if LeftTime =< 0 -> + stop; + true -> + timer:sleep(erlang:min(Pause, LeftTime)), + client_try_check(Retry, State#client{client = Client2}) + end + end. + + +%% XXX not a god test, because client's rate maybe bigger than global rate +%% so if client' rate = infinity +%% client's divisible should be true or capacity must be bigger than number of each comsume +client_random_val(infinity) -> + 1000; + +%% random in 0.5Range ~ 1Range +client_random_val(Range) -> + Half = erlang:floor(Range) div 2, + Rand = rand:uniform(Half + 1) + Half, + erlang:max(1, Rand). + +to_rate(Str) -> + {ok, Rate} = emqx_limiter_schema:to_rate(Str), + Rate. + +with_global(Modifier, ZoneName, ZoneModifier, Buckets, Case) -> + Path = [emqx_limiter, message_routing], + #{global := Global} = Cfg = emqx_config:get(Path), + Cfg2 = Cfg#{global := Modifier(Global)}, + with_zone(Cfg2, ZoneName, ZoneModifier, Buckets, Case). + +with_zone(Name, Modifier, Buckets, Case) -> + Path = [emqx_limiter, message_routing], + Cfg = emqx_config:get(Path), + with_zone(Cfg, Name, Modifier, Buckets, Case). + +with_zone(Cfg, Name, Modifier, Buckets, Case) -> + Path = [emqx_limiter, message_routing], + #{zone := ZoneCfgs, + bucket := BucketCfgs} = Cfg, + ZoneCfgs2 = apply_modifier(Name, Modifier, ZoneCfgs), + BucketCfgs2 = apply_modifier(Buckets, BucketCfgs), + Cfg2 = Cfg#{zone := ZoneCfgs2, bucket := BucketCfgs2}, + with_config(Path, fun(_) -> Cfg2 end, Case). + +with_bucket(Bucket, Modifier, Case) -> + Path = [emqx_limiter, message_routing, bucket, Bucket], + with_config(Path, Modifier, Case). + +with_per_client(Bucket, Modifier, Case) -> + Path = [emqx_limiter, message_routing, bucket, Bucket, per_client], + with_config(Path, Modifier, Case). + +with_config(Path, Modifier, Case) -> + Cfg = emqx_config:get(Path), + NewCfg = Modifier(Cfg), + ct:pal("test with config:~p~n", [NewCfg]), + emqx_config:put(Path, NewCfg), + emqx_limiter_manager:restart_server(message_routing), + timer:sleep(100), + DelayReturn + = try + Return = Case(), + fun() -> Return end + catch Type:Reason:Trace -> + fun() -> erlang:raise(Type, Reason, Trace) end + end, + emqx_config:put(Path, Cfg), + DelayReturn(). + +connect(Name) -> + emqx_limiter_server:connect(message_routing, Name). + +check_average_rate(Counter, Second, Rate, Margin) -> + Cost = counters:get(Counter, 1), + PerSec = Cost / Second, + ?LOGT(">>>> Cost:~p PerSec:~p Rate:~p ~n", [Cost, PerSec, Rate]), + ?assert(in_range(PerSec, Rate - Margin, Rate + Margin)). + +print_average_rate(Counter, Second) -> + Cost = counters:get(Counter, 1), + PerSec = Cost / Second, + ct:pal(">>>> Cost:~p PerSec:~p ~n", [Cost, PerSec]). + +in_range(Val, Min, _Max) when Val < Min -> + ct:pal("Val:~p smaller than min bound:~p~n", [Val, Min]), + false; +in_range(Val, _Min, Max) when Val > Max-> + ct:pal("Val:~p bigger than max bound:~p~n", [Val, Max]), + false; +in_range(_, _, _) -> + true. + +apply_modifier(Name, Modifier, Cfg) when is_list(Name) -> + Pairs = lists:zip(Name, Modifier), + apply_modifier(Pairs, Cfg); + +apply_modifier(Name, Modifier, #{default := Template} = Cfg) -> + Cfg#{Name => Modifier(Template)}. + +apply_modifier(Pairs, #{default := Template}) -> + Fun = fun({N, M}, Acc) -> + Acc#{N => M(Template)} + end, + lists:foldl(Fun, #{}, Pairs). diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index d554d3c8c..d69a9a321 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -105,6 +105,15 @@ end_per_testcase(_, Config) -> emqx_common_test_helpers:stop_apps([]), Config. +init_per_suite(Config) -> + emqx_channel_SUITE:set_test_listener_confs(), + emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_) -> + emqx_common_test_helpers:stop_apps([]), + ok. + %%-------------------------------------------------------------------- %% Test Cases %%-------------------------------------------------------------------- @@ -131,7 +140,9 @@ t_header(_) -> (<<"x-forwarded-port">>, _, _) -> <<"1000">> end), set_ws_opts(proxy_address_header, <<"x-forwarded-for">>), set_ws_opts(proxy_port_header, <<"x-forwarded-port">>), - {ok, St, _} = ?ws_conn:websocket_init([req, #{zone => default, listener => {ws, default}}]), + {ok, St, _} = ?ws_conn:websocket_init([req, #{zone => default, + limiter => limiter_cfg(), + listener => {ws, default}}]), WsPid = spawn(fun() -> receive {call, From, info} -> gen_server:reply(From, ?ws_conn:info(St)) @@ -143,8 +154,9 @@ t_header(_) -> } = SockInfo. t_info_limiter(_) -> - St = st(#{limiter => emqx_limiter:init(external, [])}), - ?assertEqual(undefined, ?ws_conn:info(limiter, St)). + Limiter = init_limiter(), + St = st(#{limiter => Limiter}), + ?assertEqual(Limiter, ?ws_conn:info(limiter, St)). t_info_channel(_) -> #{conn_state := connected} = ?ws_conn:info(channel, st()). @@ -249,7 +261,7 @@ t_ws_non_check_origin(_) -> headers => [{<<"origin">>, <<"http://localhost:18080">>}]})). t_init(_) -> - Opts = #{listener => {ws, default}, zone => default}, + Opts = #{listener => {ws, default}, zone => default, limiter => limiter_cfg()}, ok = meck:expect(cowboy_req, parse_header, fun(_, req) -> undefined end), ok = meck:expect(cowboy_req, reply, fun(_, Req) -> Req end), {ok, req, _} = ?ws_conn:init(req, Opts), @@ -329,8 +341,11 @@ t_websocket_info_deliver(_) -> t_websocket_info_timeout_limiter(_) -> Ref = make_ref(), + LimiterT = init_limiter(), + Next = fun emqx_ws_connection:when_msg_in/3, + Limiter = emqx_limiter_container:set_retry_context({retry, [], [], Next}, LimiterT), Event = {timeout, Ref, limit_timeout}, - {[{active, true}], St} = websocket_info(Event, st(#{limit_timer => Ref})), + {ok, St} = websocket_info(Event, st(#{limiter => Limiter})), ?assertEqual([], ?ws_conn:info(postponed, St)). t_websocket_info_timeout_keepalive(_) -> @@ -389,23 +404,27 @@ t_handle_timeout_emit_stats(_) -> ?assertEqual(undefined, ?ws_conn:info(stats_timer, St)). t_ensure_rate_limit(_) -> - Limiter = emqx_limiter:init(external, {1, 10}, {100, 1000}, []), + Limiter = init_limiter(), St = st(#{limiter => Limiter}), - St1 = ?ws_conn:ensure_rate_limit(#{cnt => 0, oct => 0}, St), - St2 = ?ws_conn:ensure_rate_limit(#{cnt => 11, oct => 1200}, St1), - ?assertEqual(blocked, ?ws_conn:info(sockstate, St2)), - ?assertEqual([{active, false}], ?ws_conn:info(postponed, St2)). + {ok, Need} = emqx_limiter_schema:to_capacity("1GB"), %% must bigger than value in emqx_ratelimit_SUITE + St1 = ?ws_conn:check_limiter([{Need, bytes_in}], + [], + fun(_, _, S) -> S end, + [], + St), + ?assertEqual(blocked, ?ws_conn:info(sockstate, St1)), + ?assertEqual([{active, false}], ?ws_conn:info(postponed, St1)). t_parse_incoming(_) -> - St = ?ws_conn:parse_incoming(<<48,3>>, st()), - St1 = ?ws_conn:parse_incoming(<<0,1,116>>, St), + {Packets, St} = ?ws_conn:parse_incoming(<<48,3>>, [], st()), + {Packets1, _} = ?ws_conn:parse_incoming(<<0,1,116>>, Packets, St), Packet = ?PUBLISH_PACKET(?QOS_0, <<"t">>, undefined, <<>>), - ?assertMatch([{incoming, Packet}], ?ws_conn:info(postponed, St1)). + ?assertMatch([{incoming, Packet}], Packets1). t_parse_incoming_frame_error(_) -> - St = ?ws_conn:parse_incoming(<<3,2,1,0>>, st()), + {Packets, _St} = ?ws_conn:parse_incoming(<<3,2,1,0>>, [], st()), FrameError = {frame_error, function_clause}, - [{incoming, FrameError}] = ?ws_conn:info(postponed, St). + [{incoming, FrameError}] = Packets. t_handle_incomming_frame_error(_) -> FrameError = {frame_error, bad_qos}, @@ -440,7 +459,9 @@ t_shutdown(_) -> st() -> st(#{}). st(InitFields) when is_map(InitFields) -> - {ok, St, _} = ?ws_conn:websocket_init([req, #{zone => default, listener => {ws, default}}]), + {ok, St, _} = ?ws_conn:websocket_init([req, #{zone => default, + listener => {ws, default}, + limiter => limiter_cfg()}]), maps:fold(fun(N, V, S) -> ?ws_conn:set_field(N, V, S) end, ?ws_conn:set_field(channel, channel(), St), InitFields @@ -474,7 +495,9 @@ channel(InitFields) -> maps:fold(fun(Field, Value, Channel) -> emqx_channel:set_field(Field, Value, Channel) end, - emqx_channel:init(ConnInfo, #{zone => default, listener => {ws, default}}), + emqx_channel:init(ConnInfo, #{zone => default, + listener => {ws, default}, + limiter => limiter_cfg()}), maps:merge(#{clientinfo => ClientInfo, session => Session, conn_state => connected @@ -533,3 +556,8 @@ ws_client(State) -> after 1000 -> ct:fail(ws_timeout) end. + +limiter_cfg() -> #{}. + +init_limiter() -> + emqx_limiter_container:get_limiter_by_names([bytes_in, message_in], limiter_cfg()). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index a722872a3..5b9c9d588 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -434,8 +434,15 @@ typename_to_spec("log_level()", _Mod) -> }; typename_to_spec("rate()", _Mod) -> #{type => string, example => <<"10M/s">>}; -typename_to_spec("bucket_rate()", _Mod) -> - #{type => string, example => <<"10M/s, 100M">>}; +typename_to_spec("capacity()", _Mod) -> + #{type => string, example => <<"100M">>}; +typename_to_spec("burst_rate()", _Mod) -> + %% 0/0s = no burst + #{type => string, example => <<"10M/1s">>}; +typename_to_spec("failure_strategy()", _Mod) -> + #{type => string, example => <<"force">>}; +typename_to_spec("initial()", _Mod) -> + #{type => string, example => <<"0M">>}; typename_to_spec(Name, Mod) -> Spec = range(Name), Spec1 = remote_module_type(Spec, Name, Mod), diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl index 1e53d0486..0782ab1b3 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl @@ -70,13 +70,14 @@ all() -> init_per_suite(Config) -> ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), - emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_gateway]), + application:load(emqx_gateway), + emqx_mgmt_api_test_util:init_suite([emqx_conf]), Config. end_per_suite(Config) -> timer:sleep(300), {ok, _} = emqx_conf:remove([<<"gateway">>,<<"lwm2m">>], #{}), - emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_conf]), + emqx_mgmt_api_test_util:end_suite([emqx_conf]), Config. init_per_testcase(_AllTestCase, Config) -> diff --git a/apps/emqx_limiter/etc/emqx_limiter.conf b/apps/emqx_limiter/etc/emqx_limiter.conf deleted file mode 100644 index 7298931e3..000000000 --- a/apps/emqx_limiter/etc/emqx_limiter.conf +++ /dev/null @@ -1,50 +0,0 @@ -##-------------------------------------------------------------------- -## Emq X Rate Limiter -##-------------------------------------------------------------------- -emqx_limiter { - bytes_in { - global = "100KB/10s" # token generation rate - zone.default = "100kB/10s" - zone.external = "20kB/10s" - bucket.tcp { - zone = default - aggregated = "100kB/10s,1Mb" - per_client = "100KB/10s,10Kb" - } - bucket.ssl { - zone = external - aggregated = "100kB/10s,1Mb" - per_client = "100KB/10s,10Kb" - } - } - - message_in { - global = "100/10s" - zone.default = "100/10s" - bucket.bucket1 { - zone = default - aggregated = "100/10s,1000" - per_client = "100/10s,100" - } - } - - connection { - global = "100/10s" - zone.default = "100/10s" - bucket.bucket1 { - zone = default - aggregated = "100/10s,1000" - per_client = "100/10s,100" - } - } - - message_routing { - global = "100/10s" - zone.default = "100/10s" - bucket.bucket1 { - zone = default - aggregated = "100/10s,100" - per_client = "100/10s,10" - } - } -} diff --git a/apps/emqx_limiter/src/emqx_limiter_client.erl b/apps/emqx_limiter/src/emqx_limiter_client.erl deleted file mode 100644 index eb7c768ff..000000000 --- a/apps/emqx_limiter/src/emqx_limiter_client.erl +++ /dev/null @@ -1,144 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_limiter_client). - -%% API --export([create/5, make_ref/3, consume/2]). --export_type([limiter/0]). - -%% tocket bucket algorithm --record(limiter, { tokens :: non_neg_integer() - , rate :: float() - , capacity :: decimal() - , lasttime :: millisecond() - , ref :: ref_limiter() - }). - --record(ref, { counter :: counters:counters_ref() - , index :: index() - , rate :: decimal() - , obtained :: non_neg_integer() - }). - -%% TODO -%% we should add a nop-limiter, when all the upper layers (global, zone, and buckets ) are infinity - --type limiter() :: #limiter{}. --type ref_limiter() :: #ref{}. --type client() :: limiter() | ref_limiter(). --type millisecond() :: non_neg_integer(). --type pause_result(Client) :: {pause, millisecond(), Client}. --type consume_result(Client) :: {ok, Client} - | pause_result(Client). --type decimal() :: emqx_limiter_decimal:decimal(). --type index() :: emqx_limiter_server:index(). - --define(NOW, erlang:monotonic_time(millisecond)). --define(MINIUMN_PAUSE, 100). - --import(emqx_limiter_decimal, [sub/2]). -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- --spec create(float(), - decimal(), - counters:counters_ref(), - index(), - decimal()) -> limiter(). -create(Rate, Capacity, Counter, Index, CounterRate) -> - #limiter{ tokens = Capacity - , rate = Rate - , capacity = Capacity - , lasttime = ?NOW - , ref = make_ref(Counter, Index, CounterRate) - }. - --spec make_ref(counters:counters_ref(), index(), decimal()) -> ref_limiter(). -make_ref(Counter, Idx, Rate) -> - #ref{counter = Counter, index = Idx, rate = Rate, obtained = 0}. - --spec consume(pos_integer(), Client) -> consume_result(Client) - when Client :: client(). -consume(Need, #limiter{tokens = Tokens, - capacity = Capacity} = Limiter) -> - if Need =< Tokens -> - try_consume_counter(Need, Limiter); - Need > Capacity -> - %% FIXME - %% The client should be able to send 4kb data if the rate is configured to be 2kb/s, it just needs 2s to complete. - throw("too big request"); %% FIXME how to deal this? - true -> - try_reset(Need, Limiter) - end; - -consume(Need, #ref{counter = Counter, - index = Index, - rate = Rate, - obtained = Obtained} = Ref) -> - Tokens = counters:get(Counter, Index), - if Tokens >= Need -> - counters:sub(Counter, Index, Need), - {ok, Ref#ref{obtained = Obtained + Need}}; - true -> - return_pause(Need - Tokens, Rate, Ref) - end. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- --spec try_consume_counter(pos_integer(), limiter()) -> consume_result(limiter()). -try_consume_counter(Need, - #limiter{tokens = Tokens, - ref = #ref{counter = Counter, - index = Index, - obtained = Obtained, - rate = CounterRate} = Ref} = Limiter) -> - CT = counters:get(Counter, Index), - if CT >= Need -> - counters:sub(Counter, Index, Need), - {ok, Limiter#limiter{tokens = sub(Tokens, Need), - ref = Ref#ref{obtained = Obtained + Need}}}; - true -> - return_pause(Need - CT, CounterRate, Limiter) - end. - --spec try_reset(pos_integer(), limiter()) -> consume_result(limiter()). -try_reset(Need, - #limiter{tokens = Tokens, - rate = Rate, - lasttime = LastTime, - capacity = Capacity} = Limiter) -> - Now = ?NOW, - Inc = erlang:floor((Now - LastTime) * Rate / emqx_limiter_schema:minimum_period()), - Tokens2 = erlang:min(Tokens + Inc, Capacity), - if Need > Tokens2 -> - return_pause(Need, Rate, Limiter); - true -> - Limiter2 = Limiter#limiter{tokens = Tokens2, - lasttime = Now}, - try_consume_counter(Need, Limiter2) - end. - --spec return_pause(pos_integer(), decimal(), Client) -> pause_result(Client) - when Client :: client(). -return_pause(_, infinity, Limiter) -> - %% workaround when emqx_limiter_server's rate is infinity - {pause, ?MINIUMN_PAUSE, Limiter}; - -return_pause(Diff, Rate, Limiter) -> - Pause = erlang:round(Diff * emqx_limiter_schema:minimum_period() / Rate), - {pause, erlang:max(Pause, ?MINIUMN_PAUSE), Limiter}. diff --git a/apps/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx_limiter/src/emqx_limiter_schema.erl deleted file mode 100644 index 0e2977025..000000000 --- a/apps/emqx_limiter/src/emqx_limiter_schema.erl +++ /dev/null @@ -1,140 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_limiter_schema). - --include_lib("typerefl/include/types.hrl"). - --export([ roots/0, fields/1, to_rate/1 - , to_bucket_rate/1, minimum_period/0]). - --define(KILOBYTE, 1024). - --type limiter_type() :: bytes_in - | message_in - | connection - | message_routing. - --type bucket_name() :: atom(). --type zone_name() :: atom(). --type rate() :: infinity | float(). --type bucket_rate() :: list(infinity | number()). - --typerefl_from_string({rate/0, ?MODULE, to_rate}). --typerefl_from_string({bucket_rate/0, ?MODULE, to_bucket_rate}). - --reflect_type([ rate/0 - , bucket_rate/0 - ]). - --export_type([limiter_type/0, bucket_name/0, zone_name/0]). - --import(emqx_schema, [sc/2, map/2]). - -roots() -> [emqx_limiter]. - -fields(emqx_limiter) -> - [ {bytes_in, sc(ref(limiter), #{})} - , {message_in, sc(ref(limiter), #{})} - , {connection, sc(ref(limiter), #{})} - , {message_routing, sc(ref(limiter), #{})} - ]; - -fields(limiter) -> - [ {global, sc(rate(), #{})} - , {zone, sc(map("zone name", rate()), #{})} - , {bucket, sc(map("bucket id", ref(bucket)), - #{desc => "Token Buckets"})} - ]; - -fields(bucket) -> - [ {zone, sc(atom(), #{desc => "the zone which the bucket in"})} - , {aggregated, sc(bucket_rate(), #{})} - , {per_client, sc(bucket_rate(), #{})} - ]. - -%% minimum period is 100ms -minimum_period() -> - 100. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- -ref(Field) -> hoconsc:ref(?MODULE, Field). - -to_rate(Str) -> - Tokens = [string:trim(T) || T <- string:tokens(Str, "/")], - case Tokens of - ["infinity"] -> - {ok, infinity}; - [Quota, Interval] -> - {ok, Val} = to_quota(Quota), - case emqx_schema:to_duration_ms(Interval) of - {ok, Ms} when Ms > 0 -> - {ok, Val * minimum_period() / Ms}; - _ -> - {error, Str} - end; - _ -> - {error, Str} - end. - -to_bucket_rate(Str) -> - Tokens = [string:trim(T) || T <- string:tokens(Str, "/,")], - case Tokens of - [Rate, Capa] -> - {ok, infinity} = to_quota(Rate), - {ok, CapaVal} = to_quota(Capa), - if CapaVal =/= infinity -> - {ok, [infinity, CapaVal]}; - true -> - {error, Str} - end; - [Quota, Interval, Capacity] -> - {ok, Val} = to_quota(Quota), - case emqx_schema:to_duration_ms(Interval) of - {ok, Ms} when Ms > 0 -> - {ok, CapaVal} = to_quota(Capacity), - {ok, [Val * minimum_period() / Ms, CapaVal]}; - _ -> - {error, Str} - end; - _ -> - {error, Str} - end. - - -to_quota(Str) -> - {ok, MP} = re:compile("^\s*(?:(?:([1-9][0-9]*)([a-zA-z]*))|infinity)\s*$"), - Result = re:run(Str, MP, [{capture, all_but_first, list}]), - case Result of - {match, [Quota, Unit]} -> - Val = erlang:list_to_integer(Quota), - Unit2 = string:to_lower(Unit), - {ok, apply_unit(Unit2, Val)}; - {match, [Quota]} -> - {ok, erlang:list_to_integer(Quota)}; - {match, []} -> - {ok, infinity}; - _ -> - {error, Str} - end. - -apply_unit("", Val) -> Val; -apply_unit("kb", Val) -> Val * ?KILOBYTE; -apply_unit("mb", Val) -> Val * ?KILOBYTE * ?KILOBYTE; -apply_unit("gb", Val) -> Val * ?KILOBYTE * ?KILOBYTE * ?KILOBYTE; -apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). diff --git a/apps/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx_limiter/src/emqx_limiter_server.erl deleted file mode 100644 index 8a712db2e..000000000 --- a/apps/emqx_limiter/src/emqx_limiter_server.erl +++ /dev/null @@ -1,426 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% A hierachical token bucket algorithm -%% Note: this is not the linux HTB algorithm(http://luxik.cdi.cz/~devik/qos/htb/manual/theory.htm) -%% Algorithm: -%% 1. the root node periodically generates tokens and then distributes them -%% just like the oscillation of water waves -%% 2. the leaf node has a counter, which is the place where the token is actually held. -%% 3. other nodes only play the role of transmission, and the rate of the node is like a valve, -%% limiting the oscillation transmitted from the parent node - --module(emqx_limiter_server). - --behaviour(gen_server). - --include_lib("emqx/include/logger.hrl"). - -%% gen_server callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3, format_status/2]). - --export([ start_link/1, connect/2, info/2 - , name/1]). - --record(root, { rate :: rate() %% number of tokens generated per period - , period :: pos_integer() %% token generation interval(second) - , childs :: list(node_id()) %% node children - , consumed :: non_neg_integer() - }). - --record(zone, { id :: pos_integer() - , name :: zone_name() - , rate :: rate() - , obtained :: non_neg_integer() %% number of tokens obtained - , childs :: list(node_id()) - }). - --record(bucket, { id :: pos_integer() - , name :: bucket_name() - , rate :: rate() - , obtained :: non_neg_integer() - , correction :: emqx_limiter_decimal:zero_or_float() %% token correction value - , capacity :: capacity() - , counter :: counters:counters_ref() - , index :: index() - }). - --record(state, { root :: undefined | root() - , counter :: undefined | counters:counters_ref() %% current counter to alloc - , index :: index() - , zones :: #{zone_name() => node_id()} - , nodes :: nodes() - , type :: limiter_type() - }). - -%% maybe use maps is better, but record is fastter --define(FIELD_OBTAINED, #zone.obtained). --define(GET_FIELD(F, Node), element(F, Node)). --define(CALL(Type, Msg), gen_server:call(name(Type), {?FUNCTION_NAME, Msg})). - --type node_id() :: pos_integer(). --type root() :: #root{}. --type zone() :: #zone{}. --type bucket() :: #bucket{}. --type node_data() :: zone() | bucket(). --type nodes() :: #{node_id() => node_data()}. --type zone_name() :: emqx_limiter_schema:zone_name(). --type limiter_type() :: emqx_limiter_schema:limiter_type(). --type bucket_name() :: emqx_limiter_schema:bucket_name(). --type rate() :: decimal(). --type flow() :: decimal(). --type capacity() :: decimal(). --type decimal() :: emqx_limiter_decimal:decimal(). --type state() :: #state{}. --type index() :: pos_integer(). - --export_type([index/0]). --import(emqx_limiter_decimal, [add/2, sub/2, mul/2, add_to_counter/3, put_to_counter/3]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- --spec connect(limiter_type(), bucket_name()) -> emqx_limiter_client:client(). -connect(Type, Bucket) -> - #{zone := Zone, - aggregated := [Aggr, Capacity], - per_client := [Client, ClientCapa]} = emqx:get_config([emqx_limiter, Type, bucket, Bucket]), - case emqx_limiter_manager:find_counter(Type, Zone, Bucket) of - {ok, Counter, Idx, Rate} -> - if Client =/= infinity andalso (Client < Aggr orelse ClientCapa < Capacity) -> - emqx_limiter_client:create(Client, ClientCapa, Counter, Idx, Rate); - true -> - emqx_limiter_client:make_ref(Counter, Idx, Rate) - end; - _ -> - ?LOG(error, "can't find the bucket:~p which type is:~p~n", [Bucket, Type]), - throw("invalid bucket") - end. - --spec info(limiter_type(), atom()) -> term(). -info(Type, Info) -> - ?CALL(Type, Info). - --spec name(limiter_type()) -> atom(). -name(Type) -> - erlang:list_to_atom(io_lib:format("~s_~s", [?MODULE, Type])). - -%%-------------------------------------------------------------------- -%% @doc -%% Starts the server -%% @end -%%-------------------------------------------------------------------- --spec start_link(limiter_type()) -> _. -start_link(Type) -> - gen_server:start_link({local, name(Type)}, ?MODULE, [Type], []). - -%%-------------------------------------------------------------------- -%%% gen_server callbacks -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Initializes the server -%% @end -%%-------------------------------------------------------------------- --spec init(Args :: term()) -> {ok, State :: term()} | - {ok, State :: term(), Timeout :: timeout()} | - {ok, State :: term(), hibernate} | - {stop, Reason :: term()} | - ignore. -init([Type]) -> - State = #state{zones = #{}, - nodes = #{}, - type = Type, - index = 1}, - State2 = init_tree(Type, State), - oscillate(State2#state.root#root.period), - {ok, State2}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Handling call messages -%% @end -%%-------------------------------------------------------------------- --spec handle_call(Request :: term(), From :: {pid(), term()}, State :: term()) -> - {reply, Reply :: term(), NewState :: term()} | - {reply, Reply :: term(), NewState :: term(), Timeout :: timeout()} | - {reply, Reply :: term(), NewState :: term(), hibernate} | - {noreply, NewState :: term()} | - {noreply, NewState :: term(), Timeout :: timeout()} | - {noreply, NewState :: term(), hibernate} | - {stop, Reason :: term(), Reply :: term(), NewState :: term()} | - {stop, Reason :: term(), NewState :: term()}. -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, State}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Handling cast messages -%% @end -%%-------------------------------------------------------------------- --spec handle_cast(Request :: term(), State :: term()) -> - {noreply, NewState :: term()} | - {noreply, NewState :: term(), Timeout :: timeout()} | - {noreply, NewState :: term(), hibernate} | - {stop, Reason :: term(), NewState :: term()}. -handle_cast(Req, State) -> - ?LOG(error, "Unexpected cast: ~p", [Req]), - {noreply, State}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Handling all non call/cast messages -%% @end -%%-------------------------------------------------------------------- --spec handle_info(Info :: timeout() | term(), State :: term()) -> - {noreply, NewState :: term()} | - {noreply, NewState :: term(), Timeout :: timeout()} | - {noreply, NewState :: term(), hibernate} | - {stop, Reason :: normal | term(), NewState :: term()}. -handle_info(oscillate, State) -> - {noreply, oscillation(State)}; - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% This function is called by a gen_server when it is about to -%% terminate. It should be the opposite of Module:init/1 and do any -%% necessary cleaning up. When it returns, the gen_server terminates -%% with Reason. The return value is ignored. -%% @end -%%-------------------------------------------------------------------- --spec terminate(Reason :: normal | shutdown | {shutdown, term()} | term(), - State :: term()) -> any(). -terminate(_Reason, _State) -> - ok. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Convert process state when code is changed -%% @end -%%-------------------------------------------------------------------- --spec code_change(OldVsn :: term() | {down, term()}, - State :: term(), - Extra :: term()) -> {ok, NewState :: term()} | - {error, Reason :: term()}. -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% This function is called for changing the form and appearance -%% of gen_server status when it is returned from sys:get_status/1,2 -%% or when it appears in termination error logs. -%% @end -%%-------------------------------------------------------------------- --spec format_status(Opt :: normal | terminate, - Status :: list()) -> Status :: term(). -format_status(_Opt, Status) -> - Status. - -%%-------------------------------------------------------------------- -%%% Internal functions -%%-------------------------------------------------------------------- -oscillate(Interval) -> - erlang:send_after(Interval, self(), ?FUNCTION_NAME). - -%% @doc generate tokens, and then spread to leaf nodes --spec oscillation(state()) -> state(). -oscillation(#state{root = #root{rate = Flow, - period = Interval, - childs = ChildIds, - consumed = Consumed} = Root, - nodes = Nodes} = State) -> - oscillate(Interval), - Childs = get_orderd_childs(ChildIds, Nodes), - {Alloced, Nodes2} = transverse(Childs, Flow, 0, Nodes), - State#state{nodes = Nodes2, - root = Root#root{consumed = Consumed + Alloced}}. - -%% @doc horizontal spread --spec transverse(list(node_data()), - flow(), - non_neg_integer(), - nodes()) -> {non_neg_integer(), nodes()}. -transverse([H | T], InFlow, Alloced, Nodes) when InFlow > 0 -> - {NodeAlloced, Nodes2} = longitudinal(H, InFlow, Nodes), - InFlow2 = sub(InFlow, NodeAlloced), - Alloced2 = Alloced + NodeAlloced, - transverse(T, InFlow2, Alloced2, Nodes2); - -transverse(_, _, Alloced, Nodes) -> - {Alloced, Nodes}. - -%% @doc vertical spread --spec longitudinal(node_data(), flow(), nodes()) -> - {non_neg_integer(), nodes()}. -longitudinal(#zone{id = Id, - rate = Rate, - obtained = Obtained, - childs = ChildIds} = Node, InFlow, Nodes) -> - Flow = erlang:min(InFlow, Rate), - - if Flow > 0 -> - Childs = get_orderd_childs(ChildIds, Nodes), - {Alloced, Nodes2} = transverse(Childs, Flow, 0, Nodes), - if Alloced > 0 -> - {Alloced, - Nodes2#{Id => Node#zone{obtained = Obtained + Alloced}}}; - true -> - %% childs are empty or all counter childs are full - {0, Nodes} - end; - true -> - {0, Nodes} - end; - -longitudinal(#bucket{id = Id, - rate = Rate, - capacity = Capacity, - correction = Correction, - counter = Counter, - index = Index, - obtained = Obtained} = Node, InFlow, Nodes) -> - Flow = add(erlang:min(InFlow, Rate), Correction), - - Tokens = counters:get(Counter, Index), - %% toknes's value mayb be a negative value(stolen from the future) - Avaiable = erlang:min(if Tokens < 0 -> - add(Capacity, Tokens); - true -> - sub(Capacity, Tokens) - end, Flow), - FixAvaiable = erlang:min(Capacity, Avaiable), - if FixAvaiable > 0 -> - {Alloced, Decimal} = add_to_counter(Counter, Index, FixAvaiable), - - {Alloced, - Nodes#{Id => Node#bucket{obtained = Obtained + Alloced, - correction = Decimal}}}; - true -> - {0, Nodes} - end. - --spec get_orderd_childs(list(node_id()), nodes()) -> list(node_data()). -get_orderd_childs(Ids, Nodes) -> - Childs = [maps:get(Id, Nodes) || Id <- Ids], - - %% sort by obtained, avoid node goes hungry - lists:sort(fun(A, B) -> - ?GET_FIELD(?FIELD_OBTAINED, A) < ?GET_FIELD(?FIELD_OBTAINED, B) - end, - Childs). - --spec init_tree(emqx_limiter_schema:limiter_type(), state()) -> state(). -init_tree(Type, State) -> - #{global := Global, - zone := Zone, - bucket := Bucket} = emqx:get_config([emqx_limiter, Type]), - {Factor, Root} = make_root(Global, Zone), - State2 = State#state{root = Root}, - {NodeId, State3} = make_zone(maps:to_list(Zone), Factor, 1, State2), - State4 = State3#state{counter = counters:new(maps:size(Bucket), - [write_concurrency])}, - make_bucket(maps:to_list(Bucket), Factor, NodeId, State4). - --spec make_root(decimal(), hocon:config()) -> {number(), root()}. -make_root(Rate, Zone) -> - ZoneNum = maps:size(Zone), - Childs = lists:seq(1, ZoneNum), - MiniPeriod = emqx_limiter_schema:minimum_period(), - if Rate >= 1 -> - {1, #root{rate = Rate, - period = MiniPeriod, - childs = Childs, - consumed = 0}}; - true -> - Factor = 1 / Rate, - {Factor, #root{rate = 1, - period = erlang:floor(Factor * MiniPeriod), - childs = Childs, - consumed = 0}} - end. - -make_zone([{Name, Rate} | T], Factor, NodeId, State) -> - #state{zones = Zones, nodes = Nodes} = State, - Zone = #zone{id = NodeId, - name = Name, - rate = mul(Rate, Factor), - obtained = 0, - childs = []}, - State2 = State#state{zones = Zones#{Name => NodeId}, - nodes = Nodes#{NodeId => Zone}}, - make_zone(T, Factor, NodeId + 1, State2); - -make_zone([], _, NodeId, State2) -> - {NodeId, State2}. - -make_bucket([{Name, Conf} | T], Factor, NodeId, State) -> - #{zone := ZoneName, - aggregated := [Rate, Capacity]} = Conf, - {Counter, Idx, State2} = alloc_counter(ZoneName, Name, Rate, State), - Node = #bucket{ id = NodeId - , name = Name - , rate = mul(Rate, Factor) - , obtained = 0 - , correction = 0 - , capacity = Capacity - , counter = Counter - , index = Idx}, - State3 = add_zone_child(NodeId, Node, ZoneName, State2), - make_bucket(T, Factor, NodeId + 1, State3); - -make_bucket([], _, _, State) -> - State. - --spec alloc_counter(zone_name(), bucket_name(), rate(), state()) -> - {counters:counters_ref(), pos_integer(), state()}. -alloc_counter(Zone, Bucket, Rate, - #state{type = Type, counter = Counter, index = Index} = State) -> - Path = emqx_limiter_manager:make_path(Type, Zone, Bucket), - case emqx_limiter_manager:find_counter(Path) of - undefined -> - init_counter(Path, Counter, Index, - Rate, State#state{index = Index + 1}); - {ok, ECounter, EIndex, _} -> - init_counter(Path, ECounter, EIndex, Rate, State) - end. - -init_counter(Path, Counter, Index, Rate, State) -> - _ = put_to_counter(Counter, Index, 0), - emqx_limiter_manager:insert_counter(Path, Counter, Index, Rate), - {Counter, Index, State}. - --spec add_zone_child(node_id(), bucket(), zone_name(), state()) -> state(). -add_zone_child(NodeId, Bucket, Name, #state{zones = Zones, nodes = Nodes} = State) -> - ZoneId = maps:get(Name, Zones), - #zone{childs = Childs} = Zone = maps:get(ZoneId, Nodes), - Nodes2 = Nodes#{ZoneId => Zone#zone{childs = [NodeId | Childs]}, - NodeId => Bucket}, - State#state{nodes = Nodes2}. diff --git a/apps/emqx_limiter/test/emqx_limiter_SUITE.erl b/apps/emqx_limiter/test/emqx_limiter_SUITE.erl deleted file mode 100644 index 499103f6d..000000000 --- a/apps/emqx_limiter/test/emqx_limiter_SUITE.erl +++ /dev/null @@ -1,272 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_limiter_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --define(APP, emqx_limiter). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - --define(BASE_CONF, <<""" -emqx_limiter { - bytes_in {global = \"100KB/10s\" - zone.default = \"100kB/10s\" - zone.external = \"20kB/10s\" - bucket.tcp {zone = default - aggregated = \"100kB/10s,1Mb\" - per_client = \"100KB/10s,10Kb\"} - bucket.ssl {zone = external - aggregated = \"100kB/10s,1Mb\" - per_client = \"100KB/10s,10Kb\"} - } - - message_in {global = \"100/10s\" - zone.default = \"100/10s\" - bucket.bucket1 {zone = default - aggregated = \"100/10s,1000\" - per_client = \"100/10s,100\"} - } - - connection {global = \"100/10s\" - zone.default = \"100/10s\" - bucket.bucket1 {zone = default - aggregated = \"100/10s,100\" - per_client = \"100/10s,10\" - } - } - - message_routing {global = \"100/10s\" - zone.default = \"100/10s\" - bucket.bucket1 {zone = default - aggregated = \"100/10s,100\" - per_client = \"100/10s,10\" - } - } -}""">>). - --define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). - --record(client_options, { interval :: non_neg_integer() - , per_cost :: non_neg_integer() - , type :: atom() - , bucket :: atom() - , lifetime :: non_neg_integer() - , rates :: list(tuple()) - }). - --record(client_state, { client :: emqx_limiter_client:limiter() - , pid :: pid() - , got :: non_neg_integer() - , options :: #client_options{}}). - -%%-------------------------------------------------------------------- -%% Setups -%%-------------------------------------------------------------------- -all() -> emqx_common_test_helpers:all(?MODULE). - -init_per_suite(Config) -> - ok = emqx_config:init_load(emqx_limiter_schema, ?BASE_CONF), - emqx_common_test_helpers:start_apps([?APP]), - Config. - -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([?APP]). - -init_per_testcase(_TestCase, Config) -> - Config. - -%%-------------------------------------------------------------------- -%% Test Cases -%%-------------------------------------------------------------------- -t_un_overload(_) -> - Conf = emqx:get_config([emqx_limiter]), - Conn = #{global => to_rate("infinity"), - zone => #{z1 => to_rate("1000/1s"), - z2 => to_rate("1000/1s")}, - bucket => #{b1 => #{zone => z1, - aggregated => to_bucket_rate("100/1s, 500"), - per_client => to_bucket_rate("10/1s, 50")}, - b2 => #{zone => z2, - aggregated => to_bucket_rate("500/1s, 500"), - per_client => to_bucket_rate("100/1s, infinity") - }}}, - Conf2 = Conf#{connection => Conn}, - emqx_config:put([emqx_limiter], Conf2), - {ok, _} = emqx_limiter_manager:restart_server(connection), - - timer:sleep(200), - - B1C = #client_options{interval = 100, - per_cost = 1, - type = connection, - bucket = b1, - lifetime = timer:seconds(3), - rates = [{fun erlang:'=<'/2, ["1000/1s", "100/1s"]}, - {fun erlang:'=:='/2, ["10/1s"]}]}, - - B2C = #client_options{interval = 100, - per_cost = 10, - type = connection, - bucket = b2, - lifetime = timer:seconds(3), - rates = [{fun erlang:'=<'/2, ["1000/1s", "500/1s"]}, - {fun erlang:'=:='/2, ["100/1s"]}]}, - - lists:foreach(fun(_) -> start_client(B1C) end, - lists:seq(1, 10)), - - - lists:foreach(fun(_) -> start_client(B2C) end, - lists:seq(1, 5)), - - ?assert(check_client_result(10 + 5)). - -t_infinity(_) -> - Conf = emqx:get_config([emqx_limiter]), - Conn = #{global => to_rate("infinity"), - zone => #{z1 => to_rate("1000/1s"), - z2 => to_rate("infinity")}, - bucket => #{b1 => #{zone => z1, - aggregated => to_bucket_rate("100/1s, infinity"), - per_client => to_bucket_rate("10/1s, 100")}, - b2 => #{zone => z2, - aggregated => to_bucket_rate("infinity, 600"), - per_client => to_bucket_rate("100/1s, infinity") - }}}, - Conf2 = Conf#{connection => Conn}, - emqx_config:put([emqx_limiter], Conf2), - {ok, _} = emqx_limiter_manager:restart_server(connection), - - timer:sleep(200), - - B1C = #client_options{interval = 100, - per_cost = 1, - type = connection, - bucket = b1, - lifetime = timer:seconds(3), - rates = [{fun erlang:'=<'/2, ["1000/1s", "100/1s"]}, - {fun erlang:'=:='/2, ["10/1s"]}]}, - - B2C = #client_options{interval = 100, - per_cost = 10, - type = connection, - bucket = b2, - lifetime = timer:seconds(3), - rates = [{fun erlang:'=:='/2, ["100/1s"]}]}, - - lists:foreach(fun(_) -> start_client(B1C) end, - lists:seq(1, 8)), - - lists:foreach(fun(_) -> start_client(B2C) end, - lists:seq(1, 4)), - - ?assert(check_client_result(8 + 4)). - -%%-------------------------------------------------------------------- -%%% Internal functions -%%-------------------------------------------------------------------- -start_client(Opts) -> - Pid = self(), - erlang:spawn(fun() -> enter_client(Opts, Pid) end). - -enter_client(#client_options{type = Type, - bucket = Bucket, - lifetime = Lifetime} = Opts, - Pid) -> - erlang:send_after(Lifetime, self(), stop), - erlang:send(self(), consume), - Client = emqx_limiter_server:connect(Type, Bucket), - client_loop(#client_state{client = Client, - pid = Pid, - got = 0, - options = Opts}). - -client_loop(#client_state{client = Client, - got = Got, - pid = Pid, - options = #client_options{interval = Interval, - per_cost = PerCost, - lifetime = Lifetime, - rates = Rates}} = State) -> - receive - consume -> - case emqx_limiter_client:consume(PerCost, Client) of - {ok, Client2} -> - erlang:send_after(Interval, self(), consume), - client_loop(State#client_state{client = Client2, - got = Got + PerCost}); - {pause, MS, Client2} -> - erlang:send_after(MS, self(), {resume, erlang:system_time(millisecond)}), - client_loop(State#client_state{client = Client2}) - end; - stop -> - Rate = Got * emqx_limiter_schema:minimum_period() / Lifetime, - ?LOGT("Got:~p, Rate is:~p Checks:~p~n", [Got, Rate, Rate]), - Check = check_rates(Rate, Rates), - erlang:send(Pid, {client, Check}); - {resume, Begin} -> - case emqx_limiter_client:consume(PerCost, Client) of - {ok, Client2} -> - Now = erlang:system_time(millisecond), - Diff = erlang:max(0, Interval - (Now - Begin)), - erlang:send_after(Diff, self(), consume), - client_loop(State#client_state{client = Client2, - got = Got + PerCost}); - {pause, MS, Client2} -> - erlang:send_after(MS, self(), {resume, Begin}), - client_loop(State#client_state{client = Client2}) - end - end. - -check_rates(Rate, [{Fun, Rates} | T]) -> - case lists:all(fun(E) -> Fun(Rate, to_rate(E)) end, Rates) of - true -> - check_rates(Rate, T); - false -> - false - end; -check_rates(_, _) -> - true. - -check_client_result(0) -> - true; - -check_client_result(N) -> - ?LOGT("check_client_result:~p~n", [N]), - receive - {client, true} -> - check_client_result(N - 1); - {client, false} -> - false; - Any -> - ?LOGT(">>>> other:~p~n", [Any]) - - after 3500 -> - ?LOGT(">>>> timeout~n", []), - false - end. - -to_rate(Str) -> - {ok, Rate} = emqx_limiter_schema:to_rate(Str), - Rate. - -to_bucket_rate(Str) -> - {ok, Result} = emqx_limiter_schema:to_bucket_rate(Str), - Result. diff --git a/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl index 691828ffd..b961ed391 100644 --- a/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl @@ -177,6 +177,6 @@ t_keepalive(_Config) -> [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), State = sys:get_state(Pid), ct:pal("~p~n", [State]), - ?assertEqual(11000, element(2, element(5, element(11, State)))), + ?assertEqual(11000, element(2, element(5, element(9, State)))), emqtt:disconnect(C1), ok. diff --git a/rebar.config.erl b/rebar.config.erl index a66d29333..2beabd147 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -305,7 +305,6 @@ relx_apps(ReleaseType, Edition) -> , emqx_statsd , emqx_prometheus , emqx_psk - , emqx_limiter ] ++ [quicer || is_quicer_supported()] %++ [emqx_license || is_enterprise(Edition)] diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript index 71bddb7ae..3ef9e033f 100755 --- a/scripts/merge-config.escript +++ b/scripts/merge-config.escript @@ -12,16 +12,54 @@ main(_) -> {ok, BaseConf} = file:read_file("apps/emqx_conf/etc/emqx_conf.conf"), - Apps = filelib:wildcard("*", "apps/") -- ["emqx_machine", "emqx_conf"], - Conf = lists:foldl(fun(App, Acc) -> - Filename = filename:join([apps, App, "etc", App]) ++ ".conf", - case filelib:is_regular(Filename) of - true -> - {ok, Bin1} = file:read_file(Filename), - [Acc, io_lib:nl(), Bin1]; - false -> Acc - end - end, BaseConf, Apps), + Cfgs = get_all_cfgs("apps/"), + Conf = lists:foldl(fun(CfgFile, Acc) -> + case filelib:is_regular(CfgFile) of + true -> + {ok, Bin1} = file:read_file(CfgFile), + [Acc, io_lib:nl(), Bin1]; + false -> Acc + end + end, BaseConf, Cfgs), ClusterInc = "include \"cluster-override.conf\"\n", LocalInc = "include \"local-override.conf\"\n", ok = file:write_file("apps/emqx_conf/etc/emqx.conf.all", [Conf, ClusterInc, LocalInc]). + +get_all_cfgs(Root) -> + Apps = filelib:wildcard("*", Root) -- ["emqx_machine", "emqx_conf"], + Dirs = [filename:join([Root, App]) || App <- Apps], + lists:foldl(fun get_cfgs/2, [], Dirs). + +get_all_cfgs(Dir, Cfgs) -> + Fun = fun(E, Acc) -> + Path = filename:join([Dir, E]), + get_cfgs(Path, Acc) + end, + lists:foldl(Fun, Cfgs, filelib:wildcard("*", Dir)). + +get_cfgs(Dir, Cfgs) -> + case filelib:is_dir(Dir) of + false -> + Cfgs; + _ -> + Files = filelib:wildcard("*", Dir), + case lists:member("etc", Files) of + false -> + try_enter_child(Dir, Files, Cfgs); + true -> + EtcDir = filename:join([Dir, "etc"]), + %% the conf name must start with emqx + %% because there are some other conf, and these conf don't start with emqx + Confs = filelib:wildcard("emqx*.conf", EtcDir), + NewCfgs = [filename:join([EtcDir, Name]) || Name <- Confs], + try_enter_child(Dir, Files, NewCfgs ++ Cfgs) + end + end. + +try_enter_child(Dir, Files, Cfgs) -> + case lists:member("src", Files) of + false -> + Cfgs; + true -> + get_all_cfgs(filename:join([Dir, "src"]), Cfgs) + end. From 82746134bba960fd1c5fda63a165de1a1c01a447 Mon Sep 17 00:00:00 2001 From: lafirest Date: Thu, 2 Dec 2021 10:48:27 +0800 Subject: [PATCH 16/31] feat(emqx_slow_sus): ported emqx_slow_subs from v4.4 --- apps/emqx/src/emqx_cm.erl | 8 +- apps/emqx/src/emqx_schema.erl | 8 + apps/emqx/src/emqx_session.erl | 84 ++++- .../emqx_message_latency_stats.erl | 120 +++++++ .../emqx_slow_subs/emqx_moving_average.erl | 90 +++++ apps/emqx/src/emqx_zone_schema.erl | 2 +- apps/emqx/test/emqx_proper_types.erl | 28 +- apps/emqx/test/emqx_session_SUITE.erl | 14 +- apps/emqx_conf/src/emqx_conf_schema.erl | 1 + apps/emqx_management/src/emqx_mgmt_api.erl | 18 +- apps/emqx_slow_subs/etc/emqx_slow_subs.conf | 40 +++ .../emqx_slow_subs/include/emqx_slow_subs.hrl | 28 ++ .../emqx_slow_subs/src/emqx_slow_subs.app.src | 12 + apps/emqx_slow_subs/src/emqx_slow_subs.erl | 318 ++++++++++++++++++ .../emqx_slow_subs/src/emqx_slow_subs_api.erl | 108 ++++++ .../emqx_slow_subs/src/emqx_slow_subs_app.erl | 30 ++ .../src/emqx_slow_subs_schema.erl | 44 +++ .../emqx_slow_subs/src/emqx_slow_subs_sup.erl | 36 ++ .../test/emqx_slow_subs_SUITE.erl | 124 +++++++ .../test/emqx_slow_subs_api_SUITE.erl | 174 ++++++++++ rebar.config.erl | 1 + 21 files changed, 1260 insertions(+), 28 deletions(-) create mode 100644 apps/emqx/src/emqx_slow_subs/emqx_message_latency_stats.erl create mode 100644 apps/emqx/src/emqx_slow_subs/emqx_moving_average.erl create mode 100644 apps/emqx_slow_subs/etc/emqx_slow_subs.conf create mode 100644 apps/emqx_slow_subs/include/emqx_slow_subs.hrl create mode 100644 apps/emqx_slow_subs/src/emqx_slow_subs.app.src create mode 100644 apps/emqx_slow_subs/src/emqx_slow_subs.erl create mode 100644 apps/emqx_slow_subs/src/emqx_slow_subs_api.erl create mode 100644 apps/emqx_slow_subs/src/emqx_slow_subs_app.erl create mode 100644 apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl create mode 100644 apps/emqx_slow_subs/src/emqx_slow_subs_sup.erl create mode 100644 apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl create mode 100644 apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index fef3e4f19..162cff2e0 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -291,8 +291,9 @@ create_session(ClientInfo, ConnInfo) -> ok = emqx_hooks:run('session.created', [ClientInfo, emqx_session:info(Session)]), Session. -get_session_confs(#{zone := Zone}, #{receive_maximum := MaxInflight, expiry_interval := EI}) -> - #{max_subscriptions => get_mqtt_conf(Zone, max_subscriptions), +get_session_confs(#{zone := Zone, clientid := ClientId}, #{receive_maximum := MaxInflight, expiry_interval := EI}) -> + #{clientid => ClientId, + max_subscriptions => get_mqtt_conf(Zone, max_subscriptions), upgrade_qos => get_mqtt_conf(Zone, upgrade_qos), max_inflight => MaxInflight, retry_interval => get_mqtt_conf(Zone, retry_interval), @@ -301,7 +302,8 @@ get_session_confs(#{zone := Zone}, #{receive_maximum := MaxInflight, expiry_inte %% TODO: Add conf for allowing/disallowing persistent sessions. %% Note that the connection info is already enriched to have %% default config values for session expiry. - is_persistent => EI > 0 + is_persistent => EI > 0, + latency_stats => emqx_config:get_zone_conf(Zone, [latency_stats]) }. mqueue_confs(Zone) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index c4f77b9a0..f8da72ad9 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -175,6 +175,9 @@ roots(low) -> , {"persistent_session_store", sc(ref("persistent_session_store"), #{})} + , {"latency_stats", + sc(ref("latency_stats"), + #{})} ]. fields("persistent_session_store") -> @@ -974,6 +977,11 @@ when deactivated, but after the retention time. """ }) } + ]; + +fields("latency_stats") -> + [ {"samples", sc(integer(), #{default => 10, + desc => "the number of smaples for calculate the average latency of delivery"})} ]. mqtt_listener() -> diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 610555819..bf79085af 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -98,7 +98,8 @@ ]). -record(session, { - %% sessionID, fresh for all new sessions unless it is a resumed persistent session + %% Client's id + clientid :: emqx_types:clientid(), id :: sessionID(), %% Is this session a persistent session i.e. was it started with Session-Expiry > 0 is_persistent :: boolean(), @@ -128,9 +129,16 @@ %% Awaiting PUBREL Timeout (Unit: millsecond) await_rel_timeout :: timeout(), %% Created at - created_at :: pos_integer() + created_at :: pos_integer(), + %% Message deliver latency stats + latency_stats :: emqx_message_latency_stats:stats() }). +%% in the previous code, we will replace the message record with the pubrel atom +%% in the pubrec function, this will lose the creation time of the message, +%% but now we need this time to calculate latency, so now pubrel atom is changed to this record +-record(pubrel_await, {timestamp :: non_neg_integer()}). + -type(session() :: #session{}). -type(publish() :: {maybe(emqx_types:packet_id()), emqx_types:message()}). @@ -157,7 +165,8 @@ mqueue_dropped, next_pkt_id, awaiting_rel_cnt, - awaiting_rel_max + awaiting_rel_max, + latency_stats ]). -define(DEFAULT_BATCH_N, 1000). @@ -170,6 +179,8 @@ , max_inflight => integer() , mqueue => emqx_mqueue:options() , is_persistent => boolean() + , clientid => emqx_types:clientid() + , latency_stats => emqx_message_latency_stats:create_options() }. %%-------------------------------------------------------------------- @@ -185,6 +196,7 @@ init(Opts) -> }, maps:get(mqueue, Opts, #{})), #session{ id = emqx_guid:gen(), + clientid = maps:get(clientid, Opts, <<>>), is_persistent = maps:get(is_persistent, Opts, false), max_subscriptions = maps:get(max_subscriptions, Opts, infinity), subscriptions = #{}, @@ -196,7 +208,8 @@ init(Opts) -> awaiting_rel = #{}, max_awaiting_rel = maps:get(max_awaiting_rel, Opts, 100), await_rel_timeout = maps:get(await_rel_timeout, Opts, 300000), - created_at = erlang:system_time(millisecond) + created_at = erlang:system_time(millisecond), + latency_stats = emqx_message_latency_stats:new(maps:get(latency_stats, Opts, #{})) }. %%-------------------------------------------------------------------- @@ -252,7 +265,9 @@ info(awaiting_rel_max, #session{max_awaiting_rel = Max}) -> info(await_rel_timeout, #session{await_rel_timeout = Timeout}) -> Timeout; info(created_at, #session{created_at = CreatedAt}) -> - CreatedAt. + CreatedAt; +info(latency_stats, #session{latency_stats = Stats}) -> + emqx_message_latency_stats:latency(Stats). %% @doc Get stats of the session. -spec(stats(session()) -> emqx_types:stats()). @@ -365,7 +380,8 @@ puback(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> Inflight1 = emqx_inflight:delete(PacketId, Inflight), - return_with(Msg, dequeue(Session#session{inflight = Inflight1})); + Session2 = update_latency(Msg, Session), + return_with(Msg, dequeue(Session2#session{inflight = Inflight1})); {value, {_Pubrel, _Ts}} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> @@ -388,9 +404,10 @@ return_with(Msg, {ok, Publishes, Session}) -> pubrec(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of {value, {Msg, _Ts}} when is_record(Msg, message) -> - Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight), + Update = with_ts(#pubrel_await{timestamp = Msg#message.timestamp}), + Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), {ok, Msg, Session#session{inflight = Inflight1}}; - {value, {pubrel, _Ts}} -> + {value, {_Pubrel, _Ts}} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} @@ -419,9 +436,10 @@ pubrel(PacketId, Session = #session{awaiting_rel = AwaitingRel}) -> | {error, emqx_types:reason_code()}). pubcomp(PacketId, Session = #session{inflight = Inflight}) -> case emqx_inflight:lookup(PacketId, Inflight) of - {value, {pubrel, _Ts}} -> + {value, {Pubrel, _Ts}} when is_record(Pubrel, pubrel_await) -> + Session2 = update_latency(Pubrel, Session), Inflight1 = emqx_inflight:delete(PacketId, Inflight), - dequeue(Session#session{inflight = Inflight1}); + dequeue(Session2#session{inflight = Inflight1}); {value, _Other} -> {error, ?RC_PACKET_IDENTIFIER_IN_USE}; none -> @@ -588,11 +606,16 @@ await(PacketId, Msg, Session = #session{inflight = Inflight}) -> %%-------------------------------------------------------------------- -spec(retry(session()) -> {ok, session()} | {ok, replies(), timeout(), session()}). -retry(Session = #session{inflight = Inflight}) -> +retry(Session = #session{inflight = Inflight, retry_interval = RetryInterval}) -> case emqx_inflight:is_empty(Inflight) of true -> {ok, Session}; - false -> retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight), - [], erlang:system_time(millisecond), Session) + false -> + Now = erlang:system_time(millisecond), + Session2 = check_expire_latency(Now, RetryInterval, Session), + retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight), + [], + Now, + Session2) end. retry_delivery([], Acc, _Now, Session = #session{retry_interval = Interval}) -> @@ -619,8 +642,8 @@ retry_delivery(PacketId, Msg, Now, Acc, Inflight) when is_record(Msg, message) - {[{PacketId, Msg1}|Acc], Inflight1} end; -retry_delivery(PacketId, pubrel, Now, Acc, Inflight) -> - Inflight1 = emqx_inflight:update(PacketId, {pubrel, Now}, Inflight), +retry_delivery(PacketId, Pubrel, Now, Acc, Inflight) -> + Inflight1 = emqx_inflight:update(PacketId, {Pubrel, Now}, Inflight), {[{pubrel, PacketId}|Acc], Inflight1}. %%-------------------------------------------------------------------- @@ -664,7 +687,7 @@ resume(ClientInfo = #{clientid := ClientId}, Session = #session{subscriptions = -spec(replay(session()) -> {ok, replies(), session()}). replay(Session = #session{inflight = Inflight}) -> - Pubs = lists:map(fun({PacketId, {pubrel, _Ts}}) -> + Pubs = lists:map(fun({PacketId, {Pubrel, _Ts}}) when is_record(Pubrel, pubrel_await) -> {pubrel, PacketId}; ({PacketId, {Msg, _Ts}}) -> {PacketId, emqx_message:set_flag(dup, true, Msg)} @@ -715,6 +738,35 @@ next_pkt_id(Session = #session{next_pkt_id = ?MAX_PACKET_ID}) -> next_pkt_id(Session = #session{next_pkt_id = Id}) -> Session#session{next_pkt_id = Id + 1}. +%%-------------------------------------------------------------------- +%% Message Latency Stats +%%-------------------------------------------------------------------- +update_latency(Msg, + #session{clientid = ClientId, + latency_stats = Stats, + created_at = CreateAt} = S) -> + case get_birth_timestamp(Msg, CreateAt) of + 0 -> S; + Ts -> + Latency = erlang:system_time(millisecond) - Ts, + Stats2 = emqx_message_latency_stats:update(ClientId, Latency, Stats), + S#session{latency_stats = Stats2} + end. + +check_expire_latency(Now, Interval, + #session{clientid = ClientId, latency_stats = Stats} = S) -> + Stats2 = emqx_message_latency_stats:check_expire(ClientId, Now, Interval, Stats), + S#session{latency_stats = Stats2}. + +get_birth_timestamp(#message{timestamp = Ts}, CreateAt) when CreateAt =< Ts -> + Ts; + +get_birth_timestamp(#pubrel_await{timestamp = Ts}, CreateAt) when CreateAt =< Ts -> + Ts; + +get_birth_timestamp(_, _) -> + 0. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_slow_subs/emqx_message_latency_stats.erl b/apps/emqx/src/emqx_slow_subs/emqx_message_latency_stats.erl new file mode 100644 index 000000000..237e1b08d --- /dev/null +++ b/apps/emqx/src/emqx_slow_subs/emqx_message_latency_stats.erl @@ -0,0 +1,120 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_message_latency_stats). + +%% API +-export([new/1, update/3, check_expire/4, latency/1]). + +-export([get_threshold/0, update_threshold/1]). + +-define(NOW, erlang:system_time(millisecond)). +-define(MINIMUM_INSERT_INTERVAL, 1000). +-define(MINIMUM_THRESHOLD, 100). +-define(DEFAULT_THRESHOLD, 500). +-define(DEFAULT_SAMPLES, 10). +-define(THRESHOLD_KEY, {?MODULE, threshold}). + +-opaque stats() :: #{ ema := emqx_moving_average:ema() + , last_update_time := timestamp() + , last_access_time := timestamp() %% timestamp of last access top-k + , last_insert_value := non_neg_integer() + }. + +-type timestamp() :: non_neg_integer(). +-type timespan() :: number(). + +-type latency_type() :: average + | expire. + +-type create_options() :: #{samples => pos_integer()}. + +-export_type([stats/0, latency_type/0, create_options/0]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new(non_neg_integer() | create_options()) -> stats(). +new(SamplesT) when is_integer(SamplesT) -> + Samples = erlang:max(1, SamplesT), + #{ ema => emqx_moving_average:new(exponential, #{period => Samples}) + , last_update_time => 0 + , last_access_time => 0 + , last_insert_value => 0 + }; + +new(OptsT) -> + Opts = maps:merge(#{samples => ?DEFAULT_SAMPLES}, OptsT), + #{samples := Samples} = Opts, + new(Samples). + +-spec update(emqx_types:clientid(), number(), stats()) -> stats(). +update(ClientId, Val, #{ema := EMA} = Stats) -> + Now = ?NOW, + #{average := Latency} = EMA2 = emqx_moving_average:update(Val, EMA), + Stats2 = call_hook(ClientId, Now, average, Latency, Stats), + Stats2#{ ema := EMA2 + , last_update_time := ?NOW}. + +-spec check_expire(emqx_types:clientid(), timestamp(), timespan(), stats()) -> stats(). +check_expire(_, Now, Interval, #{last_update_time := LUT} = S) + when LUT >= Now - Interval -> + S; + +check_expire(ClientId, Now, _Interval, #{last_update_time := LUT} = S) -> + Latency = Now - LUT, + call_hook(ClientId, Now, expire, Latency, S). + +-spec latency(stats()) -> number(). +latency(#{ema := #{average := Average}}) -> + Average. + +-spec update_threshold(pos_integer()) -> pos_integer(). +update_threshold(Threshold) -> + Val = erlang:max(Threshold, ?MINIMUM_THRESHOLD), + persistent_term:put(?THRESHOLD_KEY, Val), + Val. + +get_threshold() -> + persistent_term:get(?THRESHOLD_KEY, ?DEFAULT_THRESHOLD). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +-spec call_hook(emqx_types:clientid(), timestamp(), latency_type(), timespan(), stats()) -> stats(). +call_hook(_, _, _, Latency, S) + when Latency =< ?MINIMUM_THRESHOLD -> + S; + +call_hook(_, Now, _, _, #{last_access_time := LIT} = S) + when Now =< LIT + ?MINIMUM_INSERT_INTERVAL -> + S; + +call_hook(ClientId, Now, Type, Latency, #{last_insert_value := LIV} = Stats) -> + case Latency =< get_threshold() of + true -> + Stats#{last_access_time := Now}; + _ -> + ToInsert = erlang:floor(Latency), + Arg = #{clientid => ClientId, + latency => ToInsert, + type => Type, + last_insert_value => LIV, + update_time => Now}, + emqx:run_hook('message.slow_subs_stats', [Arg]), + Stats#{last_insert_value := ToInsert, + last_access_time := Now} + end. diff --git a/apps/emqx/src/emqx_slow_subs/emqx_moving_average.erl b/apps/emqx/src/emqx_slow_subs/emqx_moving_average.erl new file mode 100644 index 000000000..64c73f987 --- /dev/null +++ b/apps/emqx/src/emqx_slow_subs/emqx_moving_average.erl @@ -0,0 +1,90 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @see https://en.wikipedia.org/wiki/Moving_average + +-module(emqx_moving_average). + +%% API +-export([new/0, new/1, new/2, update/2]). + +-type type() :: cumulative + | exponential. + +-type ema() :: #{ type := exponential + , average := 0 | float() + , coefficient := float() + }. + +-type cma() :: #{ type := cumulative + , average := 0 | float() + , count := non_neg_integer() + }. + +-type moving_average() :: ema() + | cma(). + +-define(DEF_EMA_ARG, #{period => 10}). +-define(DEF_AVG_TYPE, exponential). + +-export_type([type/0, moving_average/0, ema/0, cma/0]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new() -> moving_average(). +new() -> + new(?DEF_AVG_TYPE, #{}). + +-spec new(type()) -> moving_average(). +new(Type) -> + new(Type, #{}). + +-spec new(type(), Args :: map()) -> moving_average(). +new(cumulative, _) -> + #{ type => cumulative + , average => 0 + , count => 0 + }; + +new(exponential, Arg) -> + #{period := Period} = maps:merge(?DEF_EMA_ARG, Arg), + #{ type => exponential + , average => 0 + %% coefficient = 2/(N+1) is a common convention, see the wiki link for details + , coefficient => 2 / (Period + 1) + }. + +-spec update(number(), moving_average()) -> moving_average(). + +update(Val, #{average := 0} = Avg) -> + Avg#{average := Val}; + +update(Val, #{ type := cumulative + , average := Average + , count := Count} = CMA) -> + NewCount = Count + 1, + CMA#{average := (Count * Average + Val) / NewCount, + count := NewCount}; + +update(Val, #{ type := exponential + , average := Average + , coefficient := Coefficient} = EMA) -> + EMA#{average := Coefficient * Val + (1 - Coefficient) * Average}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_zone_schema.erl b/apps/emqx/src/emqx_zone_schema.erl index 1d24f9481..9d8348c49 100644 --- a/apps/emqx/src/emqx_zone_schema.erl +++ b/apps/emqx/src/emqx_zone_schema.erl @@ -24,7 +24,7 @@ namespace() -> zone. %% roots are added only for document generation. roots() -> ["mqtt", "stats", "flapping_detect", "force_shutdown", "conn_congestion", "rate_limit", "quota", "force_gc", - "overload_protection" + "overload_protection", "latency_stats" ]. %% zone schemas are clones from the same name from root level diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index 6462fffed..32aba9674 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -100,6 +100,7 @@ clientinfo() -> %% See emqx_session:session() type define sessioninfo() -> ?LET(Session, {session, + clientid(), sessionid(), % id boolean(), % is_persistent subscriptions(), % subscriptions @@ -112,7 +113,8 @@ sessioninfo() -> awaiting_rel(), % awaiting_rel non_neg_integer(), % max_awaiting_rel safty_timeout(), % await_rel_timeout - timestamp() % created_at + timestamp(), % created_at + latency_stats() }, emqx_session:info(Session)). @@ -336,6 +338,30 @@ normal_topic_filter() -> end end). +%% Type defined emqx_message_lantency_stats.erl - stats() +latency_stats() -> + Keys = [{threshold, number()}, + {ema, exp_moving_average()}, + {last_update_time, non_neg_integer()}, + {last_access_time, non_neg_integer()}, + {last_insert_value, non_neg_integer()} + ], + ?LET({Ks, M}, {Keys, map(limited_atom(), limited_any_term())}, + begin + maps:merge(maps:from_list(Ks), M) + end). + +%% Type defined emqx_moving_average.erl - ema() +exp_moving_average() -> + Keys = [{type, exponential}, + {average, number()}, + {coefficient, float()} + ], + ?LET({Ks, M}, {Keys, map(limited_atom(), limited_any_term())}, + begin + maps:merge(maps:from_list(Ks), M) + end). + %%-------------------------------------------------------------------- %% Basic Types %%-------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_session_SUITE.erl b/apps/emqx/test/emqx_session_SUITE.erl index 8e29b8201..e4a4945ce 100644 --- a/apps/emqx/test/emqx_session_SUITE.erl +++ b/apps/emqx/test/emqx_session_SUITE.erl @@ -24,6 +24,9 @@ all() -> emqx_common_test_helpers:all(?MODULE). +-define(NOW, erlang:system_time(millisecond)). +-record(pubrel_await, {timestamp :: non_neg_integer()}). + %%-------------------------------------------------------------------- %% CT callbacks %%-------------------------------------------------------------------- @@ -181,7 +184,7 @@ t_puback_with_dequeue(_) -> ?assertEqual(<<"t2">>, emqx_message:topic(Msg3)). t_puback_error_packet_id_in_use(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Inflight = emqx_inflight:insert(1, {#pubrel_await{timestamp = ?NOW}, ts(millisecond)}, emqx_inflight:new()), {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:puback(1, session(#{inflight => Inflight})). @@ -193,10 +196,10 @@ t_pubrec(_) -> Inflight = emqx_inflight:insert(2, {Msg, ts(millisecond)}, emqx_inflight:new()), Session = session(#{inflight => Inflight}), {ok, Msg, Session1} = emqx_session:pubrec(2, Session), - ?assertMatch([{pubrel, _}], emqx_inflight:values(emqx_session:info(inflight, Session1))). + ?assertMatch([{{pubrel_await, _}, _}], emqx_inflight:values(emqx_session:info(inflight, Session1))). t_pubrec_packet_id_in_use_error(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Inflight = emqx_inflight:insert(1, {#pubrel_await{timestamp = ?NOW}, ts(millisecond)}, emqx_inflight:new()), {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:pubrec(1, session(#{inflight => Inflight})). @@ -212,7 +215,7 @@ t_pubrel_error_packetid_not_found(_) -> {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubrel(1, session()). t_pubcomp(_) -> - Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()), + Inflight = emqx_inflight:insert(1, {#pubrel_await{timestamp = ?NOW}, ts(millisecond)}, emqx_inflight:new()), Session = session(#{inflight => Inflight}), {ok, Session1} = emqx_session:pubcomp(1, Session), ?assertEqual(0, emqx_session:info(inflight_cnt, Session1)). @@ -261,7 +264,7 @@ t_deliver_qos0(_) -> t_deliver_qos1(_) -> ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), {ok, Session} = emqx_session:subscribe( - clientinfo(), <<"t1">>, subopts(#{qos => ?QOS_1}), session()), + clientinfo(), <<"t1">>, subopts(#{qos => ?QOS_1}), session()), Delivers = [delivery(?QOS_1, T) || T <- [<<"t1">>, <<"t2">>]], {ok, [{1, Msg1}, {2, Msg2}], Session1} = emqx_session:deliver(Delivers, Session), ?assertEqual(2, emqx_session:info(inflight_cnt, Session1)), @@ -399,4 +402,3 @@ ts(second) -> erlang:system_time(second); ts(millisecond) -> erlang:system_time(millisecond). - diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 9f03d1e7f..5511a6dd0 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -58,6 +58,7 @@ , emqx_psk_schema , emqx_limiter_schema , emqx_connector_schema + , emqx_slow_subs_schema ]). namespace() -> undefined. diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 8fd0a29a1..a2450f988 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -61,10 +61,18 @@ do_paginate(Qh, Count, Params, {Module, FormatFun}) -> query_handle(Table) when is_atom(Table) -> qlc:q([R || R <- ets:table(Table)]); + +query_handle({Table, Opts}) when is_atom(Table) -> + qlc:q([R || R <- ets:table(Table, Opts)]); + query_handle([Table]) when is_atom(Table) -> qlc:q([R || R <- ets:table(Table)]); + +query_handle([{Table, Opts}]) when is_atom(Table) -> + qlc:q([R || R <- ets:table(Table, Opts)]); + query_handle(Tables) -> - qlc:append([qlc:q([E || E <- ets:table(T)]) || T <- Tables]). + qlc:append([query_handle(T) || T <- Tables]). % query_handle(Table, MatchSpec) when is_atom(Table) -> Options = {traverse, {select, MatchSpec}}, @@ -78,8 +86,16 @@ query_handle(Tables, MatchSpec) -> count(Table) when is_atom(Table) -> ets:info(Table, size); + +count({Table, _}) when is_atom(Table) -> + ets:info(Table, size); + count([Table]) when is_atom(Table) -> ets:info(Table, size); + +count([{Table, _}]) when is_atom(Table) -> + ets:info(Table, size); + count(Tables) -> lists:sum([count(T) || T <- Tables]). diff --git a/apps/emqx_slow_subs/etc/emqx_slow_subs.conf b/apps/emqx_slow_subs/etc/emqx_slow_subs.conf new file mode 100644 index 000000000..3cf8189d8 --- /dev/null +++ b/apps/emqx_slow_subs/etc/emqx_slow_subs.conf @@ -0,0 +1,40 @@ +##-------------------------------------------------------------------- +## EMQ X Slow Subscribers Statistics +##-------------------------------------------------------------------- + +emqx_slow_subs { + enable = false + + threshold = 500ms + ## The latency threshold for statistics, the minimum value is 100ms + ## + ## Default: 500ms + + ## The eviction time of the record, which in the statistics record table + ## + ## Default: 5m + expire_interval = 5m + + ## The maximum number of records in the slow subscription statistics record table + ## + ## Value: 10 + top_k_num = 10 + + ## The interval for pushing statistics table records to the system topic. When set to 0, push is disabled + ## publish topk list to $SYS/brokers/${node}/slow_subs per notice_interval + ## publish is disabled if set to 0s. + ## + ## Value: 0s + expire_interval = 0s + + ## QoS of notification message + ## + ## Defaut: 0 + notice_qos = 0 + + ## Maximum information number in one notification + ## + ## Default: 100 + notice_batch_size = 100 + +} diff --git a/apps/emqx_slow_subs/include/emqx_slow_subs.hrl b/apps/emqx_slow_subs/include/emqx_slow_subs.hrl new file mode 100644 index 000000000..0b5e3a035 --- /dev/null +++ b/apps/emqx_slow_subs/include/emqx_slow_subs.hrl @@ -0,0 +1,28 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(TOPK_TAB, emqx_slow_subs_topk). + +-define(INDEX(Latency, ClientId), {Latency, ClientId}). + +-record(top_k, { index :: index() + , type :: emqx_message_latency_stats:latency_type() + , last_update_time :: pos_integer() + , extra = [] + }). + +-type top_k() :: #top_k{}. +-type index() :: ?INDEX(non_neg_integer(), emqx_types:clientid()). diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src new file mode 100644 index 000000000..593170c37 --- /dev/null +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src @@ -0,0 +1,12 @@ +{application, emqx_slow_subs, + [{description, "EMQ X Slow Subscribers Statistics"}, + {vsn, "1.0.0"}, % strict semver, bump manually! + {modules, []}, + {registered, [emqx_slow_subs_sup]}, + {applications, [kernel,stdlib]}, + {mod, {emqx_slow_subs_app,[]}}, + {env, []}, + {licenses, ["Apache-2.0"]}, + {maintainers, ["EMQ X Team "]}, + {links, []} + ]}. diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.erl b/apps/emqx_slow_subs/src/emqx_slow_subs.erl new file mode 100644 index 000000000..acb4ea441 --- /dev/null +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.erl @@ -0,0 +1,318 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs). + +-behaviour(gen_server). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_slow_subs/include/emqx_slow_subs.hrl"). + +-export([ start_link/0, on_stats_update/2, update_settings/1 + , clear_history/0, init_topk_tab/0 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-compile(nowarn_unused_type). + +-type state() :: #{ enable := boolean() + , last_tick_at := pos_integer() + }. + +-type log() :: #{ rank := pos_integer() + , clientid := emqx_types:clientid() + , latency := non_neg_integer() + , type := emqx_message_latency_stats:latency_type() + }. + +-type window_log() :: #{ last_tick_at := pos_integer() + , logs := [log()] + }. + +-type message() :: #message{}. + +-type stats_update_args() :: #{ clientid := emqx_types:clientid() + , latency := non_neg_integer() + , type := emqx_message_latency_stats:latency_type() + , last_insert_value := non_neg_integer() + , update_time := timer:time() + }. + +-type stats_update_env() :: #{max_size := pos_integer()}. + +-ifdef(TEST). +-define(EXPIRE_CHECK_INTERVAL, timer:seconds(1)). +-else. +-define(EXPIRE_CHECK_INTERVAL, timer:seconds(10)). +-endif. + +-define(NOW, erlang:system_time(millisecond)). +-define(NOTICE_TOPIC_NAME, "slow_subs"). +-define(DEF_CALL_TIMEOUT, timer:seconds(10)). + +%% erlang term order +%% number < atom < reference < fun < port < pid < tuple < list < bit string + +%% ets ordered_set is ascending by term order + +%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- +%% @doc Start the st_statistics +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%% XXX NOTE:pay attention to the performance here +-spec on_stats_update(stats_update_args(), stats_update_env()) -> true. +on_stats_update(#{clientid := ClientId, + latency := Latency, + type := Type, + last_insert_value := LIV, + update_time := Ts}, + #{max_size := MaxSize}) -> + + LastIndex = ?INDEX(LIV, ClientId), + Index = ?INDEX(Latency, ClientId), + + %% check whether the client is in the table + case ets:lookup(?TOPK_TAB, LastIndex) of + [#top_k{index = Index}] -> + %% if last value == the new value, update the type and last_update_time + %% XXX for clients whose latency are stable for a long time, is it possible to reduce updates? + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}); + [_] -> + %% if Latency > minimum value, we should update it + %% if Latency < minimum value, maybe it can replace the minimum value + %% so alwyas update at here + %% do we need check if Latency == minimum ??? + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}), + ets:delete(?TOPK_TAB, LastIndex); + [] -> + %% try to insert + try_insert_to_topk(MaxSize, Index, Latency, Type, Ts) + end. + +clear_history() -> + gen_server:call(?MODULE, ?FUNCTION_NAME, ?DEF_CALL_TIMEOUT). + +update_settings(Enable) -> + gen_server:call(?MODULE, {?FUNCTION_NAME, Enable}, ?DEF_CALL_TIMEOUT). + +init_topk_tab() -> + case ets:whereis(?TOPK_TAB) of + undefined -> + ?TOPK_TAB = ets:new(?TOPK_TAB, + [ ordered_set, public, named_table + , {keypos, #top_k.index}, {write_concurrency, true} + , {read_concurrency, true} + ]); + _ -> + ?TOPK_TAB + end. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + Enable = emqx:get_config([emqx_slow_subs, enable]), + {ok, check_enable(Enable, #{enable => false})}. + +handle_call({update_settings, Enable}, _From, State) -> + State2 = check_enable(Enable, State), + {reply, ok, State2}; + +handle_call(clear_history, _, State) -> + ets:delete_all_objects(?TOPK_TAB), + {reply, ok, State}; + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(expire_tick, State) -> + expire_tick(), + Logs = ets:tab2list(?TOPK_TAB), + do_clear(Logs), + {noreply, State}; + +handle_info(notice_tick, State) -> + notice_tick(), + Logs = ets:tab2list(?TOPK_TAB), + do_notification(Logs, State), + {noreply, State#{last_tick_at := ?NOW}}; + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _) -> + unload(), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +expire_tick() -> + erlang:send_after(?EXPIRE_CHECK_INTERVAL, self(), ?FUNCTION_NAME). + +notice_tick() -> + case emqx:get_config([emqx_slow_subs, notice_interval]) of + 0 -> ok; + Interval -> + erlang:send_after(Interval, self(), ?FUNCTION_NAME), + ok + end. + +-spec do_notification(list(), state()) -> ok. +do_notification([], _) -> + ok; + +do_notification(Logs, #{last_tick_at := LastTickTime}) -> + start_publish(Logs, LastTickTime), + ok. + +start_publish(Logs, TickTime) -> + emqx_pool:async_submit({fun do_publish/3, [Logs, erlang:length(Logs), TickTime]}). + +do_publish([], _, _) -> + ok; + +do_publish(Logs, Rank, TickTime) -> + BatchSize = emqx:get_config([emqx_slow_subs, notice_batch_size]), + do_publish(Logs, BatchSize, Rank, TickTime, []). + +do_publish([Log | T], Size, Rank, TickTime, Cache) when Size > 0 -> + Cache2 = [convert_to_notice(Rank, Log) | Cache], + do_publish(T, Size - 1, Rank - 1, TickTime, Cache2); + +do_publish(Logs, Size, Rank, TickTime, Cache) when Size =:= 0 -> + publish(TickTime, Cache), + do_publish(Logs, Rank, TickTime); + +do_publish([], _, _Rank, TickTime, Cache) -> + publish(TickTime, Cache), + ok. + +convert_to_notice(Rank, #top_k{index = ?INDEX(Latency, ClientId), + type = Type, + last_update_time = Ts}) -> + #{rank => Rank, + clientid => ClientId, + latency => Latency, + type => Type, + timestamp => Ts}. + +publish(TickTime, Notices) -> + WindowLog = #{last_tick_at => TickTime, + logs => lists:reverse(Notices)}, + Payload = emqx_json:encode(WindowLog), + Msg = #message{ id = emqx_guid:gen() + , qos = emqx:get_config([emqx_slow_subs, notice_qos]) + , from = ?MODULE + , topic = emqx_topic:systop(?NOTICE_TOPIC_NAME) + , payload = Payload + , timestamp = ?NOW + }, + _ = emqx_broker:safe_publish(Msg), + ok. + +load() -> + MaxSize = emqx:get_config([emqx_slow_subs, top_k_num]), + _ = emqx:hook('message.slow_subs_stats', + {?MODULE, on_stats_update, [#{max_size => MaxSize}]} + ), + ok. + +unload() -> + emqx:unhook('message.slow_subs_stats', {?MODULE, on_stats_update}). + +do_clear(Logs) -> + Now = ?NOW, + Interval = emqx:get_config([emqx_slow_subs, expire_interval]), + Each = fun(#top_k{index = Index, last_update_time = Ts}) -> + case Now - Ts >= Interval of + true -> + ets:delete(?TOPK_TAB, Index); + _ -> + true + end + end, + lists:foreach(Each, Logs). + +try_insert_to_topk(MaxSize, Index, Latency, Type, Ts) -> + case ets:info(?TOPK_TAB, size) of + Size when Size < MaxSize -> + %% if the size is under limit, insert it directly + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}); + _Size -> + %% find the minimum value + ?INDEX(Min, _) = First = + case ets:first(?TOPK_TAB) of + ?INDEX(_, _) = I -> I; + _ -> ?INDEX(Latency - 1, <<>>) + end, + + case Latency =< Min of + true -> true; + _ -> + ets:insert(?TOPK_TAB, + #top_k{index = Index, type = Type, last_update_time = Ts}), + + ets:delete(?TOPK_TAB, First) + end + end. + +check_enable(Enable, #{enable := IsEnable} = State) -> + update_threshold(), + case Enable of + IsEnable -> + State; + true -> + notice_tick(), + expire_tick(), + load(), + State#{enable := true, last_tick_at => ?NOW}; + _ -> + unload(), + State#{enable := false} + end. + +update_threshold() -> + Threshold = emqx:get_config([emqx_slow_subs, threshold]), + emqx_message_latency_stats:update_threshold(Threshold), + ok. diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl new file mode 100644 index 000000000..8af4f14ea --- /dev/null +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl @@ -0,0 +1,108 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx_slow_subs/include/emqx_slow_subs.hrl"). + +-export([api_spec/0, paths/0, schema/1, fields/1, namespace/0]). + +-export([slow_subs/2, encode_record/1, settings/2]). + +-import(hoconsc, [mk/2, ref/1]). +-import(emqx_mgmt_util, [bad_request/0]). + +-define(FORMAT_FUN, {?MODULE, encode_record}). +-define(APP, emqx_slow_subs). +-define(APP_NAME, <<"emqx_slow_subs">>). + +namespace() -> "slow_subscribers_statistics". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE). + +paths() -> ["/slow_subscriptions", "/slow_subscriptions/settings"]. + +schema(("/slow_subscriptions")) -> + #{ + 'operationId' => slow_subs, + delete => #{tags => [<<"slow subs">>], + description => <<"Clear current data and re count slow topic">>, + parameters => [], + 'requestBody' => [], + responses => #{204 => <<"No Content">>} + }, + get => #{tags => [<<"slow subs">>], + description => <<"Get slow topics statistics record data">>, + parameters => [ {page, mk(integer(), #{in => query})} + , {limit, mk(integer(), #{in => query})} + ], + 'requestBody' => [], + responses => #{200 => [{data, mk(hoconsc:array(ref(record)), #{})}]} + } + }; + +schema("/slow_subscriptions/settings") -> + #{'operationId' => settings, + get => #{tags => [<<"slow subs">>], + description => <<"Get slow subs settings">>, + responses => #{200 => conf_schema()} + }, + put => #{tags => [<<"slow subs">>], + description => <<"Update slow subs settings">>, + 'requestBody' => conf_schema(), + responses => #{200 => conf_schema()} + } + }. + +fields(record) -> + [ + {clientid, mk(string(), #{desc => <<"the clientid">>})}, + {latency, mk(integer(), #{desc => <<"average time for message delivery or time for message expire">>})}, + {type, mk(string(), #{desc => <<"type of the latency, could be average or expire">>})}, + {last_update_time, mk(integer(), #{desc => <<"the timestamp of last update">>})} + ]. + +conf_schema() -> + Ref = hoconsc:ref(emqx_slow_subs_schema, "emqx_slow_subs"), + hoconsc:mk(Ref, #{}). + +slow_subs(delete, _) -> + ok = emqx_slow_subs:clear_history(), + {204}; + +slow_subs(get, #{query_string := QS}) -> + Data = emqx_mgmt_api:paginate({?TOPK_TAB, [{traverse, last_prev}]}, QS, ?FORMAT_FUN), + {200, Data}. + +encode_record(#top_k{index = ?INDEX(Latency, ClientId), + type = Type, + last_update_time = Ts}) -> + #{clientid => ClientId, + latency => Latency, + type => Type, + last_update_time => Ts}. + +settings(get, _) -> + {200, emqx:get_raw_config([?APP_NAME], #{})}; + +settings(put, #{body := Body}) -> + {ok, #{config := #{enable := Enable}}} = emqx:update_config([?APP], Body), + _ = emqx_slow_subs:update_settings(Enable), + {200, emqx:get_raw_config([?APP_NAME], #{})}. diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_app.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_app.erl new file mode 100644 index 000000000..d171b0a4f --- /dev/null +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_app.erl @@ -0,0 +1,30 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs_app). + +-behaviour(application). + +-export([ start/2 + , stop/1 + ]). + +start(_Type, _Args) -> + {ok, Sup} = emqx_slow_subs_sup:start_link(), + {ok, Sup}. + +stop(_State) -> + ok. diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl new file mode 100644 index 000000000..7bdcb16f3 --- /dev/null +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -0,0 +1,44 @@ +-module(emqx_slow_subs_schema). + +-include_lib("typerefl/include/types.hrl"). + +-export([roots/0, fields/1]). + +roots() -> ["emqx_slow_subs"]. + +fields("emqx_slow_subs") -> + [ {enable, sc(boolean(), false, "switch of this function")} + , {threshold, + sc(emqx_schema:duration_ms(), + "500ms", + "The latency threshold for statistics, the minimum value is 100ms")} + , {expire_interval, + sc(emqx_schema:duration_ms(), + "5m", + "The eviction time of the record, which in the statistics record table")} + , {top_k_num, + sc(integer(), + 10, + "The maximum number of records in the slow subscription statistics record table")} + , {notice_interval, + sc(emqx_schema:duration_ms(), + "0s", + "The interval for pushing statistics table records to the system topic. When set to 0, push is disabled" + "publish topk list to $SYS/brokers/${node}/slow_subs per notice_interval" + "publish is disabled if set to 0s." + )} + , {notice_qos, + sc(range(0, 2), + 0, + "QoS of notification message in notice topic")} + , {notice_batch_size, + sc(integer(), + 0, + "Maximum information number in one notification")} + ]. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +sc(Type, Default, Desc) -> + hoconsc:mk(Type, #{default => Default, desc => Desc}). diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_sup.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_sup.erl new file mode 100644 index 000000000..a6ad72c74 --- /dev/null +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_sup.erl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + emqx_slow_subs:init_topk_tab(), + {ok, {{one_for_one, 10, 3600}, + [#{id => st_statistics, + start => {emqx_slow_subs, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_slow_subs]}]}}. diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl new file mode 100644 index 000000000..f66122775 --- /dev/null +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl @@ -0,0 +1,124 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-define(TOPK_TAB, emqx_slow_subs_topk). +-define(NOW, erlang:system_time(millisecond)). + +-define(BASE_CONF, <<""" +emqx_slow_subs { + enable = true + top_k_num = 5, + expire_interval = 3000 + notice_interval = 1500 + notice_qos = 0 + notice_batch_size = 3 +}""">>). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_config:init_load(emqx_slow_subs_schema, ?BASE_CONF), + emqx_common_test_helpers:start_apps([emqx_slow_subs]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_slow_subs]). + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _) -> + ok. + +%%-------------------------------------------------------------------- +%% Test Cases +%%-------------------------------------------------------------------- +t_log_and_pub(_) -> + %% Sub topic first + Subs = [{<<"/test1/+">>, ?QOS_1}, {<<"/test2/+">>, ?QOS_2}], + Clients = start_client(Subs), + emqx:subscribe("$SYS/brokers/+/slow_subs"), + timer:sleep(1000), + Now = ?NOW, + %% publish + + lists:foreach(fun(I) -> + Topic = list_to_binary(io_lib:format("/test1/~p", [I])), + Msg = emqx_message:make(undefined, ?QOS_1, Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 500}) + end, + lists:seq(1, 10)), + + lists:foreach(fun(I) -> + Topic = list_to_binary(io_lib:format("/test2/~p", [I])), + Msg = emqx_message:make(undefined, ?QOS_2, Topic, <<"Hello">>), + emqx:publish(Msg#message{timestamp = Now - 500}) + end, + lists:seq(1, 10)), + + timer:sleep(1000), + Size = ets:info(?TOPK_TAB, size), + %% some time record maybe delete due to it expired + ?assert(Size =< 6 andalso Size >= 4), + + timer:sleep(1500), + Recs = try_receive([]), + RecSum = lists:sum(Recs), + ?assert(RecSum >= 5), + ?assert(lists:all(fun(E) -> E =< 3 end, Recs)), + + timer:sleep(2000), + ?assert(ets:info(?TOPK_TAB, size) =:= 0), + [Client ! stop || Client <- Clients], + ok. + +start_client(Subs) -> + [spawn(fun() -> client(I, Subs) end) || I <- lists:seq(1, 10)]. + +client(I, Subs) -> + {ok, C} = emqtt:start_link([{host, "localhost"}, + {clientid, io_lib:format("slow_subs_~p", [I])}, + {username, <<"plain">>}, + {password, <<"plain">>}]), + {ok, _} = emqtt:connect(C), + + Len = erlang:length(Subs), + Sub = lists:nth(I rem Len + 1, Subs), + _ = emqtt:subscribe(C, Sub), + + receive + stop -> + ok + end. + +try_receive(Acc) -> + receive + {deliver, _, #message{payload = Payload}} -> + #{<<"logs">> := Logs} = emqx_json:decode(Payload, [return_maps]), + try_receive([length(Logs) | Acc]) + after 500 -> + Acc + end. diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl new file mode 100644 index 000000000..009feda01 --- /dev/null +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl @@ -0,0 +1,174 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_slow_subs_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx_management/include/emqx_mgmt.hrl"). +-include_lib("emqx_slow_subs/include/emqx_slow_subs.hrl"). + +-define(HOST, "http://127.0.0.1:18083/"). + +-define(API_VERSION, "v5"). + +-define(BASE_PATH, "api"). +-define(NOW, erlang:system_time(millisecond)). + +-define(CONF_DEFAULT, <<""" +emqx_slow_subs +{ + enable = true + top_k_num = 5, + expire_interval = 60000 + notice_interval = 0 + notice_qos = 0 + notice_batch_size = 3 +}""">>). + + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_config:init_load(emqx_slow_subs_schema, ?CONF_DEFAULT), + emqx_mgmt_api_test_util:init_suite([emqx_slow_subs]), + {ok, _} = application:ensure_all_started(emqx_authn), + Config. + +end_per_suite(Config) -> + application:stop(emqx_authn), + emqx_mgmt_api_test_util:end_suite([emqx_slow_subs]), + Config. + +init_per_testcase(_, Config) -> + application:ensure_all_started(emqx_slow_subs), + timer:sleep(500), + Config. + +end_per_testcase(_, Config) -> + application:stop(emqx_slow_subs), + Config. + +t_get_history(_) -> + Now = ?NOW, + Each = fun(I) -> + ClientId = erlang:list_to_binary(io_lib:format("test_~p", [I])), + ets:insert(?TOPK_TAB, #top_k{index = ?INDEX(I, ClientId), + type = average, + last_update_time = Now}) + end, + + lists:foreach(Each, lists:seq(1, 5)), + + {ok, Data} = request_api(get, api_path(["slow_subscriptions"]), "_page=1&_limit=10", + auth_header_()), + #{<<"data">> := [First | _]} = emqx_json:decode(Data, [return_maps]), + + RFirst = #{<<"clientid">> => <<"test_5">>, + <<"latency">> => 5, + <<"type">> => <<"average">>, + <<"last_update_time">> => Now}, + + ?assertEqual(RFirst, First). + +t_clear(_) -> + ets:insert(?TOPK_TAB, #top_k{index = ?INDEX(1, <<"test">>), + type = average, + last_update_time = ?NOW}), + + {ok, _} = request_api(delete, api_path(["slow_subscriptions"]), [], + auth_header_()), + + ?assertEqual(0, ets:info(?TOPK_TAB, size)). + +t_settting(_) -> + Conf = emqx:get_config([emqx_slow_subs]), + Conf2 = Conf#{threshold => 1000}, + {ok, Data} = request_api(put, + api_path(["slow_subscriptions", "settings"]), + [], + auth_header_(), + Conf2), + + Return = decode_json(Data), + + ?assertEqual(Conf2, Return), + + {ok, GetData} = request_api(get, + api_path(["slow_subscriptions", "settings"]), + [], + auth_header_() + ), + + GetReturn = decode_json(GetData), + + ?assertEqual(Conf2, GetReturn), + + ?assertEqual(1000, + emqx_message_latency_stats:get_threshold()). + +decode_json(Data) -> + BinJosn = emqx_json:decode(Data, [return_maps]), + emqx_map_lib:unsafe_atom_key_map(BinJosn). + +request_api(Method, Url, Auth) -> + request_api(Method, Url, [], Auth, []). + +request_api(Method, Url, QueryParams, Auth) -> + request_api(Method, Url, QueryParams, Auth, []). + +request_api(Method, Url, QueryParams, Auth, []) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth]}); +request_api(Method, Url, QueryParams, Auth, Body) -> + NewUrl = case QueryParams of + "" -> Url; + _ -> Url ++ "?" ++ QueryParams + end, + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + +do_request_api(Method, Request)-> + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], [{body_format, binary}]) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _, Return} } + when Code =:= 200 orelse Code =:= 204 -> + {ok, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +auth_header_() -> + AppId = <<"admin">>, + AppSecret = <<"public">>, + auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)). + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User,":",Pass])), + {"Authorization","Basic " ++ Encoded}. + +api_path(Parts)-> + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts). diff --git a/rebar.config.erl b/rebar.config.erl index 2beabd147..a97d77333 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -305,6 +305,7 @@ relx_apps(ReleaseType, Edition) -> , emqx_statsd , emqx_prometheus , emqx_psk + , emqx_slow_subs ] ++ [quicer || is_quicer_supported()] %++ [emqx_license || is_enterprise(Edition)] From fddb28a4b08f5da5462240138626fdc43b45a22c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 10 Dec 2021 17:12:04 +0300 Subject: [PATCH 17/31] fix(wss): update cowboy & ranch for OTP24 compatibility --- apps/emqx/rebar.config | 2 +- apps/emqx/test/emqx_listeners_SUITE.erl | 46 +++++++++++++++++++------ rebar.config | 2 +- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index f8c6c23ce..330291def 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -13,7 +13,7 @@ , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} - , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} + , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 0733bad53..a9ee0d7d9 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -24,6 +24,8 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-define(CERTS_PATH(CertName), filename:join(["../../lib/emqx/etc/certs/", CertName])). + all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> @@ -43,20 +45,34 @@ init_per_testcase(Case, Config) {ok, _} = emqx_config_handler:start_link(), PrevListeners = emqx_config:get([listeners, tcp], #{}), PrevRateLimit = emqx_config:get([rate_limit], #{}), - emqx_config:put([listeners, tcp], #{ listener_test => - #{ bind => {"127.0.0.1", 9999} - , max_connections => 4321 - , limiter => #{} - } - }), + emqx_config:put( + [listeners, tcp], + #{listener_test => #{bind => {"127.0.0.1", 9999}, + max_connections => 4321, + limiter => #{} + } + }), emqx_config:put([rate_limit], #{max_conn_rate => 1000}), - ListenerConf = #{ bind => {"127.0.0.1", 9999} - }, ok = emqx_listeners:start(), - [ {listener_conf, ListenerConf} - , {prev_listener_conf, PrevListeners} + [ {prev_listener_conf, PrevListeners} , {prev_rate_limit_conf, PrevRateLimit} | Config]; +init_per_testcase(t_wss_conn, Config) -> + {ok, _} = emqx_config_handler:start_link(), + PrevListeners = emqx_config:get([listeners, wss], #{}), + emqx_config:put( + [listeners, wss], + #{listener_test => #{bind => {{127,0,0,1}, 9998}, + limiter => #{}, + ssl => #{cacertfile => ?CERTS_PATH("cacert.pem"), + certfile => ?CERTS_PATH("cert.pem"), + keyfile => ?CERTS_PATH("key.pem") + } + } + }), + ok = emqx_listeners:start(), + [ {prev_listener_conf, PrevListeners} + | Config]; init_per_testcase(_, Config) -> {ok, _} = emqx_config_handler:start_link(), Config. @@ -70,6 +86,12 @@ end_per_testcase(Case, Config) emqx_listeners:stop(), _ = emqx_config_handler:stop(), ok; +end_per_testcase(t_wss_conn, Config) -> + PrevListener = ?config(prev_listener_conf, Config), + emqx_config:put([listeners, wss], PrevListener), + emqx_listeners:stop(), + _ = emqx_config_handler:stop(), + ok; end_per_testcase(_, _Config) -> _ = emqx_config_handler:stop(), ok. @@ -93,6 +115,10 @@ t_max_conns_tcp(_) -> t_current_conns_tcp(_) -> ?assertEqual(0, emqx_listeners:current_conns('tcp:listener_test', {{127,0,0,1}, 9999})). +t_wss_conn(_) -> + {ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000), + ok = ssl:close(Socket). + render_config_file() -> Path = local_path(["etc", "emqx.conf"]), {ok, Temp} = file:read_file(Path), diff --git a/rebar.config b/rebar.config index 9d923da9d..1604c7696 100644 --- a/rebar.config +++ b/rebar.config @@ -50,7 +50,7 @@ , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.12"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} - , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} + , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} , {mria, {git, "https://github.com/emqx/mria", {tag, "0.1.4"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} From 7d36079f0aeebe203c116c608d4cf708bea1cdd2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 11 Dec 2021 22:17:52 +0100 Subject: [PATCH 18/31] docs(connector): better namespace for http schema --- apps/emqx_connector/src/emqx_connector_http.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index bae4e334b..0249d51b1 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -36,7 +36,9 @@ -export([ roots/0 , fields/1 - , validations/0]). + , validations/0 + , namespace/0 + ]). -export([ check_ssl_opts/2 ]). @@ -50,6 +52,9 @@ %%===================================================================== %% Hocon schema + +namespace() -> "connector-http". + roots() -> fields(config). From 41aa958ac948e30fa4e2e908eeb07f8d07a735bc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 11 Dec 2021 22:20:53 +0100 Subject: [PATCH 19/31] refactor(authn): root type as array of union Deleted the wrapping union. i.e. previously it's union([union([...]), array(union([...]))]). not it's array(union([...])) It is after-all a lazy type, and the dynamic check allows single-elemented arrays not to have `[]` around it. the old union of array of union schema was just adding confusion --- apps/emqx_authn/src/emqx_authn_schema.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index c2e963ec4..7ec318803 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -50,8 +50,7 @@ config_refs(Modules) -> %% in emqx_schema, 'authentication' is a map() type which is to allow %% EMQ X more plugable. root_type() -> - T = authenticator_type(), - hoconsc:union([T, hoconsc:array(T)]). + hoconsc:array(authenticator_type()). mechanism(Name) -> hoconsc:mk(hoconsc:enum([Name]), From fcb7c2c70ff602f623f0eca07c3efc4299b77747 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 12 Dec 2021 19:25:20 +0100 Subject: [PATCH 20/31] docs: change to {module} so markdown render will not crash is treated as an HTML tag --- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 93661ab53..36cf48da0 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -118,7 +118,8 @@ fields("user_provided_function") -> [ {function, sc(binary(), #{ desc => """ The user provided function. Should be in the format: '{module}:{function}'.
-Where the is the erlang callback module and the {function} is the erlang function.
+Where {module} is the Erlang callback module and {function} is the Erlang function. +
To write your own function, checkout the function console and republish in the source file: apps/emqx_rule_engine/src/emqx_rule_outputs.erl as an example. From 38ac10d3e20323038105277dbebb46ce1c72fc4e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 11 Dec 2021 20:22:32 +0100 Subject: [PATCH 21/31] feat(plugins): add emqx_plugins skeleton * added emqx_plugins app. * emqx_plugins.erl is moved from emqx app to emqx_plugins app same for the test SUITE --- apps/emqx/src/emqx_schema.erl | 10 -- apps/emqx_conf/src/emqx_conf_schema.erl | 1 + apps/emqx_plugins/etc/emqx_plugins.conf | 7 ++ apps/emqx_plugins/src/emqx_plugins.app.src | 9 ++ apps/emqx_plugins/src/emqx_plugins.appup.src | 8 ++ .../src/emqx_plugins.erl | 7 +- apps/emqx_plugins/src/emqx_plugins_app.erl | 30 ++++++ apps/emqx_plugins/src/emqx_plugins_schema.erl | 94 +++++++++++++++++++ apps/emqx_plugins/src/emqx_plugins_sup.erl | 30 ++++++ .../test/emqx_plugins_SUITE.erl | 4 +- .../emqx_hocon_plugin/Makefile | 0 .../etc/emqx_hocon_plugin.conf | 0 .../emqx_hocon_plugin/rebar.config | 0 .../src/emqx_hocon_plugin.app.src | 0 .../src/emqx_hocon_plugin_app.erl | 0 .../src/emqx_hocon_plugin_schema.erl | 0 .../emqx_mini_plugin/Makefile | 0 .../etc/emqx_mini_plugin.conf | 0 .../priv/emqx_mini_plugin.schema | 0 .../emqx_mini_plugin/rebar.config | 0 .../src/emqx_mini_plugin.app.src | 0 .../src/emqx_mini_plugin_app.erl | 0 rebar.config.erl | 1 + 23 files changed, 185 insertions(+), 16 deletions(-) create mode 100644 apps/emqx_plugins/etc/emqx_plugins.conf create mode 100644 apps/emqx_plugins/src/emqx_plugins.app.src create mode 100644 apps/emqx_plugins/src/emqx_plugins.appup.src rename apps/{emqx => emqx_plugins}/src/emqx_plugins.erl (98%) create mode 100644 apps/emqx_plugins/src/emqx_plugins_app.erl create mode 100644 apps/emqx_plugins/src/emqx_plugins_schema.erl create mode 100644 apps/emqx_plugins/src/emqx_plugins_sup.erl rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE.erl (97%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/Makefile (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/etc/emqx_hocon_plugin.conf (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/rebar.config (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_app.erl (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_mini_plugin/etc/emqx_mini_plugin.conf (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_mini_plugin/priv/emqx_mini_plugin.schema (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src (100%) rename apps/{emqx => emqx_plugins}/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin_app.erl (100%) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index c4f77b9a0..4a15597f0 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -157,9 +157,6 @@ roots(low) -> , {"quota", sc(ref("quota"), #{})} - , {"plugins", %% TODO: move to emqx_conf_schema - sc(ref("plugins"), - #{})} , {"stats", sc(ref("stats"), #{})} @@ -797,13 +794,6 @@ fields("deflate_opts") -> } ]; -fields("plugins") -> - [ {"expand_plugins_dir", - sc(string(), - #{}) - } - ]; - fields("broker") -> [ {"sys_msg_interval", sc(hoconsc:union([disabled, duration()]), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 9f03d1e7f..3136bf557 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -50,6 +50,7 @@ , emqx_authz_schema , emqx_auto_subscribe_schema , emqx_modules_schema + , emqx_plugins_schema , emqx_dashboard_schema , emqx_gateway_schema , emqx_prometheus_schema diff --git a/apps/emqx_plugins/etc/emqx_plugins.conf b/apps/emqx_plugins/etc/emqx_plugins.conf new file mode 100644 index 000000000..35f02a9f8 --- /dev/null +++ b/apps/emqx_plugins/etc/emqx_plugins.conf @@ -0,0 +1,7 @@ +plugins { + prebuilt { + } + external { + } + install_dir = "plugins" +} diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src new file mode 100644 index 000000000..a772f219f --- /dev/null +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -0,0 +1,9 @@ +%% -*- mode: erlang -*- +{application, emqx_plugins, + [{description, "EMQ X Plugin Management"}, + {vsn, "0.1.0"}, + {modules, []}, + {mod, {emqx_plugins_app,[]}}, + {applications, [kernel,stdlib,emqx]}, + {env, []} + ]}. diff --git a/apps/emqx_plugins/src/emqx_plugins.appup.src b/apps/emqx_plugins/src/emqx_plugins.appup.src new file mode 100644 index 000000000..f9474dd33 --- /dev/null +++ b/apps/emqx_plugins/src/emqx_plugins.appup.src @@ -0,0 +1,8 @@ +%% -*- mode: erlang -*- +{"0.1.0", + [ {<<".*">>, []} + ], + [ + {<<".*">>, []} + ] +}. diff --git a/apps/emqx/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl similarity index 98% rename from apps/emqx/src/emqx_plugins.erl rename to apps/emqx_plugins/src/emqx_plugins.erl index c28ead717..003ca7ec3 100644 --- a/apps/emqx/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -16,9 +16,8 @@ -module(emqx_plugins). --include("emqx.hrl"). --include("logger.hrl"). - +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). -export([ load/0 , load/1 @@ -41,7 +40,7 @@ %% @doc Load all plugins when the broker started. -spec(load() -> ok | ignore | {error, term()}). load() -> - ok = load_ext_plugins(emqx:get_config([plugins, expand_plugins_dir], undefined)). + ok = load_ext_plugins(emqx:get_config([plugins, install_dir], undefined)). %% @doc Load a Plugin -spec(load(atom()) -> ok | {error, term()}). diff --git a/apps/emqx_plugins/src/emqx_plugins_app.erl b/apps/emqx_plugins/src/emqx_plugins_app.erl new file mode 100644 index 000000000..c04fbb445 --- /dev/null +++ b/apps/emqx_plugins/src/emqx_plugins_app.erl @@ -0,0 +1,30 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_plugins_app). + +-behaviour(application). + +-export([ start/2 + , stop/1 + ]). + +start(_Type, _Args) -> + {ok, Sup} = emqx_plugins_sup:start_link(), + {ok, Sup}. + +stop(_State) -> + ok. diff --git a/apps/emqx_plugins/src/emqx_plugins_schema.erl b/apps/emqx_plugins/src/emqx_plugins_schema.erl new file mode 100644 index 000000000..7b7b3c15e --- /dev/null +++ b/apps/emqx_plugins/src/emqx_plugins_schema.erl @@ -0,0 +1,94 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_plugins_schema). + +-behaviour(hocon_schema). + +-export([ roots/0 + , fields/1 + ]). + +-include_lib("typerefl/include/types.hrl"). + +roots() -> ["plugins"]. + +fields("plugins") -> + #{fields => fields(), + desc => """ +Manage EMQ X plugins. +
+Plugins can be pre-built as a part of EMQ X package, +or installed as a standalone package to the specific directory. +The standalone-installed plugins are referred to as 'external' plugins. +""" + }. + +fields() -> + [ {prebuilt, fun prebuilt/1} + , {external, fun external/1} + , {install_dir, fun install_dir/1} + ]. + +prebuilt(type) -> hoconsc:map("name", boolean()); +prebuilt(nullable) -> true; +prebuilt(T) when T=/= desc -> undefined; +prebuilt(desc) -> """ +A map() from plugin name to a boolean (true | false) flag to indicate +whether or not to enable the prebuilt plugin. +
+Most of the prebuilt plugins from 4.x are converted into features since 5.0. +""" ++ prebuilt_plugins() ++ +""" +
+Enabled plugins are loaded (started) as a part of EMQ X node's boot sequence. +Plugins can be loaded on the fly, and enabled from dashbaord UI and/or CLI. +
+Example config: {emqx_foo_bar: true, emqx_bazz: false} +""". + +external(type) -> hoconsc:map("name", string()); +external(nullable) -> true; +external(T) when T =/= desc -> undefined; +external(desc) -> +""" +A map from plugin name to a version number string for enabled ones. +To disable an external plugin, set the value to 'false'. +
+Enabled plugins are loaded (started) as a part of EMQ X node's boot sequence. +Plugins can be loaded on the fly, and enabled from dashbaord UI and/or CLI. +
+Example config: {emqx_extplug1: \"0.1.0\", emqx_extplug2: false} +""". + +install_dir(type) -> string(); +install_dir(nullable) -> true; +install_dir(default) -> "plugins"; %% runner's root dir +install_dir(T) when T =/= desc -> undefined; +install_dir(desc) -> """ +In which directory are the external plugins installed. +The plugin beam files and configuration files should reside in +the sub-directory named as emqx_foo_bar-0.1.0. +""". + +%% TODO: when we have some prebuilt plugins, change this function to: +%% """ +%% The names should be one of +%% - name1 +%% - name2 +%% """ +prebuilt_plugins() -> + "So far, we do not have any prebuilt plugins". diff --git a/apps/emqx_plugins/src/emqx_plugins_sup.erl b/apps/emqx_plugins/src/emqx_plugins_sup.erl new file mode 100644 index 000000000..c1e26752e --- /dev/null +++ b/apps/emqx_plugins/src/emqx_plugins_sup.erl @@ -0,0 +1,30 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_plugins_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + Children = [], + {ok, {{one_for_one, 10, 10}, Children}}. diff --git a/apps/emqx/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl similarity index 97% rename from apps/emqx/test/emqx_plugins_SUITE.erl rename to apps/emqx_plugins/test/emqx_plugins_SUITE.erl index 3aba5a997..2281f09dc 100644 --- a/apps/emqx/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -42,7 +42,7 @@ init_per_suite(Config) -> emqx_common_test_helpers:boot_modules([]), emqx_common_test_helpers:start_apps([]), - emqx_config:put([plugins, expand_plugins_dir], DataPath), + emqx_config:put([plugins, install_dir], DataPath), ?assertEqual(ok, emqx_plugins:load()), Config. @@ -57,7 +57,7 @@ t_load(_) -> ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_mini_plugin)), ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_hocon_plugin)), - emqx_config:put([plugins, expand_plugins_dir], undefined). + emqx_config:put([plugins, install_dir], undefined). t_load_ext_plugin(_) -> ?assertError({plugin_app_file_not_found, _}, diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/Makefile b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/Makefile similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/Makefile rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/Makefile diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/etc/emqx_hocon_plugin.conf b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/etc/emqx_hocon_plugin.conf similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/etc/emqx_hocon_plugin.conf rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/etc/emqx_hocon_plugin.conf diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/rebar.config b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/rebar.config similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/rebar.config rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/rebar.config diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_app.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_app.erl similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_app.erl rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_app.erl diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/etc/emqx_mini_plugin.conf b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/etc/emqx_mini_plugin.conf similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/etc/emqx_mini_plugin.conf rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/etc/emqx_mini_plugin.conf diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/priv/emqx_mini_plugin.schema b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/priv/emqx_mini_plugin.schema similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/priv/emqx_mini_plugin.schema rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/priv/emqx_mini_plugin.schema diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin_app.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin_app.erl similarity index 100% rename from apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin_app.erl rename to apps/emqx_plugins/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin_app.erl diff --git a/rebar.config.erl b/rebar.config.erl index 2beabd147..f63fedd84 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -305,6 +305,7 @@ relx_apps(ReleaseType, Edition) -> , emqx_statsd , emqx_prometheus , emqx_psk + , emqx_plugins ] ++ [quicer || is_quicer_supported()] %++ [emqx_license || is_enterprise(Edition)] From d3d8b3a01f146e87b911c832fc9422342ea705ae Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 12 Dec 2021 19:54:13 +0100 Subject: [PATCH 22/31] fix: allow listener not_found before restart --- apps/emqx/src/emqx_listeners.erl | 5 +++-- apps/emqx/test/emqx_common_test_helpers.erl | 4 ++-- apps/emqx/test/emqx_listeners_SUITE.erl | 7 +------ apps/emqx/test/emqx_persistent_session_SUITE.erl | 3 --- apps/emqx_plugins/etc/emqx_plugins.conf | 2 +- apps/emqx_plugins/test/emqx_plugins_SUITE.erl | 6 ++++-- apps/emqx_retainer/test/emqx_retainer_SUITE.erl | 1 + rebar.config.erl | 2 +- 8 files changed, 13 insertions(+), 17 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 2af2673d1..55c7d2715 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -175,9 +175,10 @@ restart_listener(Type, ListenerName, Conf) -> restart_listener(Type, ListenerName, Conf, Conf). restart_listener(Type, ListenerName, OldConf, NewConf) -> - case stop_listener(Type, ListenerName, OldConf) of + case do_stop_listener(Type, ListenerName, OldConf) of ok -> start_listener(Type, ListenerName, NewConf); - Error -> Error + {error, not_found} -> start_listener(Type, ListenerName, NewConf); + {error, Reason} -> {error, Reason} end. %% @doc Stop all listeners. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index a5d80bab5..c4336b855 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -120,6 +120,7 @@ all(Suite) -> string:substr(atom_to_list(F), 1, 2) == "t_" ]). +%% set emqx app boot modules -spec(boot_modules(all|list(atom())) -> ok). boot_modules(Mods) -> application:set_env(emqx, boot_modules, Mods). @@ -162,8 +163,7 @@ app_schema(App) -> mustache_vars(App) -> [{platform_data_dir, app_path(App, "data")}, {platform_etc_dir, app_path(App, "etc")}, - {platform_log_dir, app_path(App, "log")}, - {platform_plugins_dir, app_path(App, "plugins")} + {platform_log_dir, app_path(App, "log")} ]. start_app(App, Schema, ConfigFile, SpecAppConfig) -> diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index a9ee0d7d9..7411ca1fd 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -132,8 +132,7 @@ render_config_file() -> mustache_vars() -> [{platform_data_dir, local_path(["data"])}, {platform_etc_dir, local_path(["etc"])}, - {platform_log_dir, local_path(["log"])}, - {platform_plugins_dir, local_path(["plugins"])} + {platform_log_dir, local_path(["log"])} ]. generate_config() -> @@ -144,10 +143,6 @@ generate_config() -> set_app_env({App, Lists}) -> lists:foreach(fun({authz_file, _Var}) -> application:set_env(App, authz_file, local_path(["etc", "authz.conf"])); - ({plugins_loaded_file, _Var}) -> - application:set_env(App, - plugins_loaded_file, - local_path(["test", "emqx_SUITE_data","loaded_plugins"])); ({Par, Var}) -> application:set_env(App, Par, Var) end, Lists). diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index 0fa73ebe2..cf7579ba4 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -160,9 +160,6 @@ init_per_group(gc_tests, Config) -> init_per_suite(Config) -> Config. -set_special_confs(emqx) -> - Path = emqx_common_test_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins"), - application:set_env(emqx, plugins_loaded_file, Path); set_special_confs(_) -> ok. diff --git a/apps/emqx_plugins/etc/emqx_plugins.conf b/apps/emqx_plugins/etc/emqx_plugins.conf index 35f02a9f8..0a1dfb72d 100644 --- a/apps/emqx_plugins/etc/emqx_plugins.conf +++ b/apps/emqx_plugins/etc/emqx_plugins.conf @@ -3,5 +3,5 @@ plugins { } external { } - install_dir = "plugins" + install_dir = "{{ platform_plugins_dir }}" } diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index 2281f09dc..013544e32 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -47,7 +47,9 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:stop_apps([]), + emqx_config:erase(plugins). t_load(_) -> ?assertEqual(ok, emqx_plugins:load()), @@ -57,7 +59,7 @@ t_load(_) -> ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_mini_plugin)), ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_hocon_plugin)), - emqx_config:put([plugins, install_dir], undefined). + emqx_config:erase(plugins). t_load_ext_plugin(_) -> ?assertError({plugin_app_file_not_found, _}, diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 5596e9539..7191bacc0 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -55,6 +55,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([emqx_retainer]). + %%-------------------------------------------------------------------- %% Test Cases %%-------------------------------------------------------------------- diff --git a/rebar.config.erl b/rebar.config.erl index f63fedd84..5187379b1 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -257,7 +257,7 @@ overlay_vars_pkg(pkg) -> , {platform_etc_dir, "/etc/emqx"} , {platform_lib_dir, ""} , {platform_log_dir, "/var/log/emqx"} - , {platform_plugins_dir, "/var/lib/enqx/plugins"} + , {platform_plugins_dir, "/var/lib/emqx/plugins"} , {runner_root_dir, "/usr/lib/emqx"} , {runner_bin_dir, "/usr/bin"} , {runner_etc_dir, "/etc/emqx"} From 5785b0c20b2f4b841c1b81fad86b614e3d39fe46 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 13 Dec 2021 15:39:59 +0800 Subject: [PATCH 23/31] fix(ssl): file content return type --- apps/emqx/src/emqx_tls_lib.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 0d64f3003..ec339a968 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -353,11 +353,12 @@ is_valid_pem_file(Path) -> %% @doc This is to return SSL file content in management APIs. file_content_as_options(undefined) -> undefined; file_content_as_options(#{<<"enable">> := false} = SSL) -> - maps:without(?SSL_FILE_OPT_NAMES, SSL); + {ok, maps:without(?SSL_FILE_OPT_NAMES, SSL)}; file_content_as_options(#{<<"enable">> := true} = SSL) -> file_content_as_options(?SSL_FILE_OPT_NAMES, SSL). -file_content_as_options([], SSL) -> {ok, SSL}; +file_content_as_options([], SSL) -> + {ok, SSL}; file_content_as_options([Key | Keys], SSL) -> case maps:get(Key, SSL, undefined) of undefined -> file_content_as_options(Keys, SSL); From 69800a9e88f4b5ea321e4173a4a9d78cc6f1145e Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 13 Dec 2021 16:16:57 +0800 Subject: [PATCH 24/31] chore(otp): be compatibility with crypto in otp 24 --- apps/emqx_connector/rebar.config | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 4773d0859..58706e950 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -6,7 +6,7 @@ {deps, [ {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, - {epgsql, {git, "https://github.com/epgsql/epgsql", {tag, "4.4.0"}}}, + {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.6.0"}}}, %% NOTE: mind poolboy version when updating mongodb-erlang version {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.10"}}}, %% NOTE: mind poolboy version when updating eredis_cluster version diff --git a/rebar.config b/rebar.config index 1604c7696..c4a3320b3 100644 --- a/rebar.config +++ b/rebar.config @@ -68,7 +68,7 @@ , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} - , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} + , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} ]}. {xref_ignores, From abdb98ffa26d6300d829bb3bac24aec811156c70 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 13 Dec 2021 18:26:41 +0300 Subject: [PATCH 25/31] refactor(authz): hide mnesia authz implementation details * Eliminate type and record sharing through `emqx_authz.hrl`. * Hide all mria/mnesia interactions inside mnesia authz backend. --- apps/emqx_authz/include/emqx_authz.hrl | 29 ----- apps/emqx_authz/src/emqx_authz.erl | 24 +++- apps/emqx_authz/src/emqx_authz_api_mnesia.erl | 66 ++++------ apps/emqx_authz/src/emqx_authz_app.erl | 6 +- apps/emqx_authz/src/emqx_authz_http.erl | 2 +- apps/emqx_authz/src/emqx_authz_mnesia.erl | 114 +++++++++++++++++- apps/emqx_authz/src/emqx_authz_rule.erl | 7 +- .../test/emqx_authz_api_mnesia_SUITE.erl | 2 +- .../test/emqx_authz_mnesia_SUITE.erl | 29 ++--- 9 files changed, 177 insertions(+), 102 deletions(-) diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index de5dca73e..ae9249bb3 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -8,29 +8,6 @@ (A =:= all) orelse (A =:= <<"all">>) )). --define(ACL_SHARDED, emqx_acl_sharded). - --define(ACL_TABLE, emqx_acl). - -%% To save some space, use an integer for label, 0 for 'all', {1, Username} and {2, ClientId}. --define(ACL_TABLE_ALL, 0). --define(ACL_TABLE_USERNAME, 1). --define(ACL_TABLE_CLIENTID, 2). - --type(action() :: subscribe | publish | all). --type(permission() :: allow | deny). - --record(emqx_acl, { - who :: ?ACL_TABLE_ALL| {?ACL_TABLE_USERNAME, binary()} | {?ACL_TABLE_CLIENTID, binary()}, - rules :: [ {permission(), action(), emqx_topic:topic()} ] - }). - --record(authz_metrics, { - allow = 'client.authorize.allow', - deny = 'client.authorize.deny', - ignore = 'client.authorize.ignore' - }). - -define(CMD_REPLACE, replace). -define(CMD_DELETE, delete). -define(CMD_PREPEND, prepend). @@ -42,12 +19,6 @@ -define(CMD_MOVE_BEFORE(Before), {<<"before">>, Before}). -define(CMD_MOVE_AFTER(After), {<<"after">>, After}). --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTHZ_METRICS, ?METRICS(authz_metrics)). --define(AUTHZ_METRICS(K), ?METRICS(authz_metrics, K)). - -define(CONF_KEY_PATH, [authorization, sources]). -define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}"). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index af16c3892..d80253c4d 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -53,15 +53,32 @@ -type(sources() :: [source()]). +-define(METRIC_ALLOW, 'client.authorize.allow'). +-define(METRIC_DENY, 'client.authorize.deny'). +-define(METRIC_NOMATCH, 'client.authorize.nomatch'). +-define(METRICS, [?METRIC_ALLOW, ?METRIC_DENY, ?METRIC_NOMATCH]). + +%% Initialize authz backend. +%% Populate the passed configuration map with necessary data, +%% like `ResourceID`s -callback(init(source()) -> source()). +%% Get authz text description. -callback(description() -> string()). +%% Destroy authz backend. +%% Make cleanup of all allocated data. +%% An authz backend will not be used after `destroy`. -callback(destroy(source()) -> ok). +%% Check if a configuration map is valid for further +%% authz backend initialization. +%% The callback must deallocate all resources allocated +%% during verification. -callback(dry_run(source()) -> ok | {error, term()}). +%% Authorize client action. -callback(authorize( emqx_types:clientinfo(), emqx_types:pubsub(), @@ -70,7 +87,7 @@ -spec(register_metrics() -> ok). register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTHZ_METRICS). + lists:foreach(fun emqx_metrics:ensure/1, ?METRICS). init() -> ok = register_metrics(), @@ -273,14 +290,14 @@ authorize(#{username := Username, username => Username, ipaddr => IpAddress, topic => Topic}), - emqx_metrics:inc(?AUTHZ_METRICS(allow)), + emqx_metrics:inc(?METRIC_ALLOW), {stop, allow}; {matched, deny} -> ?SLOG(info, #{msg => "authorization_permission_denied", username => Username, ipaddr => IpAddress, topic => Topic}), - emqx_metrics:inc(?AUTHZ_METRICS(deny)), + emqx_metrics:inc(?METRIC_DENY), {stop, deny}; nomatch -> ?SLOG(info, #{msg => "authorization_failed_nomatch", @@ -288,6 +305,7 @@ authorize(#{username := Username, ipaddr => IpAddress, topic => Topic, reason => "no-match rule"}), + emqx_metrics:inc(?METRIC_NOMATCH), {stop, DefaultResult} end. diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl index 5448cbfd8..a92ce88a7 100644 --- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl @@ -20,7 +20,6 @@ -include("emqx_authz.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("stdlib/include/ms_transform.hrl"). -include_lib("typerefl/include/types.hrl"). -define(FORMAT_USERNAME_FUN, {?MODULE, format_by_username}). @@ -269,39 +268,27 @@ fields(meta) -> %%-------------------------------------------------------------------- users(get, #{query_string := PageParams}) -> - MatchSpec = ets:fun2ms( - fun({?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}, Rules}) -> - [{username, Username}, {rules, Rules}] - end), - {200, emqx_mgmt_api:paginate(?ACL_TABLE, MatchSpec, PageParams, ?FORMAT_USERNAME_FUN)}; + {Table, MatchSpec} = emqx_authz_mnesia:list_username_rules(), + {200, emqx_mgmt_api:paginate(Table, MatchSpec, PageParams, ?FORMAT_USERNAME_FUN)}; users(post, #{body := Body}) when is_list(Body) -> lists:foreach(fun(#{<<"username">> := Username, <<"rules">> := Rules}) -> - mria:dirty_write(#emqx_acl{ - who = {?ACL_TABLE_USERNAME, Username}, - rules = format_rules(Rules) - }) + emqx_authz_mnesia:store_rules({username, Username}, format_rules(Rules)) end, Body), {204}. clients(get, #{query_string := PageParams}) -> - MatchSpec = ets:fun2ms( - fun({?ACL_TABLE, {?ACL_TABLE_CLIENTID, Clientid}, Rules}) -> - [{clientid, Clientid}, {rules, Rules}] - end), - {200, emqx_mgmt_api:paginate(?ACL_TABLE, MatchSpec, PageParams, ?FORMAT_CLIENTID_FUN)}; + {Table, MatchSpec} = emqx_authz_mnesia:list_clientid_rules(), + {200, emqx_mgmt_api:paginate(Table, MatchSpec, PageParams, ?FORMAT_CLIENTID_FUN)}; clients(post, #{body := Body}) when is_list(Body) -> lists:foreach(fun(#{<<"clientid">> := Clientid, <<"rules">> := Rules}) -> - mria:dirty_write(#emqx_acl{ - who = {?ACL_TABLE_CLIENTID, Clientid}, - rules = format_rules(Rules) - }) + emqx_authz_mnesia:store_rules({clientid, Clientid}, format_rules(Rules)) end, Body), {204}. user(get, #{bindings := #{username := Username}}) -> - case mnesia:dirty_read(?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}) of - [] -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}}; - [#emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = Rules}] -> + case emqx_authz_mnesia:get_rules({username, Username}) of + not_found -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}}; + {ok, Rules} -> {200, #{username => Username, rules => [ #{topic => Topic, action => Action, @@ -311,19 +298,16 @@ user(get, #{bindings := #{username := Username}}) -> end; user(put, #{bindings := #{username := Username}, body := #{<<"username">> := Username, <<"rules">> := Rules}}) -> - mria:dirty_write(#emqx_acl{ - who = {?ACL_TABLE_USERNAME, Username}, - rules = format_rules(Rules) - }), + emqx_authz_mnesia:store_rules({username, Username}, format_rules(Rules)), {204}; user(delete, #{bindings := #{username := Username}}) -> - mria:dirty_delete({?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}}), + emqx_authz_mnesia:delete_rules({username, Username}), {204}. client(get, #{bindings := #{clientid := Clientid}}) -> - case mnesia:dirty_read(?ACL_TABLE, {?ACL_TABLE_CLIENTID, Clientid}) of - [] -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}}; - [#emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = Rules}] -> + case emqx_authz_mnesia:get_rules({clientid, Clientid}) of + not_found -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}}; + {ok, Rules} -> {200, #{clientid => Clientid, rules => [ #{topic => Topic, action => Action, @@ -333,20 +317,17 @@ client(get, #{bindings := #{clientid := Clientid}}) -> end; client(put, #{bindings := #{clientid := Clientid}, body := #{<<"clientid">> := Clientid, <<"rules">> := Rules}}) -> - mria:dirty_write(#emqx_acl{ - who = {?ACL_TABLE_CLIENTID, Clientid}, - rules = format_rules(Rules) - }), + emqx_authz_mnesia:store_rules({clientid, Clientid}, format_rules(Rules)), {204}; client(delete, #{bindings := #{clientid := Clientid}}) -> - mria:dirty_delete({?ACL_TABLE, {?ACL_TABLE_CLIENTID, Clientid}}), + emqx_authz_mnesia:delete_rules({clientid, Clientid}), {204}. all(get, _) -> - case mnesia:dirty_read(?ACL_TABLE, ?ACL_TABLE_ALL) of - [] -> + case emqx_authz_mnesia:get_rules(all) of + not_found -> {200, #{rules => []}}; - [#emqx_acl{who = ?ACL_TABLE_ALL, rules = Rules}] -> + {ok, Rules} -> {200, #{rules => [ #{topic => Topic, action => Action, permission => Permission @@ -354,18 +335,13 @@ all(get, _) -> } end; all(put, #{body := #{<<"rules">> := Rules}}) -> - mria:dirty_write(#emqx_acl{ - who = ?ACL_TABLE_ALL, - rules = format_rules(Rules) - }), + emqx_authz_mnesia:store_rules(all, format_rules(Rules)), {204}. purge(delete, _) -> case emqx_authz_api_sources:get_raw_source(<<"built-in-database">>) of [#{<<"enable">> := false}] -> - ok = lists:foreach(fun(Key) -> - ok = mria:dirty_delete(?ACL_TABLE, Key) - end, mnesia:dirty_all_keys(?ACL_TABLE)), + ok = emqx_authz_mnesia:purge_rules(), {204}; [#{<<"enable">> := true}] -> {400, #{code => <<"BAD_REQUEST">>, diff --git a/apps/emqx_authz/src/emqx_authz_app.erl b/apps/emqx_authz/src/emqx_authz_app.erl index 0fb5c4e02..623853631 100644 --- a/apps/emqx_authz/src/emqx_authz_app.erl +++ b/apps/emqx_authz/src/emqx_authz_app.erl @@ -23,12 +23,10 @@ -behaviour(application). --include("emqx_authz.hrl"). - -export([start/2, stop/1]). start(_StartType, _StartArgs) -> - ok = mria_rlog:wait_for_shards([?ACL_SHARDED], infinity), + ok = emqx_authz_mnesia:init_tables(), {ok, Sup} = emqx_authz_sup:start_link(), ok = emqx_authz:init(), {ok, Sup}. @@ -36,5 +34,3 @@ start(_StartType, _StartArgs) -> stop(_State) -> ok = emqx_authz:deinit(), ok. - -%% internal functions diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 62719b9ed..c2ee96594 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -41,7 +41,7 @@ description() -> "AuthZ with http". init(#{url := Url} = Source) -> - NSource= maps:put(base_url, maps:remove(query, Url), Source), + NSource = maps:put(base_url, maps:remove(query, Url), Source), case emqx_authz_utils:create_resource(emqx_connector_http, NSource) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> Source#{annotations => #{id => Id}} diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl index d652c6731..2ce8215cd 100644 --- a/apps/emqx_authz/src/emqx_authz_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl @@ -16,21 +16,53 @@ -module(emqx_authz_mnesia). --include("emqx_authz.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). -include_lib("emqx/include/logger.hrl"). +-define(ACL_SHARDED, emqx_acl_sharded). + +-define(ACL_TABLE, emqx_acl). + +%% To save some space, use an integer for label, 0 for 'all', {1, Username} and {2, ClientId}. +-define(ACL_TABLE_ALL, 0). +-define(ACL_TABLE_USERNAME, 1). +-define(ACL_TABLE_CLIENTID, 2). + +-type(username() :: {username, binary()}). +-type(clientid() :: {clientid, binary()}). +-type(who() :: username() | clientid() | all). + +-type(rule() :: {emqx_authz_rule:permission(), emqx_authz_rule:action(), emqx_topic:topic()}). +-type(rules() :: [rule()]). + +-record(emqx_acl, { + who :: ?ACL_TABLE_ALL | {?ACL_TABLE_USERNAME, binary()} | {?ACL_TABLE_CLIENTID, binary()}, + rules :: rules() + }). + -behaviour(emqx_authz). %% AuthZ Callbacks --export([ mnesia/1 - , description/0 +-export([ description/0 , init/1 , destroy/1 , dry_run/1 , authorize/4 ]). +%% Management API +-export([ mnesia/1 + , init_tables/0 + , store_rules/2 + , purge_rules/0 + , get_rules/1 + , delete_rules/1 + , list_clientid_rules/0 + , list_username_rules/0 + , record_count/0 + ]). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -47,6 +79,10 @@ mnesia(boot) -> {attributes, record_info(fields, ?ACL_TABLE)}, {storage_properties, [{ets, [{read_concurrency, true}]}]}]). +%%-------------------------------------------------------------------- +%% emqx_authz callbacks +%%-------------------------------------------------------------------- + description() -> "AuthZ with Mnesia". @@ -74,6 +110,78 @@ authorize(#{username := Username, end, do_authorize(Client, PubSub, Topic, Rules). +%%-------------------------------------------------------------------- +%% Management API +%%-------------------------------------------------------------------- + +init_tables() -> + ok = mria_rlog:wait_for_shards([?ACL_SHARDED], infinity). + +-spec(store_rules(who(), rules()) -> ok). +store_rules({username, Username}, Rules) -> + Record = #emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = Rules}, + mria:dirty_write(Record); +store_rules({clientid, Clientid}, Rules) -> + Record = #emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = Rules}, + mria:dirty_write(Record); +store_rules(all, Rules) -> + Record = #emqx_acl{who = ?ACL_TABLE_ALL, rules = Rules}, + mria:dirty_write(Record). + +-spec(purge_rules() -> ok). +purge_rules() -> + ok = lists:foreach( + fun(Key) -> + ok = mria:dirty_delete(?ACL_TABLE, Key) + end, + mnesia:dirty_all_keys(?ACL_TABLE)). + +-spec(get_rules(who()) -> {ok, rules()} | not_found). +get_rules({username, Username}) -> + do_get_rules({?ACL_TABLE_USERNAME, Username}); +get_rules({clientid, Clientid}) -> + do_get_rules({?ACL_TABLE_CLIENTID, Clientid}); +get_rules(all) -> + do_get_rules(?ACL_TABLE_ALL). + +-spec(delete_rules(who()) -> ok). +delete_rules({username, Username}) -> + mria:dirty_delete(?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}); +delete_rules({clientid, Clientid}) -> + mria:dirty_delete(?ACL_TABLE, {?ACL_TABLE_CLIENTID, Clientid}); +delete_rules(all) -> + mria:dirty_delete(?ACL_TABLE, ?ACL_TABLE_ALL). + +-spec(list_username_rules() -> {mria:table(), ets:match_spec()}). +list_username_rules() -> + MatchSpec = ets:fun2ms( + fun(#emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = Rules}) -> + [{username, Username}, {rules, Rules}] + end), + {?ACL_TABLE, MatchSpec}. + +-spec(list_clientid_rules() -> {mria:table(), ets:match_spec()}). +list_clientid_rules() -> + MatchSpec = ets:fun2ms( + fun(#emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = Rules}) -> + [{clientid, Clientid}, {rules, Rules}] + end), + {?ACL_TABLE, MatchSpec}. + +-spec(record_count() -> non_neg_integer()). +record_count() -> + mnesia:table_info(?ACL_TABLE, size). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +do_get_rules(Key) -> + case mnesia:dirty_read(?ACL_TABLE, Key) of + [#emqx_acl{rules = Rules}] -> {ok, Rules}; + [] -> not_found + end. + do_authorize(_Client, _PubSub, _Topic, []) -> nomatch; do_authorize(Client, PubSub, Topic, [ {Permission, Action, TopicFilter} | Tail]) -> case emqx_authz_rule:match(Client, PubSub, Topic, diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index 952d6b7a5..da8894edd 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -43,9 +43,14 @@ {'or', [ipaddress() | username() | clientid()]} | all). +-type(action() :: subscribe | publish | all). +-type(permission() :: allow | deny). + -type(rule() :: {permission(), who(), action(), list(emqx_types:topic())}). --export_type([rule/0]). +-export_type([ action/0 + , permission/0 + ]). compile({Permission, all}) when ?ALLOW_DENY(Permission) -> {Permission, all, all, [compile_topic(<<"#">>)]}; diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index b4a8f2756..ccf6cc2c9 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -217,7 +217,7 @@ t_api(_) -> request( delete , uri(["authorization", "sources", "built-in-database", "purge-all"]) , []), - ?assertEqual([], mnesia:dirty_all_keys(?ACL_TABLE)), + ?assertEqual(0, emqx_authz_mnesia:record_count()), ok. %%-------------------------------------------------------------------- diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl index 2bca1793d..dd98f77d3 100644 --- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -55,24 +55,25 @@ set_special_configs(_App) -> ok. init_per_testcase(t_authz, Config) -> - mria:dirty_write(#emqx_acl{who = {?ACL_TABLE_USERNAME, <<"test_username">>}, - rules = [{allow, publish, <<"test/", ?PH_S_USERNAME>>}, - {allow, subscribe, <<"eq #">>} - ] - }), - mria:dirty_write(#emqx_acl{who = {?ACL_TABLE_CLIENTID, <<"test_clientid">>}, - rules = [{allow, publish, <<"test/", ?PH_S_CLIENTID>>}, - {deny, subscribe, <<"eq #">>} - ] - }), - mria:dirty_write(#emqx_acl{who = ?ACL_TABLE_ALL, - rules = [{deny, all, <<"#">>}] - }), + emqx_authz_mnesia:store_rules( + {username, <<"test_username">>}, + [{allow, publish, <<"test/", ?PH_S_USERNAME>>}, + {allow, subscribe, <<"eq #">>}]), + + emqx_authz_mnesia:store_rules( + {clientid, <<"test_clientid">>}, + [{allow, publish, <<"test/", ?PH_S_CLIENTID>>}, + {deny, subscribe, <<"eq #">>}]), + + emqx_authz_mnesia:store_rules( + all, + [{deny, all, <<"#">>}]), + Config; init_per_testcase(_, Config) -> Config. end_per_testcase(t_authz, Config) -> - [ mria:dirty_delete(?ACL_TABLE, K) || K <- mnesia:dirty_all_keys(?ACL_TABLE)], + ok = emqx_authz_mnesia:purge_rules(), Config; end_per_testcase(_, Config) -> Config. From 708d9cfc6c04d84c42c7786f4cab8416f15070ca Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 13 Dec 2021 14:53:34 +0300 Subject: [PATCH 26/31] refactor(authn): restore pbkdf2 password hashing functionality --- apps/emqx/src/emqx_passwd.erl | 51 +++++++++---- apps/emqx/test/emqx_passwd_SUITE.erl | 14 +++- .../src/emqx_authn_password_hashing.erl | 38 ++++++++-- .../emqx_authn_password_hashing_SUITE.erl | 26 ++++++- .../test/emqx_authn_redis_SUITE.erl | 75 ++++++++++++------- 5 files changed, 152 insertions(+), 52 deletions(-) diff --git a/apps/emqx/src/emqx_passwd.erl b/apps/emqx/src/emqx_passwd.erl index f729ce49e..2f9775d6d 100644 --- a/apps/emqx/src/emqx_passwd.erl +++ b/apps/emqx/src/emqx_passwd.erl @@ -34,18 +34,34 @@ -type(password_hash() :: binary()). -type(hash_type_simple() :: plain | md5 | sha | sha256 | sha512). --type(hash_type() :: hash_type_simple() | bcrypt). +-type(hash_type() :: hash_type_simple() | bcrypt | pbkdf2). -type(salt_position() :: prefix | suffix). -type(salt() :: binary()). --type(hash_params() :: {bcrypt, salt()} | {hash_type_simple(), salt(), salt_position()}). +-type(pbkdf2_mac_fun() :: md4 | md5 | ripemd160 | sha | sha224 | sha256 | sha384 | sha512). +-type(pbkdf2_iterations() :: pos_integer()). +-type(pbkdf2_dk_length() :: pos_integer() | undefined). + +-type(hash_params() :: + {bcrypt, salt()} | + {pbkdf2, pbkdf2_mac_fun(), salt(), pbkdf2_iterations(), pbkdf2_dk_length()} | + {hash_type_simple(), salt(), salt_position()}). + +-export_type([pbkdf2_mac_fun/0]). %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- -spec(check_pass(hash_params(), password_hash(), password()) -> boolean()). +check_pass({pbkdf2, MacFun, Salt, Iterations, DKLength}, PasswordHash, Password) -> + case pbkdf2(MacFun, Password, Salt, Iterations, DKLength) of + {ok, HashPasswd} -> + compare_secure(hex(HashPasswd), PasswordHash); + {error, _Reason}-> + false + end; check_pass({bcrypt, Salt}, PasswordHash, Password) -> case bcrypt:hashpw(Password, Salt) of {ok, HashPasswd} -> @@ -58,6 +74,13 @@ check_pass({_SimpleHash, _Salt, _SaltPosition} = HashParams, PasswordHash, Passw compare_secure(Hash, PasswordHash). -spec(hash(hash_params(), password()) -> password_hash()). +hash({pbkdf2, MacFun, Salt, Iterations, DKLength}, Password) -> + case pbkdf2(MacFun, Password, Salt, Iterations, DKLength) of + {ok, HashPasswd} -> + hex(HashPasswd); + {error, Reason}-> + error(Reason) + end; hash({bcrypt, Salt}, Password) -> case bcrypt:hashpw(Password, Salt) of {ok, HashPasswd} -> @@ -75,13 +98,13 @@ hash({SimpleHash, Salt, suffix}, Password) when is_binary(Password), is_binary(S hash_data(plain, Data) when is_binary(Data) -> Data; hash_data(md5, Data) when is_binary(Data) -> - hexstring(crypto:hash(md5, Data)); + hex(crypto:hash(md5, Data)); hash_data(sha, Data) when is_binary(Data) -> - hexstring(crypto:hash(sha, Data)); + hex(crypto:hash(sha, Data)); hash_data(sha256, Data) when is_binary(Data) -> - hexstring(crypto:hash(sha256, Data)); + hex(crypto:hash(sha256, Data)); hash_data(sha512, Data) when is_binary(Data) -> - hexstring(crypto:hash(sha512, Data)). + hex(crypto:hash(sha512, Data)). %%-------------------------------------------------------------------- %% Internal functions @@ -103,11 +126,11 @@ compare_secure([], [], Result) -> Result == 0. -hexstring(<>) -> - iolist_to_binary(io_lib:format("~32.16.0b", [X])); -hexstring(<>) -> - iolist_to_binary(io_lib:format("~40.16.0b", [X])); -hexstring(<>) -> - iolist_to_binary(io_lib:format("~64.16.0b", [X])); -hexstring(<>) -> - iolist_to_binary(io_lib:format("~128.16.0b", [X])). +pbkdf2(MacFun, Password, Salt, Iterations, undefined) -> + pbkdf2:pbkdf2(MacFun, Password, Salt, Iterations); +pbkdf2(MacFun, Password, Salt, Iterations, DKLength) -> + pbkdf2:pbkdf2(MacFun, Password, Salt, Iterations, DKLength). + + +hex(X) when is_binary(X) -> + pbkdf2:to_hex(X). diff --git a/apps/emqx/test/emqx_passwd_SUITE.erl b/apps/emqx/test/emqx_passwd_SUITE.erl index e3c904a2e..066912ba1 100644 --- a/apps/emqx/test/emqx_passwd_SUITE.erl +++ b/apps/emqx/test/emqx_passwd_SUITE.erl @@ -88,4 +88,16 @@ t_hash(_) -> false = emqx_passwd:check_pass({bcrypt, <<>>}, <<>>, WrongPassword), %% Invalid salt, bcrypt fails - ?assertException(error, _, emqx_passwd:hash({bcrypt, Salt}, Password)). + ?assertException(error, _, emqx_passwd:hash({bcrypt, Salt}, Password)), + + BadDKlen = 1 bsl 32, + Pbkdf2Salt = <<"ATHENA.MIT.EDUraeburn">>, + Pbkdf2 = <<"01dbee7f4a9e243e988b62c73cda935d" + "a05378b93244ec8f48a99e61ad799d86">>, + Pbkdf2 = emqx_passwd:hash({pbkdf2, sha, Pbkdf2Salt, 2, 32}, Password), + true = emqx_passwd:check_pass({pbkdf2, sha, Pbkdf2Salt, 2, 32}, Pbkdf2, Password), + false = emqx_passwd:check_pass({pbkdf2, sha, Pbkdf2Salt, 2, 32}, Pbkdf2, WrongPassword), + false = emqx_passwd:check_pass({pbkdf2, sha, Pbkdf2Salt, 2, BadDKlen}, Pbkdf2, Password), + + %% Invalid derived_length, pbkdf2 fails + ?assertException(error, _, emqx_passwd:hash({pbkdf2, sha, Pbkdf2Salt, 2, BadDKlen}, Password)). diff --git a/apps/emqx_authn/src/emqx_authn_password_hashing.erl b/apps/emqx_authn/src/emqx_authn_password_hashing.erl index 4cb1ad401..9e3637285 100644 --- a/apps/emqx_authn/src/emqx_authn_password_hashing.erl +++ b/apps/emqx_authn/src/emqx_authn_password_hashing.erl @@ -27,8 +27,12 @@ -type(bcrypt_algorithm() :: #{name := bcrypt}). -type(bcrypt_algorithm_rw() :: #{name := bcrypt, salt_rounds := integer()}). --type(algorithm() :: simple_algorithm() | bcrypt_algorithm()). --type(algorithm_rw() :: simple_algorithm() | bcrypt_algorithm_rw()). +-type(pbkdf2_algorithm() :: #{name := pbkdf2, + mac_fun := emqx_passwd:pbkdf2_mac_fun(), + iterations := pos_integer()}). + +-type(algorithm() :: simple_algorithm() | pbkdf2_algorithm() | bcrypt_algorithm()). +-type(algorithm_rw() :: simple_algorithm() | pbkdf2_algorithm() | bcrypt_algorithm_rw()). %%------------------------------------------------------------------------------ %% Hocon Schema @@ -47,7 +51,7 @@ hash/2, check_password/4]). -roots() -> [bcrypt, bcrypt_rw, other_algorithms]. +roots() -> [pbkdf2, bcrypt, bcrypt_rw, other_algorithms]. fields(bcrypt_rw) -> fields(bcrypt) ++ @@ -56,6 +60,12 @@ fields(bcrypt_rw) -> fields(bcrypt) -> [{name, {enum, [bcrypt]}}]; +fields(pbkdf2) -> + [{name, {enum, [pbkdf2]}}, + {mac_fun, {enum, [md4, md5, ripemd160, sha, sha224, sha256, sha384, sha512]}}, + {iterations, integer()}, + {dk_length, fun dk_length/1}]; + fields(other_algorithms) -> [{name, {enum, [plain, md5, sha, sha256, sha512]}}, {salt_position, fun salt_position/1}]. @@ -68,6 +78,11 @@ salt_rounds(type) -> integer(); salt_rounds(default) -> 10; salt_rounds(_) -> undefined. +dk_length(type) -> integer(); +dk_length(nullable) -> true; +dk_length(default) -> undefined; +dk_length(_) -> undefined. + type_rw(type) -> hoconsc:union(rw_refs()); type_rw(default) -> #{<<"name">> => sha256, <<"salt_position">> => prefix}; @@ -108,7 +123,13 @@ hash(#{name := bcrypt, salt_rounds := _} = Algorithm, Password) -> Hash = emqx_passwd:hash({bcrypt, Salt0}, Password), Salt = Hash, {Hash, Salt}; - +hash(#{name := pbkdf2, + mac_fun := MacFun, + iterations := Iterations} = Algorithm, Password) -> + Salt = gen_salt(Algorithm), + DKLength = maps:get(dk_length, Algorithm, undefined), + Hash = emqx_passwd:hash({pbkdf2, MacFun, Salt, Iterations, DKLength}, Password), + {Hash, Salt}; hash(#{name := Other, salt_position := SaltPosition} = Algorithm, Password) -> Salt = gen_salt(Algorithm), Hash = emqx_passwd:hash({Other, Salt, SaltPosition}, Password), @@ -122,7 +143,12 @@ hash(#{name := Other, salt_position := SaltPosition} = Algorithm, Password) -> emqx_passwd:password()) -> boolean()). check_password(#{name := bcrypt}, _Salt, PasswordHash, Password) -> emqx_passwd:check_pass({bcrypt, PasswordHash}, PasswordHash, Password); - +check_password(#{name := pbkdf2, + mac_fun := MacFun, + iterations := Iterations} = Algorithm, + Salt, PasswordHash, Password) -> + DKLength = maps:get(dk_length, Algorithm, undefined), + emqx_passwd:check_pass({pbkdf2, MacFun, Salt, Iterations, DKLength}, PasswordHash, Password); check_password(#{name := Other, salt_position := SaltPosition}, Salt, PasswordHash, Password) -> emqx_passwd:check_pass({Other, Salt, SaltPosition}, PasswordHash, Password). @@ -132,8 +158,10 @@ check_password(#{name := Other, salt_position := SaltPosition}, Salt, PasswordHa rw_refs() -> [hoconsc:ref(?MODULE, bcrypt_rw), + hoconsc:ref(?MODULE, pbkdf2), hoconsc:ref(?MODULE, other_algorithms)]. ro_refs() -> [hoconsc:ref(?MODULE, bcrypt), + hoconsc:ref(?MODULE, pbkdf2), hoconsc:ref(?MODULE, other_algorithms)]. diff --git a/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl b/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl index e0273e24f..8832c551d 100644 --- a/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_password_hashing_SUITE.erl @@ -116,9 +116,8 @@ hash_examples() -> salt_position => prefix} }, #{ - password_hash => iolist_to_binary( - [<<"a1509ab67bfacbad020927b5ac9d91e9100a82e33a0ebb01459367ce921c0aa8">>, - <<"157aa5652f94bc84fa3babc08283e44887d61c48bcf8ad7bcb3259ee7d0eafcd">>]), + password_hash => <<"a1509ab67bfacbad020927b5ac9d91e9100a82e33a0ebb01459367ce921c0aa8" + "157aa5652f94bc84fa3babc08283e44887d61c48bcf8ad7bcb3259ee7d0eafcd">>, salt => <<"salt">>, password => <<"sha512">>, password_hash_algorithm => #{name => sha512, @@ -131,5 +130,26 @@ hash_examples() -> password_hash_algorithm => #{name => bcrypt, salt_rounds => 10} + }, + + #{ + password_hash => <<"01dbee7f4a9e243e988b62c73cda935d" + "a05378b93244ec8f48a99e61ad799d86">>, + salt => <<"ATHENA.MIT.EDUraeburn">>, + password => <<"password">>, + + password_hash_algorithm => #{name => pbkdf2, + iterations => 2, + dk_length => 32, + mac_fun => sha} + }, + #{ + password_hash => <<"01dbee7f4a9e243e988b62c73cda935da05378b9">>, + salt => <<"ATHENA.MIT.EDUraeburn">>, + password => <<"password">>, + + password_hash_algorithm => #{name => pbkdf2, + iterations => 2, + mac_fun => sha} } ]. diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 938ca8714..de556a7bd 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -222,28 +222,28 @@ raw_redis_auth_config() -> user_seeds() -> [#{data => #{ - password_hash => "plainsalt", - salt => "salt", - is_superuser => "1" + password_hash => <<"plainsalt">>, + salt => <<"salt">>, + is_superuser => <<"1">> }, credentials => #{ username => <<"plain">>, password => <<"plain">>}, - key => "mqtt_user:plain", + key => <<"mqtt_user:plain">>, config_params => #{}, result => {ok,#{is_superuser => true}} }, #{data => #{ - password_hash => "9b4d0c43d206d48279e69b9ad7132e22", - salt => "salt", - is_superuser => "0" + password_hash => <<"9b4d0c43d206d48279e69b9ad7132e22">>, + salt => <<"salt">>, + is_superuser => <<"0">> }, credentials => #{ username => <<"md5">>, password => <<"md5">> }, - key => "mqtt_user:md5", + key => <<"mqtt_user:md5">>, config_params => #{ password_hash_algorithm => #{name => <<"md5">>, salt_position => <<"suffix">>} @@ -252,15 +252,15 @@ user_seeds() -> }, #{data => #{ - password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf", - salt => "salt", - is_superuser => "1" + password_hash => <<"ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf">>, + salt => <<"salt">>, + is_superuser => <<"1">> }, credentials => #{ clientid => <<"sha256">>, password => <<"sha256">> }, - key => "mqtt_user:sha256", + key => <<"mqtt_user:sha256">>, config_params => #{ cmd => <<"HMGET mqtt_user:${clientid} password_hash salt is_superuser">>, password_hash_algorithm => #{name => <<"sha256">>, @@ -270,31 +270,48 @@ user_seeds() -> }, #{data => #{ - password_hash => "$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u", - salt => "$2b$12$wtY3h20mUjjmeaClpqZVve", - is_superuser => "0" + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => <<"0">> }, credentials => #{ username => <<"bcrypt">>, password => <<"bcrypt">> }, - key => "mqtt_user:bcrypt", + key => <<"mqtt_user:bcrypt">>, config_params => #{ password_hash_algorithm => #{name => <<"bcrypt">>} }, result => {ok,#{is_superuser => false}} }, - #{data => #{ - password_hash => "$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u", - salt => "$2b$12$wtY3h20mUjjmeaClpqZVve", - is_superuser => "0" + password_hash => <<"01dbee7f4a9e243e988b62c73cda935da05378b9">>, + salt => <<"ATHENA.MIT.EDUraeburn">>, + is_superuser => <<"0">> + }, + credentials => #{ + username => <<"pbkdf2">>, + password => <<"password">> + }, + key => <<"mqtt_user:pbkdf2">>, + config_params => #{ + password_hash_algorithm => #{name => <<"pbkdf2">>, + iterations => 2, + mac_fun => sha + } + }, + result => {ok,#{is_superuser => false}} + }, + #{data => #{ + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => <<"0">> }, credentials => #{ username => <<"bcrypt0">>, password => <<"bcrypt">> }, - key => "mqtt_user:bcrypt0", + key => <<"mqtt_user:bcrypt0">>, config_params => #{ % clientid variable & username credentials cmd => <<"HMGET mqtt_client:${clientid} password_hash salt is_superuser">>, @@ -304,15 +321,15 @@ user_seeds() -> }, #{data => #{ - password_hash => "$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u", - salt => "$2b$12$wtY3h20mUjjmeaClpqZVve", - is_superuser => "0" + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => <<"0">> }, credentials => #{ username => <<"bcrypt1">>, password => <<"bcrypt">> }, - key => "mqtt_user:bcrypt1", + key => <<"mqtt_user:bcrypt1">>, config_params => #{ % Bad key in cmd cmd => <<"HMGET badkey:${username} password_hash salt is_superuser">>, @@ -322,16 +339,16 @@ user_seeds() -> }, #{data => #{ - password_hash => "$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u", - salt => "$2b$12$wtY3h20mUjjmeaClpqZVve", - is_superuser => "0" + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => <<"0">> }, credentials => #{ username => <<"bcrypt2">>, % Wrong password password => <<"wrongpass">> }, - key => "mqtt_user:bcrypt2", + key => <<"mqtt_user:bcrypt2">>, config_params => #{ cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, password_hash_algorithm => #{name => <<"bcrypt">>} From 47e1875345e4cdbab487bb52d91ab66806f96a8d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 13 Dec 2021 20:50:28 +0100 Subject: [PATCH 27/31] build: add back emqx_slow_subs to release app list accidently lost it during rebase --- rebar.config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.erl b/rebar.config.erl index 0ee5aaeb2..b73a7a740 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -305,7 +305,7 @@ relx_apps(ReleaseType, Edition) -> , emqx_statsd , emqx_prometheus , emqx_psk - + , emqx_slow_subs , emqx_plugins ] ++ [quicer || is_quicer_supported()] From 867ebbc8a0e4c7ce67e3d0b1452939dd68ab2129 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 13 Dec 2021 20:51:00 +0100 Subject: [PATCH 28/31] docs: fix review comments for plugins config doc --- apps/emqx_plugins/src/emqx_plugins_schema.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_plugins/src/emqx_plugins_schema.erl b/apps/emqx_plugins/src/emqx_plugins_schema.erl index 7b7b3c15e..8d04923ff 100644 --- a/apps/emqx_plugins/src/emqx_plugins_schema.erl +++ b/apps/emqx_plugins/src/emqx_plugins_schema.erl @@ -32,7 +32,9 @@ fields("plugins") -> Manage EMQ X plugins.
Plugins can be pre-built as a part of EMQ X package, -or installed as a standalone package to the specific directory. +or installed as a standalone package in a location specified by +install_dir config key +
The standalone-installed plugins are referred to as 'external' plugins. """ }. @@ -82,6 +84,9 @@ install_dir(desc) -> """ In which directory are the external plugins installed. The plugin beam files and configuration files should reside in the sub-directory named as emqx_foo_bar-0.1.0. +
+NOTE: For security reasons, this directory should **NOT** be writable +by anyone expect for emqx (or any user which runs EMQ X) """. %% TODO: when we have some prebuilt plugins, change this function to: From 0300403fc0fa9b7a83eadef2e969dd7489625127 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 14 Dec 2021 17:51:50 -0300 Subject: [PATCH 29/31] fix(admin): inconsistent return type on empty inputs If an empty username or password are given during the initialization of `emqx_dashboard_admin`, it'll return just the atom `ok`, which breaks the expected `{ok, _}` in `emqx_dashboard_app:start/2`. ``` {'EXIT', {{failed_to_start_app,emqx_dashboard, {emqx_dashboard, {bad_return, {{emqx_dashboard_app,start,[normal,[]]}, {'EXIT', {{badmatch,ok}, [{emqx_dashboard_app,start,2, [{file, "/home/thales/dev/emqx/emqx/apps/emqx_dashboard/src/emqx_dashboard_app.erl"}, {line,32}]}, {application_master,start_it_old,4, [{file,"application_master.erl"},{line,293}]}]}}}}}}}} ``` --- apps/emqx_dashboard/src/emqx_dashboard_admin.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 8f40427e5..d95f0276c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -211,6 +211,7 @@ hash(Password) -> sha256(SaltBin, Password) -> crypto:hash('sha256', <>). +-spec(add_default_user() -> {ok, map() | empty | default_user_exists } | {error, any()}). add_default_user() -> add_default_user(binenv(default_username), binenv(default_password)). @@ -218,7 +219,7 @@ binenv(Key) -> iolist_to_binary(emqx_conf:get([emqx_dashboard, Key], "")). add_default_user(Username, Password) when ?EMPTY_KEY(Username) orelse ?EMPTY_KEY(Password) -> - ok; + {ok, empty}; add_default_user(Username, Password) -> case lookup_user(Username) of From 95de2d3467e7156936d27d3d6486d462ebdf2698 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 14 Dec 2021 17:56:32 -0300 Subject: [PATCH 30/31] fix(monitor): fix return value on badrpc If the rpc in `emqx_dashboard_monitor_api:get_collect/1` fails, an empty map is return. But the current function expects a 4-tuple, which results in a 500 error returned on such occasions. ``` curl -su admin:public localhost:8888/api/v5/monitor/current | jq . { "code": "INTERNAL_ERROR", "message": "error, function_clause, [{emqx_dashboard_monitor_api,format_current_metrics,[[#{},{0,0,0,0},{0,0,0,0}],{0,0,0,0}],[{file,\"emqx_dashboard_monitor_api.erl\"},{line,179}]},{emqx_dashboard_monitor_api,current_counters,2,[{file,\"emqx_dashboard_monitor_api.erl\"},{line,167}]},{minirest_handler,apply_callback,3,[{file,\"minirest_handler.erl\"},{line,112}]},{minirest_handler,init,2,[{file,\"minirest_handler.erl\"},{line,38}]},{cowboy_handler,execute,2,[{file,\"cowboy_handler.erl\"},{line,41}]},{cowboy_stream_h,execute,3,[{file,\"cowboy_stream_h.erl\"},{line,318}]},{cowboy_stream_h,request_process,3,[{file,\"cowboy_stream_h.erl\"},{line,302}]},{proc_lib,init_p_do_apply,3,[{file,\"proc_lib.erl\"},{line,226}]}]" } ``` --- apps/emqx/src/emqx_config.erl | 2 +- .../src/emqx_dashboard_monitor_api.erl | 6 +- .../test/emqx_dashboard_monitor_api_SUITE.erl | 121 ++++++++++++++++++ 3 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 apps/emqx_dashboard/test/emqx_dashboard_monitor_api_SUITE.erl diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 9979629bf..f3e6e1366 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -248,7 +248,7 @@ init_load(SchemaMod) -> init_load(SchemaMod, ConfFiles). %% @doc Initial load of the given config files. -%% NOTE: The order of the files is significant, configs from files orderd +%% NOTE: The order of the files is significant, configs from files ordered %% in the rear of the list overrides prior values. -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok. init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index efbb973da..a05746811 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -29,6 +29,8 @@ , sent , dropped]). +-define(EMPTY_COLLECTION, {0, 0, 0, 0}). + api_spec() -> {[ monitor_api() , monitor_nodes_api() @@ -175,7 +177,7 @@ current_counters(get, _Params) -> {200, Response}. format_current_metrics(Collects) -> - format_current_metrics(Collects, {0,0,0,0}). + format_current_metrics(Collects, ?EMPTY_COLLECTION). format_current_metrics([], Acc) -> Acc; format_current_metrics([{Received, Sent, Sub, Conn} | Collects], @@ -217,7 +219,7 @@ get_collect(Node) when Node =:= node() -> emqx_dashboard_collection:get_collect(); get_collect(Node) -> case rpc:call(Node, emqx_dashboard_collection, get_collect, []) of - {badrpc, _Reason} -> #{}; + {badrpc, _Reason} -> ?EMPTY_COLLECTION; Res -> Res end. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_api_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_api_SUITE.erl new file mode 100644 index 000000000..d6dc6e970 --- /dev/null +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_api_SUITE.erl @@ -0,0 +1,121 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_dashboard_monitor_api_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include("emqx_dashboard.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_testcase(t_badrpc_collect, Config) -> + Cluster = cluster_specs(2), + Apps = [emqx_modules, emqx_dashboard], + Nodes = [N1, N2] = lists:map(fun(Spec) -> start_slave(Spec, Apps) end, Cluster), + %% form the cluster + ok = rpc:call(N2, mria, join, [N1]), + %% Wait until all nodes are healthy: + [rpc:call(Node, mria_rlog, wait_for_shards, [[?DASHBOARD_SHARD], 5000]) + || Node <- Nodes], + [ {nodes, Nodes} + , {apps, Apps} + | Config]; +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(t_badrpc_collect, Config) -> + Apps = ?config(apps, Config), + Nodes = ?config(nodes, Config), + lists:foreach(fun(Node) -> stop_slave(Node, Apps) end, Nodes), + ok; +end_per_testcase(_, _Config) -> + ok. + +t_badrpc_collect(Config) -> + [N1, N2] = ?config(nodes, Config), + %% simulate badrpc on one node + ok = rpc:call(N2, meck, new, [emqx_dashboard_collection, [no_history, no_link]]), + %% we don't mock the `emqx_dashboard_collection:get_collect/0' to + %% provoke the `badrpc' error. + ?assertMatch( + {200, #{nodes := 2}}, + rpc:call(N1, emqx_dashboard_monitor_api, current_counters, [get, #{}])), + ok = rpc:call(N2, meck, unload, [emqx_dashboard_collection]), + ok. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +cluster_specs(NumNodes) -> + BaseGenRpcPort = 9000, + Specs0 = [#{ name => node_name(N) + , num => N + } + || N <- lists:seq(1, NumNodes)], + GenRpcPorts = maps:from_list([{node_id(Name), {tcp, BaseGenRpcPort + N}} + || #{name := Name, num := N} <- Specs0]), + [ Spec#{env => [ {gen_rpc, tcp_server_port, BaseGenRpcPort + N} + , {gen_rpc, client_config_per_node, {internal, GenRpcPorts}} + ]} + || Spec = #{num := N} <- Specs0]. + +node_name(N) -> + list_to_atom("n" ++ integer_to_list(N)). + +node_id(Name) -> + list_to_atom(lists:concat([Name, "@", host()])). + +start_slave(Spec = #{ name := Name}, Apps) -> + CommonBeamOpts = "+S 1:1 ", % We want VMs to only occupy a single core + {ok, Node} = slave:start_link(host(), Name, CommonBeamOpts ++ ebin_path()), + setup_node(Node, Spec, Apps), + Node. + +stop_slave(Node, Apps) -> + ok = rpc:call(Node, emqx_common_test_helpers, start_apps, [Apps]), + slave:stop(Node). + +host() -> + [_, Host] = string:tokens(atom_to_list(node()), "@"), Host. + +ebin_path() -> + string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " "). + +is_lib(Path) -> + string:prefix(Path, code:lib_dir()) =:= nomatch. + +setenv(Node, Env) -> + [rpc:call(Node, application, set_env, [App, Key, Val]) || {App, Key, Val} <- Env]. + +setup_node(Node, _Spec = #{env := Env}, Apps) -> + %% load these before starting ekka and such + [rpc:call(Node, application, load, [App]) || App <- [gen_rpc, emqx_conf, emqx]], + setenv(Node, Env), + EnvHandler = + fun(emqx) -> + application:set_env(emqx, boot_modules, [router, broker]); + (_) -> + ok + end, + ok = rpc:call(Node, emqx_common_test_helpers, start_apps, [Apps, EnvHandler]), + ok. From 29ccc98a42911be37b384fec75a41cea33b13bc5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 15 Dec 2021 10:36:13 +0800 Subject: [PATCH 31/31] fix: slow_sub api return field's notice_interval is missing --- apps/emqx_slow_subs/etc/emqx_slow_subs.conf | 4 ++-- apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_slow_subs/etc/emqx_slow_subs.conf b/apps/emqx_slow_subs/etc/emqx_slow_subs.conf index 3cf8189d8..a2ceb4cbc 100644 --- a/apps/emqx_slow_subs/etc/emqx_slow_subs.conf +++ b/apps/emqx_slow_subs/etc/emqx_slow_subs.conf @@ -25,11 +25,11 @@ emqx_slow_subs { ## publish is disabled if set to 0s. ## ## Value: 0s - expire_interval = 0s + notice_interval = 0s ## QoS of notification message ## - ## Defaut: 0 + ## Default: 0 notice_qos = 0 ## Maximum information number in one notification diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index 7bdcb16f3..c187a091e 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -33,7 +33,7 @@ fields("emqx_slow_subs") -> "QoS of notification message in notice topic")} , {notice_batch_size, sc(integer(), - 0, + 100, "Maximum information number in one notification")} ].