From d3f965dfe793e279fdf77afe7fe833b5eda1008c Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 15 Jul 2022 17:57:26 +0800 Subject: [PATCH 01/63] refactor(limiter): refactor the user interface --- apps/emqx/i18n/emqx_limiter_i18n.conf | 22 +- apps/emqx/src/emqx_channel.erl | 10 +- apps/emqx/src/emqx_connection.erl | 8 +- .../src/emqx_esockd_htb_limiter.erl | 16 +- .../src/emqx_limiter_container.erl | 44 +-- .../emqx_limiter/src/emqx_limiter_manager.erl | 50 +-- .../emqx_limiter/src/emqx_limiter_schema.erl | 83 ++--- .../emqx_limiter/src/emqx_limiter_server.erl | 204 +++++------ apps/emqx/src/emqx_listeners.erl | 59 +++- apps/emqx/src/emqx_schema.erl | 6 +- apps/emqx/src/emqx_ws_connection.erl | 11 +- apps/emqx/test/emqx_channel_SUITE.erl | 113 ++---- apps/emqx/test/emqx_connection_SUITE.erl | 38 +- apps/emqx/test/emqx_ratelimiter_SUITE.erl | 331 +++++++++--------- apps/emqx/test/emqx_ws_connection_SUITE.erl | 61 +++- apps/emqx_retainer/src/emqx_retainer.erl | 7 +- .../src/emqx_retainer_dispatcher.erl | 8 +- .../src/emqx_retainer_schema.erl | 2 +- .../test/emqx_retainer_SUITE.erl | 66 ++-- 19 files changed, 577 insertions(+), 562 deletions(-) diff --git a/apps/emqx/i18n/emqx_limiter_i18n.conf b/apps/emqx/i18n/emqx_limiter_i18n.conf index 6fbc923b7..b75435225 100644 --- a/apps/emqx/i18n/emqx_limiter_i18n.conf +++ b/apps/emqx/i18n/emqx_limiter_i18n.conf @@ -124,20 +124,6 @@ the check/consume will succeed, but it will be forced to wait for a short period } } - batch { - desc { - en: """The batch limiter. -This is used for EMQX internal batch operation -e.g. limit the retainer's deliver rate""" - zh: """批量操作速率控制器。 -这是给 EMQX 内部的批量操作使用的，比如用来控制保留消息的派发速率""" - } - label: { - en: """Batch""" - zh: """批量操作""" - } - } - message_routing { desc { en: """The message routing limiter. @@ -193,4 +179,12 @@ Once the limit is reached, the restricted client will be slow down even be hung zh: """流入字节率""" } } + + internal { + desc { + en: """Limiter for EMQX internal app.""" + zh: """EMQX 内部功能所用限制器。""" + + } + } } diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index d6f2b87ea..cdd4b1a9e 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -252,11 +252,12 @@ init( <<>> -> undefined; MP -> MP end, + ListenerId = emqx_listeners:listener_id(Type, Listener), ClientInfo = set_peercert_infos( Peercert, #{ zone => Zone, - listener => emqx_listeners:listener_id(Type, Listener), + listener => ListenerId, protocol => Protocol, peerhost => PeerHost, sockport => SockPort, @@ -278,7 +279,9 @@ init( outbound => #{} }, auth_cache = #{}, - quota = emqx_limiter_container:get_limiter_by_names([?LIMITER_ROUTING], LimiterCfg), + quota = emqx_limiter_container:get_limiter_by_types( + ListenerId, [?LIMITER_ROUTING], LimiterCfg + ), timers = #{}, conn_state = idle, takeover = false, @@ -1199,9 +1202,6 @@ handle_call( disconnect_and_shutdown(takenover, AllPendings, Channel); handle_call(list_authz_cache, Channel) -> {reply, emqx_authz_cache:list_authz_cache(), Channel}; -handle_call({quota, Bucket}, #channel{quota = Quota} = Channel) -> - Quota2 = emqx_limiter_container:update_by_name(message_routing, Bucket, Quota), - reply(ok, Channel#channel{quota = Quota2}); handle_call( {keepalive, Interval}, Channel = #channel{ diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 59248a0b8..1caf345e6 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -321,7 +321,7 @@ init_state( }, LimiterTypes = [?LIMITER_BYTES_IN, ?LIMITER_MESSAGE_IN], - Limiter = emqx_limiter_container:get_limiter_by_names(LimiterTypes, LimiterCfg), + Limiter = emqx_limiter_container:get_limiter_by_types(Listener, LimiterTypes, LimiterCfg), FrameOpts = #{ strict_mode => emqx_config:get_zone_conf(Zone, [mqtt, strict_mode]), @@ -672,12 +672,6 @@ handle_call(_From, info, State) -> {reply, info(State), State}; handle_call(_From, stats, State) -> {reply, stats(State), State}; -handle_call(_From, {ratelimit, Changes}, State = #state{limiter = Limiter}) -> - Fun = fun({Type, Bucket}, Acc) -> - emqx_limiter_container:update_by_name(Type, Bucket, Acc) - end, - Limiter2 = lists:foldl(Fun, Limiter, Changes), - {reply, ok, State#state{limiter = Limiter2}}; handle_call(_From, Req, State = #state{channel = Channel}) -> case emqx_channel:handle_call(Req, Channel) of {reply, Reply, NChannel} -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl index c39cf2728..16f7b03c8 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl @@ -19,12 +19,13 @@ -behaviour(esockd_generic_limiter). %% API --export([new_create_options/2, create/1, delete/1, consume/2]). +-export([new_create_options/3, create/1, delete/1, consume/2]). -type create_options() :: #{ module := ?MODULE, + id := emqx_limiter_schema:limiter_id(), type := emqx_limiter_schema:limiter_type(), - bucket := emqx_limiter_schema:bucket_name() + bucket := hocons:config() }. %%-------------------------------------------------------------------- @@ -32,15 +33,16 @@ %%-------------------------------------------------------------------- -spec new_create_options( + emqx_limiter_schema:limiter_id(), emqx_limiter_schema:limiter_type(), - emqx_limiter_schema:bucket_name() + hocons:config() ) -> create_options(). -new_create_options(Type, BucketName) -> - #{module => ?MODULE, type => Type, bucket => BucketName}. +new_create_options(Id, Type, BucketCfg) -> + #{module => ?MODULE, id => Id, type => Type, bucket => BucketCfg}. -spec create(create_options()) -> esockd_generic_limiter:limiter(). -create(#{module := ?MODULE, type := Type, bucket := BucketName}) -> - {ok, Limiter} = emqx_limiter_server:connect(Type, BucketName), +create(#{module := ?MODULE, id := Id, type := Type, bucket := BucketCfg}) -> + {ok, Limiter} = emqx_limiter_server:connect(Id, Type, BucketCfg), #{module => ?MODULE, name => Type, limiter => Limiter}. delete(_GLimiter) -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl index f82a97a5a..74b6c7b87 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl @@ -22,10 +22,8 @@ %% API -export([ - new/0, new/1, new/2, - get_limiter_by_names/2, + get_limiter_by_types/3, add_new/3, - update_by_name/3, set_retry_context/2, check/3, retry/2, @@ -48,10 +46,10 @@ }. -type future() :: pos_integer(). +-type limiter_id() :: emqx_limiter_schema:limiter_id(). -type limiter_type() :: emqx_limiter_schema:limiter_type(). -type limiter() :: emqx_htb_limiter:limiter(). -type retry_context() :: emqx_htb_limiter:retry_context(). --type bucket_name() :: emqx_limiter_schema:bucket_name(). -type millisecond() :: non_neg_integer(). -type check_result() :: {ok, container()} @@ -64,46 +62,24 @@ %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- --spec new() -> container(). -new() -> - new([]). - -%% @doc generate default data according to the type of limiter --spec new(list(limiter_type())) -> container(). -new(Types) -> - new(Types, #{}). - --spec new( - list(limiter_type()), - #{limiter_type() => emqx_limiter_schema:bucket_name()} -) -> container(). -new(Types, Names) -> - get_limiter_by_names(Types, Names). - %% @doc generate a container %% according to the type of limiter and the bucket name configuration of the limiter %% @end --spec get_limiter_by_names( +-spec get_limiter_by_types( + limiter_id() | {atom(), atom()}, list(limiter_type()), - #{limiter_type() => emqx_limiter_schema:bucket_name()} + #{limiter_type() => hocons:config()} ) -> container(). -get_limiter_by_names(Types, BucketNames) -> +get_limiter_by_types({Type, Listener}, Types, BucketCfgs) -> + Id = emqx_listeners:listener_id(Type, Listener), + get_limiter_by_types(Id, Types, BucketCfgs); +get_limiter_by_types(Id, Types, BucketCfgs) -> Init = fun(Type, Acc) -> - {ok, Limiter} = emqx_limiter_server:connect(Type, BucketNames), + {ok, Limiter} = emqx_limiter_server:connect(Id, Type, BucketCfgs), add_new(Type, Limiter, Acc) end, lists:foldl(Init, #{retry_ctx => undefined}, Types). -%% @doc add the specified type of limiter to the container --spec update_by_name( - limiter_type(), - bucket_name() | #{limiter_type() => bucket_name()}, - container() -) -> container(). -update_by_name(Type, Buckets, Container) -> - {ok, Limiter} = emqx_limiter_server:connect(Type, Buckets), - add_new(Type, Limiter, Container). - -spec add_new(limiter_type(), limiter(), container()) -> container(). add_new(Type, Limiter, Container) -> Container#{ diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl index 89148a12c..aca27a6ff 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl @@ -24,11 +24,9 @@ %% API -export([ start_link/0, - find_bucket/1, find_bucket/2, - insert_bucket/2, insert_bucket/3, - make_path/2, + delete_bucket/2, post_config_update/5 ]). @@ -50,20 +48,19 @@ format_status/2 ]). --export_type([path/0]). - --type path() :: list(atom()). +-type limiter_id() :: emqx_limiter_schema:limiter_id(). -type limiter_type() :: emqx_limiter_schema:limiter_type(). --type bucket_name() :: emqx_limiter_schema:bucket_name(). +-type uid() :: {limiter_id(), limiter_type()}. %% counter record in ets table -record(bucket, { - path :: path(), + uid :: uid(), bucket :: bucket_ref() }). -type bucket_ref() :: emqx_limiter_bucket_ref:bucket_ref(). +-define(UID(Id, Type), {Id, Type}). -define(TAB, emqx_limiter_counters). %%-------------------------------------------------------------------- @@ -85,14 +82,10 @@ restart_server(Type) -> stop_server(Type) -> emqx_limiter_server_sup:stop(Type). --spec find_bucket(limiter_type(), bucket_name()) -> +-spec find_bucket(limiter_id(), limiter_type()) -> {ok, bucket_ref()} | undefined. -find_bucket(Type, BucketName) -> - find_bucket(make_path(Type, BucketName)). - --spec find_bucket(path()) -> {ok, bucket_ref()} | undefined. -find_bucket(Path) -> - case ets:lookup(?TAB, Path) of +find_bucket(Id, Type) -> + case ets:lookup(?TAB, ?UID(Id, Type)) of [#bucket{bucket = Bucket}] -> {ok, Bucket}; _ -> @@ -100,20 +93,19 @@ find_bucket(Path) -> end. -spec insert_bucket( + limiter_id(), limiter_type(), - bucket_name(), bucket_ref() ) -> boolean(). -insert_bucket(Type, BucketName, Bucket) -> - inner_insert_bucket(make_path(Type, BucketName), Bucket). +insert_bucket(Id, Type, Bucket) -> + ets:insert( + ?TAB, + #bucket{uid = ?UID(Id, Type), bucket = Bucket} + ). --spec insert_bucket(path(), bucket_ref()) -> true. -insert_bucket(Path, Bucket) -> - inner_insert_bucket(Path, Bucket). - --spec make_path(limiter_type(), bucket_name()) -> path(). -make_path(Type, BucketName) -> - [Type | BucketName]. +-spec delete_bucket(limiter_id(), limiter_type()) -> true. +delete_bucket(Type, Id) -> + ets:delete(?TAB, ?UID(Id, Type)). post_config_update([limiter, Type], _Config, NewConf, _OldConf, _AppEnvs) -> Config = maps:get(Type, NewConf), @@ -159,7 +151,7 @@ init([]) -> set, public, named_table, - {keypos, #bucket.path}, + {keypos, #bucket.uid}, {write_concurrency, true}, {read_concurrency, true}, {heir, erlang:whereis(emqx_limiter_sup), none} @@ -266,9 +258,3 @@ format_status(_Opt, Status) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- --spec inner_insert_bucket(path(), bucket_ref()) -> true. -inner_insert_bucket(Path, Bucket) -> - ets:insert( - ?TAB, - #bucket{path = Path, bucket = Bucket} - ). diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 1e4679ee3..61dc95bc7 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -31,7 +31,9 @@ get_bucket_cfg_path/2, desc/1, types/0, - infinity_value/0 + infinity_value/0, + bucket_opts/0, + bucket_opts_meta/0 ]). -define(KILOBYTE, 1024). @@ -41,8 +43,10 @@ | message_in | connection | message_routing - | batch. + %% internal limiter for unclassified resources + | internal. +-type limiter_id() :: atom(). -type bucket_name() :: atom(). -type rate() :: infinity | float(). -type burst_rate() :: 0 | float(). @@ -76,7 +80,7 @@ bucket_name/0 ]). --export_type([limiter_type/0, bucket_path/0]). +-export_type([limiter_id/0, limiter_type/0, bucket_path/0]). -define(UNIT_TIME_IN_MS, 1000). @@ -87,13 +91,13 @@ roots() -> [limiter]. fields(limiter) -> [ {Type, - ?HOCON(?R_REF(limiter_opts), #{ + ?HOCON(?R_REF(node_opts), #{ desc => ?DESC(Type), default => make_limiter_default(Type) })} || Type <- types() ]; -fields(limiter_opts) -> +fields(node_opts) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, {burst, @@ -101,38 +105,16 @@ fields(limiter_opts) -> desc => ?DESC(burst), default => 0 })}, - {bucket, - ?HOCON( - ?MAP("bucket_name", ?R_REF(bucket_opts)), - #{ - desc => ?DESC(bucket_cfg), - default => #{<<"default">> => #{}}, - example => #{ - <<"mybucket-name">> => #{ - <<"rate">> => <<"infinity">>, - <<"capcity">> => <<"infinity">>, - <<"initial">> => <<"100">>, - <<"per_client">> => #{<<"rate">> => <<"infinity">>} - } - } - } - )} + {client, ?HOCON(?R_REF(client_opts), #{default => #{}})} ]; fields(bucket_opts) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => "infinity"})}, {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})}, - {per_client, - ?HOCON( - ?R_REF(client_bucket), - #{ - default => #{}, - desc => ?DESC(per_client) - } - )} + {client, ?HOCON(?R_REF(client_opts), #{required => false})} ]; -fields(client_bucket) -> +fields(client_opts) -> [ {rate, ?HOCON(rate(), #{default => "infinity", desc => ?DESC(rate)})}, {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})}, @@ -181,15 +163,35 @@ fields(client_bucket) -> desc(limiter) -> "Settings for the rate limiter."; -desc(limiter_opts) -> - "Settings for the limiter."; +desc(node_opts) -> + "Settings for the limiter of the node level."; +desc(node_client_opts) -> + "Settings for the client in the node level."; desc(bucket_opts) -> "Settings for the bucket."; -desc(client_bucket) -> - "Settings for the client bucket."; +desc(client_opts) -> + "Settings for the client in bucket level."; desc(_) -> undefined. +bucket_opts() -> + ?HOCON( + ?MAP("bucket_name", ?R_REF(bucket_opts)), + bucket_opts_meta() + ). + +bucket_opts_meta() -> + #{ + default => #{}, + example => + #{ + <<"rate">> => <<"infinity">>, + <<"capcity">> => <<"infinity">>, + <<"initial">> => <<"100">>, + <<"client">> => #{<<"rate">> => <<"infinity">>} + } + }. + %% default period is 100ms default_period() -> 100. @@ -202,7 +204,7 @@ get_bucket_cfg_path(Type, BucketName) -> [limiter, Type, bucket, BucketName]. types() -> - [bytes_in, message_in, connection, message_routing, batch]. + [bytes_in, message_in, connection, message_routing, internal]. %%-------------------------------------------------------------------- %% Internal functions @@ -323,15 +325,6 @@ apply_unit("gb", Val) -> Val * ?KILOBYTE * ?KILOBYTE * ?KILOBYTE; apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). make_limiter_default(connection) -> - #{ - <<"rate">> => <<"1000/s">>, - <<"bucket">> => #{ - <<"default">> => - #{ - <<"rate">> => <<"1000/s">>, - <<"capacity">> => 1000 - } - } - }; + #{<<"rate">> => <<"1000/s">>}; make_limiter_default(_) -> #{}. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 519b32eca..939824e02 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -42,11 +42,13 @@ -export([ start_link/2, - connect/2, + connect/3, + add_bucket/3, + del_bucket/2, + get_initial_val/1, whereis/1, info/1, name/1, - get_initial_val/1, restart/1, update_config/2 ]). @@ -83,6 +85,7 @@ -type buckets() :: #{bucket_name() => bucket()}. -type limiter_type() :: emqx_limiter_schema:limiter_type(). -type bucket_name() :: emqx_limiter_schema:bucket_name(). +-type limiter_id() :: emqx_limiter_schema:limiter_id(). -type rate() :: decimal(). -type flow() :: decimal(). -type capacity() :: decimal(). @@ -94,7 +97,7 @@ %% minimum coefficient for overloaded limiter -define(OVERLOAD_MIN_ALLOC, 0.3). --define(CURRYING(X, F2), fun(Y) -> F2(X, Y) end). +-define(COUNTER_SIZE, 8). -export_type([index/0]). -import(emqx_limiter_decimal, [add/2, sub/2, mul/2, put_to_counter/3]). @@ -105,39 +108,53 @@ %% API %%-------------------------------------------------------------------- -spec connect( + limiter_id(), limiter_type(), bucket_name() | #{limiter_type() => bucket_name() | undefined} ) -> {ok, emqx_htb_limiter:limiter()} | {error, _}. %% If no bucket path is set in config, there will be no limit -connect(_Type, undefined) -> +connect(_Id, _Type, undefined) -> {ok, emqx_htb_limiter:make_infinity_limiter()}; -connect(Type, BucketName) when is_atom(BucketName) -> - case get_bucket_cfg(Type, BucketName) of - undefined -> - ?SLOG(error, #{msg => "bucket_config_not_found", type => Type, bucket => BucketName}), - {error, config_not_found}; - #{ - rate := BucketRate, - capacity := BucketSize, - per_client := #{rate := CliRate, capacity := CliSize} = Cfg - } -> - case emqx_limiter_manager:find_bucket(Type, BucketName) of - {ok, Bucket} -> +connect( + Id, + Type, + #{ + rate := BucketRate, + capacity := BucketSize + } = BucketCfg +) -> + case emqx_limiter_manager:find_bucket(Id, Type) of + {ok, Bucket} -> + case find_client_cfg(Type, BucketCfg) of + #{rate := CliRate, capacity := CliSize} = ClientCfg -> {ok, if CliRate < BucketRate orelse CliSize < BucketSize -> - emqx_htb_limiter:make_token_bucket_limiter(Cfg, Bucket); + emqx_htb_limiter:make_token_bucket_limiter(ClientCfg, Bucket); true -> - emqx_htb_limiter:make_ref_limiter(Cfg, Bucket) + emqx_htb_limiter:make_ref_limiter(ClientCfg, Bucket) end}; - undefined -> - ?SLOG(error, #{msg => "bucket_not_found", type => Type, bucket => BucketName}), - {error, invalid_bucket} - end + {error, invalid_node_cfg} = Error -> + ?SLOG(error, #{msg => "invalid_node_cfg", type => Type, id => Id}), + Error + end; + undefined -> + ?SLOG(error, #{msg => "bucket_not_found", type => Type, id => Id}), + {error, invalid_bucket} end; -connect(Type, Paths) -> - connect(Type, maps:get(Type, Paths, undefined)). +connect(Id, Type, Paths) -> + connect(Id, Type, maps:get(Type, Paths, undefined)). + +-spec add_bucket(limiter_id(), limiter_type(), hocons:config() | undefined) -> ok. +add_bucket(_Id, _Type, undefine) -> + ok; +add_bucket(Id, Type, Cfg) -> + ?CALL(Type, {add_bucket, Id, Cfg}). + +-spec del_bucket(limiter_id(), limiter_type()) -> ok. +del_bucket(Id, Type) -> + ?CALL(Type, {del_bucket, Id}). -spec info(limiter_type()) -> state() | {error, _}. info(Type) -> @@ -213,6 +230,12 @@ handle_call(restart, _From, #{type := Type}) -> handle_call({update_config, Type, Config}, _From, #{type := Type}) -> NewState = init_tree(Type, Config), {reply, ok, NewState}; +handle_call({add_bucket, Id, Cfg}, _From, State) -> + NewState = do_add_bucket(Id, Cfg, State), + {reply, ok, NewState}; +handle_call({del_bucket, Id}, _From, State) -> + NewState = do_del_bucket(Id, State), + {reply, ok, NewState}; handle_call(Req, _From, State) -> ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. @@ -456,24 +479,14 @@ init_tree(Type) when is_atom(Type) -> Cfg = emqx:get_config([limiter, Type]), init_tree(Type, Cfg). -init_tree(Type, #{bucket := Buckets} = Cfg) -> - State = #{ +init_tree(Type, Cfg) -> + #{ type => Type, - root => undefined, - counter => undefined, - index => 1, + root => make_root(Cfg), + counter => counters:new(?COUNTER_SIZE, [write_concurrency]), + index => 0, buckets => #{} - }, - - Root = make_root(Cfg), - {CounterNum, DelayBuckets} = make_bucket(maps:to_list(Buckets), Type, Cfg, 1, []), - - State2 = State#{ - root := Root, - counter := counters:new(CounterNum, [write_concurrency]) - }, - - lists:foldl(fun(F, Acc) -> F(Acc) end, State2, DelayBuckets). + }. -spec make_root(hocons:confg()) -> root(). make_root(#{rate := Rate, burst := Burst}) -> @@ -484,79 +497,50 @@ make_root(#{rate := Rate, burst := Burst}) -> produced => 0.0 }. -make_bucket([{Name, Conf} | T], Type, GlobalCfg, CounterNum, DelayBuckets) -> - Path = emqx_limiter_manager:make_path(Type, Name), - Rate = get_counter_rate(Conf, GlobalCfg), - #{capacity := Capacity} = Conf, - Initial = get_initial_val(Conf), - CounterNum2 = CounterNum + 1, - InitFun = fun(#{name := BucketName} = Bucket, #{buckets := Buckets} = State) -> - {Counter, Idx, State2} = alloc_counter(Path, Rate, Initial, State), - Bucket2 = Bucket#{counter := Counter, index := Idx}, - State2#{buckets := Buckets#{BucketName => Bucket2}} - end, +do_add_bucket(Id, #{rate := Rate, capacity := Capacity} = Cfg, #{buckets := Buckets} = State) -> + case maps:get(Id, Buckets, undefined) of + undefined -> + make_bucket(Id, Cfg, State); + Bucket -> + Bucket2 = Bucket#{rate := Rate, capacity := Capacity}, + State#{buckets := Buckets#{Id := Bucket2}} + end. +make_bucket(Id, Cfg, #{index := ?COUNTER_SIZE} = State) -> + add_bucket(Id, Cfg, State#{ + counter => counters:new(?COUNTER_SIZE, [write_concurrency]), + index => 0 + }); +make_bucket( + Id, + #{rate := Rate, capacity := Capacity} = Cfg, + #{type := Type, counter := Counter, index := Index, buckets := Buckets} = State +) -> + NewIndex = Index + 1, + Initial = get_initial_val(Cfg), Bucket = #{ - name => Name, + name => Id, rate => Rate, obtained => Initial, correction => 0, capacity => Capacity, - counter => undefined, - index => undefined + counter => Counter, + index => NewIndex }, + _ = put_to_counter(Counter, NewIndex, Initial), + Ref = emqx_limiter_bucket_ref:new(Counter, NewIndex, Rate), + emqx_limiter_manager:insert_bucket(Id, Type, Ref), + State#{buckets := Buckets#{Id => Bucket}}. - DelayInit = ?CURRYING(Bucket, InitFun), - - make_bucket( - T, - Type, - GlobalCfg, - CounterNum2, - [DelayInit | DelayBuckets] - ); -make_bucket([], _Type, _Global, CounterNum, DelayBuckets) -> - {CounterNum, DelayBuckets}. - --spec alloc_counter(emqx_limiter_manager:path(), rate(), capacity(), state()) -> - {counters:counters_ref(), pos_integer(), state()}. -alloc_counter( - Path, - Rate, - Initial, - #{counter := Counter, index := Index} = State -) -> - case emqx_limiter_manager:find_bucket(Path) of - {ok, #{ - counter := ECounter, - index := EIndex - }} when ECounter =/= undefined -> - init_counter(Path, ECounter, EIndex, Rate, Initial, State); +do_del_bucket(Id, #{type := Type, buckets := Buckets} = State) -> + case maps:get(Id, Buckets, undefined) of + undefined -> + State; _ -> - init_counter( - Path, - Counter, - Index, - Rate, - Initial, - State#{index := Index + 1} - ) + emqx_limiter_manager:delete_bucket(Id, Type), + State#{buckets := maps:remove(Id, Buckets)} end. -init_counter(Path, Counter, Index, Rate, Initial, State) -> - _ = put_to_counter(Counter, Index, Initial), - Ref = emqx_limiter_bucket_ref:new(Counter, Index, Rate), - emqx_limiter_manager:insert_bucket(Path, Ref), - {Counter, Index, State}. - -%% @doc find first limited node -get_counter_rate(#{rate := Rate}, _GlobalCfg) when Rate =/= infinity -> - Rate; -get_counter_rate(_Cfg, #{rate := Rate}) when Rate =/= infinity -> - Rate; -get_counter_rate(_Cfg, _GlobalCfg) -> - emqx_limiter_schema:infinity_value(). - -spec get_initial_val(hocons:config()) -> decimal(). get_initial_val( #{ @@ -587,8 +571,14 @@ call(Type, Msg) -> gen_server:call(Pid, Msg) end. --spec get_bucket_cfg(limiter_type(), bucket_name()) -> - undefined | limiter_not_started | hocons:config(). -get_bucket_cfg(Type, Bucket) -> - Path = emqx_limiter_schema:get_bucket_cfg_path(Type, Bucket), - emqx:get_config(Path, undefined). +find_client_cfg(Type, Cfg) -> + NodeCfg = emqx:get_config([limiter, Type, client], undefined), + BucketCfg = maps:get(client, Cfg, undefined), + merge_client_cfg(NodeCfg, BucketCfg). + +merge_client_cfg(undefined, BucketCfg) -> + BucketCfg; +merge_client_cfg(NodeCfg, undefined) -> + NodeCfg; +merge_client_cfg(NodeCfg, BucketCfg) -> + maps:merge(NodeCfg, BucketCfg). diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 3c508bacc..dcb35adbb 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -279,12 +279,19 @@ stop_listener(Type, ListenerName, #{bind := Bind} = Conf) -> end. -spec do_stop_listener(atom(), atom(), map()) -> ok | {error, term()}. -do_stop_listener(Type, ListenerName, #{bind := ListenOn}) when Type == tcp; Type == ssl -> - esockd:close(listener_id(Type, ListenerName), ListenOn); -do_stop_listener(Type, ListenerName, _Conf) when Type == ws; Type == wss -> - cowboy:stop_listener(listener_id(Type, ListenerName)); -do_stop_listener(quic, ListenerName, _Conf) -> - quicer:stop_listener(listener_id(quic, ListenerName)). + +do_stop_listener(Type, ListenerName, #{bind := ListenOn} = Conf) when Type == tcp; Type == ssl -> + Id = listener_id(Type, ListenerName), + del_limiter_bucket(Id, Conf), + esockd:close(Id, ListenOn); +do_stop_listener(Type, ListenerName, Conf) when Type == ws; Type == wss -> + Id = listener_id(Type, ListenerName), + del_limiter_bucket(Id, Conf), + cowboy:stop_listener(Id); +do_stop_listener(quic, ListenerName, Conf) -> + Id = listener_id(quic, ListenerName), + del_limiter_bucket(Id, Conf), + quicer:stop_listener(Id). -ifndef(TEST). console_print(Fmt, Args) -> ?ULOG(Fmt, Args). @@ -300,10 +307,12 @@ do_start_listener(_Type, _ListenerName, #{enabled := false}) -> do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when Type == tcp; Type == ssl -> + Id = listener_id(Type, ListenerName), + add_limiter_bucket(Id, Opts), esockd:open( - listener_id(Type, ListenerName), + Id, ListenOn, - merge_default(esockd_opts(Type, Opts)), + merge_default(esockd_opts(Id, Type, Opts)), {emqx_connection, start_link, [ #{ listener => {Type, ListenerName}, @@ -318,6 +327,7 @@ do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when Type == ws; Type == wss -> Id = listener_id(Type, ListenerName), + add_limiter_bucket(Id, Opts), RanchOpts = ranch_opts(Type, ListenOn, Opts), WsOpts = ws_opts(Type, ListenerName, Opts), case Type of @@ -352,8 +362,10 @@ do_start_listener(quic, ListenerName, #{bind := ListenOn} = Opts) -> limiter => limiter(Opts) }, StreamOpts = [{stream_callback, emqx_quic_stream}], + Id = listener_id(quic, ListenerName), + add_limiter_bucket(Id, Opts), quicer:start_listener( - listener_id(quic, ListenerName), + Id, port(ListenOn), {ListenOpts, ConnectionOpts, StreamOpts} ); @@ -410,16 +422,18 @@ post_config_update([listeners, Type, Name], {action, _Action, _}, NewConf, OldCo post_config_update(_Path, _Request, _NewConf, _OldConf, _AppEnvs) -> ok. -esockd_opts(Type, Opts0) -> +esockd_opts(ListenerId, Type, Opts0) -> Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0), Limiter = limiter(Opts0), Opts2 = case maps:get(connection, Limiter, undefined) of undefined -> Opts1; - BucketName -> + BucketCfg -> Opts1#{ - limiter => emqx_esockd_htb_limiter:new_create_options(connection, BucketName) + limiter => emqx_esockd_htb_limiter:new_create_options( + ListenerId, connection, BucketCfg + ) } end, Opts3 = Opts2#{ @@ -524,6 +538,27 @@ zone(Opts) -> limiter(Opts) -> maps:get(limiter, Opts, #{}). +add_limiter_bucket(Id, #{limiter := Limiters}) -> + maps:fold( + fun(Type, Cfg, _) -> + emqx_limiter_server:add_bucket(Id, Type, Cfg) + end, + ok, + Limiters + ); +add_limiter_bucket(_Id, _cfg) -> + ok. + +del_limiter_bucket(Id, #{limiter := Limiters}) -> + lists:foreach( + fun(Type) -> + emqx_limiter_server:del_bucket(Id, Type) + end, + maps:keys(Limiters) + ); +del_limiter_bucket(_Id, _cfg) -> + ok. + enable_authn(Opts) -> maps:get(enable_authn, Opts, true). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 5652b37f7..0c8a073a0 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1619,10 +1619,10 @@ base_listener(Bind) -> )}, {"limiter", sc( - map("ratelimit_name", emqx_limiter_schema:bucket_name()), + map("ratelimit_name", ?R_REF(emqx_limiter_schema, bucket_opts)), #{ - desc => ?DESC(base_listener_limiter), - default => #{<<"connection">> => <<"default">>} + desc => ?DESC(base_listener_limiter) + %% TODO default => #{<<"connection">> => <<"default">>} } )}, {"enable_authn", diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 0134810c1..c7c31a2d8 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -273,7 +273,7 @@ check_origin_header(Req, #{listener := {Type, Listener}} = Opts) -> end. websocket_init([Req, Opts]) -> - #{zone := Zone, limiter := LimiterCfg, listener := {Type, Listener}} = Opts, + #{zone := Zone, limiter := LimiterCfg, listener := {Type, Listener} = ListenerCfg} = Opts, case check_max_connection(Type, Listener) of allow -> {Peername, PeerCert} = get_peer_info(Type, Listener, Req, Opts), @@ -287,8 +287,10 @@ websocket_init([Req, Opts]) -> ws_cookie => WsCookie, conn_mod => ?MODULE }, - Limiter = emqx_limiter_container:get_limiter_by_names( - [?LIMITER_BYTES_IN, ?LIMITER_MESSAGE_IN], LimiterCfg + Limiter = emqx_limiter_container:get_limiter_by_types( + ListenerCfg, + [?LIMITER_BYTES_IN, ?LIMITER_MESSAGE_IN], + LimiterCfg ), MQTTPiggyback = get_ws_opts(Type, Listener, mqtt_piggyback), FrameOpts = #{ @@ -487,9 +489,6 @@ handle_call(From, info, State) -> handle_call(From, stats, State) -> gen_server:reply(From, stats(State)), return(State); -handle_call(_From, {ratelimit, Type, Bucket}, State = #state{limiter = Limiter}) -> - Limiter2 = emqx_limiter_container:update_by_name(Type, Bucket, Limiter), - {reply, ok, State#state{limiter = Limiter2}}; handle_call(From, Req, State = #state{channel = Channel}) -> case emqx_channel:handle_call(Req, Channel) of {reply, Reply, NChannel} -> diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 40bf6ff45..03be7448c 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -33,18 +33,6 @@ force_gc_conf() -> force_shutdown_conf() -> #{enable => true, max_heap_size => 4194304, max_message_queue_len => 1000}. -rate_limit_conf() -> - #{ - conn_bytes_in => ["100KB", "10s"], - conn_messages_in => ["100", "10s"], - max_conn_rate => 1000, - quota => - #{ - conn_messages_routing => infinity, - overall_messages_routing => infinity - } - }. - rpc_conf() -> #{ async_batch_size => 256, @@ -173,27 +161,9 @@ listeners_conf() -> limiter_conf() -> Make = fun() -> #{ - bucket => - #{ - default => - #{ - capacity => infinity, - initial => 0, - rate => infinity, - per_client => - #{ - capacity => infinity, - divisible => false, - failure_strategy => force, - initial => 0, - low_watermark => 0, - max_retry_time => 5000, - rate => infinity - } - } - }, burst => 0, - rate => infinity + rate => infinity, + capacity => infinity } end, @@ -202,7 +172,7 @@ limiter_conf() -> Acc#{Name => Make()} end, #{}, - [bytes_in, message_in, message_routing, connection, batch] + [bytes_in, message_in, message_routing, connection, internal] ). stats_conf() -> @@ -213,7 +183,6 @@ zone_conf() -> basic_conf() -> #{ - rate_limit => rate_limit_conf(), force_gc => force_gc_conf(), force_shutdown => force_shutdown_conf(), mqtt => mqtt_conf(), @@ -274,10 +243,9 @@ end_per_suite(_Config) -> emqx_banned ]). -init_per_testcase(TestCase, Config) -> +init_per_testcase(_TestCase, Config) -> OldConf = set_test_listener_confs(), emqx_common_test_helpers:start_apps([]), - check_modify_limiter(TestCase), [{config, OldConf} | Config]. end_per_testcase(_TestCase, Config) -> @@ -285,41 +253,6 @@ end_per_testcase(_TestCase, Config) -> emqx_common_test_helpers:stop_apps([]), Config. -check_modify_limiter(TestCase) -> - Checks = [t_quota_qos0, t_quota_qos1, t_quota_qos2], - case lists:member(TestCase, Checks) of - true -> - modify_limiter(); - _ -> - ok - end. - -%% per_client 5/1s,5 -%% aggregated 10/1s,10 -modify_limiter() -> - Limiter = emqx_config:get([limiter]), - #{message_routing := #{bucket := Bucket} = Routing} = Limiter, - #{default := #{per_client := Client} = Default} = Bucket, - Client2 = Client#{ - rate := 5, - initial := 0, - capacity := 5, - low_watermark := 1 - }, - Default2 = Default#{ - per_client := Client2, - rate => 10, - initial => 0, - capacity => 10 - }, - Bucket2 = Bucket#{default := Default2}, - Routing2 = Routing#{bucket := Bucket2}, - - emqx_config:put([limiter], Limiter#{message_routing := Routing2}), - emqx_limiter_manager:restart_server(message_routing), - timer:sleep(100), - ok. - %%-------------------------------------------------------------------- %% Test cases for channel info/stats/caps %%-------------------------------------------------------------------- @@ -729,6 +662,7 @@ t_process_unsubscribe(_) -> t_quota_qos0(_) -> esockd_limiter:start_link(), + add_bucket(), Cnter = counters:new(1, []), ok = meck:expect(emqx_broker, publish, fun(_) -> [{node(), <<"topic">>, {ok, 4}}] end), ok = meck:expect( @@ -755,10 +689,12 @@ t_quota_qos0(_) -> ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end), ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end), + del_bucket(), esockd_limiter:stop(). t_quota_qos1(_) -> esockd_limiter:start_link(), + add_bucket(), ok = meck:expect(emqx_broker, publish, fun(_) -> [{node(), <<"topic">>, {ok, 4}}] end), Chann = channel(#{conn_state => connected, quota => quota()}), Pub = ?PUBLISH_PACKET(?QOS_1, <<"topic">>, 1, <<"payload">>), @@ -769,10 +705,12 @@ t_quota_qos1(_) -> {ok, ?PUBACK_PACKET(1, ?RC_SUCCESS), Chann4} = emqx_channel:handle_in(Pub, Chann3), %% Quota in overall {ok, ?PUBACK_PACKET(1, ?RC_QUOTA_EXCEEDED), _} = emqx_channel:handle_in(Pub, Chann4), + del_bucket(), esockd_limiter:stop(). t_quota_qos2(_) -> esockd_limiter:start_link(), + add_bucket(), ok = meck:expect(emqx_broker, publish, fun(_) -> [{node(), <<"topic">>, {ok, 4}}] end), Chann = channel(#{conn_state => connected, quota => quota()}), Pub1 = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 1, <<"payload">>), @@ -786,6 +724,7 @@ t_quota_qos2(_) -> {ok, ?PUBREC_PACKET(3, ?RC_SUCCESS), Chann4} = emqx_channel:handle_in(Pub3, Chann3), %% Quota in overall {ok, ?PUBREC_PACKET(4, ?RC_QUOTA_EXCEEDED), _} = emqx_channel:handle_in(Pub4, Chann4), + del_bucket(), esockd_limiter:stop(). %%-------------------------------------------------------------------- @@ -952,12 +891,6 @@ t_handle_call_takeover_end(_) -> {shutdown, takenover, [], _, _Chan} = emqx_channel:handle_call({takeover, 'end'}, channel()). -t_handle_call_quota(_) -> - {reply, ok, _Chan} = emqx_channel:handle_call( - {quota, default}, - channel() - ). - t_handle_call_unexpected(_) -> {reply, ignored, _Chan} = emqx_channel:handle_call(unexpected_req, channel()). @@ -1176,7 +1109,7 @@ t_ws_cookie_init(_) -> ConnInfo, #{ zone => default, - limiter => limiter_cfg(), + limiter => undefined, listener => {tcp, default} } ), @@ -1210,7 +1143,7 @@ channel(InitFields) -> ConnInfo, #{ zone => default, - limiter => limiter_cfg(), + limiter => undefined, listener => {tcp, default} } ), @@ -1270,9 +1203,27 @@ session(InitFields) when is_map(InitFields) -> %% conn: 5/s; overall: 10/s quota() -> - emqx_limiter_container:get_limiter_by_names([message_routing], limiter_cfg()). + emqx_limiter_container:get_limiter_by_types(?MODULE, [message_routing], limiter_cfg()). -limiter_cfg() -> #{message_routing => default}. +limiter_cfg() -> #{message_routing => make_limiter_cfg()}. + +make_limiter_cfg() -> + Client = #{ + rate => 5, + initial => 0, + capacity => 5, + low_watermark => 1, + divisible => false, + max_retry_time => timer:seconds(5), + failure_strategy => force + }, + #{client => Client, rate => 10, initial => 0, capacity => 10}. + +add_bucket() -> + emqx_limiter_server:add_bucket(?MODULE, message_routing, make_limiter_cfg()). + +del_bucket() -> + emqx_limiter_server:del_bucket(?MODULE, message_routing). v4(Channel) -> ConnInfo = emqx_channel:info(conninfo, Channel), diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index b199565c2..141dbdad6 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -78,6 +78,7 @@ end_per_suite(_Config) -> init_per_testcase(TestCase, Config) when TestCase =/= t_ws_pingreq_before_connected -> + add_bucket(), ok = meck:expect(emqx_transport, wait, fun(Sock) -> {ok, Sock} end), ok = meck:expect(emqx_transport, type, fun(_Sock) -> tcp end), ok = meck:expect( @@ -104,9 +105,11 @@ init_per_testcase(TestCase, Config) when _ -> Config end; init_per_testcase(_, Config) -> + add_bucket(), Config. end_per_testcase(TestCase, Config) -> + del_bucket(), case erlang:function_exported(?MODULE, TestCase, 2) of true -> ?MODULE:TestCase('end', Config); false -> ok @@ -291,11 +294,6 @@ t_handle_call(_) -> ?assertMatch({ok, _St}, handle_msg({event, undefined}, St)), ?assertMatch({reply, _Info, _NSt}, handle_call(self(), info, St)), ?assertMatch({reply, _Stats, _NSt}, handle_call(self(), stats, St)), - ?assertMatch({reply, ok, _NSt}, handle_call(self(), {ratelimit, []}, St)), - ?assertMatch( - {reply, ok, _NSt}, - handle_call(self(), {ratelimit, [{bytes_in, default}]}, St) - ), ?assertEqual({reply, ignored, St}, handle_call(self(), for_testing, St)), ?assertMatch( {stop, {shutdown, kicked}, ok, _NSt}, @@ -704,7 +702,33 @@ handle_msg(Msg, St) -> emqx_connection:handle_msg(Msg, St). handle_call(Pid, Call, St) -> emqx_connection:handle_call(Pid, Call, St). -limiter_cfg() -> #{}. +-define(LIMITER_ID, 'tcp:default'). init_limiter() -> - emqx_limiter_container:get_limiter_by_names([bytes_in, message_in], limiter_cfg()). + emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], limiter_cfg()). + +limiter_cfg() -> + Cfg = make_limiter_cfg(), + #{bytes_in => Cfg, message_in => Cfg}. + +make_limiter_cfg() -> + Infinity = emqx_limiter_schema:infinity_value(), + Client = #{ + rate => Infinity, + initial => 0, + capacity => Infinity, + low_watermark => 1, + divisible => false, + max_retry_time => timer:seconds(5), + failure_strategy => force + }, + #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + +add_bucket() -> + Cfg = make_limiter_cfg(), + emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), + emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). + +del_bucket() -> + emqx_limiter_server:del_bucket(?LIMITER_ID, bytes_in), + emqx_limiter_server:del_bucket(?LIMITER_ID, message_in). diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index 1251278f2..e31a220e9 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -24,48 +24,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(BASE_CONF, << - "" - "\n" - "limiter {\n" - " bytes_in {\n" - " bucket.default {\n" - " rate = infinity\n" - " capacity = infinity\n" - " }\n" - " }\n" - "\n" - " message_in {\n" - " bucket.default {\n" - " rate = infinity\n" - " capacity = infinity\n" - " }\n" - " }\n" - "\n" - " connection {\n" - " bucket.default {\n" - " rate = infinity\n" - " capacity = infinity\n" - " }\n" - " }\n" - "\n" - " message_routing {\n" - " bucket.default {\n" - " rate = infinity\n" - " capacity = infinity\n" - " }\n" - " }\n" - "\n" - " batch {\n" - " bucket.retainer {\n" - " rate = infinity\n" - " capacity = infinity\n" - " }\n" - " }\n" - "}\n" - "\n" - "" ->>). +-define(BASE_CONF, <<"">>). -record(client, { counter :: counters:counter_ref(), @@ -97,6 +56,9 @@ end_per_suite(_Config) -> init_per_testcase(_TestCase, Config) -> Config. +end_per_testcase(_TestCase, Config) -> + Config. + load_conf() -> emqx_common_test_helpers:load_config(emqx_limiter_schema, ?BASE_CONF). @@ -116,12 +78,12 @@ t_consume(_) -> failure_strategy := force } end, - Case = fun() -> - Client = connect(default), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), {ok, L2} = emqx_htb_limiter:consume(50, Client), {ok, _L3} = emqx_htb_limiter:consume(150, L2) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_retry(_) -> Cfg = fun(Cfg) -> @@ -133,15 +95,15 @@ t_retry(_) -> failure_strategy := force } end, - Case = fun() -> - Client = connect(default), - {ok, Client} = emqx_htb_limiter:retry(Client), - {_, _, Retry, L2} = emqx_htb_limiter:check(150, Client), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), + {ok, Client2} = emqx_htb_limiter:retry(Client), + {_, _, Retry, L2} = emqx_htb_limiter:check(150, Client2), L3 = emqx_htb_limiter:set_retry(Retry, L2), timer:sleep(500), {ok, _L4} = emqx_htb_limiter:retry(L3) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_restore(_) -> Cfg = fun(Cfg) -> @@ -153,15 +115,15 @@ t_restore(_) -> failure_strategy := force } end, - Case = fun() -> - Client = connect(default), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), {_, _, Retry, L2} = emqx_htb_limiter:check(150, Client), timer:sleep(200), {ok, L3} = emqx_htb_limiter:check(Retry, L2), Avaiable = emqx_htb_limiter:available(L3), ?assert(Avaiable >= 50) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_max_retry_time(_) -> Cfg = fun(Cfg) -> @@ -172,15 +134,15 @@ t_max_retry_time(_) -> failure_strategy := drop } end, - Case = fun() -> - Client = connect(default), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), Begin = ?NOW, Result = emqx_htb_limiter:consume(101, Client), ?assertMatch({drop, _}, Result), Time = ?NOW - Begin, ?assert(Time >= 500 andalso Time < 550) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_divisible(_) -> Cfg = fun(Cfg) -> @@ -191,8 +153,8 @@ t_divisible(_) -> capacity := 600 } end, - Case = fun() -> - Client = connect(default), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), Result = emqx_htb_limiter:check(1000, Client), ?assertMatch( {partial, 400, @@ -206,7 +168,7 @@ t_divisible(_) -> Result ) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_low_watermark(_) -> Cfg = fun(Cfg) -> @@ -217,8 +179,8 @@ t_low_watermark(_) -> capacity := 1000 } end, - Case = fun() -> - Client = connect(default), + Case = fun(BucketCfg) -> + Client = connect(BucketCfg), Result = emqx_htb_limiter:check(500, Client), ?assertMatch({ok, _}, Result), {_, Client2} = Result, @@ -233,28 +195,21 @@ t_low_watermark(_) -> Result2 ) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). t_infinity_client(_) -> - Fun = fun(#{per_client := Cli} = Bucket) -> - Bucket2 = Bucket#{ - rate := infinity, - capacity := infinity - }, - Cli2 = Cli#{rate := infinity, capacity := infinity}, - Bucket2#{per_client := Cli2} - end, - Case = fun() -> - Client = connect(default), + Fun = fun(Cfg) -> Cfg end, + Case = fun(Cfg) -> + Client = connect(Cfg), InfVal = emqx_limiter_schema:infinity_value(), ?assertMatch(#{bucket := #{rate := InfVal}}, Client), Result = emqx_htb_limiter:check(100000, Client), ?assertEqual({ok, Client}, Result) end, - with_bucket(default, Fun, Case). + with_per_client(Fun, Case). t_try_restore_agg(_) -> - Fun = fun(#{per_client := Cli} = Bucket) -> + Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := 1, capacity := 200, @@ -267,20 +222,20 @@ t_try_restore_agg(_) -> max_retry_time := 100, failure_strategy := force }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, - Case = fun() -> - Client = connect(default), + Case = fun(Cfg) -> + Client = connect(Cfg), {_, _, Retry, L2} = emqx_htb_limiter:check(150, Client), timer:sleep(200), {ok, L3} = emqx_htb_limiter:check(Retry, L2), Avaiable = emqx_htb_limiter:available(L3), ?assert(Avaiable >= 50) end, - with_bucket(default, Fun, Case). + with_bucket(Fun, Case). t_short_board(_) -> - Fun = fun(#{per_client := Cli} = Bucket) -> + Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("100/1s"), initial := 0, @@ -291,18 +246,18 @@ t_short_board(_) -> capacity := 600, initial := 600 }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, - Case = fun() -> + Case = fun(Cfg) -> Counter = counters:new(1, []), - start_client(default, ?NOW + 2000, Counter, 20), + start_client(Cfg, ?NOW + 2000, Counter, 20), timer:sleep(2100), check_average_rate(Counter, 2, 100) end, - with_bucket(default, Fun, Case). + with_bucket(Fun, Case). t_rate(_) -> - Fun = fun(#{per_client := Cli} = Bucket) -> + Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("100/100ms"), initial := 0, @@ -313,10 +268,10 @@ t_rate(_) -> capacity := infinity, initial := 0 }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, - Case = fun() -> - Client = connect(default), + Case = fun(Cfg) -> + Client = connect(Cfg), Ts1 = erlang:system_time(millisecond), C1 = emqx_htb_limiter:available(Client), timer:sleep(1000), @@ -326,11 +281,11 @@ t_rate(_) -> Inc = C2 - C1, ?assert(in_range(Inc, ShouldInc - 100, ShouldInc + 100), "test bucket rate") end, - with_bucket(default, Fun, Case). + with_bucket(Fun, Case). t_capacity(_) -> Capacity = 600, - Fun = fun(#{per_client := Cli} = Bucket) -> + Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("100/100ms"), initial := 0, @@ -341,15 +296,15 @@ t_capacity(_) -> capacity := infinity, initial := 0 }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, - Case = fun() -> - Client = connect(default), + Case = fun(Cfg) -> + Client = connect(Cfg), timer:sleep(1000), C1 = emqx_htb_limiter:available(Client), ?assertEqual(Capacity, C1, "test bucket capacity") end, - with_bucket(default, Fun, Case). + with_bucket(Fun, Case). %%-------------------------------------------------------------------- %% Test Cases Global Level @@ -359,7 +314,7 @@ t_collaborative_alloc(_) -> Cfg#{rate := ?RATE("600/1s")} end, - Bucket1 = fun(#{per_client := Cli} = Bucket) -> + Bucket1 = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("400/1s"), initial := 0, @@ -370,7 +325,7 @@ t_collaborative_alloc(_) -> capacity := 100, initial := 100 }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, Bucket2 = fun(Bucket) -> @@ -381,8 +336,8 @@ t_collaborative_alloc(_) -> Case = fun() -> C1 = counters:new(1, []), C2 = counters:new(1, []), - start_client(b1, ?NOW + 2000, C1, 20), - start_client(b2, ?NOW + 2000, C2, 30), + start_client({b1, Bucket1}, ?NOW + 2000, C1, 20), + start_client({b2, Bucket2}, ?NOW + 2000, C2, 30), timer:sleep(2100), check_average_rate(C1, 2, 300), check_average_rate(C2, 2, 300) @@ -402,7 +357,7 @@ t_burst(_) -> } end, - Bucket = fun(#{per_client := Cli} = Bucket) -> + Bucket = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("200/1s"), initial := 0, @@ -413,16 +368,16 @@ t_burst(_) -> capacity := 200, divisible := true }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, Case = fun() -> C1 = counters:new(1, []), C2 = counters:new(1, []), C3 = counters:new(1, []), - start_client(b1, ?NOW + 2000, C1, 20), - start_client(b2, ?NOW + 2000, C2, 30), - start_client(b3, ?NOW + 2000, C3, 30), + start_client({b1, Bucket}, ?NOW + 2000, C1, 20), + start_client({b2, Bucket}, ?NOW + 2000, C2, 30), + start_client({b3, Bucket}, ?NOW + 2000, C3, 30), timer:sleep(2100), Total = lists:sum([counters:get(X, 1) || X <- [C1, C2, C3]]), @@ -440,7 +395,7 @@ t_limit_global_with_unlimit_other(_) -> Cfg#{rate := ?RATE("600/1s")} end, - Bucket = fun(#{per_client := Cli} = Bucket) -> + Bucket = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := infinity, initial := 0, @@ -451,12 +406,12 @@ t_limit_global_with_unlimit_other(_) -> capacity := infinity, initial := 0 }, - Bucket2#{per_client := Cli2} + Bucket2#{client := Cli2} end, Case = fun() -> C1 = counters:new(1, []), - start_client(b1, ?NOW + 2000, C1, 20), + start_client({b1, Bucket}, ?NOW + 2000, C1, 20), timer:sleep(2100), check_average_rate(C1, 2, 600) end, @@ -470,28 +425,6 @@ t_limit_global_with_unlimit_other(_) -> %%-------------------------------------------------------------------- %% Test Cases container %%-------------------------------------------------------------------- -t_new_container(_) -> - C1 = emqx_limiter_container:new(), - C2 = emqx_limiter_container:new([message_routing]), - C3 = emqx_limiter_container:update_by_name(message_routing, default, C1), - ?assertMatch( - #{ - message_routing := _, - retry_ctx := undefined, - {retry, message_routing} := _ - }, - C2 - ), - ?assertMatch( - #{ - message_routing := _, - retry_ctx := undefined, - {retry, message_routing} := _ - }, - C3 - ), - ok. - t_check_container(_) -> Cfg = fun(Cfg) -> Cfg#{ @@ -500,10 +433,11 @@ t_check_container(_) -> capacity := 1000 } end, - Case = fun() -> - C1 = emqx_limiter_container:new( + Case = fun(BucketCfg) -> + C1 = emqx_limiter_container:get_limiter_by_types( + ?MODULE, [message_routing], - #{message_routing => default} + #{message_routing => BucketCfg} ), {ok, C2} = emqx_limiter_container:check(1000, message_routing, C1), {pause, Pause, C3} = emqx_limiter_container:check(1000, message_routing, C2), @@ -514,7 +448,39 @@ t_check_container(_) -> RetryData = emqx_limiter_container:get_retry_context(C5), ?assertEqual(Context, RetryData) end, - with_per_client(default, Cfg, Case). + with_per_client(Cfg, Case). + +%%-------------------------------------------------------------------- +%% Test Override +%%-------------------------------------------------------------------- +t_bucket_no_client(_) -> + Rate = ?RATE("1/s"), + GlobalMod = fun(#{client := Client} = Cfg) -> + Cfg#{client := Client#{rate := Rate}} + end, + BucketMod = fun(Bucket) -> + maps:remove(client, Bucket) + end, + Case = fun() -> + Limiter = connect(BucketMod(make_limiter_cfg())), + ?assertMatch(#{rate := Rate}, Limiter) + end, + with_global(GlobalMod, [BucketMod], Case). + +t_bucket_client(_) -> + GlobalRate = ?RATE("1/s"), + BucketRate = ?RATE("10/s"), + GlobalMod = fun(#{client := Client} = Cfg) -> + Cfg#{client := Client#{rate := GlobalRate}} + end, + BucketMod = fun(#{client := Client} = Bucket) -> + Bucket#{client := Client#{rate := BucketRate}} + end, + Case = fun() -> + Limiter = connect(BucketMod(make_limiter_cfg())), + ?assertMatch(#{rate := BucketRate}, Limiter) + end, + with_global(GlobalMod, [BucketMod], Case). %%-------------------------------------------------------------------- %% Test Cases misc @@ -607,19 +573,23 @@ t_schema_unit(_) -> %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -start_client(Name, EndTime, Counter, Number) -> +start_client(Cfg, EndTime, Counter, Number) -> lists:foreach( fun(_) -> spawn(fun() -> - start_client(Name, EndTime, Counter) + do_start_client(Cfg, EndTime, Counter) end) end, lists:seq(1, Number) ). -start_client(Name, EndTime, Counter) -> - #{per_client := PerClient} = - emqx_config:get([limiter, message_routing, bucket, Name]), +do_start_client({Name, CfgFun}, EndTime, Counter) -> + do_start_client(Name, CfgFun(make_limiter_cfg()), EndTime, Counter); +do_start_client(Cfg, EndTime, Counter) -> + do_start_client(?MODULE, Cfg, EndTime, Counter). + +do_start_client(Name, Cfg, EndTime, Counter) -> + #{client := PerClient} = Cfg, #{rate := Rate} = PerClient, Client = #client{ start = ?NOW, @@ -627,7 +597,7 @@ start_client(Name, EndTime, Counter) -> counter = Counter, obtained = 0, rate = Rate, - client = connect(Name) + client = connect(Name, Cfg) }, client_loop(Client). @@ -711,35 +681,50 @@ to_rate(Str) -> {ok, Rate} = emqx_limiter_schema:to_rate(Str), Rate. -with_global(Modifier, BuckeTemps, Case) -> - Fun = fun(Cfg) -> - #{bucket := #{default := BucketCfg}} = Cfg2 = Modifier(Cfg), - Fun = fun({Name, BMod}, Acc) -> - Acc#{Name => BMod(BucketCfg)} - end, - Buckets = lists:foldl(Fun, #{}, BuckeTemps), - Cfg2#{bucket := Buckets} - end, +with_global(Modifier, Buckets, Case) -> + with_config([limiter, message_routing], Modifier, Buckets, Case). - with_config([limiter, message_routing], Fun, Case). +with_bucket(Modifier, Case) -> + Cfg = Modifier(make_limiter_cfg()), + add_bucket(Cfg), + Case(Cfg), + del_bucket(). -with_bucket(Bucket, Modifier, Case) -> - Path = [limiter, message_routing, bucket, Bucket], - with_config(Path, Modifier, Case). +with_per_client(Modifier, Case) -> + #{client := Client} = Cfg = make_limiter_cfg(), + Cfg2 = Cfg#{client := Modifier(Client)}, + add_bucket(Cfg2), + Case(Cfg2), + del_bucket(). -with_per_client(Bucket, Modifier, Case) -> - Path = [limiter, message_routing, bucket, Bucket, per_client], - with_config(Path, Modifier, Case). - -with_config(Path, Modifier, Case) -> +with_config(Path, Modifier, Buckets, Case) -> Cfg = emqx_config:get(Path), NewCfg = Modifier(Cfg), - ct:pal("test with config:~p~n", [NewCfg]), emqx_config:put(Path, NewCfg), emqx_limiter_server:restart(message_routing), timer:sleep(500), + BucketCfg = make_limiter_cfg(), + lists:foreach( + fun + ({Name, BucketFun}) -> + add_bucket(Name, BucketFun(BucketCfg)); + (BucketFun) -> + add_bucket(BucketFun(BucketCfg)) + end, + Buckets + ), DelayReturn = delay_return(Case), + lists:foreach( + fun + ({Name, _Cfg}) -> + del_bucket(Name); + (_Cfg) -> + del_bucket() + end, + Buckets + ), emqx_config:put(Path, Cfg), + emqx_limiter_server:restart(message_routing), DelayReturn(). delay_return(Case) -> @@ -751,10 +736,40 @@ delay_return(Case) -> fun() -> erlang:raise(Type, Reason, Trace) end end. -connect(Name) -> - {ok, Limiter} = emqx_limiter_server:connect(message_routing, Name), +connect({Name, CfgFun}) -> + connect(Name, CfgFun(make_limiter_cfg())); +connect(Cfg) -> + connect(?MODULE, Cfg). + +connect(Name, Cfg) -> + {ok, Limiter} = emqx_limiter_server:connect(Name, message_routing, Cfg), Limiter. +make_limiter_cfg() -> + Infinity = emqx_limiter_schema:infinity_value(), + Client = #{ + rate => Infinity, + initial => 0, + capacity => Infinity, + low_watermark => 0, + divisible => false, + max_retry_time => timer:seconds(5), + failure_strategy => force + }, + #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + +add_bucket(Cfg) -> + add_bucket(?MODULE, Cfg). + +add_bucket(Name, Cfg) -> + emqx_limiter_server:add_bucket(Name, message_routing, Cfg). + +del_bucket() -> + del_bucket(?MODULE). + +del_bucket(Name) -> + emqx_limiter_server:del_bucket(Name, message_routing). + check_average_rate(Counter, Second, Rate) -> Cost = counters:get(Counter, 1), PerSec = Cost / Second, diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 89d892c67..47591bf64 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -59,6 +59,7 @@ init_per_testcase(TestCase, Config) when TestCase =/= t_ws_pingreq_before_connected, TestCase =/= t_ws_non_check_origin -> + add_bucket(), %% Meck Cm ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), @@ -96,6 +97,7 @@ init_per_testcase(TestCase, Config) when | Config ]; init_per_testcase(t_ws_non_check_origin, Config) -> + add_bucket(), ok = emqx_common_test_helpers:start_apps([]), PrevConfig = emqx_config:get_listener_conf(ws, default, [websocket]), emqx_config:put_listener_conf(ws, default, [websocket, check_origin_enable], false), @@ -105,6 +107,7 @@ init_per_testcase(t_ws_non_check_origin, Config) -> | Config ]; init_per_testcase(_, Config) -> + add_bucket(), PrevConfig = emqx_config:get_listener_conf(ws, default, [websocket]), ok = emqx_common_test_helpers:start_apps([]), [ @@ -119,6 +122,7 @@ end_per_testcase(TestCase, _Config) when TestCase =/= t_ws_non_check_origin, TestCase =/= t_ws_pingreq_before_connected -> + del_bucket(), lists:foreach( fun meck:unload/1, [ @@ -131,11 +135,13 @@ end_per_testcase(TestCase, _Config) when ] ); end_per_testcase(t_ws_non_check_origin, Config) -> + del_bucket(), PrevConfig = ?config(prev_config, Config), emqx_config:put_listener_conf(ws, default, [websocket], PrevConfig), emqx_common_test_helpers:stop_apps([]), ok; end_per_testcase(_, Config) -> + del_bucket(), PrevConfig = ?config(prev_config, Config), emqx_config:put_listener_conf(ws, default, [websocket], PrevConfig), emqx_common_test_helpers:stop_apps([]), @@ -501,15 +507,11 @@ t_handle_timeout_emit_stats(_) -> ?assertEqual(undefined, ?ws_conn:info(stats_timer, St)). t_ensure_rate_limit(_) -> - %% XXX In the future, limiter should provide API for config update - Path = [limiter, bytes_in, bucket, default, per_client], - PerClient = emqx_config:get(Path), {ok, Rate} = emqx_limiter_schema:to_rate("50MB"), - emqx_config:put(Path, PerClient#{rate := Rate}), - emqx_limiter_server:restart(bytes_in), - timer:sleep(100), - - Limiter = init_limiter(), + Limiter = init_limiter(#{ + bytes_in => make_limiter_cfg(Rate), + message_in => make_limiter_cfg() + }), St = st(#{limiter => Limiter}), %% must bigger than value in emqx_ratelimit_SUITE @@ -522,11 +524,7 @@ t_ensure_rate_limit(_) -> St ), ?assertEqual(blocked, ?ws_conn:info(sockstate, St1)), - ?assertEqual([{active, false}], ?ws_conn:info(postponed, St1)), - - emqx_config:put(Path, PerClient), - emqx_limiter_server:restart(bytes_in), - timer:sleep(100). + ?assertEqual([{active, false}], ?ws_conn:info(postponed, St1)). t_parse_incoming(_) -> {Packets, St} = ?ws_conn:parse_incoming(<<48, 3>>, [], st()), @@ -691,7 +689,40 @@ ws_client(State) -> ct:fail(ws_timeout) end. -limiter_cfg() -> #{bytes_in => default, message_in => default}. +-define(LIMITER_ID, 'ws:default'). init_limiter() -> - emqx_limiter_container:get_limiter_by_names([bytes_in, message_in], limiter_cfg()). + init_limiter(limiter_cfg()). + +init_limiter(LimiterCfg) -> + emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], LimiterCfg). + +limiter_cfg() -> + Cfg = make_limiter_cfg(), + #{bytes_in => Cfg, message_in => Cfg}. + +make_limiter_cfg() -> + Infinity = emqx_limiter_schema:infinity_value(), + make_limiter_cfg(Infinity). + +make_limiter_cfg(ClientRate) -> + Infinity = emqx_limiter_schema:infinity_value(), + Client = #{ + rate => ClientRate, + initial => 0, + capacity => Infinity, + low_watermark => 1, + divisible => false, + max_retry_time => timer:seconds(5), + failure_strategy => force + }, + #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + +add_bucket() -> + Cfg = make_limiter_cfg(), + emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), + emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). + +del_bucket() -> + emqx_limiter_server:del_bucket(?LIMITER_ID, bytes_in), + emqx_limiter_server:del_bucket(?LIMITER_ID, message_in). diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 5d911b5f4..f5a3ad403 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -348,12 +348,16 @@ enable_retainer( #{context_id := ContextId} = State, #{ msg_clear_interval := ClearInterval, - backend := BackendCfg + backend := BackendCfg, + flow_control := FlowControl } ) -> NewContextId = ContextId + 1, Context = create_resource(new_context(NewContextId), BackendCfg), load(Context), + emqx_limiter_server:add_bucket( + ?APP, internal, maps:get(batch_deliver_limiter, FlowControl, undefined) + ), State#{ enable := true, context_id := NewContextId, @@ -369,6 +373,7 @@ disable_retainer( } = State ) -> unload(), + emqx_limiter_server:del_bucket(?APP, internal), ok = close_resource(Context), State#{ enable := false, diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl index 29818481d..f52fd982c 100644 --- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl +++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl @@ -115,8 +115,8 @@ start_link(Pool, Id) -> init([Pool, Id]) -> erlang:process_flag(trap_exit, true), true = gproc_pool:connect_worker(Pool, {Pool, Id}), - BucketName = emqx:get_config([retainer, flow_control, batch_deliver_limiter], undefined), - {ok, Limiter} = emqx_limiter_server:connect(batch, BucketName), + BucketCfg = emqx:get_config([retainer, flow_control, batch_deliver_limiter], undefined), + {ok, Limiter} = emqx_limiter_server:connect(?APP, internal, BucketCfg), {ok, #{pool => Pool, id => Id, limiter => Limiter}}. %%-------------------------------------------------------------------- @@ -155,8 +155,8 @@ handle_cast({dispatch, Context, Pid, Topic}, #{limiter := Limiter} = State) -> {ok, Limiter2} = dispatch(Context, Pid, Topic, undefined, Limiter), {noreply, State#{limiter := Limiter2}}; handle_cast({refresh_limiter, Conf}, State) -> - BucketName = emqx_map_lib:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), - {ok, Limiter} = emqx_limiter_server:connect(batch, BucketName), + BucketCfg = emqx_map_lib:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), + {ok, Limiter} = emqx_limiter_server:connect(?APP, internal, BucketCfg), {noreply, State#{limiter := Limiter}}; handle_cast(Msg, State) -> ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 526059c9e..22083ba2c 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -86,7 +86,7 @@ fields(flow_control) -> )}, {batch_deliver_limiter, sc( - emqx_limiter_schema:bucket_name(), + ?R_REF(emqx_limiter_schema, bucket_opts), batch_deliver_limiter, undefined )} diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index ed49f6f5c..d7ddc2424 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -368,27 +368,16 @@ t_stop_publish_clear_msg(_) -> ok = emqtt:disconnect(C1). t_flow_control(_) -> - #{per_client := PerClient} = RetainerCfg = emqx_config:get([limiter, batch, bucket, retainer]), - RetainerCfg2 = RetainerCfg#{ - per_client := - PerClient#{ - rate := emqx_ratelimiter_SUITE:to_rate("1/1s"), - capacity := 1 - } - }, - emqx_config:put([limiter, batch, bucket, retainer], RetainerCfg2), - emqx_limiter_manager:restart_server(batch), - timer:sleep(500), - - emqx_retainer_dispatcher:refresh_limiter(), - timer:sleep(500), - + Rate = emqx_ratelimiter_SUITE:to_rate("1/1s"), + LimiterCfg = make_limiter_cfg(Rate), + JsonCfg = make_limiter_json(<<"1/1s">>), + emqx_limiter_server:add_bucket(emqx_retainer, internal, LimiterCfg), emqx_retainer:update_config(#{ <<"flow_control">> => #{ <<"batch_read_number">> => 1, <<"batch_deliver_number">> => 1, - <<"batch_deliver_limiter">> => retainer + <<"batch_deliver_limiter">> => JsonCfg } }), {ok, C1} = emqtt:start_link([{clean_start, true}, {proto_ver, v5}]), @@ -424,13 +413,14 @@ t_flow_control(_) -> ok = emqtt:disconnect(C1), - %% recover the limiter - emqx_config:put([limiter, batch, bucket, retainer], RetainerCfg), - emqx_limiter_manager:restart_server(batch), - timer:sleep(500), - - emqx_retainer_dispatcher:refresh_limiter(), - timer:sleep(500), + emqx_limiter_server:del_bucket(emqx_retainer, internal), + emqx_retainer:update_config(#{ + <<"flow_control">> => + #{ + <<"batch_read_number">> => 1, + <<"batch_deliver_number">> => 1 + } + }), ok. t_clear_expired(_) -> @@ -684,3 +674,33 @@ with_conf(ConfMod, Case) -> emqx_retainer:update_config(Conf), erlang:raise(Type, Error, Strace) end. + +make_limiter_cfg(Rate) -> + Infinity = emqx_limiter_schema:infinity_value(), + Client = #{ + rate => Rate, + initial => 0, + capacity => Infinity, + low_watermark => 1, + divisible => false, + max_retry_time => timer:seconds(5), + failure_strategy => force + }, + #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + +make_limiter_json(Rate) -> + Client = #{ + <<"rate">> => Rate, + <<"initial">> => 0, + <<"capacity">> => <<"infinity">>, + <<"low_watermark">> => 0, + <<"divisible">> => <<"false">>, + <<"max_retry_time">> => <<"5s">>, + <<"failure_strategy">> => <<"force">> + }, + #{ + <<"client">> => Client, + <<"rate">> => <<"infinity">>, + <<"initial">> => 0, + <<"capacity">> => <<"infinity">> + }. From dbab1bc96aeeb8dab40ab31e86bc7ad1c97fd92e Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 20 Jul 2022 15:00:12 +0800 Subject: [PATCH 02/63] fix(limiter): fix elvis && dialyzer error --- .../emqx_limiter/src/emqx_limiter_server.erl | 8 ++--- apps/emqx/src/emqx_listeners.erl | 4 +-- apps/emqx_retainer/src/emqx_retainer_api.erl | 36 ++----------------- 3 files changed, 8 insertions(+), 40 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 939824e02..997f6b788 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -75,11 +75,11 @@ -type state() :: #{ type := limiter_type(), - root := undefined | root(), + root := root(), buckets := buckets(), %% current counter to alloc - counter := undefined | counters:counters_ref(), - index := index() + counter := counters:counters_ref(), + index := 0 | index() }. -type buckets() :: #{bucket_name() => bucket()}. @@ -507,7 +507,7 @@ do_add_bucket(Id, #{rate := Rate, capacity := Capacity} = Cfg, #{buckets := Buck end. make_bucket(Id, Cfg, #{index := ?COUNTER_SIZE} = State) -> - add_bucket(Id, Cfg, State#{ + make_bucket(Id, Cfg, State#{ counter => counters:new(?COUNTER_SIZE, [write_concurrency]), index => 0 }); diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index dcb35adbb..fdbd5350e 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -546,7 +546,7 @@ add_limiter_bucket(Id, #{limiter := Limiters}) -> ok, Limiters ); -add_limiter_bucket(_Id, _cfg) -> +add_limiter_bucket(_Id, _Cfg) -> ok. del_limiter_bucket(Id, #{limiter := Limiters}) -> @@ -556,7 +556,7 @@ del_limiter_bucket(Id, #{limiter := Limiters}) -> end, maps:keys(Limiters) ); -del_limiter_bucket(_Id, _cfg) -> +del_limiter_bucket(_Id, _Cfg) -> ok. enable_authn(Opts) -> diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index 7d085b422..2c0bd725c 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -151,13 +151,8 @@ config(get, _) -> {200, emqx:get_raw_config([retainer])}; config(put, #{body := Body}) -> try - check_bucket_exists( - Body, - fun(Conf) -> - {ok, _} = emqx_retainer:update_config(Conf), - {200, emqx:get_raw_config([retainer])} - end - ) + {ok, _} = emqx_retainer:update_config(Body), + {200, emqx:get_raw_config([retainer])} catch _:Reason:_ -> {400, #{ @@ -237,30 +232,3 @@ check_backend(Type, Params, Cont) -> _ -> {400, 'BAD_REQUEST', <<"This API only support built in database">>} end. - -check_bucket_exists( - #{ - <<"flow_control">> := - #{<<"batch_deliver_limiter">> := Name} = Flow - } = Conf, - Cont -) -> - case erlang:binary_to_atom(Name) of - '' -> - %% workaround, empty string means set the value to undefined, - %% but now, we can't store `undefined` in the config file correct, - %% but, we can delete this field - Cont(Conf#{ - <<"flow_control">> := maps:remove(<<"batch_deliver_limiter">>, Flow) - }); - Bucket -> - Path = emqx_limiter_schema:get_bucket_cfg_path(batch, Bucket), - case emqx:get_config(Path, undefined) of - undefined -> - {400, 'BAD_REQUEST', <<"The limiter bucket not exists">>}; - _ -> - Cont(Conf) - end - end; -check_bucket_exists(Conf, Cont) -> - Cont(Conf). From 15c8110af219582949304f62436a16eac17545b4 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 22 Jul 2022 17:14:24 +0800 Subject: [PATCH 03/63] fix(limiter): lift the level of the `client` field --- apps/emqx/i18n/emqx_limiter_i18n.conf | 6 +- .../emqx_limiter/src/emqx_limiter_schema.erl | 93 +++++++++++++------ .../emqx_limiter/src/emqx_limiter_server.erl | 58 ++++++------ apps/emqx/src/emqx_schema.erl | 9 +- .../src/emqx_dashboard_swagger.erl | 2 + .../src/emqx_retainer_schema.erl | 2 +- 6 files changed, 107 insertions(+), 63 deletions(-) diff --git a/apps/emqx/i18n/emqx_limiter_i18n.conf b/apps/emqx/i18n/emqx_limiter_i18n.conf index b75435225..99ecc9e1e 100644 --- a/apps/emqx/i18n/emqx_limiter_i18n.conf +++ b/apps/emqx/i18n/emqx_limiter_i18n.conf @@ -89,10 +89,10 @@ the check/consume will succeed, but it will be forced to wait for a short period } } - per_client { + client { desc { - en: """The rate limit for each user of the bucket, this field is not required""" - zh: """对桶的每个使用者的速率控制设置，这个不是必须的""" + en: """The rate limit for each user of the bucket""" + zh: """对桶的每个使用者的速率控制设置""" } label: { en: """Per Client""" diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 61dc95bc7..f5e90a7e8 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -32,8 +32,7 @@ desc/1, types/0, infinity_value/0, - bucket_opts/0, - bucket_opts_meta/0 + bucket_fields/1 ]). -define(KILOBYTE, 1024). @@ -96,7 +95,17 @@ fields(limiter) -> default => make_limiter_default(Type) })} || Type <- types() - ]; + ] ++ + [ + {client, + ?HOCON( + ?R_REF(client_fields), + #{ + desc => ?DESC(client), + default => #{} + } + )} + ]; fields(node_opts) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, @@ -104,15 +113,22 @@ fields(node_opts) -> ?HOCON(burst_rate(), #{ desc => ?DESC(burst), default => 0 - })}, - {client, ?HOCON(?R_REF(client_opts), #{default => #{}})} + })} + ]; +fields(client_fields) -> + [ + {Type, + ?HOCON(?R_REF(client_opts), #{ + desc => ?DESC(Type), + default => #{} + })} + || Type <- types() ]; fields(bucket_opts) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => "infinity"})}, - {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})}, - {client, ?HOCON(?R_REF(client_opts), #{required => false})} + {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})} ]; fields(client_opts) -> [ @@ -159,14 +175,23 @@ fields(client_opts) -> default => force } )} - ]. + ]; +fields({client_fields, Types}) -> + [ + {Type, + ?HOCON(?R_REF(client_opts), #{ + desc => ?DESC(Type), + required => false + })} + || Type <- Types + ]; +fields({bucket_fields, Types}) -> + bucket_fields(Types). desc(limiter) -> "Settings for the rate limiter."; desc(node_opts) -> "Settings for the limiter of the node level."; -desc(node_client_opts) -> - "Settings for the client in the node level."; desc(bucket_opts) -> "Settings for the bucket."; desc(client_opts) -> @@ -174,23 +199,37 @@ desc(client_opts) -> desc(_) -> undefined. -bucket_opts() -> - ?HOCON( - ?MAP("bucket_name", ?R_REF(bucket_opts)), - bucket_opts_meta() - ). - -bucket_opts_meta() -> - #{ - default => #{}, - example => - #{ - <<"rate">> => <<"infinity">>, - <<"capcity">> => <<"infinity">>, - <<"initial">> => <<"100">>, - <<"client">> => #{<<"rate">> => <<"infinity">>} - } - }. +bucket_fields(Type) when is_atom(Type) -> + fields(bucket_opts) ++ + [ + {client, + ?HOCON( + ?R_REF(?MODULE, client_opts), + #{ + desc => ?DESC(client), + required => false + } + )} + ]; +bucket_fields(Types) -> + [ + {Type, + ?HOCON(?R_REF(?MODULE, bucket_opts), #{ + desc => ?DESC(?MODULE, Type), + required => false + })} + || Type <- Types + ] ++ + [ + {client, + ?HOCON( + ?R_REF(?MODULE, {client_fields, Types}), + #{ + desc => ?DESC(client), + required => false + } + )} + ]. %% default period is 100ms default_period() -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 997f6b788..66cafa7dc 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -116,35 +116,28 @@ %% If no bucket path is set in config, there will be no limit connect(_Id, _Type, undefined) -> {ok, emqx_htb_limiter:make_infinity_limiter()}; -connect( - Id, - Type, - #{ - rate := BucketRate, - capacity := BucketSize - } = BucketCfg -) -> - case emqx_limiter_manager:find_bucket(Id, Type) of - {ok, Bucket} -> - case find_client_cfg(Type, BucketCfg) of - #{rate := CliRate, capacity := CliSize} = ClientCfg -> - {ok, - if - CliRate < BucketRate orelse CliSize < BucketSize -> - emqx_htb_limiter:make_token_bucket_limiter(ClientCfg, Bucket); - true -> - emqx_htb_limiter:make_ref_limiter(ClientCfg, Bucket) - end}; - {error, invalid_node_cfg} = Error -> - ?SLOG(error, #{msg => "invalid_node_cfg", type => Type, id => Id}), - Error - end; +connect(Id, Type, Cfg) -> + case find_limiter_cfg(Type, Cfg) of + {undefined, _} -> + {ok, emqx_htb_limiter:make_infinity_limiter()}; + { + #{ + rate := BucketRate, + capacity := BucketSize + } = BucketCfg, + #{rate := CliRate, capacity := CliSize} = ClientCfg + } -> + {ok, + if + CliRate < BucketRate orelse CliSize < BucketSize -> + emqx_htb_limiter:make_token_bucket_limiter(ClientCfg, BucketCfg); + true -> + emqx_htb_limiter:make_ref_limiter(ClientCfg, BucketCfg) + end}; undefined -> ?SLOG(error, #{msg => "bucket_not_found", type => Type, id => Id}), {error, invalid_bucket} - end; -connect(Id, Type, Paths) -> - connect(Id, Type, maps:get(Type, Paths, undefined)). + end. -spec add_bucket(limiter_id(), limiter_type(), hocons:config() | undefined) -> ok. add_bucket(_Id, _Type, undefine) -> @@ -571,9 +564,16 @@ call(Type, Msg) -> gen_server:call(Pid, Msg) end. -find_client_cfg(Type, Cfg) -> - NodeCfg = emqx:get_config([limiter, Type, client], undefined), - BucketCfg = maps:get(client, Cfg, undefined), +find_limiter_cfg(Type, #{rate := _} = Cfg) -> + {Cfg, find_client_cfg(Type, maps:get(client, Cfg, undefined))}; +find_limiter_cfg(Type, Cfg) -> + { + maps:get(Type, Cfg), + find_client_cfg(Type, emqx_map_lib:deep_get([client, Type], Cfg, undefined)) + }. + +find_client_cfg(Type, BucketCfg) -> + NodeCfg = emqx:get_config([limiter, client, Type], undefined), merge_client_cfg(NodeCfg, BucketCfg). merge_client_cfg(undefined, BucketCfg) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 0c8a073a0..81f5c922a 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1619,10 +1619,13 @@ base_listener(Bind) -> )}, {"limiter", sc( - map("ratelimit_name", ?R_REF(emqx_limiter_schema, bucket_opts)), + ?R_REF( + emqx_limiter_schema, + {bucket_fields, [bytes_in, message_in, connection, message_routing]} + ), #{ - desc => ?DESC(base_listener_limiter) - %% TODO default => #{<<"connection">> => <<"default">>} + desc => ?DESC(base_listener_limiter), + default => #{} } )}, {"enable_authn", diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 80b4e9624..b9c1e5c61 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -755,6 +755,8 @@ to_bin(List) when is_list(List) -> end; to_bin(Boolean) when is_boolean(Boolean) -> Boolean; to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +to_bin({Type, Args}) -> + unicode:characters_to_binary(io_lib:format("~p(~p)", [Type, Args])); to_bin(X) -> X. diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 22083ba2c..986eb4105 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -86,7 +86,7 @@ fields(flow_control) -> )}, {batch_deliver_limiter, sc( - ?R_REF(emqx_limiter_schema, bucket_opts), + ?R_REF(emqx_limiter_schema, {bucket_fields, internal}), batch_deliver_limiter, undefined )} From ce46cb9216efe56f875db5b7ef1a2498e06ee771 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 22 Jul 2022 18:59:02 +0800 Subject: [PATCH 04/63] fix(limiter): fix test case error --- .../emqx_limiter/src/emqx_limiter_schema.erl | 5 +++- .../emqx_limiter/src/emqx_limiter_server.erl | 29 ++++++++++-------- apps/emqx/src/emqx_listeners.erl | 4 +-- apps/emqx/test/emqx_channel_SUITE.erl | 14 +++++---- apps/emqx/test/emqx_connection_SUITE.erl | 13 ++++---- apps/emqx/test/emqx_ratelimiter_SUITE.erl | 30 ++++++++++--------- apps/emqx/test/emqx_ws_connection_SUITE.erl | 29 ++++++++++-------- 7 files changed, 71 insertions(+), 53 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index f5e90a7e8..23d4d4e4c 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -102,7 +102,10 @@ fields(limiter) -> ?R_REF(client_fields), #{ desc => ?DESC(client), - default => #{} + default => maps:from_list([ + {erlang:atom_to_binary(Type), #{}} + || Type <- types() + ]) } )} ]; diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 66cafa7dc..c5e919296 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -124,19 +124,22 @@ connect(Id, Type, Cfg) -> #{ rate := BucketRate, capacity := BucketSize - } = BucketCfg, + }, #{rate := CliRate, capacity := CliSize} = ClientCfg } -> - {ok, - if - CliRate < BucketRate orelse CliSize < BucketSize -> - emqx_htb_limiter:make_token_bucket_limiter(ClientCfg, BucketCfg); - true -> - emqx_htb_limiter:make_ref_limiter(ClientCfg, BucketCfg) - end}; - undefined -> - ?SLOG(error, #{msg => "bucket_not_found", type => Type, id => Id}), - {error, invalid_bucket} + case emqx_limiter_manager:find_bucket(Id, Type) of + {ok, Bucket} -> + {ok, + if + CliRate < BucketRate orelse CliSize < BucketSize -> + emqx_htb_limiter:make_token_bucket_limiter(ClientCfg, Bucket); + true -> + emqx_htb_limiter:make_ref_limiter(ClientCfg, Bucket) + end}; + undefined -> + ?SLOG(error, #{msg => "bucket_not_found", type => Type, id => Id}), + {error, invalid_bucket} + end end. -spec add_bucket(limiter_id(), limiter_type(), hocons:config() | undefined) -> ok. @@ -523,7 +526,7 @@ make_bucket( _ = put_to_counter(Counter, NewIndex, Initial), Ref = emqx_limiter_bucket_ref:new(Counter, NewIndex, Rate), emqx_limiter_manager:insert_bucket(Id, Type, Ref), - State#{buckets := Buckets#{Id => Bucket}}. + State#{buckets := Buckets#{Id => Bucket}, index := NewIndex}. do_del_bucket(Id, #{type := Type, buckets := Buckets} = State) -> case maps:get(Id, Buckets, undefined) of @@ -568,7 +571,7 @@ find_limiter_cfg(Type, #{rate := _} = Cfg) -> {Cfg, find_client_cfg(Type, maps:get(client, Cfg, undefined))}; find_limiter_cfg(Type, Cfg) -> { - maps:get(Type, Cfg), + maps:get(Type, Cfg, undefined), find_client_cfg(Type, emqx_map_lib:deep_get([client, Type], Cfg, undefined)) }. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index fdbd5350e..9ccc5f2df 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -538,13 +538,13 @@ zone(Opts) -> limiter(Opts) -> maps:get(limiter, Opts, #{}). -add_limiter_bucket(Id, #{limiter := Limiters}) -> +add_limiter_bucket(Id, #{limiter := Limiter}) -> maps:fold( fun(Type, Cfg, _) -> emqx_limiter_server:add_bucket(Id, Type, Cfg) end, ok, - Limiters + maps:without([client], Limiter) ); add_limiter_bucket(_Id, _Cfg) -> ok. diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 03be7448c..df1720772 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -1205,9 +1205,7 @@ session(InitFields) when is_map(InitFields) -> quota() -> emqx_limiter_container:get_limiter_by_types(?MODULE, [message_routing], limiter_cfg()). -limiter_cfg() -> #{message_routing => make_limiter_cfg()}. - -make_limiter_cfg() -> +limiter_cfg() -> Client = #{ rate => 5, initial => 0, @@ -1217,10 +1215,16 @@ make_limiter_cfg() -> max_retry_time => timer:seconds(5), failure_strategy => force }, - #{client => Client, rate => 10, initial => 0, capacity => 10}. + #{ + message_routing => bucket_cfg(), + client => #{message_routing => Client} + }. + +bucket_cfg() -> + #{rate => 10, initial => 0, capacity => 10}. add_bucket() -> - emqx_limiter_server:add_bucket(?MODULE, message_routing, make_limiter_cfg()). + emqx_limiter_server:add_bucket(?MODULE, message_routing, bucket_cfg()). del_bucket() -> emqx_limiter_server:del_bucket(?MODULE, message_routing). diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 141dbdad6..c5dfdf34a 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -708,11 +708,8 @@ init_limiter() -> emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], limiter_cfg()). limiter_cfg() -> - Cfg = make_limiter_cfg(), - #{bytes_in => Cfg, message_in => Cfg}. - -make_limiter_cfg() -> Infinity = emqx_limiter_schema:infinity_value(), + Cfg = bucket_cfg(), Client = #{ rate => Infinity, initial => 0, @@ -722,10 +719,14 @@ make_limiter_cfg() -> max_retry_time => timer:seconds(5), failure_strategy => force }, - #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + #{bytes_in => Cfg, message_in => Cfg, client => #{bytes_in => Client, message_in => Client}}. + +bucket_cfg() -> + Infinity = emqx_limiter_schema:infinity_value(), + #{rate => Infinity, initial => 0, capacity => Infinity}. add_bucket() -> - Cfg = make_limiter_cfg(), + Cfg = bucket_cfg(), emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index e31a220e9..7efcbaa18 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -310,8 +310,8 @@ t_capacity(_) -> %% Test Cases Global Level %%-------------------------------------------------------------------- t_collaborative_alloc(_) -> - GlobalMod = fun(Cfg) -> - Cfg#{rate := ?RATE("600/1s")} + GlobalMod = fun(#{message_routing := MR} = Cfg) -> + Cfg#{message_routing := MR#{rate := ?RATE("600/1s")}} end, Bucket1 = fun(#{client := Cli} = Bucket) -> @@ -350,10 +350,12 @@ t_collaborative_alloc(_) -> ). t_burst(_) -> - GlobalMod = fun(Cfg) -> + GlobalMod = fun(#{message_routing := MR} = Cfg) -> Cfg#{ - rate := ?RATE("200/1s"), - burst := ?RATE("400/1s") + message_routing := MR#{ + rate := ?RATE("200/1s"), + burst := ?RATE("400/1s") + } } end, @@ -391,8 +393,8 @@ t_burst(_) -> ). t_limit_global_with_unlimit_other(_) -> - GlobalMod = fun(Cfg) -> - Cfg#{rate := ?RATE("600/1s")} + GlobalMod = fun(#{message_routing := MR} = Cfg) -> + Cfg#{message_routing := MR#{rate := ?RATE("600/1s")}} end, Bucket = fun(#{client := Cli} = Bucket) -> @@ -433,11 +435,11 @@ t_check_container(_) -> capacity := 1000 } end, - Case = fun(BucketCfg) -> + Case = fun(#{client := Client} = BucketCfg) -> C1 = emqx_limiter_container:get_limiter_by_types( ?MODULE, [message_routing], - #{message_routing => BucketCfg} + #{message_routing => BucketCfg, client => #{message_routing => Client}} ), {ok, C2} = emqx_limiter_container:check(1000, message_routing, C1), {pause, Pause, C3} = emqx_limiter_container:check(1000, message_routing, C2), @@ -455,8 +457,8 @@ t_check_container(_) -> %%-------------------------------------------------------------------- t_bucket_no_client(_) -> Rate = ?RATE("1/s"), - GlobalMod = fun(#{client := Client} = Cfg) -> - Cfg#{client := Client#{rate := Rate}} + GlobalMod = fun(#{client := #{message_routing := MR} = Client} = Cfg) -> + Cfg#{client := Client#{message_routing := MR#{rate := Rate}}} end, BucketMod = fun(Bucket) -> maps:remove(client, Bucket) @@ -470,8 +472,8 @@ t_bucket_no_client(_) -> t_bucket_client(_) -> GlobalRate = ?RATE("1/s"), BucketRate = ?RATE("10/s"), - GlobalMod = fun(#{client := Client} = Cfg) -> - Cfg#{client := Client#{rate := GlobalRate}} + GlobalMod = fun(#{client := #{message_routing := MR} = Client} = Cfg) -> + Cfg#{client := Client#{message_routing := MR#{rate := GlobalRate}}} end, BucketMod = fun(#{client := Client} = Bucket) -> Bucket#{client := Client#{rate := BucketRate}} @@ -682,7 +684,7 @@ to_rate(Str) -> Rate. with_global(Modifier, Buckets, Case) -> - with_config([limiter, message_routing], Modifier, Buckets, Case). + with_config([limiter], Modifier, Buckets, Case). with_bucket(Modifier, Case) -> Cfg = Modifier(make_limiter_cfg()), diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 47591bf64..47efc1829 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -509,8 +509,9 @@ t_handle_timeout_emit_stats(_) -> t_ensure_rate_limit(_) -> {ok, Rate} = emqx_limiter_schema:to_rate("50MB"), Limiter = init_limiter(#{ - bytes_in => make_limiter_cfg(Rate), - message_in => make_limiter_cfg() + bytes_in => bucket_cfg(), + message_in => bucket_cfg(), + client => #{bytes_in => client_cfg(Rate)} }), St = st(#{limiter => Limiter}), @@ -698,28 +699,32 @@ init_limiter(LimiterCfg) -> emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], LimiterCfg). limiter_cfg() -> - Cfg = make_limiter_cfg(), - #{bytes_in => Cfg, message_in => Cfg}. + Cfg = bucket_cfg(), + Client = client_cfg(), + #{bytes_in => Cfg, message_in => Cfg, client => #{bytes_in => Client, message_in => Client}}. -make_limiter_cfg() -> +client_cfg() -> Infinity = emqx_limiter_schema:infinity_value(), - make_limiter_cfg(Infinity). + client_cfg(Infinity). -make_limiter_cfg(ClientRate) -> +client_cfg(Rate) -> Infinity = emqx_limiter_schema:infinity_value(), - Client = #{ - rate => ClientRate, + #{ + rate => Rate, initial => 0, capacity => Infinity, low_watermark => 1, divisible => false, max_retry_time => timer:seconds(5), failure_strategy => force - }, - #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + }. + +bucket_cfg() -> + Infinity = emqx_limiter_schema:infinity_value(), + #{rate => Infinity, initial => 0, capacity => Infinity}. add_bucket() -> - Cfg = make_limiter_cfg(), + Cfg = bucket_cfg(), emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). From bcc78950a93cc227b4288562b108915760ace19d Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 26 Jul 2022 14:33:04 +0800 Subject: [PATCH 05/63] fix(limiter): fix schema error --- .../emqx_limiter/src/emqx_limiter_schema.erl | 99 +++++++++---------- apps/emqx/src/emqx_schema.erl | 6 +- .../src/emqx_retainer_schema.erl | 2 +- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 23d4d4e4c..89c927950 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -31,8 +31,7 @@ get_bucket_cfg_path/2, desc/1, types/0, - infinity_value/0, - bucket_fields/1 + infinity_value/0 ]). -define(KILOBYTE, 1024). @@ -92,7 +91,7 @@ fields(limiter) -> {Type, ?HOCON(?R_REF(node_opts), #{ desc => ?DESC(Type), - default => make_limiter_default(Type) + default => #{} })} || Type <- types() ] ++ @@ -179,17 +178,12 @@ fields(client_opts) -> } )} ]; -fields({client_fields, Types}) -> - [ - {Type, - ?HOCON(?R_REF(client_opts), #{ - desc => ?DESC(Type), - required => false - })} - || Type <- Types - ]; -fields({bucket_fields, Types}) -> - bucket_fields(Types). +fields(listener_fields) -> + bucket_fields([bytes_in, message_in, connection, message_routing], listener_client_fields); +fields(listener_client_fields) -> + client_fields([bytes_in, message_in, connection, message_routing]); +fields(Type) -> + bucket_field(Type). desc(limiter) -> "Settings for the rate limiter."; @@ -202,38 +196,6 @@ desc(client_opts) -> desc(_) -> undefined. -bucket_fields(Type) when is_atom(Type) -> - fields(bucket_opts) ++ - [ - {client, - ?HOCON( - ?R_REF(?MODULE, client_opts), - #{ - desc => ?DESC(client), - required => false - } - )} - ]; -bucket_fields(Types) -> - [ - {Type, - ?HOCON(?R_REF(?MODULE, bucket_opts), #{ - desc => ?DESC(?MODULE, Type), - required => false - })} - || Type <- Types - ] ++ - [ - {client, - ?HOCON( - ?R_REF(?MODULE, {client_fields, Types}), - #{ - desc => ?DESC(client), - required => false - } - )} - ]. - %% default period is 100ms default_period() -> 100. @@ -366,7 +328,44 @@ apply_unit("mb", Val) -> Val * ?KILOBYTE * ?KILOBYTE; apply_unit("gb", Val) -> Val * ?KILOBYTE * ?KILOBYTE * ?KILOBYTE; apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). -make_limiter_default(connection) -> - #{<<"rate">> => <<"1000/s">>}; -make_limiter_default(_) -> - #{}. +bucket_field(Type) when is_atom(Type) -> + fields(bucket_opts) ++ + [ + {client, + ?HOCON( + ?R_REF(?MODULE, client_opts), + #{ + desc => ?DESC(client), + required => false + } + )} + ]. +bucket_fields(Types, ClientRef) -> + [ + {Type, + ?HOCON(?R_REF(?MODULE, bucket_opts), #{ + desc => ?DESC(?MODULE, Type), + required => false + })} + || Type <- Types + ] ++ + [ + {client, + ?HOCON( + ?R_REF(?MODULE, ClientRef), + #{ + desc => ?DESC(client), + required => false + } + )} + ]. + +client_fields(Types) -> + [ + {Type, + ?HOCON(?R_REF(client_opts), #{ + desc => ?DESC(Type), + required => false + })} + || Type <- Types + ]. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 81f5c922a..f7c18ae8e 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1621,11 +1621,13 @@ base_listener(Bind) -> sc( ?R_REF( emqx_limiter_schema, - {bucket_fields, [bytes_in, message_in, connection, message_routing]} + listener_fields ), #{ desc => ?DESC(base_listener_limiter), - default => #{} + default => #{ + <<"connection">> => #{<<"rate">> => <<"1000/s">>, <<"capacity">> => 1000} + } } )}, {"enable_authn", diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 986eb4105..51dbf496b 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -86,7 +86,7 @@ fields(flow_control) -> )}, {batch_deliver_limiter, sc( - ?R_REF(emqx_limiter_schema, {bucket_fields, internal}), + ?R_REF(emqx_limiter_schema, internal), batch_deliver_limiter, undefined )} From bc716884e09b31c75c5629fe0313ecc46910d605 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 26 Jul 2022 15:16:53 +0800 Subject: [PATCH 06/63] fix(limiter): fix spellcheck error --- apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 89c927950..bce87e2ba 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -193,6 +193,14 @@ desc(bucket_opts) -> "Settings for the bucket."; desc(client_opts) -> "Settings for the client in bucket level."; +desc(client_fields) -> + "Fields of the client level."; +desc(listener_fields) -> + "Fields of the listener."; +desc(listener_client_fields) -> + "Fields of the client level of the listener."; +desc(internal) -> + "Internal limiter."; desc(_) -> undefined. From c56f84b997517ee5efb5ba038668121f980f0c52 Mon Sep 17 00:00:00 2001 From: lukas Date: Thu, 21 Jul 2022 19:18:42 +0200 Subject: [PATCH 07/63] feat(helm): add ssl support for helm chart --- deploy/charts/emqx/README.md | 167 ++++++++++-------- deploy/charts/emqx/templates/StatefulSet.yaml | 14 +- deploy/charts/emqx/templates/certificate.yaml | 16 ++ deploy/charts/emqx/values.yaml | 9 + 4 files changed, 135 insertions(+), 71 deletions(-) create mode 100644 deploy/charts/emqx/templates/certificate.yaml diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 496d52061..ed331619d 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -1,92 +1,121 @@ # Introduction -This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager. + +This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager. # Prerequisites + + Kubernetes 1.6+ + Helm # Installing the Chart + To install the chart with the release name `my-emqx`: -+ From github - ``` - $ git clone https://github.com/emqx/emqx.git - $ cd emqx/deploy/charts/emqx - $ helm install my-emqx . - ``` ++ From github + ``` + $ git clone https://github.com/emqx/emqx.git + $ cd emqx/deploy/charts/emqx + $ helm install my-emqx . + ``` -+ From chart repos - ``` - helm repo add emqx https://repos.emqx.io/charts - helm install my-emqx emqx/emqx - ``` - > If you want to install an unstable version, you need to add `--devel` when you execute the `helm install` command. ++ From chart repos + ``` + helm repo add emqx https://repos.emqx.io/charts + helm install my-emqx emqx/emqx + ``` + > If you want to install an unstable version, you need to add `--devel` when you execute the `helm install` command. # Uninstalling the Chart + To uninstall/delete the `my-emqx` deployment: + ``` $ helm del my-emqx ``` # Configuration + The following table lists the configurable parameters of the emqx chart and their default values. -| Parameter | Description | Default Value | -| --- | --- | --- | -| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. |3| -| `image.repository` | EMQX Image name |emqx/emqx| -| `image.pullPolicy` | The image pull policy |IfNotPresent| -| `image.pullSecrets ` | The image pull secrets |`[]` (does not add image pull secrets to deployed pods)| -| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | -| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | -`podAnnotations ` | Annotations for pod | `{}` -`podManagementPolicy`| To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock | `Parallel` -| `persistence.enabled` | Enable EMQX persistence using PVC |false| -| `persistence.storageClass` | Storage class of backing PVC |`nil` (uses alpha storage class annotation)| -| `persistence.existingClaim` | EMQX data Persistent Volume existing claim name, evaluated as a template |""| -| `persistence.accessMode` | PVC Access Mode for EMQX volume |ReadWriteOnce| -| `persistence.size` | PVC Storage Request for EMQX volume |20Mi| -| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`| -| `resources` | CPU/Memory resource requests/limits |{}| -| `nodeSelector` | Node labels for pod assignment |`{}`| -| `tolerations` | Toleration labels for pod assignment |`[]`| -| `affinity` | Map of node/pod affinities |`{}`| -| `service.type` | Kubernetes Service type. |ClusterIP| -| `service.mqtt` | Port for MQTT. |1883| -| `service.mqttssl` | Port for MQTT(SSL). |8883| -| `service.mgmt` | Port for mgmt API. |8081| -| `service.ws` | Port for WebSocket/HTTP. |8083| -| `service.wss` | Port for WSS/HTTPS. |8084| -| `service.dashboard` | Port for dashboard. |18083| -| `service.nodePorts.mqtt` | Kubernetes node port for MQTT. |nil| -| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). |nil| -| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. |nil| -| `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. |nil| -| `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. |nil| -| `service.nodePorts.dashboard` | Kubernetes node port for dashboard. |nil| -| `service.loadBalancerIP` | loadBalancerIP for Service | nil | -| `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | -| `service.externalIPs` | ExternalIPs for the service | [] | -| `service.annotations` | Service annotations | {}(evaluated as a template)| -| `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | -| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | -| `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | -| `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` -| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | -| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | -| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | -| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | -| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | -| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | -| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | -| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | -| `metrics.type` | Now we only supported "prometheus" | "prometheus" | +| Parameter | Description | Default Value | +|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | 3 | +| `image.repository` | EMQX Image name | emqx/emqx | +| `image.pullPolicy` | The image pull policy | IfNotPresent | +| `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | +| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | +| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | +| `podAnnotations ` | Annotations for pod | `{}` | +| `podManagementPolicy` | To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock | `Parallel` | +| `persistence.enabled` | Enable EMQX persistence using PVC | false | +| `persistence.storageClass` | Storage class of backing PVC | `nil` (uses alpha storage class annotation) | +| `persistence.existingClaim` | EMQX data Persistent Volume existing claim name, evaluated as a template | "" | +| `persistence.accessMode` | PVC Access Mode for EMQX volume | ReadWriteOnce | +| `persistence.size` | PVC Storage Request for EMQX volume | 20Mi | +| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. | `{}` | +| `resources` | CPU/Memory resource requests/limits | {} | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Map of node/pod affinities | `{}` | +| `service.type` | Kubernetes Service type. | ClusterIP | +| `service.mqtt` | Port for MQTT. | 1883 | +| `service.mqttssl` | Port for MQTT(SSL). | 8883 | +| `service.mgmt` | Port for mgmt API. | 8081 | +| `service.ws` | Port for WebSocket/HTTP. | 8083 | +| `service.wss` | Port for WSS/HTTPS. | 8084 | +| `service.dashboard` | Port for dashboard. | 18083 | +| `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil | +| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil | +| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil | +| `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | nil | +| `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | nil | +| `service.nodePorts.dashboard` | Kubernetes node port for dashboard. | nil | +| `service.loadBalancerIP` | loadBalancerIP for Service | nil | +| `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | +| `service.externalIPs` | ExternalIPs for the service | [] | +| `service.annotations` | Service annotations | {}(evaluated as a template) | +| `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | +| `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | +| `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` | +| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | +| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | +| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | +| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | +| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | +| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | +| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | +| `metrics.type` | Now we only supported "prometheus" | "prometheus" | +| `ssl.enabled` | Enable SSL support | false | +| `ssl.useExisting` | Use existing certificate or let cert-manager generate one | false | +| `ssl.existingName` | Name of existing certificate | emqx-tls | +| `ssl.dnsnames` | DNS name(s) for certificate to be generated | {} | +| `ssl.issuer.name` | Issuer name for certificate generation | letsencrypt-dns | +| `ssl.issuer.kind` | Issuer kind for certificate generation | ClusterIssuer | ## EMQX specific settings -The following table lists the configurable [EMQX](https://www.emqx.io/)-specific parameters of the chart and their default values. -Parameter | Description | Default Value ---- | --- | --- -`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix can be omitted) or using the configuration files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil` + +The following table lists the configurable [EMQX](https://www.emqx.io/)-specific parameters of the chart and their +default values. +Parameter | Description | Default Value +--- | --- | --- +`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items +expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix +can be omitted) or using the configuration +files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil` `emqxLicenseSecretName` | Name of the secret that holds the license information | `nil` + +## SSL settings +`cert-manager` generates secrets with certificate data using the keys `tls.crt` and `tls.key`. The helm chart always mounts those keys as files to `/tmp/ssl/` +which needs to explicitly configured by either changing the emqx config file or by passing the following environment variables: + +``` + EMQX_LISTENERS__SSL__DEFAULT__SSL_OPTIONS__CERTFILE: /tmp/ssl/tls.crt + EMQX_LISTENERS__SSL__DEFAULT__SSL_OPTIONS__KEYFILE: /tmp/ssl/tls.key +``` + +If you chose to use an existing certificate, make sure, you update the filenames accordingly. + diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 80f7e2c0a..d44c88a86 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -53,6 +53,11 @@ spec: {{- end }} spec: volumes: + {{- if .Values.ssl.enabled }} + - name: ssl-cert + secret: + secretName: {{ include "emqx.fullname" . }}-tls + {{- end }} {{- if not .Values.persistence.enabled }} - name: emqx-data emptyDir: {} @@ -124,12 +129,17 @@ spec: volumeMounts: - name: emqx-data mountPath: "/opt/emqx/data" - {{ if .Values.emqxLicenseSecretName }} + {{- if .Values.ssl.enabled }} + - name: ssl-cert + mountPath: /tmp/ssl + readOnly: true + {{- end}} + {{ if .Values.emqxLicenseSecretName }} - name: emqx-license mountPath: "/opt/emqx/etc/emqx.lic" subPath: "emqx.lic" readOnly: true - {{ end }} + {{- end }} readinessProbe: httpGet: path: /api/v5/status diff --git a/deploy/charts/emqx/templates/certificate.yaml b/deploy/charts/emqx/templates/certificate.yaml new file mode 100644 index 000000000..36b7f6521 --- /dev/null +++ b/deploy/charts/emqx/templates/certificate.yaml @@ -0,0 +1,16 @@ +{{- if and (.Values.ssl.enable) (not .Values.ssl.useExisting) -}} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "emqx.fullname" . }}-tls +spec: + secretName: {{ include "emqx.fullname" . }}-tls + issuerRef: + name: {{ default "letsencrypt-staging" .Values.ssl.issuer.name }} + kind: {{ default "ClusterIssuer" .Values.ssl.issuer.kind }} + dnsNames: + {{- range .Values.ssl.dnsnames }} + - {{ . }} + {{- end }} +{{- end -}} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 7ed4f4995..94e7eeb3c 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -203,3 +203,12 @@ containerSecurityContext: metrics: enabled: false type: prometheus + +ssl: + enabled: false + useExisting: false + existingName: emqx-tls + dnsnames: {} + issuer: + name: letsencrypt-dns + kind: ClusterIssuer From 78deee68460a6b93dfec61e94052ab6484914242 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 1 Aug 2022 17:35:48 +0800 Subject: [PATCH 08/63] fix(channel): Adjust the timing of the `client.connected` event --- apps/emqx/src/emqx_channel.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index cdd4b1a9e..39bd81b40 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1,4 +1,4 @@ -%%-------------------------------------------------------------------- +%-------------------------------------------------------------------- %% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -357,7 +357,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) -> }, case authenticate(?CONNECT_PACKET(NConnPkt), NChannel1) of {ok, Properties, NChannel2} -> - process_connect(Properties, ensure_connected(NChannel2)); + process_connect(Properties, NChannel2); {continue, Properties, NChannel2} -> handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, Properties}, NChannel2); {error, ReasonCode} -> @@ -381,7 +381,7 @@ handle_in( {ok, NProperties, NChannel} -> case ConnState of connecting -> - process_connect(NProperties, ensure_connected(NChannel)); + process_connect(NProperties, NChannel); _ -> handle_out( auth, @@ -611,7 +611,7 @@ process_connect( case emqx_cm:open_session(CleanStart, ClientInfo, ConnInfo) of {ok, #{session := Session, present := false}} -> NChannel = Channel#channel{session = Session}, - handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, NChannel); + handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, ensure_connected(NChannel)); {ok, #{session := Session, present := true, pendings := Pendings}} -> Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), NChannel = Channel#channel{ @@ -619,7 +619,7 @@ process_connect( resuming = true, pendings = Pendings1 }, - handle_out(connack, {?RC_SUCCESS, sp(true), AckProps}, NChannel); + handle_out(connack, {?RC_SUCCESS, sp(true), AckProps}, ensure_connected(NChannel)); {error, client_id_unavailable} -> handle_out(connack, ?RC_CLIENT_IDENTIFIER_NOT_VALID, Channel); {error, Reason} -> From a78760dbac4de67e2c1c0ff92e33e6100b934836 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 1 Aug 2022 09:59:23 -0300 Subject: [PATCH 09/63] chore: bump app vsns --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index fed9e6bc2..b7e65a042 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index c3b7b4f13..4e1a3518f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.3"}, + {vsn, "5.0.4"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx]}, diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 1640f4cc9..5a823067a 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.2"}, + {vsn, "5.0.3"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx]}, From 4a1ede7e7ec2e565e4ff94a11a2a4d879ed5b498 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 29 Jul 2022 15:18:31 -0300 Subject: [PATCH 10/63] test: fix flaky gateway tests --- apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl | 4 ++++ apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index fb3207944..aac140d3e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -239,6 +239,7 @@ t_gateway_exproto_with_ssl(_) -> t_authn(_) -> GwConf = #{name => <<"stomp">>}, {201, _} = request(post, "/gateway", GwConf), + ct:sleep(500), {204, _} = request(get, "/gateway/stomp/authentication"), AuthConf = #{ @@ -263,6 +264,7 @@ t_authn(_) -> t_authn_data_mgmt(_) -> GwConf = #{name => <<"stomp">>}, {201, _} = request(post, "/gateway", GwConf), + ct:sleep(500), {204, _} = request(get, "/gateway/stomp/authentication"), AuthConf = #{ @@ -271,6 +273,7 @@ t_authn_data_mgmt(_) -> user_id_type => <<"clientid">> }, {201, _} = request(post, "/gateway/stomp/authentication", AuthConf), + ct:sleep(500), {200, ConfResp} = request(get, "/gateway/stomp/authentication"), assert_confs(AuthConf, ConfResp), @@ -374,6 +377,7 @@ t_listeners_authn(_) -> ] }, {201, _} = request(post, "/gateway", GwConf), + ct:sleep(500), {200, ConfResp} = request(get, "/gateway/stomp"), assert_confs(GwConf, ConfResp), diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index 4633b421e..dff7b3fd4 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -47,6 +47,7 @@ end_per_suite(_Conf) -> init_per_testcase(_CaseName, Conf) -> _ = emqx_gateway_conf:unload_gateway(stomp), + ct:sleep(500), Conf. %%-------------------------------------------------------------------- @@ -282,6 +283,7 @@ t_load_remove_authn(_) -> {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), + ct:sleep(500), {ok, _} = emqx_gateway_conf:add_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_1), assert_confs( @@ -314,6 +316,7 @@ t_load_remove_listeners(_) -> {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), + ct:sleep(500), {ok, _} = emqx_gateway_conf:add_listener( <<"stomp">>, @@ -371,6 +374,7 @@ t_load_remove_listener_authn(_) -> {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), + ct:sleep(500), {ok, _} = emqx_gateway_conf:add_authn( <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_1 From cffaf95d00a39908fa4687afa4e21cd08cb86949 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 2 Aug 2022 17:20:58 +0800 Subject: [PATCH 11/63] chore: update apps/emqx/src/emqx_channel.erl --- apps/emqx/src/emqx_channel.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 39bd81b40..eff03e8ed 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1,4 +1,4 @@ -%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- %% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); From 8e53a617910888d214ec73dad0d6fe587a8f4b83 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 2 Aug 2022 19:49:06 +0800 Subject: [PATCH 12/63] fix: return 503 instead of crash when dashboard generate router timeout --- apps/emqx_dashboard/src/emqx_dashboard_listener.erl | 7 ++++++- apps/emqx_dashboard/src/emqx_dashboard_middleware.erl | 3 ++- mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl index 8bd5a4eb3..3c53c2e3f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl @@ -38,7 +38,12 @@ ]). is_ready(Timeout) -> - ready =:= gen_server:call(?MODULE, is_ready, Timeout). + try + ready =:= gen_server:call(?MODULE, is_ready, Timeout) + catch + exit:{timeout, _} -> + false + end. start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl index a2cf9db6f..67f907bbb 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl @@ -43,5 +43,6 @@ check_dispatch_ready(Env) -> true; true -> %% dashboard should always ready, if not, is_ready/1 will block until ready. - emqx_dashboard_listener:is_ready(timer:seconds(15)) + %% if not ready, dashboard will return 503. + emqx_dashboard_listener:is_ready(timer:seconds(20)) end. diff --git a/mix.exs b/mix.exs index 714279cfd..a218e091d 100644 --- a/mix.exs +++ b/mix.exs @@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.13.2", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.6", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.5", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.6", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, {:replayq, "0.3.4", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, diff --git a/rebar.config b/rebar.config index 58b4b079b..854ba0d59 100644 --- a/rebar.config +++ b/rebar.config @@ -57,7 +57,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.6"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.5"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.6"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} , {replayq, "0.3.4"} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} From b19e8fb3cd9e5685a424d74d8978e6e817cdb3e0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 29 Jul 2022 13:42:24 -0300 Subject: [PATCH 13/63] feat(license): add HTTP API for license --- .../i18n/emqx_license_http_api.conf | 23 ++ .../src/emqx_license_http_api.erl | 142 ++++++++++++ .../emqx_license/test/emqx_license_SUITE.erl | 1 - .../test/emqx_license_http_api_SUITE.erl | 210 ++++++++++++++++++ .../test/emqx_license_test_lib.erl | 26 +++ scripts/merge-i18n.escript | 8 +- 6 files changed, 406 insertions(+), 4 deletions(-) create mode 100644 lib-ee/emqx_license/i18n/emqx_license_http_api.conf create mode 100644 lib-ee/emqx_license/src/emqx_license_http_api.erl create mode 100644 lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl diff --git a/lib-ee/emqx_license/i18n/emqx_license_http_api.conf b/lib-ee/emqx_license/i18n/emqx_license_http_api.conf new file mode 100644 index 000000000..6b2c7a687 --- /dev/null +++ b/lib-ee/emqx_license/i18n/emqx_license_http_api.conf @@ -0,0 +1,23 @@ +emqx_license_http_api { + desc_license_info_api { + desc { + en: "Get license info" + zh: "获取许可证信息" + } + label: { + en: "License info" + zh: "许可证信息" + } + } + + desc_license_upload_api { + desc { + en: "Upload a license file or key" + zh: "上传许可证文件或钥匙" + } + label: { + en: "Update license" + zh: "更新许可证" + } + } +} diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/lib-ee/emqx_license/src/emqx_license_http_api.erl new file mode 100644 index 000000000..b204583ba --- /dev/null +++ b/lib-ee/emqx_license/src/emqx_license_http_api.erl @@ -0,0 +1,142 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_license_http_api). + +-behaviour(minirest_api). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-export([ + namespace/0, + api_spec/0, + paths/0, + schema/1 +]). + +-export([ + '/license'/2, + '/license/upload'/2 +]). + +-define(BAD_REQUEST, 'BAD_REQUEST'). +-define(NOT_FOUND, 'NOT_FOUND'). + +namespace() -> "license_http_api". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}). + +paths() -> + [ + "/license", + "/license/upload" + ]. + +schema("/license") -> + #{ + 'operationId' => '/license', + get => #{ + tags => [<<"license">>], + summary => <<"Get license info">>, + description => ?DESC("desc_license_info_api"), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + map(), + #{ + sample_license_info => #{ + value => #{ + customer => "Foo", + customer_type => 10, + deployment => "bar-deployment", + email => "contact@foo.com", + expiry => false, + expiry_at => "2295-10-27", + max_connections => 10, + start_at => "2022-01-11", + type => "trial" + } + } + } + ) + } + } + }; +schema("/license/upload") -> + #{ + 'operationId' => '/license/upload', + post => #{ + tags => [<<"license">>], + summary => <<"Upload license">>, + description => ?DESC("desc_license_upload_api"), + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_license_schema:license_type(), + #{ + license_key => #{ + summary => <<"License key string">>, + value => #{ + <<"key">> => <<"xxx">>, + <<"connection_low_watermark">> => "75%", + <<"connection_high_watermark">> => "80%" + } + }, + license_file => #{ + summary => <<"Path to a license file">>, + value => #{ + <<"file">> => <<"/path/to/license">>, + <<"connection_low_watermark">> => "75%", + <<"connection_high_watermark">> => "80%" + } + } + } + ), + responses => #{ + 200 => <<"ok">>, + 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"bad request">>), + 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"file not found">>) + } + } + }. + +'/license'(get, _Params) -> + License = maps:from_list(emqx_license_checker:dump()), + {200, License}. + +'/license/upload'(post, #{body := #{<<"file">> := Filepath}}) -> + case emqx_license:update_file(Filepath) of + {error, enoent} -> + ?SLOG(error, #{ + msg => "license_file_not_found", + path => Filepath + }), + {404, <<"file not found">>}; + {error, Error} -> + ?SLOG(error, #{ + msg => "bad_license_file", + reason => Error, + path => Filepath + }), + {400, <<"bad request">>}; + {ok, _} -> + ?SLOG(info, #{ + msg => "updated_license_file", + path => Filepath + }), + {200, <<"ok">>} + end; +'/license/upload'(post, #{body := #{<<"key">> := Key}}) -> + case emqx_license:update_key(Key) of + {error, Error} -> + ?SLOG(error, #{ + msg => "bad_license_key", + reason => Error + }), + {400, <<"bad request">>}; + {ok, _} -> + ?SLOG(info, #{msg => "updated_license_key"}), + {200, <<"ok">>} + end; +'/license/upload'(post, _Params) -> + {400, <<"bad request">>}. diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index a648595d2..08b3cb692 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -142,7 +142,6 @@ setup_test(TestCase, Config) when RawConfig = #{<<"type">> => file, <<"file">> => LicensePath}, emqx_config:put_raw([<<"license">>], RawConfig), ok = meck:new(emqx_license, [non_strict, passthrough, no_history, no_link]), - %% meck:expect(emqx_license, read_license, fun() -> {ok, License} end), meck:expect( emqx_license_parser, parse, diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl new file mode 100644 index 000000000..06bf35867 --- /dev/null +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -0,0 +1,210 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_license_http_api_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + _ = application:load(emqx_conf), + emqx_config:save_schema_mod_and_names(emqx_license_schema), + ok = meck:new(emqx_license_parser, [non_strict, passthrough, no_history, no_link]), + ok = meck:expect( + emqx_license_parser, + parse, + fun(X) -> + emqx_license_parser:parse( + X, + emqx_license_test_lib:public_key_pem() + ) + end + ), + emqx_common_test_helpers:start_apps([emqx_license, emqx_dashboard], fun set_special_configs/1), + Config. + +end_per_suite(_) -> + emqx_common_test_helpers:stop_apps([emqx_license, emqx_dashboard]), + ok = meck:unload([emqx_license_parser]), + Config = #{type => file, file => emqx_license_test_lib:default_license()}, + emqx_config:put([license], Config), + RawConfig = #{<<"type">> => file, <<"file">> => emqx_license_test_lib:default_license()}, + emqx_config:put_raw([<<"license">>], RawConfig), + ok. + +set_special_configs(emqx_dashboard) -> + emqx_dashboard_api_test_helpers:set_default_config(<<"license_admin">>); +set_special_configs(emqx_license) -> + LicenseKey = emqx_license_test_lib:make_license(#{max_connections => "100"}), + Config = #{type => key, key => LicenseKey}, + emqx_config:put([license], Config), + RawConfig = #{<<"type">> => key, <<"key">> => LicenseKey}, + emqx_config:put_raw([<<"license">>], RawConfig); +set_special_configs(_) -> + ok. + +init_per_testcase(_TestCase, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + Config. + +end_per_testcase(_TestCase, _Config) -> + {ok, _} = reset_license(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +request(Method, Uri, Body) -> + emqx_dashboard_api_test_helpers:request(<<"license_admin">>, Method, Uri, Body). + +uri(Segments) -> + emqx_dashboard_api_test_helpers:uri(Segments). + +get_license() -> + maps:from_list(emqx_license_checker:dump()). + +default_license() -> + emqx_license_test_lib:make_license(#{max_connections => "100"}). + +reset_license() -> + emqx_license:update_key(default_license()). + +assert_untouched_license() -> + ?assertMatch( + #{max_connections := 100}, + get_license() + ). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_license_info(_Config) -> + Res = request(get, uri(["license"]), []), + ?assertMatch({ok, 200, _}, Res), + {ok, 200, Payload} = Res, + ?assertEqual( + #{ + <<"customer">> => <<"Foo">>, + <<"customer_type">> => 10, + <<"deployment">> => <<"bar-deployment">>, + <<"email">> => <<"contact@foo.com">>, + <<"expiry">> => false, + <<"expiry_at">> => <<"2295-10-27">>, + <<"max_connections">> => 100, + <<"start_at">> => <<"2022-01-11">>, + <<"type">> => <<"trial">> + }, + emqx_json:decode(Payload, [return_maps]) + ), + ok. + +t_license_upload_file_success(_Config) -> + NewKey = emqx_license_test_lib:make_license(#{max_connections => "999"}), + Path = "/tmp/new.lic", + ok = file:write_file(Path, NewKey), + try + ?assertEqual( + {ok, 200, <<"ok">>}, + request( + post, + uri(["license", "upload"]), + #{file => Path} + ) + ), + ?assertMatch( + #{max_connections := 999}, + get_license() + ), + ok + after + ok = file:delete(Path), + ok + end. + +t_license_upload_file_not_found(_Config) -> + ?assertEqual( + {ok, 404, <<"file not found">>}, + request( + post, + uri(["license", "upload"]), + #{file => "/tmp/inexistent.lic"} + ) + ), + assert_untouched_license(), + ok. + +t_license_upload_file_reading_error(_Config) -> + %% eisdir + Path = "/tmp/", + ?assertEqual( + {ok, 400, <<"bad request">>}, + request( + post, + uri(["license", "upload"]), + #{file => Path} + ) + ), + assert_untouched_license(), + ok. + +t_license_upload_file_bad_license(_Config) -> + Path = "/tmp/bad.lic", + ok = file:write_file(Path, <<"bad key">>), + try + ?assertEqual( + {ok, 400, <<"bad request">>}, + request( + post, + uri(["license", "upload"]), + #{file => Path} + ) + ), + assert_untouched_license(), + ok + after + ok = file:delete(Path), + ok + end. + +t_license_upload_key_success(_Config) -> + NewKey = emqx_license_test_lib:make_license(#{max_connections => "999"}), + ?assertEqual( + {ok, 200, <<"ok">>}, + request( + post, + uri(["license", "upload"]), + #{key => NewKey} + ) + ), + ?assertMatch( + #{max_connections := 999}, + get_license() + ), + ok. + +t_license_upload_key_bad_key(_Config) -> + BadKey = <<"bad key">>, + ?assertEqual( + {ok, 400, <<"bad request">>}, + request( + post, + uri(["license", "upload"]), + #{key => BadKey} + ) + ), + assert_untouched_license(), + ok. diff --git a/lib-ee/emqx_license/test/emqx_license_test_lib.erl b/lib-ee/emqx_license/test/emqx_license_test_lib.erl index d3f2b5bd7..af3912f75 100644 --- a/lib-ee/emqx_license/test/emqx_license_test_lib.erl +++ b/lib-ee/emqx_license/test/emqx_license_test_lib.erl @@ -47,6 +47,32 @@ test_key(Filename, Format) -> public_key:pem_entry_decode(PemEntry) end. +make_license(Values0 = #{}) -> + Defaults = #{ + license_format => "220111", + license_type => "0", + customer_type => "10", + name => "Foo", + email => "contact@foo.com", + deployment => "bar-deployment", + start_date => "20220111", + days => "100000", + max_connections => "10" + }, + Values1 = maps:merge(Defaults, Values0), + Keys = [ + license_format, + license_type, + customer_type, + name, + email, + deployment, + start_date, + days, + max_connections + ], + Values = lists:map(fun(K) -> maps:get(K, Values1) end, Keys), + make_license(Values); make_license(Values) -> Key = private_key(), Text = string:join(Values, "\n"), diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript index 9f8ac91ff..e98631cfc 100755 --- a/scripts/merge-i18n.escript +++ b/scripts/merge-i18n.escript @@ -4,10 +4,12 @@ main(_) -> BaseConf = <<"">>, - Cfgs = get_all_cfgs("apps/"), - Conf = [merge(BaseConf, Cfgs), + Cfgs0 = get_all_cfgs("apps/"), + Cfgs1 = get_all_cfgs("lib-ee/"), + Conf0 = merge(BaseConf, Cfgs0), + Conf = [merge(Conf0, Cfgs1), io_lib:nl() - ], + ], ok = file:write_file("apps/emqx_dashboard/priv/i18n.conf", Conf). merge(BaseConf, Cfgs) -> From 1a66e53c4901335dbb91486314bfc0c64d72abd9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 1 Aug 2022 09:27:41 -0300 Subject: [PATCH 14/63] chore(license): change api responses after review --- .../src/emqx_license_http_api.erl | 62 ++++++--- .../test/emqx_license_http_api_SUITE.erl | 123 +++++++++++++----- 2 files changed, 129 insertions(+), 56 deletions(-) diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/lib-ee/emqx_license/src/emqx_license_http_api.erl index b204583ba..e11631004 100644 --- a/lib-ee/emqx_license/src/emqx_license_http_api.erl +++ b/lib-ee/emqx_license/src/emqx_license_http_api.erl @@ -47,17 +47,7 @@ schema("/license") -> map(), #{ sample_license_info => #{ - value => #{ - customer => "Foo", - customer_type => 10, - deployment => "bar-deployment", - email => "contact@foo.com", - expiry => false, - expiry_at => "2295-10-27", - max_connections => 10, - start_at => "2022-01-11", - type => "trial" - } + value => sample_license_info_response() } } ) @@ -93,13 +83,36 @@ schema("/license/upload") -> } ), responses => #{ - 200 => <<"ok">>, - 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"bad request">>), - 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"file not found">>) + 200 => emqx_dashboard_swagger:schema_with_examples( + map(), + #{ + sample_license_info => #{ + value => sample_license_info_response() + } + } + ), + 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad license key">>), + 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"File not found">>) } } }. +sample_license_info_response() -> + #{ + customer => "Foo", + customer_type => 10, + deployment => "bar-deployment", + email => "contact@foo.com", + expiry => false, + expiry_at => "2295-10-27", + max_connections => 10, + start_at => "2022-01-11", + type => "trial" + }. + +error_msg(Code, Msg) -> + #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + '/license'(get, _Params) -> License = maps:from_list(emqx_license_checker:dump()), {200, License}. @@ -111,20 +124,28 @@ schema("/license/upload") -> msg => "license_file_not_found", path => Filepath }), - {404, <<"file not found">>}; + {404, error_msg(?NOT_FOUND, <<"File not found">>)}; + {error, Error} when is_atom(Error) -> + ?SLOG(error, #{ + msg => "bad_license_file", + reason => Error, + path => Filepath + }), + {400, error_msg(?BAD_REQUEST, emqx_misc:explain_posix(Error))}; {error, Error} -> ?SLOG(error, #{ msg => "bad_license_file", reason => Error, path => Filepath }), - {400, <<"bad request">>}; + {400, error_msg(?BAD_REQUEST, <<"Bad license file">>)}; {ok, _} -> ?SLOG(info, #{ msg => "updated_license_file", path => Filepath }), - {200, <<"ok">>} + License = maps:from_list(emqx_license_checker:dump()), + {200, License} end; '/license/upload'(post, #{body := #{<<"key">> := Key}}) -> case emqx_license:update_key(Key) of @@ -133,10 +154,11 @@ schema("/license/upload") -> msg => "bad_license_key", reason => Error }), - {400, <<"bad request">>}; + {400, error_msg(?BAD_REQUEST, <<"Bad license key">>)}; {ok, _} -> ?SLOG(info, #{msg => "updated_license_key"}), - {200, <<"ok">>} + License = maps:from_list(emqx_license_checker:dump()), + {200, License} end; '/license/upload'(post, _Params) -> - {400, <<"bad request">>}. + {400, error_msg(?BAD_REQUEST, <<"Invalid request params">>)}. diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index 06bf35867..cb34f8f50 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -117,13 +117,26 @@ t_license_upload_file_success(_Config) -> Path = "/tmp/new.lic", ok = file:write_file(Path, NewKey), try + Res = request( + post, + uri(["license", "upload"]), + #{file => Path} + ), + ?assertMatch({ok, 200, _}, Res), + {ok, 200, Payload} = Res, ?assertEqual( - {ok, 200, <<"ok">>}, - request( - post, - uri(["license", "upload"]), - #{file => Path} - ) + #{ + <<"customer">> => <<"Foo">>, + <<"customer_type">> => 10, + <<"deployment">> => <<"bar-deployment">>, + <<"email">> => <<"contact@foo.com">>, + <<"expiry">> => false, + <<"expiry_at">> => <<"2295-10-27">>, + <<"max_connections">> => 999, + <<"start_at">> => <<"2022-01-11">>, + <<"type">> => <<"trial">> + }, + emqx_json:decode(Payload, [return_maps]) ), ?assertMatch( #{max_connections := 999}, @@ -136,13 +149,20 @@ t_license_upload_file_success(_Config) -> end. t_license_upload_file_not_found(_Config) -> + Res = request( + post, + uri(["license", "upload"]), + #{file => "/tmp/inexistent.lic"} + ), + + ?assertMatch({ok, 404, _}, Res), + {ok, 404, Payload} = Res, ?assertEqual( - {ok, 404, <<"file not found">>}, - request( - post, - uri(["license", "upload"]), - #{file => "/tmp/inexistent.lic"} - ) + #{ + <<"code">> => <<"NOT_FOUND">>, + <<"message">> => <<"File not found">> + }, + emqx_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. @@ -150,13 +170,19 @@ t_license_upload_file_not_found(_Config) -> t_license_upload_file_reading_error(_Config) -> %% eisdir Path = "/tmp/", + Res = request( + post, + uri(["license", "upload"]), + #{file => Path} + ), + ?assertMatch({ok, 400, _}, Res), + {ok, 400, Payload} = Res, ?assertEqual( - {ok, 400, <<"bad request">>}, - request( - post, - uri(["license", "upload"]), - #{file => Path} - ) + #{ + <<"code">> => <<"BAD_REQUEST">>, + <<"message">> => <<"Illegal operation on a directory">> + }, + emqx_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. @@ -165,13 +191,19 @@ t_license_upload_file_bad_license(_Config) -> Path = "/tmp/bad.lic", ok = file:write_file(Path, <<"bad key">>), try + Res = request( + post, + uri(["license", "upload"]), + #{file => Path} + ), + ?assertMatch({ok, 400, _}, Res), + {ok, 400, Payload} = Res, ?assertEqual( - {ok, 400, <<"bad request">>}, - request( - post, - uri(["license", "upload"]), - #{file => Path} - ) + #{ + <<"code">> => <<"BAD_REQUEST">>, + <<"message">> => <<"Bad license file">> + }, + emqx_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok @@ -182,13 +214,26 @@ t_license_upload_file_bad_license(_Config) -> t_license_upload_key_success(_Config) -> NewKey = emqx_license_test_lib:make_license(#{max_connections => "999"}), + Res = request( + post, + uri(["license", "upload"]), + #{key => NewKey} + ), + ?assertMatch({ok, 200, _}, Res), + {ok, 200, Payload} = Res, ?assertEqual( - {ok, 200, <<"ok">>}, - request( - post, - uri(["license", "upload"]), - #{key => NewKey} - ) + #{ + <<"customer">> => <<"Foo">>, + <<"customer_type">> => 10, + <<"deployment">> => <<"bar-deployment">>, + <<"email">> => <<"contact@foo.com">>, + <<"expiry">> => false, + <<"expiry_at">> => <<"2295-10-27">>, + <<"max_connections">> => 999, + <<"start_at">> => <<"2022-01-11">>, + <<"type">> => <<"trial">> + }, + emqx_json:decode(Payload, [return_maps]) ), ?assertMatch( #{max_connections := 999}, @@ -198,13 +243,19 @@ t_license_upload_key_success(_Config) -> t_license_upload_key_bad_key(_Config) -> BadKey = <<"bad key">>, + Res = request( + post, + uri(["license", "upload"]), + #{key => BadKey} + ), + ?assertMatch({ok, 400, _}, Res), + {ok, 400, Payload} = Res, ?assertEqual( - {ok, 400, <<"bad request">>}, - request( - post, - uri(["license", "upload"]), - #{key => BadKey} - ) + #{ + <<"code">> => <<"BAD_REQUEST">>, + <<"message">> => <<"Bad license key">> + }, + emqx_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. From 889829e47d7647d174cabfae80dda67c4b11a769 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 2 Aug 2022 08:57:00 -0300 Subject: [PATCH 15/63] docs: use better translation --- lib-ee/emqx_license/i18n/emqx_license_http_api.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_license/i18n/emqx_license_http_api.conf b/lib-ee/emqx_license/i18n/emqx_license_http_api.conf index 6b2c7a687..67ec1381f 100644 --- a/lib-ee/emqx_license/i18n/emqx_license_http_api.conf +++ b/lib-ee/emqx_license/i18n/emqx_license_http_api.conf @@ -13,7 +13,7 @@ emqx_license_http_api { desc_license_upload_api { desc { en: "Upload a license file or key" - zh: "上传许可证文件或钥匙" + zh: "上传许可证文件或密钥" } label: { en: "Update license" From f0f1a7ad05929fb4b82292c892b71a14f338a525 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 2 Aug 2022 09:34:56 -0300 Subject: [PATCH 16/63] chore: update changelog --- CHANGES-5.0.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 4dd74823c..1f237c6b1 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -1,3 +1,14 @@ +# 5.0.5 + +## Bug fixes + +* Allow changing the license type from key to file (and vice-versa). [#8598](https://github.com/emqx/emqx/pull/8598) + +## Enhancements + +* The license is now copied to all nodes in the cluster when it's reloaded. [#8598](https://github.com/emqx/emqx/pull/8598) +* Added a HTTP API to manage licenses. [#8610](https://github.com/emqx/emqx/pull/8610) + # 5.0.4 ## Bug fixes From 69ec76f586df51d472ab3701e91acb3a918bfce7 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 2 Aug 2022 17:25:33 -0300 Subject: [PATCH 17/63] chore(license): treat license file API as an upload (5.0) Improving the HTTP API for license after PR feedback. --- .../i18n/emqx_license_http_api.conf | 17 ++- lib-ee/emqx_license/src/emqx_license.erl | 18 ++- .../src/emqx_license_http_api.erl | 80 +++++----- .../emqx_license/src/emqx_license_schema.erl | 14 +- .../emqx_license/test/emqx_license_SUITE.erl | 7 +- .../test/emqx_license_http_api_SUITE.erl | 140 ++++++------------ 6 files changed, 131 insertions(+), 145 deletions(-) diff --git a/lib-ee/emqx_license/i18n/emqx_license_http_api.conf b/lib-ee/emqx_license/i18n/emqx_license_http_api.conf index 67ec1381f..59f76b7d6 100644 --- a/lib-ee/emqx_license/i18n/emqx_license_http_api.conf +++ b/lib-ee/emqx_license/i18n/emqx_license_http_api.conf @@ -10,10 +10,21 @@ emqx_license_http_api { } } - desc_license_upload_api { + desc_license_file_api { desc { - en: "Upload a license file or key" - zh: "上传许可证文件或密钥" + en: "Upload a license file" + zh: "上传一个许可证文件" + } + label: { + en: "Update license" + zh: "更新许可证" + } + } + + desc_license_key_api { + desc { + en: "Update a license key" + zh: "更新一个许可证密钥" } label: { en: "Update license" diff --git a/lib-ee/emqx_license/src/emqx_license.erl b/lib-ee/emqx_license/src/emqx_license.erl index 24b2cc709..d37d40dd3 100644 --- a/lib-ee/emqx_license/src/emqx_license.erl +++ b/lib-ee/emqx_license/src/emqx_license.erl @@ -22,6 +22,7 @@ read_license/0, read_license/1, update_file/1, + update_file_contents/1, update_key/1, license_dir/0, save_and_backup_license/1 @@ -70,16 +71,21 @@ relative_license_path() -> update_file(Filename) when is_binary(Filename); is_list(Filename) -> case file:read_file(Filename) of {ok, Contents} -> - Result = emqx_conf:update( - ?CONF_KEY_PATH, - {file, Contents}, - #{rawconf_with_defaults => true, override_to => local} - ), - handle_config_update_result(Result); + update_file_contents(Contents); {error, Error} -> {error, Error} end. +-spec update_file_contents(binary() | string()) -> + {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +update_file_contents(Contents) when is_binary(Contents) -> + Result = emqx_conf:update( + ?CONF_KEY_PATH, + {file, Contents}, + #{rawconf_with_defaults => true, override_to => local} + ), + handle_config_update_result(Result). + -spec update_key(binary() | string()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_key(Value) when is_binary(Value); is_list(Value) -> diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/lib-ee/emqx_license/src/emqx_license_http_api.erl index e11631004..e758bbf6b 100644 --- a/lib-ee/emqx_license/src/emqx_license_http_api.erl +++ b/lib-ee/emqx_license/src/emqx_license_http_api.erl @@ -18,11 +18,11 @@ -export([ '/license'/2, - '/license/upload'/2 + '/license/key'/2, + '/license/file'/2 ]). -define(BAD_REQUEST, 'BAD_REQUEST'). --define(NOT_FOUND, 'NOT_FOUND'). namespace() -> "license_http_api". @@ -32,7 +32,8 @@ api_spec() -> paths() -> [ "/license", - "/license/upload" + "/license/key", + "/license/file" ]. schema("/license") -> @@ -54,15 +55,36 @@ schema("/license") -> } } }; -schema("/license/upload") -> +schema("/license/file") -> #{ - 'operationId' => '/license/upload', + 'operationId' => '/license/file', post => #{ tags => [<<"license">>], - summary => <<"Upload license">>, - description => ?DESC("desc_license_upload_api"), + summary => <<"Upload license file">>, + description => ?DESC("desc_license_file_api"), + 'requestBody' => emqx_dashboard_swagger:file_schema(filename), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + map(), + #{ + sample_license_info => #{ + value => sample_license_info_response() + } + } + ), + 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad license file">>) + } + } + }; +schema("/license/key") -> + #{ + 'operationId' => '/license/key', + post => #{ + tags => [<<"license">>], + summary => <<"Update license key">>, + description => ?DESC("desc_license_key_api"), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_license_schema:license_type(), + emqx_license_schema:key_license(), #{ license_key => #{ summary => <<"License key string">>, @@ -71,14 +93,6 @@ schema("/license/upload") -> <<"connection_low_watermark">> => "75%", <<"connection_high_watermark">> => "80%" } - }, - license_file => #{ - summary => <<"Path to a license file">>, - value => #{ - <<"file">> => <<"/path/to/license">>, - <<"connection_low_watermark">> => "75%", - <<"connection_high_watermark">> => "80%" - } } } ), @@ -91,8 +105,7 @@ schema("/license/upload") -> } } ), - 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad license key">>), - 404 => emqx_dashboard_swagger:error_codes([?NOT_FOUND], <<"File not found">>) + 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad license file">>) } } }. @@ -117,37 +130,26 @@ error_msg(Code, Msg) -> License = maps:from_list(emqx_license_checker:dump()), {200, License}. -'/license/upload'(post, #{body := #{<<"file">> := Filepath}}) -> - case emqx_license:update_file(Filepath) of - {error, enoent} -> - ?SLOG(error, #{ - msg => "license_file_not_found", - path => Filepath - }), - {404, error_msg(?NOT_FOUND, <<"File not found">>)}; - {error, Error} when is_atom(Error) -> - ?SLOG(error, #{ - msg => "bad_license_file", - reason => Error, - path => Filepath - }), - {400, error_msg(?BAD_REQUEST, emqx_misc:explain_posix(Error))}; +'/license/file'(post, #{body := #{<<"filename">> := #{type := _} = File}}) -> + [{_Filename, Contents}] = maps:to_list(maps:without([type], File)), + case emqx_license:update_file_contents(Contents) of {error, Error} -> ?SLOG(error, #{ msg => "bad_license_file", - reason => Error, - path => Filepath + reason => Error }), {400, error_msg(?BAD_REQUEST, <<"Bad license file">>)}; {ok, _} -> ?SLOG(info, #{ - msg => "updated_license_file", - path => Filepath + msg => "updated_license_file" }), License = maps:from_list(emqx_license_checker:dump()), {200, License} end; -'/license/upload'(post, #{body := #{<<"key">> := Key}}) -> +'/license/file'(post, _Params) -> + {400, error_msg(?BAD_REQUEST, <<"Invalid request params">>)}. + +'/license/key'(post, #{body := #{<<"key">> := Key}}) -> case emqx_license:update_key(Key) of {error, Error} -> ?SLOG(error, #{ @@ -160,5 +162,5 @@ error_msg(Code, Msg) -> License = maps:from_list(emqx_license_checker:dump()), {200, License} end; -'/license/upload'(post, _Params) -> +'/license/key'(post, _Params) -> {400, error_msg(?BAD_REQUEST, <<"Invalid request params">>)}. diff --git a/lib-ee/emqx_license/src/emqx_license_schema.erl b/lib-ee/emqx_license/src/emqx_license_schema.erl index 88d245eb3..ab0da3b9a 100644 --- a/lib-ee/emqx_license/src/emqx_license_schema.erl +++ b/lib-ee/emqx_license/src/emqx_license_schema.erl @@ -15,7 +15,9 @@ -export([roots/0, fields/1, validations/0, desc/1]). -export([ - license_type/0 + license_type/0, + key_license/0, + file_license/0 ]). roots() -> @@ -99,10 +101,16 @@ validations() -> license_type() -> hoconsc:union([ - hoconsc:ref(?MODULE, key_license), - hoconsc:ref(?MODULE, file_license) + key_license(), + file_license() ]). +key_license() -> + hoconsc:ref(?MODULE, key_license). + +file_license() -> + hoconsc:ref(?MODULE, file_license). + check_license_watermark(Conf) -> case hocon_maps:get("license.connection_low_watermark", Conf) of undefined -> diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index 08b3cb692..ec92f4efe 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -16,6 +16,9 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + %% hack needed to avoid inter-suite flakiness when using meck... + ok = cover:stop(), + {ok, _} = cover:start(), _ = application:load(emqx_conf), emqx_config:save_schema_mod_and_names(emqx_license_schema), emqx_common_test_helpers:start_apps([emqx_license], fun set_special_configs/1), @@ -141,7 +144,9 @@ setup_test(TestCase, Config) when emqx_config:put([license], LicConfig), RawConfig = #{<<"type">> => file, <<"file">> => LicensePath}, emqx_config:put_raw([<<"license">>], RawConfig), - ok = meck:new(emqx_license, [non_strict, passthrough, no_history, no_link]), + ok = meck:new(emqx_license_parser, [ + non_strict, passthrough, no_history, no_link + ]), meck:expect( emqx_license_parser, parse, diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index cb34f8f50..f2feed893 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -21,23 +21,12 @@ all() -> init_per_suite(Config) -> _ = application:load(emqx_conf), emqx_config:save_schema_mod_and_names(emqx_license_schema), - ok = meck:new(emqx_license_parser, [non_strict, passthrough, no_history, no_link]), - ok = meck:expect( - emqx_license_parser, - parse, - fun(X) -> - emqx_license_parser:parse( - X, - emqx_license_test_lib:public_key_pem() - ) - end - ), emqx_common_test_helpers:start_apps([emqx_license, emqx_dashboard], fun set_special_configs/1), Config. end_per_suite(_) -> - emqx_common_test_helpers:stop_apps([emqx_license, emqx_dashboard]), ok = meck:unload([emqx_license_parser]), + emqx_common_test_helpers:stop_apps([emqx_license, emqx_dashboard]), Config = #{type => file, file => emqx_license_test_lib:default_license()}, emqx_config:put([license], Config), RawConfig = #{<<"type">> => file, <<"file">> => emqx_license_test_lib:default_license()}, @@ -51,7 +40,19 @@ set_special_configs(emqx_license) -> Config = #{type => key, key => LicenseKey}, emqx_config:put([license], Config), RawConfig = #{<<"type">> => key, <<"key">> => LicenseKey}, - emqx_config:put_raw([<<"license">>], RawConfig); + emqx_config:put_raw([<<"license">>], RawConfig), + ok = meck:new(emqx_license_parser, [non_strict, passthrough, no_history, no_link]), + ok = meck:expect( + emqx_license_parser, + parse, + fun(X) -> + emqx_license_parser:parse( + X, + emqx_license_test_lib:public_key_pem() + ) + end + ), + ok; set_special_configs(_) -> ok. @@ -88,6 +89,14 @@ assert_untouched_license() -> get_license() ). +multipart_formdata_request(Uri, File) -> + emqx_dashboard_api_test_helpers:multipart_formdata_request( + Uri, + _Username = <<"license_admin">>, + _Fields = [], + [File] + ). + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -114,109 +123,54 @@ t_license_info(_Config) -> t_license_upload_file_success(_Config) -> NewKey = emqx_license_test_lib:make_license(#{max_connections => "999"}), - Path = "/tmp/new.lic", - ok = file:write_file(Path, NewKey), - try - Res = request( - post, - uri(["license", "upload"]), - #{file => Path} - ), - ?assertMatch({ok, 200, _}, Res), - {ok, 200, Payload} = Res, - ?assertEqual( - #{ - <<"customer">> => <<"Foo">>, - <<"customer_type">> => 10, - <<"deployment">> => <<"bar-deployment">>, - <<"email">> => <<"contact@foo.com">>, - <<"expiry">> => false, - <<"expiry_at">> => <<"2295-10-27">>, - <<"max_connections">> => 999, - <<"start_at">> => <<"2022-01-11">>, - <<"type">> => <<"trial">> - }, - emqx_json:decode(Payload, [return_maps]) - ), - ?assertMatch( - #{max_connections := 999}, - get_license() - ), - ok - after - ok = file:delete(Path), - ok - end. - -t_license_upload_file_not_found(_Config) -> - Res = request( - post, - uri(["license", "upload"]), - #{file => "/tmp/inexistent.lic"} + Res = multipart_formdata_request( + uri(["license", "file"]), + {filename, "emqx.lic", NewKey} ), - - ?assertMatch({ok, 404, _}, Res), - {ok, 404, Payload} = Res, + ?assertMatch({ok, 200, _}, Res), + {ok, 200, Payload} = Res, ?assertEqual( #{ - <<"code">> => <<"NOT_FOUND">>, - <<"message">> => <<"File not found">> + <<"customer">> => <<"Foo">>, + <<"customer_type">> => 10, + <<"deployment">> => <<"bar-deployment">>, + <<"email">> => <<"contact@foo.com">>, + <<"expiry">> => false, + <<"expiry_at">> => <<"2295-10-27">>, + <<"max_connections">> => 999, + <<"start_at">> => <<"2022-01-11">>, + <<"type">> => <<"trial">> }, emqx_json:decode(Payload, [return_maps]) ), - assert_untouched_license(), + ?assertMatch( + #{max_connections := 999}, + get_license() + ), ok. -t_license_upload_file_reading_error(_Config) -> - %% eisdir - Path = "/tmp/", - Res = request( - post, - uri(["license", "upload"]), - #{file => Path} +t_license_upload_file_bad_license(_Config) -> + Res = multipart_formdata_request( + uri(["license", "file"]), + {filename, "bad.lic", <<"bad key">>} ), ?assertMatch({ok, 400, _}, Res), {ok, 400, Payload} = Res, ?assertEqual( #{ <<"code">> => <<"BAD_REQUEST">>, - <<"message">> => <<"Illegal operation on a directory">> + <<"message">> => <<"Bad license file">> }, emqx_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. -t_license_upload_file_bad_license(_Config) -> - Path = "/tmp/bad.lic", - ok = file:write_file(Path, <<"bad key">>), - try - Res = request( - post, - uri(["license", "upload"]), - #{file => Path} - ), - ?assertMatch({ok, 400, _}, Res), - {ok, 400, Payload} = Res, - ?assertEqual( - #{ - <<"code">> => <<"BAD_REQUEST">>, - <<"message">> => <<"Bad license file">> - }, - emqx_json:decode(Payload, [return_maps]) - ), - assert_untouched_license(), - ok - after - ok = file:delete(Path), - ok - end. - t_license_upload_key_success(_Config) -> NewKey = emqx_license_test_lib:make_license(#{max_connections => "999"}), Res = request( post, - uri(["license", "upload"]), + uri(["license", "key"]), #{key => NewKey} ), ?assertMatch({ok, 200, _}, Res), @@ -245,7 +199,7 @@ t_license_upload_key_bad_key(_Config) -> BadKey = <<"bad key">>, Res = request( post, - uri(["license", "upload"]), + uri(["license", "key"]), #{key => BadKey} ), ?assertMatch({ok, 400, _}, Res), From 4869225395951b8a16b120d0066abad6bc2999f9 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 3 Aug 2022 12:24:03 +0800 Subject: [PATCH 18/63] chore: update Running/Stopped to running/stopped --- CHANGES-5.0.md | 1 + apps/emqx_management/src/emqx_mgmt.erl | 4 ++-- apps/emqx_management/src/emqx_mgmt_api_nodes.erl | 11 +++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 1f237c6b1..8f5592d4a 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -8,6 +8,7 @@ * The license is now copied to all nodes in the cluster when it's reloaded. [#8598](https://github.com/emqx/emqx/pull/8598) * Added a HTTP API to manage licenses. [#8610](https://github.com/emqx/emqx/pull/8610) +* Updated `/nodes` API node_status from `Running/Stopped` to `running/stopped`. [#8642](https://github.com/emqx/emqx/pull/8642) # 5.0.4 diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index cdf3bf504..8f73d5767 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -138,7 +138,7 @@ node_info() -> max_fds, lists:usort(lists:flatten(erlang:system_info(check_io))) ), connections => ets:info(emqx_channel, size), - node_status => 'Running', + node_status => 'running', uptime => proplists:get_value(uptime, BrokerInfo), version => iolist_to_binary(proplists:get_value(version, BrokerInfo)), role => mria_rlog:role() @@ -156,7 +156,7 @@ node_info(Node) -> wrap_rpc(emqx_management_proto_v2:node_info(Node)). stopped_node_info(Node) -> - #{name => Node, node_status => 'Stopped'}. + #{name => Node, node_status => 'stopped'}. %%-------------------------------------------------------------------- %% Brokers diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index f1731db4d..dda82aeb3 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -189,8 +189,8 @@ fields(node_info) -> )}, {node_status, mk( - enum(['Running', 'Stopped']), - #{desc => <<"Node status">>, example => "Running"} + enum(['running', 'stopped']), + #{desc => <<"Node status">>, example => "running"} )}, {otp_release, mk( @@ -288,19 +288,18 @@ get_stats(Node) -> %% internal function format(_Node, Info = #{memory_total := Total, memory_used := Used}) -> - {ok, SysPathBinary} = file:get_cwd(), - SysPath = list_to_binary(SysPathBinary), + RootDir = code:root_dir(), LogPath = case log_path() of undefined -> <<"log.file_handler.default.enable is false,only log to console">>; Path -> - filename:join(SysPath, Path) + filename:join(RootDir, Path) end, Info#{ memory_total := emqx_mgmt_util:kmg(Total), memory_used := emqx_mgmt_util:kmg(Used), - sys_path => SysPath, + sys_path => RootDir, log_path => LogPath }. From a19514f0009d989fd8aa5b8846d393d780e91e39 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 3 Aug 2022 12:34:30 +0800 Subject: [PATCH 19/63] chore: bump emqx_management to 5.0.3 --- apps/emqx_management/src/emqx_management.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index e5b769b6f..9de47ca50 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.2"}, + {vsn, "5.0.3"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]}, From fd7c97735dc22145ce0c2d40e5eaca7add89ba67 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 3 Aug 2022 12:58:20 +0800 Subject: [PATCH 20/63] chore: make sure path is binary --- apps/emqx_management/src/emqx_mgmt_api_nodes.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index dda82aeb3..e0f0912df 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -288,7 +288,7 @@ get_stats(Node) -> %% internal function format(_Node, Info = #{memory_total := Total, memory_used := Used}) -> - RootDir = code:root_dir(), + RootDir = list_to_binary(code:root_dir()), LogPath = case log_path() of undefined -> From abcdad39e571728d5f2935879a7a53f65385017b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 3 Aug 2022 16:29:06 +0800 Subject: [PATCH 21/63] chore: bump fvt_test's minirest to latest(1.3.6) --- .ci/fvt_tests/http_server/README.md | 1 - .ci/fvt_tests/http_server/rebar.config | 2 +- .../http_server/src/http_server.app.src | 4 +- .ci/fvt_tests/http_server/src/http_server.erl | 130 +++++++++++++----- 4 files changed, 96 insertions(+), 41 deletions(-) diff --git a/.ci/fvt_tests/http_server/README.md b/.ci/fvt_tests/http_server/README.md index ea14939b3..93619927f 100644 --- a/.ci/fvt_tests/http_server/README.md +++ b/.ci/fvt_tests/http_server/README.md @@ -27,4 +27,3 @@ ok + POST `/counter` 计数器加一 - diff --git a/.ci/fvt_tests/http_server/rebar.config b/.ci/fvt_tests/http_server/rebar.config index 4085df159..8ddb3a7ab 100644 --- a/.ci/fvt_tests/http_server/rebar.config +++ b/.ci/fvt_tests/http_server/rebar.config @@ -3,7 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ - {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.6"}}} + {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "1.3.6"}}} ]}. {shell, [ diff --git a/.ci/fvt_tests/http_server/src/http_server.app.src b/.ci/fvt_tests/http_server/src/http_server.app.src index 420aff9d3..6be2382b6 100644 --- a/.ci/fvt_tests/http_server/src/http_server.app.src +++ b/.ci/fvt_tests/http_server/src/http_server.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, http_server, - [{description, "An OTP application"}, - {vsn, "0.1.0"}, + [{description, "An HTTP server application"}, + {vsn, "0.2.0"}, {registered, []}, % {mod, {http_server_app, []}}, {modules, []}, diff --git a/.ci/fvt_tests/http_server/src/http_server.erl b/.ci/fvt_tests/http_server/src/http_server.erl index 4aaa25b95..c7097854e 100644 --- a/.ci/fvt_tests/http_server/src/http_server.erl +++ b/.ci/fvt_tests/http_server/src/http_server.erl @@ -10,51 +10,107 @@ stop/0 ]). --rest_api(#{ - name => get_counter, - method => 'GET', - path => "/counter", - func => get_counter, - descr => "Check counter" -}). --rest_api(#{ - name => add_counter, - method => 'POST', - path => "/counter", - func => add_counter, - descr => "Counter plus one" -}). +-behavior(minirest_api). --export([ - get_counter/2, - add_counter/2 -]). +-export([api_spec/0]). +-export([counter/2]). + +api_spec() -> + { + [counter_api()], + [] + }. + +counter_api() -> + MetaData = #{ + get => #{ + description => "Get counter", + summary => "Get counter", + responses => #{ + 200 => #{ + content => #{ + 'application/json' => + #{ + type => object, + properties => #{ + code => #{type => integer, example => 0}, + data => #{type => integer, example => 0} + } + } + } + } + } + }, + post => #{ + description => "Add counter", + summary => "Add counter", + 'requestBody' => #{ + content => #{ + 'application/json' => #{ + schema => + #{ + type => object, + properties => #{ + payload => #{type => string, example => <<"sample payload">>}, + id => #{type => integer, example => 0} + } + } + } + } + }, + responses => #{ + 200 => #{ + content => #{ + 'application/json' => + #{ + type => object, + properties => #{ + code => #{type => integer, example => 0} + } + } + } + } + } + } + }, + {"/counter", MetaData, counter}. + +counter(get, _Params) -> + V = ets:info(relup_test_message, size), + {200, #{<<"content-type">> => <<"text/plain">>}, #{<<"code">> => 0, <<"data">> => V}}; +counter(post, #{body := Params}) -> + case Params of + #{<<"payload">> := _, <<"id">> := Id} -> + ets:insert(relup_test_message, {Id, maps:remove(<<"id">>, Params)}), + {200, #{<<"code">> => 0}}; + _ -> + io:format("discarded: ~p\n", [Params]), + {200, #{<<"code">> => -1}} + end. start() -> application:ensure_all_started(minirest), _ = spawn(fun ets_owner/0), - Handlers = [{"/", minirest:handler(#{modules => [?MODULE]})}], - Dispatch = [{"/[...]", minirest, Handlers}], - minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 7077}]}, Dispatch). + RanchOptions = #{ + max_connections => 512, + num_acceptors => 4, + socket_opts => [{send_timeout, 5000}, {port, 7077}, {backlog, 512}] + }, + Minirest = #{ + base_path => "", + modules => [?MODULE], + dispatch => [{"/[...]", ?MODULE, []}], + protocol => http, + ranch_options => RanchOptions, + middlewares => [cowboy_router, cowboy_handler] + }, + Res = minirest:start(?MODULE, Minirest), + minirest:update_dispatch(?MODULE), + Res. stop() -> ets:delete(relup_test_message), - minirest:stop_http(?MODULE). - -get_counter(_Binding, _Params) -> - V = ets:info(relup_test_message, size), - return({ok, V}). - -add_counter(_Binding, Params) -> - case lists:keymember(<<"payload">>, 1, Params) of - true -> - {value, {<<"id">>, ID}, Params1} = lists:keytake(<<"id">>, 1, Params), - ets:insert(relup_test_message, {ID, Params1}); - _ -> - io:format("discarded: ~p\n", [Params]), - ok - end, - return(). + minirest:stop(?MODULE). ets_owner() -> ets:new(relup_test_message, [named_table, public]), From 19e101445c5877611035e42f3023990b8e5c25e6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 2 Aug 2022 14:59:12 -0300 Subject: [PATCH 22/63] feat: add option to gc after TLS/SSL handshake --- CHANGES-5.0.md | 1 + apps/emqx/i18n/emqx_schema_i18n.conf | 17 ++++++++++++++ apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_schema.erl | 10 +++++++- apps/emqx/test/emqx_schema_tests.erl | 35 ++++++++++++++++++++++++++++ mix.exs | 2 +- rebar.config | 2 +- 7 files changed, 65 insertions(+), 4 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 4dd74823c..a7d37d5b4 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -34,6 +34,7 @@ * Improve authentication tracing. [#8554](https://github.com/emqx/emqx/pull/8554) * Standardize the '/listeners' and `/gateway//listeners` API fields. It will introduce some incompatible updates, see [#8571](https://github.com/emqx/emqx/pull/8571) +* Add option to perform GC on connection process after TLS/SSL handshake is performed. [#8637](https://github.com/emqx/emqx/pull/8637) # 5.0.3 diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 330c766d1..63667dd22 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1841,6 +1841,23 @@ Maximum time duration allowed for the handshake to complete } } +server_ssl_opts_schema_gc_after_handshake { + desc { + en: """ +Performance tuning. If enabled, will immediately perform a GC after +the TLS/SSL handshake is established. +""" + zh: """ +性能调整。 如果启用，将在TLS/SSL握手建立后立即执行GC。 +TLS/SSL握手建立后立即进行GC。 +""" + } + label: { + en: "Perform GC after handshake" + zh: "握手后执行GC" + } +} + fields_listeners_tcp { desc { en: """ diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index b33840aaa..f09092e2b 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -26,7 +26,7 @@ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, - {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}, + {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.3"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}}, diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 9f96595ed..b55e84e7b 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1953,7 +1953,15 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> } )} || IsRanchListener - ] + ] ++ + [ + {"gc_after_handshake", + sc(boolean(), #{ + default => false, + desc => ?DESC(server_ssl_opts_schema_gc_after_handshake) + })} + || not IsRanchListener + ] ]. %% @doc Make schema for SSL client. diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index 2e776a368..a40026d4c 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -141,3 +141,38 @@ bad_tls_version_test() -> validate(Sc, #{<<"versions">> => [<<"foo">>]}) ), ok. + +ssl_opts_gc_after_handshake_test_rancher_listener_test() -> + Sc = emqx_schema:server_ssl_opts_schema( + #{ + gc_after_handshake => false + }, + _IsRanchListener = true + ), + ?assertThrow( + {_Sc, [ + #{ + kind := validation_error, + reason := unknown_fields, + unknown := <<"gc_after_handshake">> + } + ]}, + validate(Sc, #{<<"gc_after_handshake">> => true}) + ), + ok. + +ssl_opts_gc_after_handshake_test_not_rancher_listener_test() -> + Sc = emqx_schema:server_ssl_opts_schema( + #{ + gc_after_handshake => false + }, + _IsRanchListener = false + ), + Checked = validate(Sc, #{<<"gc_after_handshake">> => <<"true">>}), + ?assertMatch( + #{ + gc_after_handshake := true + }, + Checked + ), + ok. diff --git a/mix.exs b/mix.exs index d49051ab7..78ba36490 100644 --- a/mix.exs +++ b/mix.exs @@ -51,7 +51,7 @@ defmodule EMQXUmbrella.MixProject do {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, - {:esockd, github: "emqx/esockd", tag: "5.9.3", override: true}, + {:esockd, github: "emqx/esockd", tag: "5.9.4", override: true}, {:ekka, github: "emqx/ekka", tag: "0.13.3", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.6", override: true}, diff --git a/rebar.config b/rebar.config index 3d3d5968f..f91841f8d 100644 --- a/rebar.config +++ b/rebar.config @@ -53,7 +53,7 @@ , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.3"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.6"}}} From 7f932d64be4a93cfc6b7cc6f82f17c12e6cae5d1 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 3 Aug 2022 09:37:27 -0300 Subject: [PATCH 23/63] docs: adjust descriptions Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/i18n/emqx_schema_i18n.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 63667dd22..1fc91c9cd 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1844,11 +1844,11 @@ Maximum time duration allowed for the handshake to complete server_ssl_opts_schema_gc_after_handshake { desc { en: """ -Performance tuning. If enabled, will immediately perform a GC after -the TLS/SSL handshake is established. +Memory usage tuning. If enabled, will immediately perform a garbage collection after +the TLS/SSL handshake. """ zh: """ -性能调整。 如果启用，将在TLS/SSL握手建立后立即执行GC。 +内存使用调优。如果启用，将在TLS/SSL握手完成后立即执行垃圾回收。 TLS/SSL握手建立后立即进行GC。 """ } From 01b9115fd8bafbebcf50b6646b303dafae9226da Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 3 Aug 2022 16:30:34 +0200 Subject: [PATCH 24/63] fix: keep alive check According to MQTT spec, MQTT Server should check a complete MQTT message recv in last keep-alive time frame instead of number of received bytes from the socket. This commit change to check the recv pkt counter from process dict instead. Also it could save some calls to erlang port. --- apps/emqx/src/emqx_connection.erl | 11 +++-------- apps/emqx/test/emqx_connection_SUITE.erl | 5 ----- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 1caf345e6..4ebc5b5e6 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -708,8 +708,6 @@ handle_timeout( TRef, keepalive, State = #state{ - transport = Transport, - socket = Socket, channel = Channel } ) -> @@ -717,12 +715,9 @@ handle_timeout( disconnected -> {ok, State}; _ -> - case Transport:getstat(Socket, [recv_oct]) of - {ok, [{recv_oct, RecvOct}]} -> - handle_timeout(TRef, {keepalive, RecvOct}, State); - {error, Reason} -> - handle_info({sock_error, Reason}, State) - end + %% recv_pkt: valid MQTT message + RecvCnt = emqx_pd:get_counter(recv_pkt), + handle_timeout(TRef, {keepalive, RecvCnt}, State) end; handle_timeout(TRef, Msg, State) -> with_channel(handle_timeout, [TRef, Msg], State). diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index c5dfdf34a..344d81f8c 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -316,11 +316,6 @@ t_handle_timeout(_) -> emqx_connection:handle_timeout(TRef, keepalive, State) ), - ok = meck:expect(emqx_transport, getstat, fun(_Sock, _Options) -> {error, for_testing} end), - ?assertMatch( - {stop, {shutdown, for_testing}, _NState}, - emqx_connection:handle_timeout(TRef, keepalive, State) - ), ?assertMatch({ok, _NState}, emqx_connection:handle_timeout(TRef, undefined, State)). t_parse_incoming(_) -> From dec892e86765da28116b4301212eefe6c73dea11 Mon Sep 17 00:00:00 2001 From: Benjamin Krenn Date: Mon, 11 Jul 2022 16:56:46 +0200 Subject: [PATCH 25/63] feat(shared-sub): add round_robin_per_group strategy add round robin per group strategy that balances load in a more predictable fashion when using no replication --- apps/emqx/i18n/emqx_schema_i18n.conf | 2 + apps/emqx/src/emqx_schema.erl | 20 +- apps/emqx/src/emqx_shared_sub.erl | 43 +++- apps/emqx/test/emqx_shared_sub_SUITE.erl | 286 ++++++++++++++++++++++- 4 files changed, 340 insertions(+), 11 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 330c766d1..64c1f7244 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1115,6 +1115,7 @@ special characters are allowed. en: """Dispatch strategy for shared subscription. - `random`: dispatch the message to a random selected subscriber - `round_robin`: select the subscribers in a round-robin manner +- `round_robin_per_group`: select the subscribers in round-robin fashion within each shared subscriber group - `sticky`: always use the last selected subscriber to dispatch, until the subscriber disconnects. - `hash`: select the subscribers by the hash of `clientIds` @@ -1124,6 +1125,7 @@ subscriber was not found, send to a random subscriber cluster-wide cn: """共享订阅的分发策略名称。 - `random`: 随机选择一个组内成员； - `round_robin`: 循环选择下一个成员； +- `round_robin_per_group`: 在共享组内循环选择下一个成员； - `sticky`: 使用上一次选中的成员； - `hash`: 根据 ClientID 哈希映射到一个成员； - `local`: 随机分发到节点本地成成员，如果本地成员不存在，则随机分发 diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 5652b37f7..1010b42ff 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1160,7 +1160,15 @@ fields("broker") -> )}, {"shared_subscription_strategy", sc( - hoconsc:enum([random, round_robin, sticky, local, hash_topic, hash_clientid]), + hoconsc:enum([ + random, + round_robin, + round_robin_per_group, + sticky, + local, + hash_topic, + hash_clientid + ]), #{ default => round_robin, desc => ?DESC(broker_shared_subscription_strategy) @@ -1200,7 +1208,15 @@ fields("shared_subscription_group") -> [ {"strategy", sc( - hoconsc:enum([random, round_robin, sticky, local, hash_topic, hash_clientid]), + hoconsc:enum([ + random, + round_robin, + round_robin_per_group, + sticky, + local, + hash_topic, + hash_clientid + ]), #{ default => random, desc => ?DESC(shared_subscription_strategy_enum) diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index bdd6d22ea..bd9c16206 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -72,6 +72,7 @@ -type strategy() :: random | round_robin + | round_robin_per_group | sticky | local %% same as hash_clientid, backward compatible @@ -81,6 +82,7 @@ -define(SERVER, ?MODULE). -define(TAB, emqx_shared_subscription). +-define(SHARED_SUBS_ROUND_ROBIN_COUNTER, emqx_shared_subscriber_round_robin_counter). -define(SHARED_SUBS, emqx_shared_subscriber). -define(ALIVE_SUBS, emqx_alive_shared_subscribers). -define(SHARED_SUB_QOS1_DISPATCH_TIMEOUT_SECONDS, 5). @@ -315,7 +317,14 @@ do_pick_subscriber(Group, Topic, round_robin, _ClientId, _SourceTopic, Count) -> N -> (N + 1) rem Count end, _ = erlang:put({shared_sub_round_robin, Group, Topic}, Rem), - Rem + 1. + Rem + 1; +do_pick_subscriber(Group, Topic, round_robin_per_group, _ClientId, _SourceTopic, Count) -> + %% reset the counter to 1 if counter > subscriber count to avoid the counter to grow larger + %% than the current subscriber count. + %% if no counter for the given group topic exists - due to a configuration change - create a new one starting at 0 + ets:update_counter(?SHARED_SUBS_ROUND_ROBIN_COUNTER, {Group, Topic}, {2, 1, Count, 1}, { + {Group, Topic}, 0 + }). subscribers(Group, Topic) -> ets:select(?TAB, [{{emqx_shared_subscription, Group, Topic, '$1'}, [], ['$1']}]). @@ -330,6 +339,7 @@ init([]) -> {atomic, PMon} = mria:transaction(?SHARED_SUB_SHARD, fun init_monitors/0), ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), + ok = emqx_tables:new(?SHARED_SUBS_ROUND_ROBIN_COUNTER, [public, set, {write_concurrency, true}]), {ok, update_stats(#state{pmon = PMon})}. init_monitors() -> @@ -348,12 +358,14 @@ handle_call({subscribe, Group, Topic, SubPid}, _From, State = #state{pmon = PMon false -> ok = emqx_router:do_add_route(Topic, {Group, node()}) end, ok = maybe_insert_alive_tab(SubPid), + ok = maybe_insert_round_robin_count({Group, Topic}), true = ets:insert(?SHARED_SUBS, {{Group, Topic}, SubPid}), {reply, ok, update_stats(State#state{pmon = emqx_pmon:monitor(SubPid, PMon)})}; handle_call({unsubscribe, Group, Topic, SubPid}, _From, State) -> mria:dirty_delete_object(?TAB, record(Group, Topic, SubPid)), true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), delete_route_if_needed({Group, Topic}), + maybe_delete_round_robin_count({Group, Topic}), {reply, ok, State}; handle_call(Req, _From, State) -> ?SLOG(error, #{msg => "unexpected_call", req => Req}), @@ -395,6 +407,25 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- +maybe_insert_round_robin_count({Group, _Topic} = GroupTopic) -> + strategy(Group) =:= round_robin_per_group andalso + ets:insert(?SHARED_SUBS_ROUND_ROBIN_COUNTER, {GroupTopic, 0}), + ok. + +maybe_delete_round_robin_count({Group, _Topic} = GroupTopic) -> + strategy(Group) =:= round_robin_per_group andalso + if_no_more_subscribers(GroupTopic, fun() -> + ets:delete(?SHARED_SUBS_ROUND_ROBIN_COUNTER, GroupTopic) + end), + ok. + +if_no_more_subscribers(GroupTopic, Fn) -> + case ets:member(?SHARED_SUBS, GroupTopic) of + true -> ok; + false -> Fn() + end, + ok. + %% keep track of alive remote pids maybe_insert_alive_tab(Pid) when ?IS_LOCAL_PID(Pid) -> ok; maybe_insert_alive_tab(Pid) when is_pid(Pid) -> @@ -407,6 +438,7 @@ cleanup_down(SubPid) -> fun(Record = #emqx_shared_subscription{topic = Topic, group = Group}) -> ok = mria:dirty_delete_object(?TAB, Record), true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), + maybe_delete_round_robin_count({Group, Topic}), delete_route_if_needed({Group, Topic}) end, mnesia:dirty_match_object(#emqx_shared_subscription{_ = '_', subpid = SubPid}) @@ -430,8 +462,7 @@ is_alive_sub(Pid) when ?IS_LOCAL_PID(Pid) -> is_alive_sub(Pid) -> [] =/= ets:lookup(?ALIVE_SUBS, Pid). -delete_route_if_needed({Group, Topic}) -> - case ets:member(?SHARED_SUBS, {Group, Topic}) of - true -> ok; - false -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) - end. +delete_route_if_needed({Group, Topic} = GroupTopic) -> + if_no_more_subscribers(GroupTopic, fun() -> + ok = emqx_router:do_delete_route(Topic, {Group, node()}) + end). diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 39ef34cf6..3b6357d7d 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -195,6 +195,266 @@ t_round_robin(_) -> ok = ensure_config(round_robin, true), test_two_messages(round_robin). +t_round_robin_per_group(_) -> + ok = ensure_config(round_robin_per_group, true), + test_two_messages(round_robin_per_group). + +%% this would fail if executed with the standard round_robin strategy +t_round_robin_per_group_even_distribution_one_group(_) -> + ok = ensure_config(round_robin_per_group, true), + Topic = <<"foo/bar">>, + Group = <<"group1">>, + {ok, ConnPid1} = emqtt:start_link([{clientid, <<"C0">>}]), + {ok, ConnPid2} = emqtt:start_link([{clientid, <<"C1">>}]), + {ok, _} = emqtt:connect(ConnPid1), + {ok, _} = emqtt:connect(ConnPid2), + + emqtt:subscribe(ConnPid1, {<<"$share/", Group/binary, "/", Topic/binary>>, 0}), + emqtt:subscribe(ConnPid2, {<<"$share/", Group/binary, "/", Topic/binary>>, 0}), + + %% publisher with persistent connection + {ok, PublisherPid} = emqtt:start_link(), + {ok, _} = emqtt:connect(PublisherPid), + + lists:foreach( + fun(I) -> + Message = erlang:integer_to_binary(I), + emqtt:publish(PublisherPid, Topic, Message) + end, + lists:seq(0, 9) + ), + + AllReceivedMessages = lists:map( + fun(#{client_pid := SubscriberPid, payload := Payload}) -> {SubscriberPid, Payload} end, + lists:reverse(recv_msgs(10)) + ), + MessagesReceivedSubscriber1 = lists:filter( + fun({P, _Payload}) -> P == ConnPid1 end, AllReceivedMessages + ), + MessagesReceivedSubscriber2 = lists:filter( + fun({P, _Payload}) -> P == ConnPid2 end, AllReceivedMessages + ), + + emqtt:stop(ConnPid1), + emqtt:stop(ConnPid2), + emqtt:stop(PublisherPid), + + %% ensure each subscriber received 5 messages in alternating fashion + %% one receives all even and the other all uneven payloads + ?assertEqual( + [ + {ConnPid1, <<"0">>}, + {ConnPid1, <<"2">>}, + {ConnPid1, <<"4">>}, + {ConnPid1, <<"6">>}, + {ConnPid1, <<"8">>} + ], + MessagesReceivedSubscriber1 + ), + + ?assertEqual( + [ + {ConnPid2, <<"1">>}, + {ConnPid2, <<"3">>}, + {ConnPid2, <<"5">>}, + {ConnPid2, <<"7">>}, + {ConnPid2, <<"9">>} + ], + MessagesReceivedSubscriber2 + ), + ok. + +t_round_robin_per_group_even_distribution_two_groups(_) -> + ok = ensure_config(round_robin_per_group, true), + Topic = <<"foo/bar">>, + {ok, ConnPid1} = emqtt:start_link([{clientid, <<"C0">>}]), + {ok, ConnPid2} = emqtt:start_link([{clientid, <<"C1">>}]), + {ok, ConnPid3} = emqtt:start_link([{clientid, <<"C2">>}]), + {ok, ConnPid4} = emqtt:start_link([{clientid, <<"C3">>}]), + ConnPids = [ConnPid1, ConnPid2, ConnPid3, ConnPid4], + lists:foreach(fun(P) -> emqtt:connect(P) end, ConnPids), + + %% group1 subscribers + emqtt:subscribe(ConnPid1, {<<"$share/group1/", Topic/binary>>, 0}), + emqtt:subscribe(ConnPid2, {<<"$share/group1/", Topic/binary>>, 0}), + %% group2 subscribers + emqtt:subscribe(ConnPid3, {<<"$share/group2/", Topic/binary>>, 0}), + emqtt:subscribe(ConnPid4, {<<"$share/group2/", Topic/binary>>, 0}), + + publish_fire_and_forget(10, Topic), + + AllReceivedMessages = lists:map( + fun(#{client_pid := SubscriberPid, payload := Payload}) -> {SubscriberPid, Payload} end, + lists:reverse(recv_msgs(20)) + ), + MessagesReceivedSubscriber1 = lists:filter( + fun({P, _Payload}) -> P == ConnPid1 end, AllReceivedMessages + ), + MessagesReceivedSubscriber2 = lists:filter( + fun({P, _Payload}) -> P == ConnPid2 end, AllReceivedMessages + ), + MessagesReceivedSubscriber3 = lists:filter( + fun({P, _Payload}) -> P == ConnPid3 end, AllReceivedMessages + ), + MessagesReceivedSubscriber4 = lists:filter( + fun({P, _Payload}) -> P == ConnPid4 end, AllReceivedMessages + ), + + lists:foreach(fun(P) -> emqtt:stop(P) end, ConnPids), + + %% ensure each subscriber received 5 messages in alternating fashion in each group + %% subscriber 1 and 3 should receive all even messages + %% subscriber 2 and 4 should receive all uneven messages + ?assertEqual( + [ + {ConnPid3, <<"0">>}, + {ConnPid3, <<"2">>}, + {ConnPid3, <<"4">>}, + {ConnPid3, <<"6">>}, + {ConnPid3, <<"8">>} + ], + MessagesReceivedSubscriber3 + ), + + ?assertEqual( + [ + {ConnPid2, <<"1">>}, + {ConnPid2, <<"3">>}, + {ConnPid2, <<"5">>}, + {ConnPid2, <<"7">>}, + {ConnPid2, <<"9">>} + ], + MessagesReceivedSubscriber2 + ), + + ?assertEqual( + [ + {ConnPid4, <<"1">>}, + {ConnPid4, <<"3">>}, + {ConnPid4, <<"5">>}, + {ConnPid4, <<"7">>}, + {ConnPid4, <<"9">>} + ], + MessagesReceivedSubscriber4 + ), + + ?assertEqual( + [ + {ConnPid1, <<"0">>}, + {ConnPid1, <<"2">>}, + {ConnPid1, <<"4">>}, + {ConnPid1, <<"6">>}, + {ConnPid1, <<"8">>} + ], + MessagesReceivedSubscriber1 + ), + ok. + +t_round_robin_per_group_two_nodes_publish_to_same_node(_) -> + ensure_config(round_robin_per_group), + Node = start_slave('rr_p_g_t_n', 31337), + ensure_node_config(Node, round_robin_per_group), + + %% connect two subscribers on each node + Topic = <<"foo/bar">>, + {ok, Subscriber0} = emqtt:start_link([{clientid, <<"C0">>}]), + {ok, Subscriber1} = emqtt:start_link([{clientid, <<"C1">>}]), + {ok, Subscriber2} = emqtt:start_link([{clientid, <<"C2">>}, {port, 31337}]), + {ok, Subscriber3} = emqtt:start_link([{clientid, <<"C3">>}, {port, 31337}]), + SubscriberPids = [Subscriber0, Subscriber1, Subscriber2, Subscriber3], + lists:foreach(fun(P) -> emqtt:connect(P) end, SubscriberPids), + + %% node 1 subscribers + emqtt:subscribe(Subscriber0, {<<"$share/group1/", Topic/binary>>, 0}), + emqtt:subscribe(Subscriber1, {<<"$share/group1/", Topic/binary>>, 0}), + %% node 2 subscribers + emqtt:subscribe(Subscriber2, {<<"$share/group1/", Topic/binary>>, 0}), + emqtt:subscribe(Subscriber3, {<<"$share/group1/", Topic/binary>>, 0}), + + publish_fire_and_forget(10, Topic), + + AllMessages = recv_msgs(10), + MessagesBySubscriber = lists:foldl( + fun(#{client_pid := Subscriber, payload := Payload}, Acc) -> + maps:update_with(Subscriber, fun(T) -> [Payload | T] end, [Payload], Acc) + end, + maps:new(), + AllMessages + ), + lists:foreach(fun(Pid) -> emqtt:stop(Pid) end, SubscriberPids), + stop_slave(Node), + + ?assertEqual( + #{ + Subscriber0 => [<<"0">>, <<"4">>, <<"8">>], + Subscriber1 => [<<"1">>, <<"5">>, <<"9">>], + Subscriber2 => [<<"2">>, <<"6">>], + Subscriber3 => [<<"3">>, <<"7">>] + }, + MessagesBySubscriber + ). + +t_round_robin_per_group_two_nodes_alternating_publish(_) -> + ensure_config(round_robin_per_group), + Node = start_slave('rr_p_g_t_n_2', 41338), + ensure_node_config(Node, round_robin_per_group), + + %% connect two subscribers on each node + Topic = <<"foo/bar">>, + {ok, Subscriber0} = emqtt:start_link([{clientid, <<"C0">>}]), + {ok, Subscriber1} = emqtt:start_link([{clientid, <<"C1">>}]), + {ok, Subscriber2} = emqtt:start_link([{clientid, <<"C2">>}, {port, 41338}]), + {ok, Subscriber3} = emqtt:start_link([{clientid, <<"C3">>}, {port, 41338}]), + SubscriberPids = [Subscriber0, Subscriber1, Subscriber2, Subscriber3], + lists:foreach(fun(P) -> emqtt:connect(P) end, SubscriberPids), + + %% node 1 subscribers + emqtt:subscribe(Subscriber0, {<<"$share/group1/", Topic/binary>>, 0}), + emqtt:subscribe(Subscriber1, {<<"$share/group1/", Topic/binary>>, 0}), + %% node 2 subscribers + emqtt:subscribe(Subscriber2, {<<"$share/group1/", Topic/binary>>, 0}), + emqtt:subscribe(Subscriber3, {<<"$share/group1/", Topic/binary>>, 0}), + + %% alternate publish messages between the nodes + lists:foreach( + fun(I) -> + Message = erlang:integer_to_binary(I), + {ok, PublisherPid} = + case I rem 2 of + 0 -> emqtt:start_link(); + 1 -> emqtt:start_link([{port, 41338}]) + end, + {ok, _} = emqtt:connect(PublisherPid), + emqtt:publish(PublisherPid, Topic, Message), + emqtt:stop(PublisherPid), + ct:sleep(50) + end, + lists:seq(0, 9) + ), + + AllMessages = recv_msgs(10), + MessagesBySubscriber = lists:foldl( + fun(#{client_pid := Subscriber, payload := Payload}, Acc) -> + maps:update_with(Subscriber, fun(T) -> [Payload | T] end, [Payload], Acc) + end, + maps:new(), + AllMessages + ), + lists:foreach(fun(Pid) -> emqtt:stop(Pid) end, SubscriberPids), + stop_slave(Node), + + %% this result show that when clustered round_robin_per_group behaves like the normal round_robin + %% strategy meaning that subscribers receive two consecutive messages which is not ideal + ?assertEqual( + #{ + Subscriber0 => [<<"0">>, <<"1">>, <<"8">>, <<"9">>], + Subscriber1 => [<<"2">>, <<"3">>], + Subscriber2 => [<<"4">>, <<"5">>], + Subscriber3 => [<<"6">>, <<"7">>] + }, + MessagesBySubscriber + ). + t_sticky(_) -> ok = ensure_config(sticky, true), test_two_messages(sticky). @@ -292,7 +552,7 @@ test_two_messages(Strategy, Group) -> emqtt:subscribe(ConnPid2, {<<"$share/", Group/binary, "/", Topic/binary>>, 0}), Message1 = emqx_message:make(ClientId1, 0, Topic, <<"hello1">>), - Message2 = emqx_message:make(ClientId1, 0, Topic, <<"hello2">>), + Message2 = emqx_message:make(ClientId2, 0, Topic, <<"hello2">>), ct:sleep(100), emqx:publish(Message1), @@ -307,6 +567,7 @@ test_two_messages(Strategy, Group) -> case Strategy of sticky -> ?assertEqual(UsedSubPid1, UsedSubPid2); round_robin -> ?assertNotEqual(UsedSubPid1, UsedSubPid2); + round_robin_per_group -> ?assertNotEqual(UsedSubPid1, UsedSubPid2); hash -> ?assertEqual(UsedSubPid1, UsedSubPid2); _ -> ok end, @@ -348,7 +609,8 @@ t_per_group_config(_) -> ok = ensure_group_config(#{ <<"local_group">> => local, <<"round_robin_group">> => round_robin, - <<"sticky_group">> => sticky + <<"sticky_group">> => sticky, + <<"round_robin_per_group_group">> => round_robin_per_group }), %% Each test is repeated 4 times because random strategy may technically pass the test %% so we run 8 tests to make random pass in only 1/256 runs @@ -360,7 +622,9 @@ t_per_group_config(_) -> test_two_messages(sticky, <<"sticky_group">>), test_two_messages(sticky, <<"sticky_group">>), test_two_messages(round_robin, <<"round_robin_group">>), - test_two_messages(round_robin, <<"round_robin_group">>). + test_two_messages(round_robin, <<"round_robin_group">>), + test_two_messages(round_robin_per_group, <<"round_robin_per_group_group">>), + test_two_messages(round_robin_per_group, <<"round_robin_per_group_group">>). t_local(_) -> GroupConfig = #{ @@ -482,6 +746,9 @@ ensure_config(Strategy, AckEnabled) -> emqx_config:put([broker, shared_dispatch_ack_enabled], AckEnabled), ok. +ensure_node_config(Node, Strategy) -> + rpc:call(Node, emqx_config, force_put, [[broker, shared_subscription_strategy], Strategy]). + ensure_group_config(Group2Strategy) -> lists:foreach( fun({Group, Strategy}) -> @@ -505,6 +772,19 @@ ensure_group_config(Node, Group2Strategy) -> maps:to_list(Group2Strategy) ). +publish_fire_and_forget(Count, Topic) when Count > 1 -> + lists:foreach( + fun(I) -> + Message = erlang:integer_to_binary(I), + {ok, PublisherPid} = emqtt:start_link(), + {ok, _} = emqtt:connect(PublisherPid), + emqtt:publish(PublisherPid, Topic, Message), + emqtt:stop(PublisherPid), + ct:sleep(50) + end, + lists:seq(0, Count - 1) + ). + subscribed(Group, Topic, Pid) -> lists:member(Pid, emqx_shared_sub:subscribers(Group, Topic)). From 7e6b94fc121b4e38b1a5d2c603bbc05c6169074e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 4 Aug 2022 10:22:04 -0300 Subject: [PATCH 26/63] test: avoid stopping cover and using meck --- .../emqx_license/src/emqx_license_parser.erl | 7 +++++++ .../emqx_license/test/emqx_license_SUITE.erl | 18 +++--------------- .../test/emqx_license_http_api_SUITE.erl | 15 ++++----------- 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/lib-ee/emqx_license/src/emqx_license_parser.erl b/lib-ee/emqx_license/src/emqx_license_parser.erl index f0ac7b8f5..469f70f2a 100644 --- a/lib-ee/emqx_license/src/emqx_license_parser.erl +++ b/lib-ee/emqx_license/src/emqx_license_parser.erl @@ -72,9 +72,16 @@ %% API %%-------------------------------------------------------------------- +-ifdef(TEST). +-spec parse(string() | binary()) -> {ok, license()} | {error, term()}. +parse(Content) -> + PubKey = persistent_term:get({emqx_license_parser_test, pubkey}, ?PUBKEY), + parse(Content, PubKey). +-else. -spec parse(string() | binary()) -> {ok, license()} | {error, term()}. parse(Content) -> parse(Content, ?PUBKEY). +-endif. parse(Content, Pem) -> [PemEntry] = public_key:pem_decode(Pem), diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index ec92f4efe..fb25bd0b9 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -16,9 +16,6 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - %% hack needed to avoid inter-suite flakiness when using meck... - ok = cover:stop(), - {ok, _} = cover:start(), _ = application:load(emqx_conf), emqx_config:save_schema_mod_and_names(emqx_license_schema), emqx_common_test_helpers:start_apps([emqx_license], fun set_special_configs/1), @@ -144,18 +141,9 @@ setup_test(TestCase, Config) when emqx_config:put([license], LicConfig), RawConfig = #{<<"type">> => file, <<"file">> => LicensePath}, emqx_config:put_raw([<<"license">>], RawConfig), - ok = meck:new(emqx_license_parser, [ - non_strict, passthrough, no_history, no_link - ]), - meck:expect( - emqx_license_parser, - parse, - fun(X) -> - emqx_license_parser:parse( - X, - emqx_license_test_lib:public_key_pem() - ) - end + ok = persistent_term:put( + {emqx_license_parser_test, pubkey}, + emqx_license_test_lib:public_key_pem() ), ok; (_) -> diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index f2feed893..867db6043 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -25,12 +25,12 @@ init_per_suite(Config) -> Config. end_per_suite(_) -> - ok = meck:unload([emqx_license_parser]), emqx_common_test_helpers:stop_apps([emqx_license, emqx_dashboard]), Config = #{type => file, file => emqx_license_test_lib:default_license()}, emqx_config:put([license], Config), RawConfig = #{<<"type">> => file, <<"file">> => emqx_license_test_lib:default_license()}, emqx_config:put_raw([<<"license">>], RawConfig), + persistent_term:erase({emqx_license_parser_test, pubkey}), ok. set_special_configs(emqx_dashboard) -> @@ -41,16 +41,9 @@ set_special_configs(emqx_license) -> emqx_config:put([license], Config), RawConfig = #{<<"type">> => key, <<"key">> => LicenseKey}, emqx_config:put_raw([<<"license">>], RawConfig), - ok = meck:new(emqx_license_parser, [non_strict, passthrough, no_history, no_link]), - ok = meck:expect( - emqx_license_parser, - parse, - fun(X) -> - emqx_license_parser:parse( - X, - emqx_license_test_lib:public_key_pem() - ) - end + ok = persistent_term:put( + {emqx_license_parser_test, pubkey}, + emqx_license_test_lib:public_key_pem() ), ok; set_special_configs(_) -> From d2e146aeffbf117655172c83a0556740c81e4706 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 4 Aug 2022 11:56:19 -0300 Subject: [PATCH 27/63] test: fix flaky gateway tests --- apps/emqx_gateway/test/emqx_gateway_auth_ct.erl | 4 +++- apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl index 16d033d41..e24764030 100644 --- a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl +++ b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl @@ -143,7 +143,9 @@ on_start_auth(authn_http) -> Setup = fun(Gateway) -> Path = io_lib:format("/gateway/~ts/authentication", [Gateway]), {204, _} = request(delete, Path), - {201, _} = request(post, Path, http_authn_config()) + timer:sleep(200), + {201, _} = request(post, Path, http_authn_config()), + timer:sleep(200) end, lists:foreach(Setup, ?GATEWAYS), diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl index 54928a792..084fd9f57 100644 --- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl @@ -103,7 +103,7 @@ init_per_suite(Config) -> end_per_suite(_) -> {ok, _} = emqx:remove_config([gateway, mqttsn]), - emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_auhtn, emqx_conf]). + emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]). restart_mqttsn_with_subs_resume_on() -> Conf = emqx:get_raw_config([gateway, mqttsn]), From 68592373e33a7d92a6c07200299bc102479626e8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 4 Aug 2022 14:01:27 -0300 Subject: [PATCH 28/63] test: add a couple more tests --- .../test/emqx_license_http_api_SUITE.erl | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index 867db6043..cb184b4ce 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -159,6 +159,24 @@ t_license_upload_file_bad_license(_Config) -> assert_untouched_license(), ok. +t_license_upload_file_not_json(_Config) -> + Res = request( + post, + uri(["license", "file"]), + <<"">> + ), + ?assertMatch({ok, 400, _}, Res), + {ok, 400, Payload} = Res, + ?assertEqual( + #{ + <<"code">> => <<"BAD_REQUEST">>, + <<"message">> => <<"Invalid request params">> + }, + emqx_json:decode(Payload, [return_maps]) + ), + assert_untouched_license(), + ok. + t_license_upload_key_success(_Config) -> NewKey = emqx_license_test_lib:make_license(#{max_connections => "999"}), Res = request( @@ -206,3 +224,21 @@ t_license_upload_key_bad_key(_Config) -> ), assert_untouched_license(), ok. + +t_license_upload_key_not_json(_Config) -> + Res = request( + post, + uri(["license", "key"]), + <<"">> + ), + ?assertMatch({ok, 400, _}, Res), + {ok, 400, Payload} = Res, + ?assertEqual( + #{ + <<"code">> => <<"BAD_REQUEST">>, + <<"message">> => <<"Invalid request params">> + }, + emqx_json:decode(Payload, [return_maps]) + ), + assert_untouched_license(), + ok. From e76682d368d438c75bca027b0208637caa8c5a33 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 4 Aug 2022 15:27:45 -0300 Subject: [PATCH 29/63] test: apply review comments --- lib-ee/emqx_license/src/emqx_license_parser.erl | 2 +- lib-ee/emqx_license/test/emqx_license_SUITE.erl | 2 +- lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib-ee/emqx_license/src/emqx_license_parser.erl b/lib-ee/emqx_license/src/emqx_license_parser.erl index 469f70f2a..e3abf1301 100644 --- a/lib-ee/emqx_license/src/emqx_license_parser.erl +++ b/lib-ee/emqx_license/src/emqx_license_parser.erl @@ -75,7 +75,7 @@ -ifdef(TEST). -spec parse(string() | binary()) -> {ok, license()} | {error, term()}. parse(Content) -> - PubKey = persistent_term:get({emqx_license_parser_test, pubkey}, ?PUBKEY), + PubKey = persistent_term:get(emqx_license_test_pubkey, ?PUBKEY), parse(Content, PubKey). -else. -spec parse(string() | binary()) -> {ok, license()} | {error, term()}. diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index fb25bd0b9..851ef30ef 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -142,7 +142,7 @@ setup_test(TestCase, Config) when RawConfig = #{<<"type">> => file, <<"file">> => LicensePath}, emqx_config:put_raw([<<"license">>], RawConfig), ok = persistent_term:put( - {emqx_license_parser_test, pubkey}, + emqx_license_test_pubkey, emqx_license_test_lib:public_key_pem() ), ok; diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index cb184b4ce..afcb85059 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -30,7 +30,7 @@ end_per_suite(_) -> emqx_config:put([license], Config), RawConfig = #{<<"type">> => file, <<"file">> => emqx_license_test_lib:default_license()}, emqx_config:put_raw([<<"license">>], RawConfig), - persistent_term:erase({emqx_license_parser_test, pubkey}), + persistent_term:erase(emqx_license_test_pubkey), ok. set_special_configs(emqx_dashboard) -> @@ -42,7 +42,7 @@ set_special_configs(emqx_license) -> RawConfig = #{<<"type">> => key, <<"key">> => LicenseKey}, emqx_config:put_raw([<<"license">>], RawConfig), ok = persistent_term:put( - {emqx_license_parser_test, pubkey}, + emqx_license_test_pubkey, emqx_license_test_lib:public_key_pem() ), ok; From 71f31ab5cd7f42bc67e716fea3239e4550edccb4 Mon Sep 17 00:00:00 2001 From: Benjamin Krenn Date: Fri, 5 Aug 2022 11:10:54 +0200 Subject: [PATCH 30/63] feat(shared-sub): add round_robin_per_group strategy remove flaky tests that provide little value --- apps/emqx/test/emqx_shared_sub_SUITE.erl | 105 ----------------------- 1 file changed, 105 deletions(-) diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 3b6357d7d..aeb44d529 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -350,111 +350,6 @@ t_round_robin_per_group_even_distribution_two_groups(_) -> ), ok. -t_round_robin_per_group_two_nodes_publish_to_same_node(_) -> - ensure_config(round_robin_per_group), - Node = start_slave('rr_p_g_t_n', 31337), - ensure_node_config(Node, round_robin_per_group), - - %% connect two subscribers on each node - Topic = <<"foo/bar">>, - {ok, Subscriber0} = emqtt:start_link([{clientid, <<"C0">>}]), - {ok, Subscriber1} = emqtt:start_link([{clientid, <<"C1">>}]), - {ok, Subscriber2} = emqtt:start_link([{clientid, <<"C2">>}, {port, 31337}]), - {ok, Subscriber3} = emqtt:start_link([{clientid, <<"C3">>}, {port, 31337}]), - SubscriberPids = [Subscriber0, Subscriber1, Subscriber2, Subscriber3], - lists:foreach(fun(P) -> emqtt:connect(P) end, SubscriberPids), - - %% node 1 subscribers - emqtt:subscribe(Subscriber0, {<<"$share/group1/", Topic/binary>>, 0}), - emqtt:subscribe(Subscriber1, {<<"$share/group1/", Topic/binary>>, 0}), - %% node 2 subscribers - emqtt:subscribe(Subscriber2, {<<"$share/group1/", Topic/binary>>, 0}), - emqtt:subscribe(Subscriber3, {<<"$share/group1/", Topic/binary>>, 0}), - - publish_fire_and_forget(10, Topic), - - AllMessages = recv_msgs(10), - MessagesBySubscriber = lists:foldl( - fun(#{client_pid := Subscriber, payload := Payload}, Acc) -> - maps:update_with(Subscriber, fun(T) -> [Payload | T] end, [Payload], Acc) - end, - maps:new(), - AllMessages - ), - lists:foreach(fun(Pid) -> emqtt:stop(Pid) end, SubscriberPids), - stop_slave(Node), - - ?assertEqual( - #{ - Subscriber0 => [<<"0">>, <<"4">>, <<"8">>], - Subscriber1 => [<<"1">>, <<"5">>, <<"9">>], - Subscriber2 => [<<"2">>, <<"6">>], - Subscriber3 => [<<"3">>, <<"7">>] - }, - MessagesBySubscriber - ). - -t_round_robin_per_group_two_nodes_alternating_publish(_) -> - ensure_config(round_robin_per_group), - Node = start_slave('rr_p_g_t_n_2', 41338), - ensure_node_config(Node, round_robin_per_group), - - %% connect two subscribers on each node - Topic = <<"foo/bar">>, - {ok, Subscriber0} = emqtt:start_link([{clientid, <<"C0">>}]), - {ok, Subscriber1} = emqtt:start_link([{clientid, <<"C1">>}]), - {ok, Subscriber2} = emqtt:start_link([{clientid, <<"C2">>}, {port, 41338}]), - {ok, Subscriber3} = emqtt:start_link([{clientid, <<"C3">>}, {port, 41338}]), - SubscriberPids = [Subscriber0, Subscriber1, Subscriber2, Subscriber3], - lists:foreach(fun(P) -> emqtt:connect(P) end, SubscriberPids), - - %% node 1 subscribers - emqtt:subscribe(Subscriber0, {<<"$share/group1/", Topic/binary>>, 0}), - emqtt:subscribe(Subscriber1, {<<"$share/group1/", Topic/binary>>, 0}), - %% node 2 subscribers - emqtt:subscribe(Subscriber2, {<<"$share/group1/", Topic/binary>>, 0}), - emqtt:subscribe(Subscriber3, {<<"$share/group1/", Topic/binary>>, 0}), - - %% alternate publish messages between the nodes - lists:foreach( - fun(I) -> - Message = erlang:integer_to_binary(I), - {ok, PublisherPid} = - case I rem 2 of - 0 -> emqtt:start_link(); - 1 -> emqtt:start_link([{port, 41338}]) - end, - {ok, _} = emqtt:connect(PublisherPid), - emqtt:publish(PublisherPid, Topic, Message), - emqtt:stop(PublisherPid), - ct:sleep(50) - end, - lists:seq(0, 9) - ), - - AllMessages = recv_msgs(10), - MessagesBySubscriber = lists:foldl( - fun(#{client_pid := Subscriber, payload := Payload}, Acc) -> - maps:update_with(Subscriber, fun(T) -> [Payload | T] end, [Payload], Acc) - end, - maps:new(), - AllMessages - ), - lists:foreach(fun(Pid) -> emqtt:stop(Pid) end, SubscriberPids), - stop_slave(Node), - - %% this result show that when clustered round_robin_per_group behaves like the normal round_robin - %% strategy meaning that subscribers receive two consecutive messages which is not ideal - ?assertEqual( - #{ - Subscriber0 => [<<"0">>, <<"1">>, <<"8">>, <<"9">>], - Subscriber1 => [<<"2">>, <<"3">>], - Subscriber2 => [<<"4">>, <<"5">>], - Subscriber3 => [<<"6">>, <<"7">>] - }, - MessagesBySubscriber - ). - t_sticky(_) -> ok = ensure_config(sticky, true), test_two_messages(sticky). From 01ba45cc373c904f13f954df28034effd2200801 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 8 Aug 2022 17:34:18 +0800 Subject: [PATCH 31/63] fix(mgmt): remove the `/api/v5` prefix of status API --- apps/emqx_dashboard/src/emqx_dashboard.erl | 1 + .../src/emqx_mgmt_api_status.erl | 48 ++++--------------- 2 files changed, 10 insertions(+), 39 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 9bf981323..6c2a02e47 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -73,6 +73,7 @@ start_listeners(Listeners) -> Dispatch = [ {"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, + {emqx_mgmt_api_status:path(), emqx_mgmt_api_status, []}, {?BASE_PATH ++ "/[...]", emqx_dashboard_bad_api, []}, {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}} ], diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl index 5ece0bda4..70ff98988 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_status.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -14,55 +14,25 @@ %% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_mgmt_api_status). -%% API --behaviour(minirest_api). -export([ - api_spec/0, - paths/0, - schema/1 + init/2, + path/0 ]). --export([running_status/2]). +path() -> + "/status". -api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). - -paths() -> - ["/status"]. - -schema("/status") -> - #{ - 'operationId' => running_status, - get => - #{ - description => <<"Node running status">>, - tags => [<<"Status">>], - security => [], - responses => - #{ - 200 => - #{ - description => <<"Node is running">>, - content => - #{ - 'text/plain' => - #{ - schema => #{type => string}, - example => - <<"Node emqx@127.0.0.1 is started\nemqx is running">> - } - } - } - } - } - }. +init(Req0, State) -> + {Code, Headers, Body} = running_status(), + Req = cowboy_req:reply(Code, Headers, Body, Req0), + {ok, Req, State}. %%-------------------------------------------------------------------- %% API Handler funcs %%-------------------------------------------------------------------- -running_status(get, _Params) -> +running_status() -> BrokerStatus = case emqx:is_running() of true -> From dee92bb8b8033e9e07873ac00bd384bef05ac7f8 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 8 Aug 2022 18:01:29 +0800 Subject: [PATCH 32/63] fix(mgmt): fix status api test case error --- apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl | 2 +- apps/emqx_management/test/emqx_mgmt_api_test_util.erl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl index 3e7f77a31..b725e37b2 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl @@ -31,7 +31,7 @@ end_per_suite(_) -> emqx_mgmt_api_test_util:end_suite(). t_status(_Config) -> - Path = emqx_mgmt_api_test_util:api_path(["status"]), + Path = emqx_mgmt_api_test_util:api_path_without_base_path(["/status"]), Status = io_lib:format("Node ~ts is ~ts~nemqx is ~ts", [node(), started, running]), {ok, Status} = emqx_mgmt_api_test_util:request_api(get, Path), ok. diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 1bdf584e5..a8b04dc80 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -110,6 +110,9 @@ build_http_header(X) -> api_path(Parts) -> ?SERVER ++ filename:join([?BASE_PATH | Parts]). +api_path_without_base_path(Parts) -> + ?SERVER ++ filename:join([Parts]). + %% Usage: %% upload_request(<<"site.com/api/upload">>, <<"path/to/file.png">>, %% <<"upload">>, <<"image/png">>, [], <<"some-token">>) From 966b508c6f1dce57620784d6d652a274e73b0580 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 9 Aug 2022 09:21:53 +0800 Subject: [PATCH 33/63] chore: fix status test error --- deploy/charts/emqx/templates/StatefulSet.yaml | 4 ++-- scripts/pkg-tests.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index d44c88a86..3af9fd62d 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -142,14 +142,14 @@ spec: {{- end }} readinessProbe: httpGet: - path: /api/v5/status + path: /status port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP | default 18083 }} initialDelaySeconds: 10 periodSeconds: 5 failureThreshold: 30 livenessProbe: httpGet: - path: /api/v5/status + path: /status port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP | default 18083 }} initialDelaySeconds: 60 periodSeconds: 30 diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh index ec3c115b0..9f3e4d7bd 100755 --- a/scripts/pkg-tests.sh +++ b/scripts/pkg-tests.sh @@ -103,7 +103,7 @@ emqx_test(){ exit 1 fi IDLE_TIME=0 - while ! curl http://127.0.0.1:18083/api/v5/status >/dev/null 2>&1; do + while ! curl http://127.0.0.1:18083/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx running error" @@ -197,7 +197,7 @@ EOF exit 1 fi IDLE_TIME=0 - while ! curl http://127.0.0.1:18083/api/v5/status >/dev/null 2>&1; do + while ! curl http://127.0.0.1:18083/status >/dev/null 2>&1; do if [ $IDLE_TIME -gt 10 ] then echo "emqx running error" From 0f8e80afb82266f4c807fd54dc8ca8fedf029a37 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 9 Aug 2022 10:39:55 +0800 Subject: [PATCH 34/63] chore: fix ci workflows error --- .github/workflows/build_packages.yaml | 2 +- .github/workflows/build_slim_packages.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 6877b25d7..a7fb86aa9 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -194,7 +194,7 @@ jobs: ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' for i in {1..18}; do - if curl -fs 127.0.0.1:18083/api/v5/status > /dev/null; then + if curl -fs 127.0.0.1:18083/status > /dev/null; then ready='yes' break fi diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 27f43d0ea..56d2a6394 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -178,7 +178,7 @@ jobs: ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' for i in {1..30}; do - if curl -fs 127.0.0.1:18083/api/v5/status > /dev/null; then + if curl -fs 127.0.0.1:18083/status > /dev/null; then ready='yes' break fi From 0f8ebbdf0f0c12eb13435560e0ae9595363bb590 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 9 Aug 2022 15:17:39 +0800 Subject: [PATCH 35/63] feat(dashboard): add bootstrap files to initialize user accounts --- .../i18n/emqx_dashboard_i18n.conf | 10 +++ .../src/emqx_dashboard_admin.erl | 62 +++++++++++++++++-- .../emqx_dashboard/src/emqx_dashboard_app.erl | 1 + .../src/emqx_dashboard_schema.erl | 3 +- 4 files changed, 69 insertions(+), 7 deletions(-) diff --git a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf index d6587c203..ad41b06a7 100644 --- a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf +++ b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf @@ -197,4 +197,14 @@ its own from which a browser should permit loading resources.""" zh: "多语言支持" } } + bootstrap_user { + desc { + en: "Initialize users file." + zh: "初始化用户文件" + } + label { + en: "Initialize users file" + zh: "初始化用户文件" + } + } } diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 0650062f8..0df1e4e9e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -50,7 +50,8 @@ -export([ add_default_user/0, - default_username/0 + default_username/0, + add_bootstrap_user/0 ]). -type emqx_admin() :: #?ADMIN{}. @@ -74,6 +75,28 @@ mnesia(boot) -> ]} ]). +%%-------------------------------------------------------------------- +%% bootstrap API +%%-------------------------------------------------------------------- + +-spec add_default_user() -> {ok, map() | empty | default_user_exists} | {error, any()}. +add_default_user() -> + add_default_user(binenv(default_username), binenv(default_password)). + +-spec add_bootstrap_user() -> ok | {error, _}. +add_bootstrap_user() -> + case emqx:get_config([dashboard, bootstrap_user], undefined) of + undefined -> + ok; + File -> + case mnesia:table_info(?ADMIN, size) of + 0 -> + add_bootstrap_user(File); + _ -> + ok + end + end. + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -272,11 +295,6 @@ destroy_token_by_username(Username, Token) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- - --spec add_default_user() -> {ok, map() | empty | default_user_exists} | {error, any()}. -add_default_user() -> - add_default_user(binenv(default_username), binenv(default_password)). - default_username() -> binenv(default_username). @@ -290,3 +308,35 @@ add_default_user(Username, Password) -> [] -> add_user(Username, Password, <<"administrator">>); _ -> {ok, default_user_exists} end. + +add_bootstrap_user(File) -> + case file:open(File, read) of + {ok, Dev} -> + {ok, MP} = re:compile(<<"(\.+):(\.+)">>), + try + load_bootstrap_user(Dev, MP) + catch + Type:Reason -> + {error, {Type, Reason}} + after + file:close(Dev) + end; + Error -> + Error + end. + +load_bootstrap_user(Dev, MP) -> + case file:read_line(Dev) of + {ok, Line} -> + case re:run(Line, MP, [global, {capture, all_but_first, binary}]) of + {match, Captured} -> + [add_user(Username, Password, <<>>) || [Username, Password] <- Captured]; + _ -> + ok + end, + load_bootstrap_user(Dev, MP); + eof -> + ok; + Error -> + Error + end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_app.erl b/apps/emqx_dashboard/src/emqx_dashboard_app.erl index 08bfe1d21..4d9c1416a 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_app.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_app.erl @@ -31,6 +31,7 @@ start(_StartType, _StartArgs) -> case emqx_dashboard:start_listeners() of ok -> emqx_dashboard_cli:load(), + ok = emqx_dashboard_admin:add_bootstrap_user(), {ok, _} = emqx_dashboard_admin:add_default_user(), {ok, Sup}; {error, Reason} -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 55a1fd38c..4bb9fb6af 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -54,7 +54,8 @@ fields("dashboard") -> } )}, {cors, fun cors/1}, - {i18n_lang, fun i18n_lang/1} + {i18n_lang, fun i18n_lang/1}, + {bootstrap_user, ?HOCON(binary(), #{desc => ?DESC(bootstrap_user), required => false})} ]; fields("listeners") -> [ From 463d255c0a5ddd024c18bd48ac56b84204346f5d Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 9 Aug 2022 16:36:46 +0800 Subject: [PATCH 36/63] fix(dashboard): make dialyzer happy --- apps/emqx_dashboard/src/emqx_dashboard_admin.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 0df1e4e9e..3d7447dc5 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -310,7 +310,7 @@ add_default_user(Username, Password) -> end. add_bootstrap_user(File) -> - case file:open(File, read) of + case file:open(File, [read]) of {ok, Dev} -> {ok, MP} = re:compile(<<"(\.+):(\.+)">>), try @@ -330,7 +330,8 @@ load_bootstrap_user(Dev, MP) -> {ok, Line} -> case re:run(Line, MP, [global, {capture, all_but_first, binary}]) of {match, Captured} -> - [add_user(Username, Password, <<>>) || [Username, Password] <- Captured]; + _ = [add_user(Username, Password, <<>>) || [Username, Password] <- Captured], + ok; _ -> ok end, From 13f5c0b6b10e6b66c7d2e1a481f9d781eaccbad3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 9 Aug 2022 11:22:19 +0200 Subject: [PATCH 37/63] fix(lwm2m): better logging for unknown object IDs Prior to this change, unknown object IDs would result in a tuple {error, no_xml_definition}, and this tuple is passed down to xmerl lib to get XML node data and eventually crash with function_clause. With this fix, the unknown object ID exception is caught and logged properly. --- CHANGES-5.0.md | 1 + .../emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl | 14 ++-- .../src/lwm2m/emqx_lwm2m_message.erl | 4 +- .../src/lwm2m/emqx_lwm2m_xml_object.erl | 9 ++- .../src/lwm2m/emqx_lwm2m_xml_object_db.erl | 7 +- apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl | 69 +++++++++++++++++++ 6 files changed, 90 insertions(+), 14 deletions(-) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 59e1cceef..e4ba22f58 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -9,6 +9,7 @@ * The license is now copied to all nodes in the cluster when it's reloaded. [#8598](https://github.com/emqx/emqx/pull/8598) * Added a HTTP API to manage licenses. [#8610](https://github.com/emqx/emqx/pull/8610) * Updated `/nodes` API node_status from `Running/Stopped` to `running/stopped`. [#8642](https://github.com/emqx/emqx/pull/8642) +* Better logging on unknown object IDs. [#8670](https://github.com/emqx/emqx/pull/8670) # 5.0.4 diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl index 46f7534f1..3fd78b383 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl @@ -221,14 +221,16 @@ read_resp_to_mqtt({ok, SuccessCode}, CoapPayload, Format, Ref) -> Result = content_to_mqtt(CoapPayload, Format, Ref), make_response(SuccessCode, Ref, Format, Result) catch - error:not_implemented -> - make_response(not_implemented, Ref); - _:Ex:_ST -> + throw:{bad_request, Reason} -> + ?SLOG(error, #{msg => "bad_request", payload => CoapPayload, reason => Reason}), + make_response(bad_request, Ref); + E:R:ST -> ?SLOG(error, #{ - msg => "bad_payload_format", + msg => "bad_request", payload => CoapPayload, - reason => Ex, - stacktrace => _ST + exception => E, + reason => R, + stacktrace => ST }), make_response(bad_request, Ref) end. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl index 4b13015f5..7e9d418f7 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl @@ -29,7 +29,7 @@ tlv_to_json(BaseName, TlvData) -> DecodedTlv = emqx_lwm2m_tlv:parse(TlvData), ObjectId = object_id(BaseName), - ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true), + ObjDefinition = emqx_lwm2m_xml_object:get_obj_def_assertive(ObjectId, true), case DecodedTlv of [#{tlv_resource_with_value := Id, value := Value}] -> TrueBaseName = basename(BaseName, undefined, undefined, Id, 3), @@ -318,7 +318,7 @@ path([H | T], Acc) -> text_to_json(BaseName, Text) -> {ObjectId, ResourceId} = object_resource_id(BaseName), - ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true), + ObjDefinition = emqx_lwm2m_xml_object:get_obj_def_assertive(ObjectId, true), Val = text_value(Text, ResourceId, ObjDefinition), [#{path => BaseName, value => Val}]. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl index cb1d0b1d1..53f9f0442 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl @@ -21,6 +21,7 @@ -export([ get_obj_def/2, + get_obj_def_assertive/2, get_object_id/1, get_object_name/1, get_object_and_resource_id/2, @@ -29,7 +30,13 @@ get_resource_operations/2 ]). -% This module is for future use. Disabled now. +get_obj_def_assertive(ObjectId, IsInt) -> + case get_obj_def(ObjectId, IsInt) of + {error, no_xml_definition} -> + erlang:throw({bad_request, {unknown_object_id, ObjectId}}); + Xml -> + Xml + end. get_obj_def(ObjectIdInt, true) -> emqx_lwm2m_xml_object_db:find_objectid(ObjectIdInt); diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl index b19e3550b..3e30cc32a 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl @@ -76,12 +76,9 @@ find_name(Name) -> end, case ets:lookup(?LWM2M_OBJECT_NAME_TO_ID_TAB, NameBinary) of [] -> - undefined; + {error, no_xml_definition}; [{NameBinary, ObjectId}] -> - case ets:lookup(?LWM2M_OBJECT_DEF_TAB, ObjectId) of - [] -> undefined; - [{ObjectId, Xml}] -> Xml - end + find_objectid(ObjectId) end. stop() -> diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl index da44beb91..2f5d71a43 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl @@ -850,6 +850,75 @@ case10_read(Config) -> ), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). +case10_read_bad_request(Config) -> + UdpSock = ?config(sock, Config), + Epn = "urn:oma:lwm2m:oma:3", + MsgId1 = 15, + RespTopic = list_to_binary("lwm2m/" ++ Epn ++ "/up/resp"), + emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), + timer:sleep(200), + % step 1, device register ... + test_send_coap_request( + UdpSock, + post, + sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + #coap_content{ + content_format = <<"text/plain">>, + payload = + <<";rt=\"oma.lwm2m\";ct=11543,,,">> + }, + [], + MsgId1 + ), + #coap_message{method = Method1} = test_recv_coap_response(UdpSock), + ?assertEqual({ok, created}, Method1), + test_recv_mqtt_response(RespTopic), + + % step2, send a READ command to device + CmdId = 206, + CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, + Command = #{ + <<"requestID">> => CmdId, + <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"path">> => <<"/3333/0/0">> + } + }, + CommandJson = emqx_json:encode(Command), + ?LOGT("CommandJson=~p", [CommandJson]), + test_mqtt_broker:publish(CommandTopic, CommandJson, 0), + timer:sleep(50), + Request2 = test_recv_coap_request(UdpSock), + #coap_message{method = Method2, payload = Payload2} = Request2, + ?LOGT("LwM2M client got ~p", [Request2]), + ?assertEqual(get, Method2), + ?assertEqual(<<>>, Payload2), + timer:sleep(50), + + test_send_coap_response( + UdpSock, + "127.0.0.1", + ?PORT, + {ok, content}, + #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, + Request2, + true + ), + timer:sleep(100), + + ReadResult = emqx_json:encode(#{ + <<"requestID">> => CmdId, + <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"code">> => <<"4.00">>, + <<"codeMsg">> => <<"bad_request">>, + <<"reqPath">> => <<"/3333/0/0">> + } + }), + ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). + case10_read_separate_ack(Config) -> UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", From dd8bcd80cb2e016a19cb60c9592bfd19e8e7fc6a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 9 Aug 2022 13:30:09 +0200 Subject: [PATCH 38/63] chore: bump app vsn for emqx_gateway to 0.1.3 --- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 963cacc2f..a740e29fc 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, grpc, emqx, emqx_authn]}, From f7016e53a47efe7e6c38fa2fd37bfcd43a14d076 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 9 Aug 2022 20:19:31 +0200 Subject: [PATCH 39/63] chore: delete hocon from emqx_prometheus application deps --- apps/emqx_prometheus/rebar.config | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 66afda52e..88b3d27a2 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -2,9 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, - %% FIXME: tag this as v3.1.3 - {prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}} + {prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}} ]}. {edoc_opts, [{preprocess, true}]}. From 18614ad291328bf1c27793638495768e02cac65f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 9 Aug 2022 20:20:22 +0200 Subject: [PATCH 40/63] fix(connector): add back deleted fields as deprecated max_retries and retry_interval were removed from 5.0.4 wihtout backward compatibliity. This commit adds the fields back as deprecated --- apps/emqx/rebar.config | 2 +- apps/emqx_connector/src/emqx_connector_http.erl | 10 ++++++++++ mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index f09092e2b..5b5b3db39 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.3"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 59b4ddffa..b12b38838 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -88,6 +88,16 @@ fields(config) -> desc => ?DESC("connect_timeout") } )}, + {max_retries, + sc( + non_neg_integer(), + #{deprecated => {since, "5.0.4"}} + )}, + {retry_interval, + sc( + emqx_schema:duration(), + #{deprecated => {since, "5.0.4"}} + )}, {pool_type, sc( pool_type(), diff --git a/mix.exs b/mix.exs index 7f8fb9fd3..8b8ded1a2 100644 --- a/mix.exs +++ b/mix.exs @@ -66,7 +66,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.29.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.30.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 98b4894cc..5f7892d87 100644 --- a/rebar.config +++ b/rebar.config @@ -67,7 +67,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.29.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From 4994731d3e5b02375a01440fe8fa193890ab7657 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 9 Aug 2022 20:24:43 +0200 Subject: [PATCH 41/63] chore: update emqx_connector app vsn --- apps/emqx_connector/src/emqx_connector.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 007962da3..cce266966 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "An OTP application"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ From ced6bb2fefbc0f7c728063670208b20936e47c70 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 10 Aug 2022 09:53:01 +0200 Subject: [PATCH 42/63] docs: update changelog for config compatibility fix --- CHANGES-5.0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index e4ba22f58..40a949d13 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -3,6 +3,8 @@ ## Bug fixes * Allow changing the license type from key to file (and vice-versa). [#8598](https://github.com/emqx/emqx/pull/8598) +* Add back http connector config keys `max_retries` `retry_interval` as deprecated fields [#8672](https://github.com/emqx/emqx/issues/8672) + This caused upgrade failure in 5.0.4, because it would fail to boot on configs created from older version. ## Enhancements From 7ca8550a5da22ef6a01db3ff67471daad9cc1e64 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 9 Aug 2022 19:09:07 +0200 Subject: [PATCH 43/63] feat(quic): more listener options --- apps/emqx/i18n/emqx_schema_i18n.conf | 35 +++++++++++++++++++++++++--- apps/emqx/src/emqx_listeners.erl | 31 +++++++++++++----------- apps/emqx/src/emqx_schema.erl | 22 ++++++++++++++--- 3 files changed, 69 insertions(+), 19 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 13aab3e01..c42ccb356 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1967,11 +1967,10 @@ Path to the secret key file. fields_mqtt_quic_listener_idle_timeout { desc { en: """ -Close transport-layer connections from the clients that have not sent MQTT CONNECT -message within this interval. +How long a connection can go idle before it is gracefully shut down. 0 to disable """ zh: """ -关闭在此间隔内未发送 MQTT CONNECT 消息的客户端的传输层连接。 +一个连接在被关闭之前可以空闲多长时间。0表示禁用 """ } label: { @@ -1980,6 +1979,36 @@ message within this interval. } } +fields_mqtt_quic_listener_handshake_idle_timeout { + desc { + en: """ +How long a handshake can idle before it is discarded. +""" + zh: """ +一个握手在被丢弃之前可以空闲多长时间。 +""" + } + label: { + en: "Handshake Idle Timeout" + zh: "握手发呆超时时间" + } +} + +fields_mqtt_quic_listener_keep_alive_interval { + desc { + en: """ +How often to send PING frames to keep a connection alive. 0 means disabled. +""" + zh: """ +发送 PING 帧的频率，以保活连接. 设为0，禁用 +""" + } + label: { + en: "Keep Alive Interval" + zh: "PING保活频率" + } +} + base_listener_bind { desc { en: """ diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 9fc82d123..326661e75 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -335,23 +335,31 @@ do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when wss -> cowboy:start_tls(Id, RanchOpts, WsOpts) end; %% Start MQTT/QUIC listener -do_start_listener(quic, ListenerName, #{bind := ListenOn} = Opts) -> +do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> + ListenOn = + case Bind of + {Addr, Port} when tuple_size(Addr) == 4 -> + %% IPv4 + lists:flatten(io_lib:format("~ts:~w", [inet:ntoa(Addr), Port])); + {Addr, Port} when tuple_size(Addr) == 8 -> + %% IPv6 + lists:flatten(io_lib:format("[~ts]:~w", [inet:ntoa(Addr), Port])); + Port -> + Port + end, + case [A || {quicer, _, _} = A <- application:which_applications()] of [_] -> DefAcceptors = erlang:system_info(schedulers_online) * 8, - IdleTimeout = timer:seconds(maps:get(idle_timeout, Opts)), ListenOpts = [ {cert, maps:get(certfile, Opts)}, {key, maps:get(keyfile, Opts)}, {alpn, ["mqtt"]}, {conn_acceptors, lists:max([DefAcceptors, maps:get(acceptors, Opts, 0)])}, - {keep_alive_interval_ms, ceil(IdleTimeout / 3)}, - {server_resumption_level, 2}, - {idle_timeout_ms, - lists:max([ - emqx_config:get_zone_conf(zone(Opts), [mqtt, idle_timeout]) * 3, - IdleTimeout - ])} + {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)}, + {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)}, + {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)}, + {server_resumption_level, 2} ], ConnectionOpts = #{ conn_callback => emqx_quic_connection, @@ -366,7 +374,7 @@ do_start_listener(quic, ListenerName, #{bind := ListenOn} = Opts) -> add_limiter_bucket(Id, Opts), quicer:start_listener( Id, - port(ListenOn), + ListenOn, {ListenOpts, ConnectionOpts, StreamOpts} ); [] -> @@ -482,9 +490,6 @@ ip_port(Port) when is_integer(Port) -> ip_port({Addr, Port}) -> [{ip, Addr}, {port, Port}]. -port(Port) when is_integer(Port) -> Port; -port({_Addr, Port}) when is_integer(Port) -> Port. - esockd_access_rules(StrRules) -> Access = fun(S) -> [A, CIDR] = string:tokens(S, " "), diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 7b329ea55..ed9821eac 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -867,11 +867,27 @@ fields("mqtt_quic_listener") -> {"ciphers", ciphers_schema(quic)}, {"idle_timeout", sc( - duration(), + duration_ms(), #{ - default => "15s", + default => "0", desc => ?DESC(fields_mqtt_quic_listener_idle_timeout) } + )}, + {"handshake_idle_timeout", + sc( + duration_ms(), + #{ + default => "10s", + desc => ?DESC(fields_mqtt_quic_listener_handshake_idle_timeout) + } + )}, + {"keep_alive_interval", + sc( + duration_ms(), + #{ + default => 0, + desc => ?DESC(fields_mqtt_quic_listener_keep_alive_interval) + } )} ] ++ base_listener(14567); fields("ws_opts") -> @@ -905,7 +921,7 @@ fields("ws_opts") -> duration(), #{ default => "7200s", - desc => ?DESC(fields_mqtt_quic_listener_idle_timeout) + desc => ?DESC(fields_ws_opts_idle_timeout) } )}, {"max_frame_size", From 64aa30ec631e37970db0568d1aea83e44987a9f3 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 2 Aug 2022 15:20:56 +0300 Subject: [PATCH 44/63] chore(authn/authz): better handling of placeholder interpolation errors --- .gitignore | 1 + CHANGES-5.0.md | 1 + apps/emqx_authn/include/emqx_authn.hrl | 4 ++ apps/emqx_authn/src/emqx_authn.app.src | 2 +- apps/emqx_authn/src/emqx_authn_utils.erl | 15 ++++- .../src/simple_authn/emqx_authn_http.erl | 42 +++++++------- .../src/simple_authn/emqx_authn_mongodb.erl | 50 +++++++++-------- .../src/simple_authn/emqx_authn_mysql.erl | 54 +++++++++--------- .../src/simple_authn/emqx_authn_pgsql.erl | 52 +++++++++-------- .../src/simple_authn/emqx_authn_redis.erl | 56 ++++++++++--------- .../emqx_authn/test/emqx_authn_http_SUITE.erl | 41 ++++++++++++++ .../test/emqx_authn_mongo_SUITE.erl | 14 +++++ .../test/emqx_authn_mysql_SUITE.erl | 14 +++++ .../test/emqx_authn_pgsql_SUITE.erl | 14 +++++ .../test/emqx_authn_redis_SUITE.erl | 14 +++++ apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_authz/src/emqx_authz.erl | 8 +++ apps/emqx_authz/src/emqx_authz_utils.erl | 8 +-- 18 files changed, 268 insertions(+), 124 deletions(-) diff --git a/.gitignore b/.gitignore index d8b3806e3..26b146cef 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,4 @@ apps/emqx/test/emqx_static_checks_data/master.bpapi # rendered configurations *.conf.rendered lux_logs/ +.ci/docker-compose-file/redis/*.log diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 40a949d13..734b88515 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -11,6 +11,7 @@ * The license is now copied to all nodes in the cluster when it's reloaded. [#8598](https://github.com/emqx/emqx/pull/8598) * Added a HTTP API to manage licenses. [#8610](https://github.com/emqx/emqx/pull/8610) * Updated `/nodes` API node_status from `Running/Stopped` to `running/stopped`. [#8642](https://github.com/emqx/emqx/pull/8642) +* Improve handling of placeholder interpolation errors [#8635](https://github.com/emqx/emqx/pull/8635) * Better logging on unknown object IDs. [#8670](https://github.com/emqx/emqx/pull/8670) # 5.0.4 diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index d59eea1af..ba5f80a74 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -38,4 +38,8 @@ -define(RESOURCE_GROUP, <<"emqx_authn">>). +-define(WITH_SUCCESSFUL_RENDER(Code), + emqx_authn_utils:with_successful_render(?MODULE, fun() -> Code end) +). + -endif. diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 8087e822f..ef67b9a14 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 994f2f275..b989da3b4 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -34,7 +34,8 @@ ensure_apps_started/1, cleanup_resources/0, make_resource_id/1, - without_password/1 + without_password/1, + with_successful_render/2 ]). -define(AUTHN_PLACEHOLDERS, [ @@ -136,6 +137,18 @@ render_sql_params(ParamList, Credential) -> #{return => rawlist, var_trans => fun handle_sql_var/2} ). +with_successful_render(Provider, Fun) when is_function(Fun, 0) -> + try + Fun() + catch + error:{cannot_get_variable, Name} -> + ?TRACE_AUTHN(error, "placeholder_interpolation_failed", #{ + provider => Provider, + placeholder => Name + }), + ignore + end. + %% true is_superuser(#{<<"is_superuser">> := <<"true">>}) -> #{is_superuser => true}; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index db3eb3c2f..2304cf1e4 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -187,25 +187,29 @@ authenticate( request_timeout := RequestTimeout } = State ) -> - Request = generate_request(Credential, State), - Response = emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}), - ?TRACE_AUTHN_PROVIDER("http_response", #{ - request => request_for_log(Credential, State), - response => response_for_log(Response), - resource => ResourceId - }), - case Response of - {ok, 204, _Headers} -> - {ok, #{is_superuser => false}}; - {ok, 200, Headers, Body} -> - handle_response(Headers, Body); - {ok, _StatusCode, _Headers} = Response -> - ignore; - {ok, _StatusCode, _Headers, _Body} = Response -> - ignore; - {error, _Reason} -> - ignore - end. + ?WITH_SUCCESSFUL_RENDER( + begin + Request = generate_request(Credential, State), + Response = emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}), + ?TRACE_AUTHN_PROVIDER("http_response", #{ + request => request_for_log(Credential, State), + response => response_for_log(Response), + resource => ResourceId + }), + case Response of + {ok, 204, _Headers} -> + {ok, #{is_superuser => false}}; + {ok, 200, Headers, Body} -> + handle_response(Headers, Body); + {ok, _StatusCode, _Headers} = Response -> + ignore; + {ok, _StatusCode, _Headers, _Body} = Response -> + ignore; + {error, _Reason} -> + ignore + end + end + ). destroy(#{resource_id := ResourceId}) -> _ = emqx_resource:remove_local(ResourceId), diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index f7249ae57..43a1ebd3b 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -162,35 +162,39 @@ authenticate( resource_id := ResourceId } = State ) -> - Filter = emqx_authn_utils:render_deep(FilterTemplate, Credential), - case emqx_resource:query(ResourceId, {find_one, Collection, Filter, #{}}) of - undefined -> - ignore; - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER(error, "mongodb_query_failed", #{ - resource => ResourceId, - collection => Collection, - filter => Filter, - reason => Reason - }), - ignore; - Doc -> - case check_password(Password, Doc, State) of - ok -> - {ok, is_superuser(Doc, State)}; - {error, {cannot_find_password_hash_field, PasswordHashField}} -> - ?TRACE_AUTHN_PROVIDER(error, "cannot_find_password_hash_field", #{ + ?WITH_SUCCESSFUL_RENDER( + begin + Filter = emqx_authn_utils:render_deep(FilterTemplate, Credential), + case emqx_resource:query(ResourceId, {find_one, Collection, Filter, #{}}) of + undefined -> + ignore; + {error, Reason} -> + ?TRACE_AUTHN_PROVIDER(error, "mongodb_query_failed", #{ resource => ResourceId, collection => Collection, filter => Filter, - document => Doc, - password_hash_field => PasswordHashField + reason => Reason }), ignore; - {error, Reason} -> - {error, Reason} + Doc -> + case check_password(Password, Doc, State) of + ok -> + {ok, is_superuser(Doc, State)}; + {error, {cannot_find_password_hash_field, PasswordHashField}} -> + ?TRACE_AUTHN_PROVIDER(error, "cannot_find_password_hash_field", #{ + resource => ResourceId, + collection => Collection, + filter => Filter, + document => Doc, + password_hash_field => PasswordHashField + }), + ignore; + {error, Reason} -> + {error, Reason} + end end - end. + end + ). %%------------------------------------------------------------------------------ %% Internal functions diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index e95302ad4..4efa62670 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -113,32 +113,36 @@ authenticate( password_hash_algorithm := Algorithm } ) -> - Params = emqx_authn_utils:render_sql_params(TmplToken, Credential), - case emqx_resource:query(ResourceId, {prepared_query, ?PREPARE_KEY, Params, Timeout}) of - {ok, _Columns, []} -> - ignore; - {ok, Columns, [Row | _]} -> - Selected = maps:from_list(lists:zip(Columns, Row)), - case - emqx_authn_utils:check_password_from_selected_map( - Algorithm, Selected, Password - ) - of - ok -> - {ok, emqx_authn_utils:is_superuser(Selected)}; + ?WITH_SUCCESSFUL_RENDER( + begin + Params = emqx_authn_utils:render_sql_params(TmplToken, Credential), + case emqx_resource:query(ResourceId, {prepared_query, ?PREPARE_KEY, Params, Timeout}) of + {ok, _Columns, []} -> + ignore; + {ok, Columns, [Row | _]} -> + Selected = maps:from_list(lists:zip(Columns, Row)), + case + emqx_authn_utils:check_password_from_selected_map( + Algorithm, Selected, Password + ) + of + ok -> + {ok, emqx_authn_utils:is_superuser(Selected)}; + {error, Reason} -> + {error, Reason} + end; {error, Reason} -> - {error, Reason} - end; - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER(error, "mysql_query_failed", #{ - resource => ResourceId, - tmpl_token => TmplToken, - params => Params, - timeout => Timeout, - reason => Reason - }), - ignore - end. + ?TRACE_AUTHN_PROVIDER(error, "mysql_query_failed", #{ + resource => ResourceId, + tmpl_token => TmplToken, + params => Params, + timeout => Timeout, + reason => Reason + }), + ignore + end + end + ). parse_config( #{ diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 2962308ab..f8b47959a 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -115,31 +115,35 @@ authenticate( password_hash_algorithm := Algorithm } ) -> - Params = emqx_authn_utils:render_sql_params(PlaceHolders, Credential), - case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Params}) of - {ok, _Columns, []} -> - ignore; - {ok, Columns, [Row | _]} -> - NColumns = [Name || #column{name = Name} <- Columns], - Selected = maps:from_list(lists:zip(NColumns, erlang:tuple_to_list(Row))), - case - emqx_authn_utils:check_password_from_selected_map( - Algorithm, Selected, Password - ) - of - ok -> - {ok, emqx_authn_utils:is_superuser(Selected)}; + ?WITH_SUCCESSFUL_RENDER( + begin + Params = emqx_authn_utils:render_sql_params(PlaceHolders, Credential), + case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Params}) of + {ok, _Columns, []} -> + ignore; + {ok, Columns, [Row | _]} -> + NColumns = [Name || #column{name = Name} <- Columns], + Selected = maps:from_list(lists:zip(NColumns, erlang:tuple_to_list(Row))), + case + emqx_authn_utils:check_password_from_selected_map( + Algorithm, Selected, Password + ) + of + ok -> + {ok, emqx_authn_utils:is_superuser(Selected)}; + {error, Reason} -> + {error, Reason} + end; {error, Reason} -> - {error, Reason} - end; - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER(error, "postgresql_query_failed", #{ - resource => ResourceId, - params => Params, - reason => Reason - }), - ignore - end. + ?TRACE_AUTHN_PROVIDER(error, "postgresql_query_failed", #{ + resource => ResourceId, + params => Params, + reason => Reason + }), + ignore + end + end + ). parse_config( #{ diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 71cd292e6..684d60e49 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -133,33 +133,37 @@ authenticate( password_hash_algorithm := Algorithm } ) -> - NKey = emqx_authn_utils:render_str(KeyTemplate, Credential), - Command = [CommandName, NKey | Fields], - case emqx_resource:query(ResourceId, {cmd, Command}) of - {ok, []} -> - ignore; - {ok, Values} -> - Selected = merge(Fields, Values), - case - emqx_authn_utils:check_password_from_selected_map( - Algorithm, Selected, Password - ) - of - ok -> - {ok, emqx_authn_utils:is_superuser(Selected)}; + ?WITH_SUCCESSFUL_RENDER( + begin + NKey = emqx_authn_utils:render_str(KeyTemplate, Credential), + Command = [CommandName, NKey | Fields], + case emqx_resource:query(ResourceId, {cmd, Command}) of + {ok, []} -> + ignore; + {ok, Values} -> + Selected = merge(Fields, Values), + case + emqx_authn_utils:check_password_from_selected_map( + Algorithm, Selected, Password + ) + of + ok -> + {ok, emqx_authn_utils:is_superuser(Selected)}; + {error, Reason} -> + {error, Reason} + end; {error, Reason} -> - {error, Reason} - end; - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER(error, "redis_query_failed", #{ - resource => ResourceId, - cmd => Command, - keys => NKey, - fields => Fields, - reason => Reason - }), - ignore - end. + ?TRACE_AUTHN_PROVIDER(error, "redis_query_failed", #{ + resource => ResourceId, + cmd => Command, + keys => NKey, + fields => Fields, + reason => Reason + }), + ignore + end + end + ). %%------------------------------------------------------------------------------ %% Internal functions diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 44ef43903..db22a6cfe 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -247,6 +247,27 @@ t_update(_Config) -> emqx_access_control:authenticate(?CREDENTIALS) ). +t_interpolation_error(_Config) -> + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, raw_http_auth_config()} + ), + + Headers = #{<<"content-type">> => <<"application/json">>}, + Response = ?SERVER_RESPONSE_JSON(allow), + + ok = emqx_authn_http_test_server:set_handler( + fun(Req0, State) -> + Req = cowboy_req:reply(200, Headers, Response, Req0), + {ok, Req, State} + end + ), + + ?assertMatch( + ?EXCEPTION_DENY, + emqx_access_control:authenticate(maps:without([username], ?CREDENTIALS)) + ). + t_is_superuser(_Config) -> Config = raw_http_auth_config(), {ok, _} = emqx:update_config( @@ -410,6 +431,26 @@ samples() -> result => {ok, #{is_superuser => false, user_property => #{}}} }, + %% simple get request, no username + #{ + handler => fun(Req0, State) -> + #{ + username := <<"plain">>, + password := <<"plain">> + } = cowboy_req:match_qs([username, password], Req0), + + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + jiffy:encode(#{result => allow, is_superuser => false}), + Req0 + ), + {ok, Req, State} + end, + config_params => #{}, + result => {ok, #{is_superuser => false, user_property => #{}}} + }, + %% get request with json body response #{ handler => fun(Req0, State) -> diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl index 2f7dd2391..ddde18c49 100644 --- a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl @@ -288,6 +288,20 @@ raw_mongo_auth_config() -> user_seeds() -> [ + #{ + data => #{ + username => <<"plain">>, + password_hash => <<"plainsalt">>, + salt => <<"salt">>, + is_superuser => <<"1">> + }, + credentials => #{ + password => <<"plain">> + }, + config_params => #{}, + result => {error, not_authorized} + }, + #{ data => #{ username => <<"plain">>, diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index 0fdba0b31..175aa7f1d 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -258,6 +258,20 @@ raw_mysql_auth_config() -> user_seeds() -> [ + #{ + data => #{ + username => "plain", + password_hash => "plainsalt", + salt => "salt", + is_superuser_str => "1" + }, + credentials => #{ + password => <<"plain">> + }, + config_params => #{}, + result => {error, not_authorized} + }, + #{ data => #{ username => "plain", diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index ff017a79e..02095c07d 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -320,6 +320,20 @@ raw_pgsql_auth_config() -> user_seeds() -> [ + #{ + data => #{ + username => "plain", + password_hash => "plainsalt", + salt => "salt", + is_superuser_str => "1" + }, + credentials => #{ + password => <<"plain">> + }, + config_params => #{}, + result => {error, not_authorized} + }, + #{ data => #{ username => "plain", diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index dde5f8188..889404c5e 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -280,6 +280,20 @@ raw_redis_auth_config() -> user_seeds() -> [ + #{ + data => #{ + password_hash => <<"plainsalt">>, + salt => <<"salt">>, + is_superuser => <<"1">> + }, + credentials => #{ + password => <<"plain">> + }, + key => <<"mqtt_user:plain">>, + config_params => #{}, + result => {error, not_authorized} + }, + #{ data => #{ password_hash => <<"plainsalt">>, diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index ed19b15a8..e40b5e64c 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index f7ebcece0..8eb3c6eae 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -402,6 +402,14 @@ do_authorize( Matched -> {Matched, Type} catch + error:{cannot_get_variable, Name} -> + emqx_metrics_worker:inc(authz_metrics, Type, nomatch), + ?SLOG(warning, #{ + msg => "placeholder_interpolation_failed", + placeholder => Name, + authorize_type => Type + }), + do_authorize(Client, PubSub, Topic, Tail); Class:Reason:Stacktrace -> emqx_metrics_worker:inc(authz_metrics, Type, nomatch), ?SLOG(warning, #{ diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index dd6f66c7f..d364bc5fa 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -181,15 +181,15 @@ convert_client_var({dn, DN}) -> {cert_subject, DN}; convert_client_var({protocol, Proto}) -> {proto_name, Proto}; convert_client_var(Other) -> Other. -handle_var({var, _Name}, undefined) -> - "undefined"; +handle_var({var, Name}, undefined) -> + error({cannot_get_variable, Name}); handle_var({var, <<"peerhost">>}, IpAddr) -> inet_parse:ntoa(IpAddr); handle_var(_Name, Value) -> emqx_placeholder:bin(Value). -handle_sql_var({var, _Name}, undefined) -> - "undefined"; +handle_sql_var({var, Name}, undefined) -> + error({cannot_get_variable, Name}); handle_sql_var({var, <<"peerhost">>}, IpAddr) -> inet_parse:ntoa(IpAddr); handle_sql_var(_Name, Value) -> From d55c9341eb7f1a8d5109bf6912ceb070fbb0b6ef Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 11 Aug 2022 17:42:22 +0800 Subject: [PATCH 45/63] fix(dashboard): fix nit codes --- apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf | 15 +++++++++++++-- apps/emqx_dashboard/src/emqx_dashboard_admin.erl | 2 ++ apps/emqx_dashboard/src/emqx_dashboard_app.erl | 10 +++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf index ad41b06a7..e404b54b4 100644 --- a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf +++ b/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf @@ -203,8 +203,19 @@ its own from which a browser should permit loading resources.""" zh: "初始化用户文件" } label { - en: "Initialize users file" - zh: "初始化用户文件" + en: """Is used to add an administrative user to Dashboard when emqx is first launched, + the format is: + ``` + username1:password1 + username2:password2 + ``` +""" + zh: """用于在首次启动 emqx 时，为 Dashboard 添加管理用户，其格式为： + ``` + username1:password1 + username2:password2 + ``` +""" } } } diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 3d7447dc5..85f145595 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -19,6 +19,7 @@ -module(emqx_dashboard_admin). -include("emqx_dashboard.hrl"). +-include_lib("emqx/include/logger.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). -boot_mnesia({mnesia, [boot]}). @@ -91,6 +92,7 @@ add_bootstrap_user() -> File -> case mnesia:table_info(?ADMIN, size) of 0 -> + ?SLOG(debug, #{msg => "Add dashboard bootstrap users", file => File}), add_bootstrap_user(File); _ -> ok diff --git a/apps/emqx_dashboard/src/emqx_dashboard_app.erl b/apps/emqx_dashboard/src/emqx_dashboard_app.erl index 4d9c1416a..5084d76c4 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_app.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_app.erl @@ -31,9 +31,13 @@ start(_StartType, _StartArgs) -> case emqx_dashboard:start_listeners() of ok -> emqx_dashboard_cli:load(), - ok = emqx_dashboard_admin:add_bootstrap_user(), - {ok, _} = emqx_dashboard_admin:add_default_user(), - {ok, Sup}; + case emqx_dashboard_admin:add_bootstrap_user() of + ok -> + {ok, _} = emqx_dashboard_admin:add_default_user(), + {ok, Sup}; + Error -> + Error + end; {error, Reason} -> {error, Reason} end. From 6dbae601772aa025d244500d263183550726983e Mon Sep 17 00:00:00 2001 From: CrazyWisdom Date: Thu, 11 Aug 2022 17:59:40 +0800 Subject: [PATCH 46/63] docs(README): add rule engine link --- README-CN.md | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README-CN.md b/README-CN.md index 772b4ba8a..205a038da 100644 --- a/README-CN.md +++ b/README-CN.md @@ -24,7 +24,7 @@ EMQX 自 2013 年在 GitHub 发布开源版本以来，获得了来自 50 多个 #### EMQX Cloud -使用 EMQX 最简单的方式是在 EMQX Cloud 上创建完全托管的 MQTT 服务。[免费试用 EMQX Cloud](https://www.emqx.com/zh/signup?continue=https%3A%2F%2Fcloud.emqx.com%2Fconsole%2F)，无需绑定信用卡。 +使用 EMQX 最简单的方式是在 EMQX Cloud 上创建完全托管的 MQTT 服务。[免费试用 EMQX Cloud](https://www.emqx.com/zh/signup?utm_source=github.com&utm_medium=referral&utm_campaign=emqx-readme-to-cloud&continue=https://cloud.emqx.com/console/deployments/0?oper=new)，无需绑定信用卡。 #### 使用 Docker 运行 EMQX diff --git a/README.md b/README.md index 6441ec986..2a9a7320a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ English | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | [рус EMQX is the most scalable and popular open-source MQTT broker with a high performance that connects 100M+ IoT devices in 1 cluster at 1ms latency. Move and process millions of MQTT messages per second. -The EMQX v5.0 has been verified in [test scenarios](https://www.emqx.com/en/blog/reaching-100m-mqtt-connections-with-emqx-5-0) to scale to 100 million concurrent device connections, which is a critically important milestone for IoT designers. It also comes with plenty of exciting new features and huge performance improvements, including a more powerful rule engine, enhanced security management, Mria database extension, and much more to enhance the scalability of IoT applications. +The EMQX v5.0 has been verified in [test scenarios](https://www.emqx.com/en/blog/reaching-100m-mqtt-connections-with-emqx-5-0) to scale to 100 million concurrent device connections, which is a critically important milestone for IoT designers. It also comes with plenty of exciting new features and huge performance improvements, including a more powerful [rule engine](https://www.emqx.com/en/solutions/iot-rule-engine), enhanced security management, Mria database extension, and much more to enhance the scalability of IoT applications. During the last several years, EMQX has gained popularity among IoT companies and is used by more than 20,000 global users from over 50 countries, with more than 100 million IoT device connections supported worldwide. @@ -25,7 +25,7 @@ For more information, please visit [EMQX homepage](https://www.emqx.io/). #### EMQX Cloud -The simplest way to set up EMQX is to create a managed deployment with EMQX Cloud. You can [try EMQX Cloud for free](https://www.emqx.com/en/signup?continue=https%3A%2F%2Fcloud-intl.emqx.com%2Fconsole%2F), no credit card required. +The simplest way to set up EMQX is to create a managed deployment with EMQX Cloud. You can [try EMQX Cloud for free](https://www.emqx.com/en/signup?utm_source=github.com&utm_medium=referral&utm_campaign=emqx-readme-to-cloud&continue=https://cloud-intl.emqx.com/console/deployments/0?oper=new), no credit card required. #### Run EMQX using Docker From 449239d0f46c56de2f60f827632f1ea6d19bda05 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 12 Aug 2022 01:29:37 +0800 Subject: [PATCH 47/63] fix: count all msg size of `event_message` for mqtt bridge --- apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index 581d2670f..43700506b 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -142,11 +142,14 @@ from_binary(Bin) -> binary_to_term(Bin). %% @doc Estimate the size of a message. %% Count only the topic length + payload size +%% There is no topic and payload for event message. So count all `Msg` term -spec estimate_size(msg()) -> integer(). estimate_size(#message{topic = Topic, payload = Payload}) -> size(Topic) + size(Payload); estimate_size(#{topic := Topic, payload := Payload}) -> - size(Topic) + size(Payload). + size(Topic) + size(Payload); +estimate_size(Term) -> + erlang:external_size(Term). set_headers(undefined, Msg) -> Msg; From c0750b9946b3972531ca95bcf7dced1c11af093d Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 11 Aug 2022 20:37:45 +0200 Subject: [PATCH 48/63] build: bump relup helper to 2.1.0 --- rebar.config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.erl b/rebar.config.erl index 6748f7ce8..976c0d3b8 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -126,7 +126,7 @@ project_app_dirs(Edition) -> plugins() -> [ - {relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.0.0"}}}, + {relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.1.0"}}}, %% emqx main project does not require port-compiler %% pin at root level for deterministic {pc, "v1.14.0"} From 809f2ede09fd931cb9b59572843409628481191d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 11 Aug 2022 22:05:03 +0200 Subject: [PATCH 49/63] fix(bin/emqx): ensure no crash dump when checking compatibility --- bin/emqx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index a70c676fd..e73240d5d 100755 --- a/bin/emqx +++ b/bin/emqx @@ -287,7 +287,8 @@ COMPATIBILITY_CHECK=' compatiblity_info() { # RELEASE_LIB is used by Elixir - "$BINDIR/$PROGNAME" \ + # set crash-dump bytes to zero to ensure no crash dump is generated when erl crashes + env ERL_CRASH_DUMP_BYTES=0 "$BINDIR/$PROGNAME" \ -noshell \ -boot_var RELEASE_LIB "$ERTS_LIB_DIR/lib" \ -boot "$REL_DIR/start_clean" \ From d4bf658e38f720703552a209a6415850357fc609 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 12 Aug 2022 16:36:54 +0800 Subject: [PATCH 50/63] fix(dashboard): add bootstrap user tag && return error when adding bootstrap user fails --- .../src/emqx_dashboard_admin.erl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 85f145595..7f5c31771 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -56,6 +56,7 @@ ]). -type emqx_admin() :: #?ADMIN{}. +-define(BOOTSTRAP_USER_TAG, <<"bootstrap user">>). %%-------------------------------------------------------------------- %% Mnesia bootstrap @@ -314,7 +315,7 @@ add_default_user(Username, Password) -> add_bootstrap_user(File) -> case file:open(File, [read]) of {ok, Dev} -> - {ok, MP} = re:compile(<<"(\.+):(\.+)">>), + {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), try load_bootstrap_user(Dev, MP) catch @@ -331,13 +332,16 @@ load_bootstrap_user(Dev, MP) -> case file:read_line(Dev) of {ok, Line} -> case re:run(Line, MP, [global, {capture, all_but_first, binary}]) of - {match, Captured} -> - _ = [add_user(Username, Password, <<>>) || [Username, Password] <- Captured], - ok; + {match, [[Username, Password]]} -> + case add_user(Username, Password, ?BOOTSTRAP_USER_TAG) of + {ok, _} -> + load_bootstrap_user(Dev, MP); + Error -> + Error + end; _ -> - ok - end, - load_bootstrap_user(Dev, MP); + load_bootstrap_user(Dev, MP) + end; eof -> ok; Error -> From c1b951cc43ecd0e323f6ba7a643562fae38fe1a8 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 12 Aug 2022 17:55:26 +0800 Subject: [PATCH 51/63] chore: update ce-dashboard vsn --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6c6c02ca1..49c2b1ed7 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.0.5 +export EMQX_DASHBOARD_VERSION ?= v1.0.6 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) From 3cb79e811f78d877d6b1d84abb3756157eb48118 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 12 Aug 2022 17:42:54 +0800 Subject: [PATCH 52/63] fix: remove the extra fields max_retries --- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index fe19ed066..3be8d38fd 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "An OTP application"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl b/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl index f11247d68..02dd0a76d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_webhook_schema.erl @@ -51,7 +51,9 @@ basic_config() -> } )} ] ++ - proplists:delete(base_url, emqx_connector_http:fields(config)). + proplists:delete( + max_retries, proplists:delete(base_url, emqx_connector_http:fields(config)) + ). request_config() -> [ From fc6d79c2da69e28f9d35d74c2d588fa8b168b66b Mon Sep 17 00:00:00 2001 From: Xinyu Liu <506895667@qq.com> Date: Fri, 12 Aug 2022 18:30:34 +0800 Subject: [PATCH 53/63] fix: typos in emqx_schema_i18n.conf --- apps/emqx/i18n/emqx_schema_i18n.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index c42ccb356..17e59a4dd 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -2005,7 +2005,7 @@ How often to send PING frames to keep a connection alive. 0 means disabled. } label: { en: "Keep Alive Interval" - zh: "PING保活频率" + zh: "PING 保活频率" } } From 18d710e9322d0111e233d3b715dd54ca2c83dad7 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 12 Aug 2022 18:37:27 +0800 Subject: [PATCH 54/63] chore: release v5.0.5-beta.1 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 75d5852c9..3bf2aabb6 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.4"). +-define(EMQX_RELEASE_CE, "5.0.5-beta.1"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.0-alpha.1"). From c246e758be780b1679867175be62f8d26865a417 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 12 Aug 2022 11:57:56 +0200 Subject: [PATCH 55/63] refactor(Makefile): always execute prepare-build-deps script --- Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 49c2b1ed7..6e68d5164 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ $(shell $(CURDIR)/scripts/git-hooks-init.sh) +$(shell $(CURDIR)/scripts/prepare-build-deps.sh) REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts @@ -111,7 +112,7 @@ cover: $(REBAR) coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send -COMMON_DEPS := $(REBAR) prepare-build-deps get-dashboard conf-segs +COMMON_DEPS := $(REBAR) get-dashboard conf-segs ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar .PHONY: $(REL_PROFILES) @@ -219,9 +220,6 @@ conf-segs: @scripts/merge-config.escript @scripts/merge-i18n.escript -prepare-build-deps: - @scripts/prepare-build-deps.sh - ## elixir target is to create release packages using Elixir's Mix .PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir) $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get From c82a05a0e73db11444260bc1de07373408ad94b3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 12 Aug 2022 13:17:22 +0200 Subject: [PATCH 56/63] refactor(Makefile): support download enterprise dashboard download --- Makefile | 16 ++++++++++------ build | 2 ++ scripts/get-dashboard.sh | 16 ++++++++++++++-- scripts/pre-compile.sh | 23 +++++++++++++++++++++++ 4 files changed, 49 insertions(+), 8 deletions(-) create mode 100755 scripts/pre-compile.sh diff --git a/Makefile b/Makefile index 6e68d5164..0ec95d4e7 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v1.0.6 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) @@ -56,10 +57,6 @@ mix-deps-get: $(ELIXIR_COMMON_DEPS) $(REBAR): ensure-rebar3 -.PHONY: get-dashboard -get-dashboard: - @$(SCRIPTS)/get-dashboard.sh - .PHONY: eunit eunit: $(REBAR) conf-segs @ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c @@ -81,7 +78,12 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh) ## app/name-ct targets are intended for local tests hence cover is not enabled .PHONY: $(APPS:%=%-ct) define gen-app-ct-target -$1-ct: $(REBAR) conf-segs +$1-ct: $(REBAR) +ifeq (,$(findstring lib-ee,$1)) + @./scripts/pre-compile.sh emqx +else + @./scripts/pre-compile.sh emqx-enterprise +endif @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) @@ -112,7 +114,8 @@ cover: $(REBAR) coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send -COMMON_DEPS := $(REBAR) get-dashboard conf-segs +COMMON_DEPS := $(REBAR) + ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar .PHONY: $(REL_PROFILES) @@ -148,6 +151,7 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%) ## which may not have the right credentials .PHONY: $(PROFILES:%=deps-%) $(PROFILES:%=deps-%): $(COMMON_DEPS) + @./scripts/pre-compile.sh $(@:deps-%=%) @$(REBAR) as $(@:deps-%=%) get-deps @rm -f rebar.lock diff --git a/build b/build index 07c16b69e..80dbe2775 100755 --- a/build +++ b/build @@ -106,6 +106,7 @@ assert_no_compile_time_only_deps() { } make_rel() { + ./scripts/pre-compile.sh $PROFILE # compile all beams ./rebar3 as "$PROFILE" compile # generate docs (require beam compiled), generated to etc and priv dirs @@ -116,6 +117,7 @@ make_rel() { } make_elixir_rel() { + ./scripts/pre-compile.sh "$PROFILE" export_release_vars "$PROFILE" mix release --overwrite assert_no_compile_time_only_deps diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index 481a932ee..0069f3cc2 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -5,8 +5,20 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." -RELEASE_ASSET_FILE="emqx-dashboard.zip" -VERSION="${EMQX_DASHBOARD_VERSION}" +VERSION="${1}" +case "$VERSION" in + v*) + RELEASE_ASSET_FILE="emqx-dashboard.zip" + ;; + e*) + RELEASE_ASSET_FILE="emqx-enterprise-dashboard.zip" + ;; + *) + echo "Unknown version $VERSION" + exit 1 + ;; +esac + DASHBOARD_PATH='apps/emqx_dashboard/priv' DASHBOARD_REPO='emqx-dashboard-web-new' DIRECT_DOWNLOAD_URL="https://github.com/emqx/${DASHBOARD_REPO}/releases/download/${VERSION}/${RELEASE_ASSET_FILE}" diff --git a/scripts/pre-compile.sh b/scripts/pre-compile.sh new file mode 100755 index 000000000..0fe99c6b2 --- /dev/null +++ b/scripts/pre-compile.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# NOTE: PROFILE_STR may not be exactly PROFILE (emqx or emqx-enterprise) +# it might be with suffix such as -pkg etc. +PROFILE_STR="${1}" + +case "$PROFILE_STR" in + *enterprise*) + dashboard_version="$EMQX_EE_DASHBOARD_VERSION" + ;; + *) + dashboard_version="$EMQX_DASHBOARD_VERSION" + ;; +esac + +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +./scripts/get-dashboard.sh "$dashboard_version" +./scripts/merge-config.escript +./scripts/merge-i18n.escript From fbb97b16be7af930c3b92acfddf5adff5120ca0d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 12 Aug 2022 14:18:55 +0200 Subject: [PATCH 57/63] ci: test all apps with dual profile no need to test emqx-enterprise profile if the EMQX_RELEASE_EDITION compilation flag is not used though --- .github/workflows/run_test_cases.yaml | 20 +++++++++++++++++++- Makefile | 6 +----- build | 2 +- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index e08e3906b..4b76938c0 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -145,6 +145,10 @@ jobs: fail-fast: false matrix: app_name: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }} + profile: + - emqx + - emqx-enterprise + runs-on: aws-amd64 container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" defaults: @@ -163,8 +167,22 @@ jobs: # produces .coverdata - name: run common test working-directory: source + env: + PROFILE: ${{ matrix.profile }} + WHICH_APP: ${{ matrix.app_name }} run: | - make ${{ matrix.app_name }}-ct + if [ "$PROFILE" = 'emqx-enterprise' ]; then + COMPILE_FLAGS="$(grep -R "EMQX_RELEASE_EDITION" "$WHICH_APP" | wc -l || true)" + if [ "$COMPILE_FLAGS" -gt 0 ]; then + # need to clean first because the default profile was + make clean + make "${WHICH_APP}-ct" + else + echo "no_common_test_run_for_app ${WHICH_APP}" + fi + else + make "${WHICH_APP}-ct" + fi - uses: actions/upload-artifact@v1 with: name: coverdata diff --git a/Makefile b/Makefile index 0ec95d4e7..5bb3f0caf 100644 --- a/Makefile +++ b/Makefile @@ -79,11 +79,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh) .PHONY: $(APPS:%=%-ct) define gen-app-ct-target $1-ct: $(REBAR) -ifeq (,$(findstring lib-ee,$1)) - @./scripts/pre-compile.sh emqx -else - @./scripts/pre-compile.sh emqx-enterprise -endif + @./scripts/pre-compile.sh $(PROFILE) @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) diff --git a/build b/build index 80dbe2775..703692000 100755 --- a/build +++ b/build @@ -106,7 +106,7 @@ assert_no_compile_time_only_deps() { } make_rel() { - ./scripts/pre-compile.sh $PROFILE + ./scripts/pre-compile.sh "$PROFILE" # compile all beams ./rebar3 as "$PROFILE" compile # generate docs (require beam compiled), generated to etc and priv dirs From b8d5ec47f7c7aa0c85de9ca25cb960bd76645931 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 12 Aug 2022 15:49:23 +0200 Subject: [PATCH 58/63] fix(Makefile): make use of the FORCE target --- Makefile | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 5bb3f0caf..2c3077acc 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,3 @@ -$(shell $(CURDIR)/scripts/git-hooks-init.sh) -$(shell $(CURDIR)/scripts/prepare-build-deps.sh) REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts @@ -32,6 +30,13 @@ export REBAR_GIT_CLONE_OPTIONS += --depth=1 .PHONY: default default: $(REBAR) $(PROFILE) +.PHONY: prepare +prepare: FORCE + @$(SCRIPTS)/git-hooks-init.sh # this is no longer needed since 5.0 but we keep it anyway + @$(SCRIPTS)/prepare-build-deps.sh + +FORCE: + .PHONY: all all: $(REBAR) $(PROFILES) @@ -55,7 +60,7 @@ ensure-mix-rebar: $(REBAR) mix-deps-get: $(ELIXIR_COMMON_DEPS) @mix deps.get -$(REBAR): ensure-rebar3 +$(REBAR): prepare ensure-rebar3 .PHONY: eunit eunit: $(REBAR) conf-segs @@ -73,14 +78,14 @@ ct: $(REBAR) conf-segs static_checks: @$(REBAR) as check do dialyzer, xref, ct --suite apps/emqx/test/emqx_static_checks --readable $(CT_READABLE) -APPS=$(shell $(CURDIR)/scripts/find-apps.sh) +APPS=$(shell $(SCRIPTS)/find-apps.sh) ## app/name-ct targets are intended for local tests hence cover is not enabled .PHONY: $(APPS:%=%-ct) define gen-app-ct-target $1-ct: $(REBAR) - @./scripts/pre-compile.sh $(PROFILE) - @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) + @$(SCRIPTS)/pre-compile.sh $(PROFILE) + @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(subst /,-,$1) --suite $(shell $(SCRIPTS)/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) @@ -88,7 +93,7 @@ $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) .PHONY: $(APPS:%=%-prop) define gen-app-prop-target $1-prop: - $(REBAR) proper -d test/props -v -m $(shell $(CURDIR)/scripts/find-props.sh $1) + $(REBAR) proper -d test/props -v -m $(shell $(SCRIPTS)/find-props.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-prop-target,$(app)))) @@ -147,7 +152,7 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%) ## which may not have the right credentials .PHONY: $(PROFILES:%=deps-%) $(PROFILES:%=deps-%): $(COMMON_DEPS) - @./scripts/pre-compile.sh $(@:deps-%=%) + @$(SCRIPTS)/pre-compile.sh $(@:deps-%=%) @$(REBAR) as $(@:deps-%=%) get-deps @rm -f rebar.lock @@ -168,7 +173,7 @@ $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS) .PHONY: $(REL_PROFILES:%=%-relup-downloads) define download-relup-packages $1-relup-downloads: - @if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-build/download-base-packages.sh $1; fi + @if [ "$${EMQX_RELUP}" = "true" ]; then $(SCRIPTS)/relup-build/download-base-packages.sh $1; fi endef ALL_ZIPS = $(REL_PROFILES) $(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt)))) @@ -217,8 +222,8 @@ $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt)))) .PHONY: conf-segs: - @scripts/merge-config.escript - @scripts/merge-i18n.escript + @$(SCRIPTS)/merge-config.escript + @$(SCRIPTS)/merge-i18n.escript ## elixir target is to create release packages using Elixir's Mix .PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir) @@ -245,6 +250,6 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt)))) .PHONY: fmt fmt: $(REBAR) - @./scripts/erlfmt -w '{apps,lib-ee}/*/{src,include,test}/**/*.{erl,hrl,app.src}' - @./scripts/erlfmt -w 'rebar.config.erl' + @$(SCRIPTS)/erlfmt -w '{apps,lib-ee}/*/{src,include,test}/**/*.{erl,hrl,app.src}' + @$(SCRIPTS)/erlfmt -w 'rebar.config.erl' @mix format From 4949fb222e8864e569cd5af39942fadb3f0c433f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 13 Aug 2022 10:12:33 +0200 Subject: [PATCH 59/63] ci(test): do not fail coverdata uppload if no file found the optional make -ct action should not fail the following upload action --- .github/workflows/run_test_cases.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 4b76938c0..50569db3e 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -178,16 +178,17 @@ jobs: make clean make "${WHICH_APP}-ct" else - echo "no_common_test_run_for_app ${WHICH_APP}" + echo "no_common_test_run_for_app ${WHICH_APP}-ct" fi else make "${WHICH_APP}-ct" fi - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 with: name: coverdata path: source/_build/test/cover - - uses: actions/upload-artifact@v1 + if-no-files-found: warn # do not fail if no coverdata found + - uses: actions/upload-artifact@v3 if: failure() with: name: logs_${{ matrix.otp_release }} From c93829c784fde0060e8b28398936ec20139e0bf3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 13 Aug 2022 10:38:22 +0200 Subject: [PATCH 60/63] build: pass build profile as environment variable --- build | 11 ++++++++++- rebar.config.erl | 41 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/build b/build index 703692000..99728d03e 100755 --- a/build +++ b/build @@ -14,9 +14,18 @@ if [ "$DEBUG" -eq 1 ]; then set -x fi -PROFILE="$1" +PROFILE_ARG="$1" ARTIFACT="$2" +if [[ "${PROFILE:-${PROFILE_ARG}}" != "$PROFILE_ARG" ]]; then + echo "PROFILE env var is set to '$PROFILE', but '$0' arg1 is '$1'" + exit 1 +fi + +# make sure PROFILE is exported, it is needed by rebar.config.erl +PROFILE=$PROFILE_ARG +export PROFILE + # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" diff --git a/rebar.config.erl b/rebar.config.erl index 976c0d3b8..18e2cafc0 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -4,6 +4,7 @@ do(Dir, CONFIG) -> ok = assert_otp(), + ok = warn_profile_env(), case iolist_to_binary(Dir) of <<".">> -> C1 = deps(CONFIG), @@ -117,6 +118,9 @@ is_raspbian() -> is_win32() -> win32 =:= element(1, os:type()). +project_app_dirs() -> + project_app_dirs(get_edition_from_profille_env()). + project_app_dirs(Edition) -> ["apps/*"] ++ case is_enterprise(Edition) of @@ -149,6 +153,9 @@ test_deps() -> {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}} ]. +common_compile_opts(Vsn) -> + common_compile_opts(get_edition_from_profille_env(), Vsn). + common_compile_opts(Edition, Vsn) -> % always include debug_info [ @@ -159,6 +166,32 @@ common_compile_opts(Edition, Vsn) -> [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"] ++ [{d, 'BUILD_WITHOUT_QUIC'} || not is_quicer_supported()]. +warn_profile_env() -> + case os:getenv("PROFILE") of + false -> + io:format( + standard_error, + "WARN: environment variable PROFILE is not set, using 'emqx-enterprise'~n", + [] + ); + _ -> + ok + end. + +%% this function is only used for test/check profiles +get_edition_from_profille_env() -> + case os:getenv("PROFILE") of + "emqx" -> + ce; + "emqx-enterprise" -> + ee; + false -> + ee; + V -> + io:format(standard_error, "ERROR: bad_PROFILE ~p~n", [V]), + exit(bad_PROFILE) + end. + prod_compile_opts(Edition, Vsn) -> [ compressed, @@ -212,14 +245,14 @@ profiles_dev() -> Vsn = get_vsn('emqx-enterprise'), [ {check, [ - {erl_opts, common_compile_opts(ee, Vsn)}, - {project_app_dirs, project_app_dirs(ee)} + {erl_opts, common_compile_opts(Vsn)}, + {project_app_dirs, project_app_dirs()} ]}, {test, [ {deps, test_deps()}, - {erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()}, + {erl_opts, common_compile_opts(Vsn) ++ erl_opts_i()}, {extra_src_dirs, [{"test", [{recursive, true}]}]}, - {project_app_dirs, project_app_dirs(ee)} + {project_app_dirs, project_app_dirs()} ]} ]. From d63d6b5f27575a39b7f4792108f0e2e61c241a52 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 13 Aug 2022 21:04:48 +0200 Subject: [PATCH 61/63] ci: do not run opensource edition test for lib-ee apps --- .github/workflows/run_test_cases.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 50569db3e..3b65fe885 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -178,10 +178,17 @@ jobs: make clean make "${WHICH_APP}-ct" else - echo "no_common_test_run_for_app ${WHICH_APP}-ct" + echo "skip_common_test_run_for_app ${WHICH_APP}-ct" fi else - make "${WHICH_APP}-ct" + case "$WHICH_APP" in + lib-ee/*) + echo "skip_opensource_edition_test_for_lib-ee" + ;; + *) + make "${WHICH_APP}-ct" + ;; + esac fi - uses: actions/upload-artifact@v3 with: From 59dfde04a99cba66989eeb780b91e6f23d333e86 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 13 Aug 2022 21:07:13 +0200 Subject: [PATCH 62/63] build: match emqx-pkg profile --- rebar.config.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rebar.config.erl b/rebar.config.erl index 18e2cafc0..a4265cae3 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -183,8 +183,12 @@ get_edition_from_profille_env() -> case os:getenv("PROFILE") of "emqx" -> ce; + "emqx-" ++ _ -> + ce; "emqx-enterprise" -> ee; + "emqx-enterprise-" ++ _ -> + ee; false -> ee; V -> From cd0114b3dc4cf344d31860c1a6333b87865335ab Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 14 Aug 2022 17:57:11 +0800 Subject: [PATCH 63/63] fix: ensure PROFILE=emqx when running make emqx-elixir --- deploy/docker/Dockerfile | 7 ++++--- deploy/docker/Dockerfile.alpine | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 5cc247977..6c5baa391 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -7,14 +7,15 @@ COPY . /emqx ARG EMQX_NAME=emqx ENV EMQX_RELUP=false -RUN export PROFILE="$EMQX_NAME" \ - && export EMQX_NAME=${EMQX_NAME%%-elixir} \ +RUN export PROFILE=${EMQX_NAME%%-elixir} \ + && export EMQX_NAME1=$EMQX_NAME \ + && export EMQX_NAME=$PROFILE \ && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \ && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \ && export EMQX_REL_FORM='docker' \ && cd /emqx \ && rm -rf $EMQX_LIB_PATH \ - && make $PROFILE \ + && make $EMQX_NAME1 \ && mkdir -p /emqx-rel \ && mv $EMQX_REL_PATH /emqx-rel diff --git a/deploy/docker/Dockerfile.alpine b/deploy/docker/Dockerfile.alpine index a8aee2f50..6368a0b51 100644 --- a/deploy/docker/Dockerfile.alpine +++ b/deploy/docker/Dockerfile.alpine @@ -28,14 +28,15 @@ COPY . /emqx ARG EMQX_NAME=emqx ENV EMQX_RELUP=false -RUN export PROFILE="$EMQX_NAME" \ - && export EMQX_NAME=${EMQX_NAME%%-elixir} \ +RUN export PROFILE=${EMQX_NAME%%-elixir} \ + && export EMQX_NAME1=$EMQX_NAME \ + && export EMQX_NAME=$PROFILE \ && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \ && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \ && export EMQX_REL_FORM='docker' \ && cd /emqx \ && rm -rf $EMQX_LIB_PATH \ - && make $PROFILE \ + && make $EMQX_NAME1 \ && mkdir -p /emqx-rel \ && mv $EMQX_REL_PATH /emqx-rel