diff --git a/.gitignore b/.gitignore index 4188126da..b7a358b31 100644 --- a/.gitignore +++ b/.gitignore @@ -50,5 +50,11 @@ _upgrade_base/ TAGS erlang_ls.config .els_cache/ +# VSCode files .vs/ .vscode/ +# Emacs Backup files +*~ +# Emacs temporary files +.#* +*# diff --git a/Makefile b/Makefile index 6d159d144..83fa2f15b 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X -export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.16 +export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.17 ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index df5ae9034..2b70d6dda 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -833,6 +833,39 @@ force_shutdown { max_heap_size = 32MB } +overload_protection { + ## React on system overload or not + ## @doc overload_protection.enable + ## ValueType: Boolean + ## Default: false + enable = false + + ## Backoff delay in ms + ## @doc overload_protection.backoff_delay + ## ValueType: Integer + ## Range: (0, infinity) + ## Default: 1 + backoff_delay = 1 + + ## Backoff GC enabled + ## @doc overload_protection.backoff_gc + ## ValueType: Boolean + ## Default: false + backoff_gc = false + + ## Backoff hibernation enabled + ## @doc overload_protection.backoff_hibernation + ## ValueType: Boolean + ## Default: true + backoff_hibernation = true + + ## Backoff hibernation enabled + ## @doc overload_protection.backoff_hibernation + ## ValueType: Boolean + ## Default: true + backoff_new_conn = true +} + force_gc { ## Force the MQTT connection process GC after this number of ## messages or bytes passed through. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 4ec7c7dc5..b851700b9 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -9,27 +9,28 @@ %% This rebar.config is necessary because the app may be used as a %% `git_subdir` dependency in other projects. {deps, - [ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} + [ {lc, {git, "https://github.com/qzhuyan/lc.git", {tag, "0.1.1"}}} + , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.3"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.9"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.5"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.6"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} ]}. -{plugins, [rebar3_proper]}. +{plugins, [{rebar3_proper, "0.12.1"}]}. {extra_src_dirs, [{"etc", [recursive]}]}. {profiles, [ {test, [{deps, - [ meck + [ {meck, "0.9.2"} + , {proper, "1.4.0"} , {bbmustache,"1.10.0"} - , {emqx_ct_helpers, {git,"https://github.com/emqx/emqx-ct-helpers.git", {tag,"2.1.0"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}} ]}, {extra_src_dirs, [{"test",[recursive]}]} diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 031e4f654..3f167d093 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -5,7 +5,7 @@ {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, - {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon,jiffy]}, + {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon,jiffy,lc]}, {mod, {emqx_app,[]}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index e64899b5b..6f0329762 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -81,7 +81,7 @@ set_debug_secret(PathToSecretFile) -> catch _ : _ -> error({badfile, PathToSecretFile}) end; {error, Reason} -> - ?ULOG("Failed to read debug_info encryption key file ~s: ~p~n", + ?ULOG("Failed to read debug_info encryption key file ~ts: ~p~n", [PathToSecretFile, Reason]), error(Reason) end, diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 14f05dc5c..2df126ea7 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -410,21 +410,23 @@ normalize(#deactivated_alarm{activate_at = ActivateAt, normalize_message(Name, no_details) -> list_to_binary(io_lib:format("~p", [Name])); +normalize_message(runq_overload, #{node := Node, runq_length := Len}) -> + list_to_binary(io_lib:format("VM is overloaded on node: ~p: ~p", [Node, Len])); normalize_message(high_system_memory_usage, #{high_watermark := HighWatermark}) -> list_to_binary(io_lib:format("System memory usage is higher than ~p%", [HighWatermark])); normalize_message(high_process_memory_usage, #{high_watermark := HighWatermark}) -> list_to_binary(io_lib:format("Process memory usage is higher than ~p%", [HighWatermark])); normalize_message(high_cpu_usage, #{usage := Usage}) -> - list_to_binary(io_lib:format("~s cpu usage", [Usage])); + list_to_binary(io_lib:format("~ts cpu usage", [Usage])); normalize_message(too_many_processes, #{usage := Usage}) -> - list_to_binary(io_lib:format("~s process usage", [Usage])); + list_to_binary(io_lib:format("~ts process usage", [Usage])); normalize_message(cluster_rpc_apply_failed, #{tnx_id := TnxId}) -> list_to_binary(io_lib:format("cluster_rpc_apply_failed:~w", [TnxId])); normalize_message(partition, #{occurred := Node}) -> - list_to_binary(io_lib:format("Partition occurs at node ~s", [Node])); + list_to_binary(io_lib:format("Partition occurs at node ~ts", [Node])); normalize_message(<<"resource", _/binary>>, #{type := Type, id := ID}) -> - list_to_binary(io_lib:format("Resource ~s(~s) is down", [Type, ID])); + list_to_binary(io_lib:format("Resource ~ts(~ts) is down", [Type, ID])); normalize_message(<<"conn_congestion/", Info/binary>>, _) -> - list_to_binary(io_lib:format("connection congested: ~s", [Info])); + list_to_binary(io_lib:format("connection congested: ~ts", [Info])); normalize_message(_Name, _UnknownDetails) -> <<"Unknown alarm">>. diff --git a/apps/emqx/src/emqx_alarm_handler.erl b/apps/emqx/src/emqx_alarm_handler.erl index 06f4e23a6..4cf699895 100644 --- a/apps/emqx/src/emqx_alarm_handler.erl +++ b/apps/emqx/src/emqx_alarm_handler.erl @@ -20,6 +20,7 @@ -include("emqx.hrl"). -include("logger.hrl"). +-include_lib("lc/include/lc.hrl"). %% gen_event callbacks @@ -74,6 +75,14 @@ handle_event({clear_alarm, process_memory_high_watermark}, State) -> emqx_alarm:deactivate(high_process_memory_usage), {ok, State}; +handle_event({set_alarm, {?LC_ALARM_ID_RUNQ, Info}}, State) -> + emqx_alarm:activate(runq_overload, Info), + {ok, State}; + +handle_event({clear_alarm, ?LC_ALARM_ID_RUNQ}, State) -> + emqx_alarm:deactivate(runq_overload), + {ok, State}; + handle_event(_, State) -> {ok, State}. diff --git a/apps/emqx/src/emqx_broker_bench.erl b/apps/emqx/src/emqx_broker_bench.erl index 5aad43cc9..f40d03fe2 100644 --- a/apps/emqx/src/emqx_broker_bench.erl +++ b/apps/emqx/src/emqx_broker_bench.erl @@ -57,8 +57,8 @@ run(#{subscribers := Subs, lists:foreach(fun(Pid) -> Pid ! start_subscribe end, SubsPids), collect_results(SubsPids, subscribe_time) end), - io:format(user, "InsertTotalTime: ~s~n", [ns(T1)]), - io:format(user, "InsertTimeAverage: ~s~n", [ns(SubsTime / Subs)]), + io:format(user, "InsertTotalTime: ~ts~n", [ns(T1)]), + io:format(user, "InsertTimeAverage: ~ts~n", [ns(SubsTime / Subs)]), io:format(user, "InsertRps: ~p~n", [rps(Subs * SubOps, T1)]), io:format(user, "lookup ...~n", []), @@ -67,8 +67,8 @@ run(#{subscribers := Subs, lists:foreach(fun(Pid) -> Pid ! start_lookup end, PubsPids), collect_results(PubsPids, lookup_time) end), - io:format(user, "LookupTotalTime: ~s~n", [ns(T2)]), - io:format(user, "LookupTimeAverage: ~s~n", [ns(PubsTime / Pubs)]), + io:format(user, "LookupTotalTime: ~ts~n", [ns(T2)]), + io:format(user, "LookupTimeAverage: ~ts~n", [ns(PubsTime / Pubs)]), io:format(user, "LookupRps: ~p~n", [rps(Pubs * PubOps, T2)]), io:format(user, "mnesia table(s) RAM: ~p~n", [ram_bytes()]), @@ -79,7 +79,7 @@ run(#{subscribers := Subs, lists:foreach(fun(Pid) -> Pid ! stop end, SubsPids), wait_until_empty() end), - io:format(user, "TimeToUnsubscribeAll: ~s~n", [ns(T3)]). + io:format(user, "TimeToUnsubscribeAll: ~ts~n", [ns(T3)]). wait_until_empty() -> case emqx_trie:empty() of diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 61ccdae16..51e1ed162 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -291,7 +291,8 @@ handle_in(?CONNECT_PACKET(), Channel = #channel{conn_state = connecting}) -> handle_out(connack, ?RC_PROTOCOL_ERROR, Channel); handle_in(?CONNECT_PACKET(ConnPkt), Channel) -> - case pipeline([fun enrich_conninfo/2, + case pipeline([fun overload_protection/2, + fun enrich_conninfo/2, fun run_conn_hooks/2, fun check_connect/2, fun enrich_client/2, @@ -1158,6 +1159,9 @@ run_terminate_hook(Reason, #channel{clientinfo = ClientInfo, session = Session}) %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +overload_protection(_, #channel{clientinfo = #{zone := Zone}}) -> + emqx_olp:backoff(Zone), + ok. %%-------------------------------------------------------------------- %% Enrich MQTT Connect Info diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 83db8e480..f53b71901 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -41,13 +41,6 @@ -define(MOD, {mod}). -define(WKEY, '?'). --define(ATOM_CONF_PATH(PATH, EXP, EXP_ON_FAIL), - try [safe_atom(Key) || Key <- PATH] of - AtomKeyPath -> EXP - catch - error:badarg -> EXP_ON_FAIL - end). - -type handler_name() :: module(). -type handlers() :: #{emqx_config:config_key() => handlers(), ?MOD => handler_name()}. @@ -76,8 +69,9 @@ stop() -> -spec update_config(module(), emqx_config:config_key_path(), emqx_config:update_args()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_config(SchemaModule, ConfKeyPath, UpdateArgs) -> - ?ATOM_CONF_PATH(ConfKeyPath, gen_server:call(?MODULE, {change_config, SchemaModule, - AtomKeyPath, UpdateArgs}), {error, {not_found, ConfKeyPath}}). + %% force covert the path to a list of atoms, as there maybe some wildcard names/ids in the path + AtomKeyPath = [atom(Key) || Key <- ConfKeyPath], + gen_server:call(?MODULE, {change_config, SchemaModule, AtomKeyPath, UpdateArgs}). -spec add_handler(emqx_config:config_key_path(), handler_name()) -> ok. add_handler(ConfKeyPath, HandlerName) -> @@ -310,9 +304,9 @@ bin_path(ConfKeyPath) -> [bin(Key) || Key <- ConfKeyPath]. bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(B) when is_binary(B) -> B. -safe_atom(Bin) when is_binary(Bin) -> - binary_to_existing_atom(Bin, latin1); -safe_atom(Str) when is_list(Str) -> - list_to_existing_atom(Str); -safe_atom(Atom) when is_atom(Atom) -> +atom(Bin) when is_binary(Bin) -> + binary_to_atom(Bin, utf8); +atom(Str) when is_list(Str) -> + list_to_atom(Str); +atom(Atom) when is_atom(Atom) -> Atom. diff --git a/apps/emqx/src/emqx_congestion.erl b/apps/emqx/src/emqx_congestion.erl index f0db5415f..170c6bc69 100644 --- a/apps/emqx/src/emqx_congestion.erl +++ b/apps/emqx/src/emqx_congestion.erl @@ -22,7 +22,7 @@ -define(ALARM_CONN_CONGEST(Channel, Reason), list_to_binary( - io_lib:format("~s/~s/~s", + io_lib:format("~ts/~ts/~ts", [Reason, emqx_channel:info(clientid, Channel), maps:get(username, emqx_channel:info(clientinfo, Channel), <<"unknown_user">>)]))). diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index cb6e2ce8f..b01aad468 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -116,7 +116,7 @@ -define(ENABLED(X), (X =/= undefined)). -define(ALARM_TCP_CONGEST(Channel), - list_to_binary(io_lib:format("mqtt_conn/congested/~s/~s", + list_to_binary(io_lib:format("mqtt_conn/congested/~ts/~ts", [emqx_channel:info(clientid, Channel), emqx_channel:info(username, Channel)]))). @@ -317,13 +317,20 @@ exit_on_sock_error(Reason) -> %%-------------------------------------------------------------------- %% Recv Loop -recvloop(Parent, State = #state{idle_timeout = IdleTimeout}) -> +recvloop(Parent, State = #state{ idle_timeout = IdleTimeout + , zone = Zone + }) -> receive Msg -> handle_recv(Msg, Parent, State) after IdleTimeout + 100 -> - hibernate(Parent, cancel_stats_timer(State)) + case emqx_olp:backoff_hibernation(Zone) of + true -> + recvloop(Parent, State); + false -> + hibernate(Parent, cancel_stats_timer(State)) + end end. handle_recv({system, From, Request}, Parent, State) -> @@ -822,8 +829,10 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> %%-------------------------------------------------------------------- %% Run GC and Check OOM -run_gc(Stats, State = #state{gc_state = GcSt}) -> - case ?ENABLED(GcSt) andalso emqx_gc:run(Stats, GcSt) of +run_gc(Stats, State = #state{gc_state = GcSt, zone = Zone}) -> + case ?ENABLED(GcSt) andalso not emqx_olp:backoff_gc(Zone) + andalso emqx_gc:run(Stats, GcSt) + of false -> State; {_IsGC, GcSt1} -> State#state{gc_state = GcSt1} diff --git a/apps/emqx/src/emqx_ctl.erl b/apps/emqx/src/emqx_ctl.erl index 52930e714..3ca8174fe 100644 --- a/apps/emqx/src/emqx_ctl.erl +++ b/apps/emqx/src/emqx_ctl.erl @@ -128,7 +128,7 @@ help() -> [] -> print("No commands available.~n"); Cmds -> - print("Usage: ~s~n", [?MODULE]), + print("Usage: ~ts~n", [?MODULE]), lists:foreach(fun({_, {Mod, Cmd}, _}) -> print("~110..-s~n", [""]), Mod:Cmd(usage) end, Cmds) @@ -136,11 +136,11 @@ help() -> -spec(print(io:format()) -> ok). print(Msg) -> - io:format("~s", [format(Msg)]). + io:format("~ts", [format(Msg)]). -spec(print(io:format(), [term()]) -> ok). print(Format, Args) -> - io:format("~s", [format(Format, Args)]). + io:format("~ts", [format(Format, Args)]). -spec(usage([cmd_usage()]) -> ok). usage(UsageList) -> @@ -152,7 +152,7 @@ usage(CmdParams, Desc) -> -spec(format(io:format()) -> string()). format(Msg) -> - lists:flatten(io_lib:format("~s", [Msg])). + lists:flatten(io_lib:format("~ts", [Msg])). -spec(format(io:format(), [term()]) -> string()). format(Format, Args) -> @@ -170,7 +170,7 @@ format_usage(CmdParams, Desc) -> CmdLines = split_cmd(CmdParams), DescLines = split_cmd(Desc), lists:foldl(fun({CmdStr, DescStr}, Usage) -> - Usage ++ format("~-70s# ~s~n", [CmdStr, DescStr]) + Usage ++ format("~-70s# ~ts~n", [CmdStr, DescStr]) end, "", zip_cmd(CmdLines, DescLines)). %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 5c4776207..187a55fdd 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -131,15 +131,15 @@ start_listener(Type, ListenerName, #{bind := Bind} = Conf) -> case do_start_listener(Type, ListenerName, Conf) of {ok, {skipped, Reason}} when Reason =:= listener_disabled; Reason =:= quic_app_missing -> - console_print("- Skip - starting listener ~s on ~s ~n due to ~p", + console_print("- Skip - starting listener ~ts on ~ts ~n due to ~p", [listener_id(Type, ListenerName), format_addr(Bind), Reason]); {ok, _} -> - console_print("Listener ~s on ~s started.~n", + console_print("Listener ~ts on ~ts started.~n", [listener_id(Type, ListenerName), format_addr(Bind)]); {error, {already_started, Pid}} -> {error, {already_started, Pid}}; {error, Reason} -> - ?ELOG("Failed to start listener ~s on ~s: ~0p~n", + ?ELOG("Failed to start listener ~ts on ~ts: ~0p~n", [listener_id(Type, ListenerName), format_addr(Bind), Reason]), error(Reason) end. @@ -180,11 +180,11 @@ stop_listener(ListenerId) -> stop_listener(Type, ListenerName, #{bind := Bind} = Conf) -> case do_stop_listener(Type, ListenerName, Conf) of ok -> - console_print("Listener ~s on ~s stopped.~n", + console_print("Listener ~ts on ~ts stopped.~n", [listener_id(Type, ListenerName), format_addr(Bind)]), ok; {error, Reason} -> - ?ELOG("Failed to stop listener ~s on ~s: ~0p~n", + ?ELOG("Failed to stop listener ~ts on ~ts: ~0p~n", [listener_id(Type, ListenerName), format_addr(Bind), Reason]), {error, Reason} end. @@ -289,7 +289,9 @@ esockd_opts(Type, Opts0) -> infinity -> Opts1; Rate -> Opts1#{max_conn_rate => Rate} end, - Opts3 = Opts2#{access_rules => esockd_access_rules(maps:get(access_rules, Opts0, []))}, + Opts3 = Opts2#{ access_rules => esockd_access_rules(maps:get(access_rules, Opts0, [])) + , tune_fun => {emqx_olp, backoff_new_conn, [zone(Opts0)]} + }, maps:to_list(case Type of tcp -> Opts3#{tcp_options => tcp_opts(Opts0)}; ssl -> Opts3#{ssl_options => ssl_opts(Opts0), tcp_options => tcp_opts(Opts0)} @@ -342,9 +344,9 @@ merge_default(Options) -> format_addr(Port) when is_integer(Port) -> io_lib:format("0.0.0.0:~w", [Port]); format_addr({Addr, Port}) when is_list(Addr) -> - io_lib:format("~s:~w", [Addr, Port]); + io_lib:format("~ts:~w", [Addr, Port]); format_addr({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). + io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]). listener_id(Type, ListenerName) -> list_to_atom(lists:append([str(Type), ":", str(ListenerName)])). diff --git a/apps/emqx/src/emqx_logger_textfmt.erl b/apps/emqx/src/emqx_logger_textfmt.erl index 94af0ca2a..986c0fd8a 100644 --- a/apps/emqx/src/emqx_logger_textfmt.erl +++ b/apps/emqx/src/emqx_logger_textfmt.erl @@ -32,7 +32,7 @@ enrich(Report, #{mfa := Mfa, line := Line}) -> enrich(Report, _) -> Report. enrich_fmt({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) -> - {Fmt ++ " mfa: ~s line: ~w", Args ++ [mfa(Mfa), Line]}; + {Fmt ++ " mfa: ~ts line: ~w", Args ++ [mfa(Mfa), Line]}; enrich_fmt(Msg, _) -> Msg. diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 740c29290..5f04cf403 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -184,6 +184,15 @@ {counter, 'session.terminated'} ]). +%% Overload protetion counters +-define(OLP_METRICS, + [{counter, 'olp.delay.ok'}, + {counter, 'olp.delay.timeout'}, + {counter, 'olp.hbn'}, + {counter, 'olp.gc'}, + {counter, 'olp.new_conn'} + ]). + -record(state, {next_idx = 1}). -record(metric, {name, type, idx}). @@ -430,7 +439,8 @@ init([]) -> ?MESSAGE_METRICS, ?DELIVERY_METRICS, ?CLIENT_METRICS, - ?SESSION_METRICS + ?SESSION_METRICS, + ?OLP_METRICS ]), % Store reserved indices ok = lists:foreach(fun({Type, Name}) -> @@ -575,5 +585,11 @@ reserved_idx('session.takeovered') -> 222; reserved_idx('session.discarded') -> 223; reserved_idx('session.terminated') -> 224; +reserved_idx('olp.delay.ok') -> 300; +reserved_idx('olp.delay.timeout') -> 301; +reserved_idx('olp.hbn') -> 302; +reserved_idx('olp.gc') -> 303; +reserved_idx('olp.new_conn') -> 304; + reserved_idx(_) -> undefined. diff --git a/apps/emqx/src/emqx_olp.erl b/apps/emqx/src/emqx_olp.erl new file mode 100644 index 000000000..df97beb35 --- /dev/null +++ b/apps/emqx/src/emqx_olp.erl @@ -0,0 +1,136 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_olp). + +-include_lib("lc/include/lc.hrl"). + +-export([ is_overloaded/0 + , backoff/1 + , backoff_gc/1 + , backoff_hibernation/1 + , backoff_new_conn/1 + ]). + + +%% exports for O&M +-export([ status/0 + , enable/0 + , disable/0 + ]). + +-type cfg_key() :: + backoff_gc | + backoff_hibernation | + backoff_new_conn. + +-type cnt_name() :: + 'olp.delay.ok' | + 'olp.delay.timeout' | + 'olp.hbn' | + 'olp.gc' | + 'olp.new_conn'. + +-define(overload_protection, overload_protection). + +%% @doc Light realtime check if system is overloaded. +-spec is_overloaded() -> boolean(). +is_overloaded() -> + load_ctl:is_overloaded(). + +%% @doc Backoff with a delay if the system is overloaded, for tasks that could be deferred. +%% returns `false' if backoff didn't happen, the system is cool. +%% returns `ok' if backoff is triggered and get unblocked when the system is cool. +%% returns `timeout' if backoff is trigged but get unblocked due to timeout as configured. +-spec backoff(Zone :: atom()) -> ok | false | timeout. +backoff(Zone) -> + case emqx_config:get_zone_conf(Zone, [?overload_protection]) of + #{enable := true, backoff_delay := Delay} -> + case load_ctl:maydelay(Delay) of + false -> false; + ok -> + emqx_metrics:inc('olp.delay.ok'), + ok; + timeout -> + emqx_metrics:inc('olp.delay.timeout'), + timeout + end; + _ -> + ok + end. + +%% @doc If forceful GC should be skipped when the system is overloaded. +-spec backoff_gc(Zone :: atom()) -> boolean(). +backoff_gc(Zone) -> + do_check(Zone, ?FUNCTION_NAME, 'olp.gc'). + +%% @doc If hibernation should be skipped when the system is overloaded. +-spec backoff_hibernation(Zone :: atom()) -> boolean(). +backoff_hibernation(Zone) -> + do_check(Zone, ?FUNCTION_NAME, 'olp.hbn'). + +%% @doc Returns {error, overloaded} if new connection should be +%% closed when system is overloaded. +-spec backoff_new_conn(Zone :: atom()) -> ok | {error, overloaded}. +backoff_new_conn(Zone) -> + case do_check(Zone, ?FUNCTION_NAME, 'olp.new_conn') of + true -> + {error, overloaded}; + false -> + ok + end. + +-spec status() -> any(). +status() -> + is_overloaded(). + +%% @doc turn off backgroud runq check. +-spec disable() -> ok | {error, timeout}. +disable() -> + load_ctl:stop_runq_flagman(5000). + +%% @doc turn on backgroud runq check. +-spec enable() -> {ok, pid()} | {error, running | restarting | disabled}. +enable() -> + case load_ctl:restart_runq_flagman() of + {error, disabled} -> + OldCfg = load_ctl:get_config(), + ok = load_ctl:put_config(OldCfg#{ ?RUNQ_MON_F0 => true }), + load_ctl:restart_runq_flagman(); + Other -> + Other + end. + +%%% Internals +-spec do_check(Zone::atom(), cfg_key(), cnt_name()) -> boolean(). +do_check(Zone, Key, CntName) -> + case load_ctl:is_overloaded() of + true -> + case emqx_config:get_zone_conf(Zone, [?overload_protection]) of + #{enable := true, Key := true} -> + emqx_metrics:inc(CntName), + true; + _ -> + false + end; + false -> false + end. + + +%%%_* Emacs ==================================================================== +%%% Local Variables: +%%% allout-layout: t +%%% erlang-indent-level: 2 +%%% End: diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index d1577e0c7..60835d4ab 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -446,14 +446,14 @@ format_header(#mqtt_packet_header{type = Type, true -> <<>>; false -> [", ", S] end, - io_lib:format("~s(Q~p, R~p, D~p~s)", [type_name(Type), QoS, i(Retain), i(Dup), S1]). + io_lib:format("~ts(Q~p, R~p, D~p~ts)", [type_name(Type), QoS, i(Retain), i(Dup), S1]). format_variable(undefined, _) -> undefined; format_variable(Variable, undefined) -> format_variable(Variable); format_variable(Variable, Payload) -> - io_lib:format("~s, Payload=~0p", [format_variable(Variable), Payload]). + io_lib:format("~ts, Payload=~0p", [format_variable(Variable), Payload]). format_variable(#mqtt_packet_connect{ proto_ver = ProtoVer, @@ -468,10 +468,10 @@ format_variable(#mqtt_packet_connect{ will_payload = WillPayload, username = Username, password = Password}) -> - Format = "ClientId=~s, ProtoName=~s, ProtoVsn=~p, CleanStart=~s, KeepAlive=~p, Username=~s, Password=~s", + Format = "ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts", Args = [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)], {Format1, Args1} = if - WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~s, Payload=~0p)", + WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~ts, Payload=~0p)", Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]}; true -> {Format, Args} end, @@ -487,7 +487,7 @@ format_variable(#mqtt_packet_connack{ack_flags = AckFlags, format_variable(#mqtt_packet_publish{topic_name = TopicName, packet_id = PacketId}) -> - io_lib:format("Topic=~s, PacketId=~p", [TopicName, PacketId]); + io_lib:format("Topic=~ts, PacketId=~p", [TopicName, PacketId]); format_variable(#mqtt_packet_puback{packet_id = PacketId, reason_code = ReasonCode}) -> diff --git a/apps/emqx/src/emqx_plugins.erl b/apps/emqx/src/emqx_plugins.erl index e334bdb4a..c28ead717 100644 --- a/apps/emqx/src/emqx_plugins.erl +++ b/apps/emqx/src/emqx_plugins.erl @@ -151,7 +151,7 @@ load_ext_plugin(PluginDir) -> % catch % throw : {conf_file_not_found, ConfFile} -> % %% this is maybe a dependency of an external plugin - % ?LOG(debug, "config_load_error_ignored for app=~p, path=~s", [AppName, ConfFile]), + % ?LOG(debug, "config_load_error_ignored for app=~p, path=~ts", [AppName, ConfFile]), % ok % end. diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index c23aec17b..cc195419c 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -35,13 +35,19 @@ init(ConnOpts) when is_map(ConnOpts) -> -spec new_conn(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}. new_conn(Conn, S) -> process_flag(trap_exit, true), - {ok, Pid} = emqx_connection:start_link(emqx_quic_stream, {self(), Conn}, S), - receive - {Pid, stream_acceptor_ready} -> - ok = quicer:async_handshake(Conn), - {ok, S}; - {'EXIT', Pid, _Reason} -> - {error, stream_accept_error} + case emqx_olp:is_overloaded() of + false -> + {ok, Pid} = emqx_connection:start_link(emqx_quic_stream, {self(), Conn}, S), + receive + {Pid, stream_acceptor_ready} -> + ok = quicer:async_handshake(Conn), + {ok, S}; + {'EXIT', Pid, _Reason} -> + {error, stream_accept_error} + end; + true -> + emqx_metrics:inc('olp.new_conn'), + {error, overloaded} end. -spec connected(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}. diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index afc1c3f87..3e5475c13 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -177,7 +177,7 @@ topics() -> -spec(print_routes(emqx_types:topic()) -> ok). print_routes(Topic) -> lists:foreach(fun(#route{topic = To, dest = Dest}) -> - io:format("~s -> ~s~n", [To, Dest]) + io:format("~ts -> ~ts~n", [To, Dest]) end, match_routes(Topic)). call(Router, Msg) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index a2fb13bab..a60ff468d 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -72,6 +72,7 @@ -export([namespace/0, roots/0, roots/1, fields/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1, default_ciphers/1]). +-export([sc/2, map/2]). namespace() -> undefined. @@ -122,6 +123,9 @@ roots(medium) -> , {"force_shutdown", sc(ref("force_shutdown"), #{})} + , {"overload_protection", + sc(ref("overload_protection"), + #{})} ]; roots(low) -> [ {"force_gc", @@ -323,7 +327,9 @@ fields("mqtt") -> fields("zone") -> Fields = ["mqtt", "stats", "flapping_detect", "force_shutdown", - "conn_congestion", "rate_limit", "quota", "force_gc"], + "conn_congestion", "rate_limit", "quota", "force_gc", + "overload_protection" + ], [{F, ref(emqx_zone_schema, F)} || F <- Fields]; fields("rate_limit") -> @@ -391,6 +397,35 @@ fields("force_shutdown") -> })} ]; +fields("overload_protection") -> + [ {"enable", + sc(boolean(), + #{ desc => "React on system overload or not" + , default => false + })} + , {"backoff_delay", + sc(range(0, inf), + #{ desc => "Some unimporant tasks could be delayed" + "for execution, here set the delays in ms" + , default => 1 + })} + , {"backoff_gc", + sc(boolean(), + #{ desc => "Skip forceful GC if necessary" + , default => false + })} + , {"backoff_hibernation", + sc(boolean(), + #{ desc => "Skip process hibernation if necessary" + , default => true + })} + , {"backoff_new_conn", + sc(boolean(), + #{ desc => "Close new incoming connections if necessary" + , default => true + })} + ]; + fields("conn_congestion") -> [ {"enable_alarm", sc(boolean(), @@ -1280,7 +1315,7 @@ validate_heap_size(Siz) -> (1 bsl 27) - 1 end, case Siz > MaxSiz of - true -> error(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); + true -> error(io_lib:format("force_shutdown_policy: heap-size ~ts is too large", [Siz])); false -> ok end. parse_user_lookup_fun(StrConf) -> diff --git a/apps/emqx/src/emqx_tracer.erl b/apps/emqx/src/emqx_tracer.erl index ab354ae21..512ef45aa 100644 --- a/apps/emqx/src/emqx_tracer.erl +++ b/apps/emqx/src/emqx_tracer.erl @@ -70,7 +70,7 @@ trace(publish, #message{from = From, topic = Topic, payload = Payload}) when is_binary(From); is_atom(From) -> emqx_logger:info(#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} }, - "PUBLISH to ~s: ~0p", [Topic, Payload]). + "PUBLISH to ~ts: ~0p", [Topic, Payload]). %% @doc Start to trace clientid or topic. -spec(start_trace(trace_who(), logger:level() | all, string()) -> ok | {error, term()}). @@ -83,8 +83,8 @@ start_trace(Who, Level, LogFile) -> try logger:compare_levels(Level, PrimaryLevel) of lt -> {error, - io_lib:format("Cannot trace at a log level (~s) " - "lower than the primary log level (~s)", + io_lib:format("Cannot trace at a log level (~ts) " + "lower than the primary log level (~ts)", [Level, PrimaryLevel])}; _GtOrEq -> install_trace_handler(Who, Level, LogFile) diff --git a/apps/emqx/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl index 537330b01..2c19df1ea 100644 --- a/apps/emqx/src/emqx_vm.erl +++ b/apps/emqx/src/emqx_vm.erl @@ -33,7 +33,6 @@ , process_gc_info_keys/0 , get_process_gc_info/0 , get_process_gc_info/1 - , get_process_group_leader_info/1 , get_process_limit/0 ]). @@ -316,9 +315,6 @@ get_process_gc_info() -> get_process_gc_info(Pid) when is_pid(Pid) -> process_info(Pid, ?PROCESS_GC_KEYS). -get_process_group_leader_info(LeaderPid) when is_pid(LeaderPid) -> - [{Key, Value}|| {Key, Value} <- process_info(LeaderPid), lists:member(Key, ?PROCESS_INFO_KEYS)]. - get_process_limit() -> erlang:system_info(process_limit). diff --git a/apps/emqx/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl index 158dce848..dca38dc36 100644 --- a/apps/emqx/test/emqx_SUITE.erl +++ b/apps/emqx/test/emqx_SUITE.erl @@ -23,14 +23,14 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_emqx_pubsub_api(_) -> true = emqx:is_running(node()), diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl index 00a1f9fbe..dc7745e42 100644 --- a/apps/emqx/test/emqx_access_control_SUITE.erl +++ b/apps/emqx/test/emqx_access_control_SUITE.erl @@ -22,15 +22,15 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules([router, broker]), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules([router, broker]), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_authenticate(_) -> ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())). diff --git a/apps/emqx/test/emqx_alarm_SUITE.erl b/apps/emqx/test/emqx_alarm_SUITE.erl index 453061f9f..0a720ffc1 100644 --- a/apps/emqx/test/emqx_alarm_SUITE.erl +++ b/apps/emqx/test/emqx_alarm_SUITE.erl @@ -23,29 +23,29 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_testcase(t_size_limit, Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), {ok, _} = emqx:update_config([alarm], #{ <<"size_limit">> => 2 }), Config; init_per_testcase(t_validity_period, Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), {ok, _} = emqx:update_config([alarm], #{ <<"validity_period">> => <<"1s">> }), Config; init_per_testcase(_, Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_testcase(_, _Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_alarm(_) -> ok = emqx_alarm:activate(unknown_alarm), diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index e4684649d..c4ed85125 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -82,15 +82,15 @@ destroy(_State) -> ok. all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> application:set_env(ekka, strict_mode, true), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_) -> - emqx_ct_helpers:stop_apps([]), + emqx_common_test_helpers:stop_apps([]), ok. init_per_testcase(Case, Config) -> @@ -297,7 +297,7 @@ update_config(Path, ConfigRequest) -> emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). certs(Certs) -> - CertsPath = emqx_ct_helpers:deps_path(emqx, "etc/certs"), + CertsPath = emqx_common_test_helpers:deps_path(emqx, "etc/certs"), lists:foldl(fun({Key, Filename}, Acc) -> {ok, Bin} = file:read_file(filename:join([CertsPath, Filename])), Acc#{Key => Bin} diff --git a/apps/emqx/test/emqx_authz_cache_SUITE.erl b/apps/emqx/test/emqx_authz_cache_SUITE.erl index 46a4d7d74..752302241 100644 --- a/apps/emqx/test/emqx_authz_cache_SUITE.erl +++ b/apps/emqx/test/emqx_authz_cache_SUITE.erl @@ -21,15 +21,15 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). %%-------------------------------------------------------------------- %% Test cases diff --git a/apps/emqx/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl index 6e69a7371..b215b5055 100644 --- a/apps/emqx/test/emqx_banned_SUITE.erl +++ b/apps/emqx/test/emqx_banned_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> application:load(emqx), diff --git a/apps/emqx/test/emqx_batch_SUITE.erl b/apps/emqx/test/emqx_batch_SUITE.erl index 856a9b1ad..5f8d01a1a 100644 --- a/apps/emqx/test/emqx_batch_SUITE.erl +++ b/apps/emqx/test/emqx_batch_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_batch_full_commit(_) -> B0 = emqx_batch:init(#{batch_size => 3, diff --git a/apps/emqx/test/emqx_boot_SUITE.erl b/apps/emqx/test/emqx_boot_SUITE.erl index f46647ad5..8b372acbd 100644 --- a/apps/emqx/test/emqx_boot_SUITE.erl +++ b/apps/emqx/test/emqx_boot_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_is_enabled(_) -> ok = application:set_env(emqx, boot_modules, all), diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index fbc374f90..b6a99b8bc 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -27,15 +27,15 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). init_per_testcase(Case, Config) -> ?MODULE:Case({init, Config}). diff --git a/apps/emqx/test/emqx_broker_helper_SUITE.erl b/apps/emqx/test/emqx_broker_helper_SUITE.erl index 31b36ecdc..59b3847b0 100644 --- a/apps/emqx/test/emqx_broker_helper_SUITE.erl +++ b/apps/emqx/test/emqx_broker_helper_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_testcase(_TestCase, Config) -> emqx_broker_helper:start_link(), diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 775b40ee8..2b9051de0 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -25,7 +25,7 @@ all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). force_gc_conf() -> #{bytes => 16777216,count => 16000,enable => true}. diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index 0a3a050ac..99dee6edc 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -77,14 +77,14 @@ groups() -> ]. init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), emqx_config:put_listener_conf(ssl, default, [ssl, verify], verify_peer), emqx_listeners:restart_listener('ssl:default'), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). %%-------------------------------------------------------------------- %% Test cases for MQTT v3 @@ -324,7 +324,7 @@ tls_certcn_as_clientid(TLSVsn) -> tls_certcn_as_clientid(TLSVsn, RequiredTLSVsn) -> CN = <<"Client">>, emqx_config:put_zone_conf(default, [mqtt, peer_cert_as_clientid], cn), - SslConf = emqx_ct_helpers:client_ssl_twoway(TLSVsn), + SslConf = emqx_common_test_helpers:client_ssl_twoway(TLSVsn), {ok, Client} = emqtt:start_link([{port, 8883}, {ssl, true}, {ssl_opts, SslConf}]), {ok, _} = emqtt:connect(Client), #{clientinfo := #{clientid := CN}} = emqx_cm:get_chan_info(CN), diff --git a/apps/emqx/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl index 512e07c9c..de143ef46 100644 --- a/apps/emqx/test/emqx_cm_SUITE.erl +++ b/apps/emqx/test/emqx_cm_SUITE.erl @@ -37,15 +37,15 @@ %%-------------------------------------------------------------------- suite() -> [{timetrap, {minutes, 2}}]. -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). %%-------------------------------------------------------------------- %% TODO: Add more test cases diff --git a/apps/emqx/test/emqx_cm_locker_SUITE.erl b/apps/emqx/test/emqx_cm_locker_SUITE.erl index 90320a811..d97e958ca 100644 --- a/apps/emqx/test/emqx_cm_locker_SUITE.erl +++ b/apps/emqx/test/emqx_cm_locker_SUITE.erl @@ -21,15 +21,15 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_start_link(_) -> emqx_cm_locker:start_link(). diff --git a/apps/emqx/test/emqx_cm_registry_SUITE.erl b/apps/emqx/test/emqx_cm_registry_SUITE.erl index 35f748477..16a315da7 100644 --- a/apps/emqx/test/emqx_cm_registry_SUITE.erl +++ b/apps/emqx/test/emqx_cm_registry_SUITE.erl @@ -25,15 +25,15 @@ %% CT callbacks %%-------------------------------------------------------------------- -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). init_per_testcase(_TestCase, Config) -> Config. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl new file mode 100644 index 000000000..119f39ae2 --- /dev/null +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -0,0 +1,435 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_common_test_helpers). + +-define(THIS_APP, ?MODULE). +-include_lib("common_test/include/ct.hrl"). + +-type(special_config_handler() :: fun()). + +-type(apps() :: list(atom())). + +-export([ all/1 + , boot_modules/1 + , start_apps/1 + , start_apps/2 + , start_app/4 + , stop_apps/1 + , reload/2 + , app_path/2 + , deps_path/2 + , flush/0 + , flush/1 + ]). + +-export([ ensure_mnesia_stopped/0 + , wait_for/4 + , change_emqx_opts/1 + , change_emqx_opts/2 + , client_ssl_twoway/0 + , client_ssl_twoway/1 + , client_ssl/0 + , client_ssl/1 + , wait_mqtt_payload/1 + , not_wait_mqtt_payload/1 + , render_config_file/2 + , read_schema_configs/2 + ]). + +-define( CERTS_PATH(CertName), filename:join( [ "etc", "certs", CertName ]) ). + +-define( MQTT_SSL_TWOWAY, [ { cacertfile, ?CERTS_PATH( "cacert.pem" ) }, + { verify, verify_peer }, + { fail_if_no_peer_cert, true } ] ). + +-define( MQTT_SSL_CLIENT_CERTS, [ { keyfile, ?CERTS_PATH( "client-key.pem" ) }, + { cacertfile, ?CERTS_PATH( "cacert.pem" ) }, + { certfile, ?CERTS_PATH( "client-cert.pem" ) } ] ). + +-define( TLS_1_3_CIPHERS, [ { versions, [ 'tlsv1.3' ] }, + { ciphers, [ "TLS_AES_256_GCM_SHA384", + "TLS_AES_128_GCM_SHA256", + "TLS_CHACHA20_POLY1305_SHA256", + "TLS_AES_128_CCM_SHA256", + "TLS_AES_128_CCM_8_SHA256" + ] } + ]). + +-define( TLS_OLD_CIPHERS, [ { versions, [ 'tlsv1.1', 'tlsv1.2' ] }, + { ciphers, [ "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES256-SHA384", + "ECDHE-RSA-AES256-SHA384", + "ECDHE-ECDSA-DES-CBC3-SHA", + "ECDH-ECDSA-AES256-GCM-SHA384", + "ECDH-RSA-AES256-GCM-SHA384", + "ECDH-ECDSA-AES256-SHA384", + "ECDH-RSA-AES256-SHA384", + "DHE-DSS-AES256-GCM-SHA384", + "DHE-DSS-AES256-SHA256", + "AES256-GCM-SHA384", + "AES256-SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-SHA256", + "ECDHE-RSA-AES128-SHA256", + "ECDH-ECDSA-AES128-GCM-SHA256", + "ECDH-RSA-AES128-GCM-SHA256", + "ECDH-ECDSA-AES128-SHA256", + "ECDH-RSA-AES128-SHA256", + "DHE-DSS-AES128-GCM-SHA256", + "DHE-DSS-AES128-SHA256", + "AES128-GCM-SHA256", + "AES128-SHA256", + "ECDHE-ECDSA-AES256-SHA", + "ECDHE-RSA-AES256-SHA", + "DHE-DSS-AES256-SHA", + "ECDH-ECDSA-AES256-SHA", + "ECDH-RSA-AES256-SHA", + "AES256-SHA", + "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", + "DHE-DSS-AES128-SHA", + "ECDH-ECDSA-AES128-SHA", + "ECDH-RSA-AES128-SHA", + "AES128-SHA" + ] } + ]). + + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +all(Suite) -> + lists:usort([F || {F, 1} <- Suite:module_info(exports), + string:substr(atom_to_list(F), 1, 2) == "t_" + ]). + +-spec(boot_modules(all|list(atom())) -> ok). +boot_modules(Mods) -> + application:set_env(emqx, boot_modules, Mods). + +-spec(start_apps(Apps :: apps()) -> ok). +start_apps(Apps) -> + start_apps(Apps, fun(_) -> ok end). + +-spec(start_apps(Apps :: apps(), Handler :: special_config_handler()) -> ok). +start_apps(Apps, Handler) when is_function(Handler) -> + %% Load all application code to beam vm first + %% Because, minirest, ekka etc.. application will scan these modules + lists:foreach(fun load/1, [emqx | Apps]), + lists:foreach(fun(App) -> start_app(App, Handler) end, [emqx | Apps]). + +load(App) -> + case application:load(App) of + ok -> ok; + {error, {already_loaded, _}} -> ok; + {error, Reason} -> error({failed_to_load_app, App, Reason}) + end. + +start_app(App, Handler) -> + start_app(App, + app_schema(App), + app_path(App, filename:join(["etc", atom_to_list(App) ++ ".conf"])), + Handler). + +%% TODO: get rid of cuttlefish +app_schema(App) -> + CuttlefishSchema = app_path(App, filename:join(["priv", atom_to_list(App) ++ ".schema"])), + case filelib:is_regular(CuttlefishSchema) of + true -> + CuttlefishSchema; + false -> + Mod = list_to_atom(atom_to_list(App) ++ "_schema"), + try + true = is_list(Mod:roots()), + Mod + catch + C : E -> + error(#{app => App, + file => CuttlefishSchema, + module => Mod, + exeption => C, + reason => E + }) + end + end. + +mustache_vars(App) -> + [{platform_data_dir, app_path(App, "data")}, + {platform_etc_dir, app_path(App, "etc")}, + {platform_log_dir, app_path(App, "log")}, + {platform_plugins_dir, app_path(App, "plugins")} + ]. + +start_app(App, Schema, ConfigFile, SpecAppConfig) -> + Vars = mustache_vars(App), + RenderedConfigFile = render_config_file(ConfigFile, Vars), + read_schema_configs(Schema, RenderedConfigFile), + force_set_config_file_paths(App, [RenderedConfigFile]), + SpecAppConfig(App), + case application:ensure_all_started(App) of + {ok, _} -> ok; + {error, Reason} -> error({failed_to_start_app, App, Reason}) + end. + +render_config_file(ConfigFile, Vars0) -> + Temp = case file:read_file(ConfigFile) of + {ok, T} -> T; + {error, Reason} -> error({failed_to_read_config_template, ConfigFile, Reason}) + end, + Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- Vars0], + Targ = bbmustache:render(Temp, Vars), + NewName = ConfigFile ++ ".rendered", + ok = file:write_file(NewName, Targ), + NewName. + +read_schema_configs(Schema, ConfigFile) -> + NewConfig = generate_config(Schema, ConfigFile), + lists:foreach( + fun({App, Configs}) -> + [application:set_env(App, Par, Value) || {Par, Value} <- Configs] + end, NewConfig). + +generate_config(SchemaModule, ConfigFile) when is_atom(SchemaModule) -> + {ok, Conf0} = hocon:load(ConfigFile, #{format => richmap}), + hocon_schema:generate(SchemaModule, Conf0); +generate_config(SchemaFile, ConfigFile) -> + {ok, Conf1} = hocon:load(ConfigFile, #{format => proplists}), + Schema = cuttlefish_schema:files([SchemaFile]), + cuttlefish_generator:map(Schema, Conf1). + +-spec(stop_apps(list()) -> ok). +stop_apps(Apps) -> + [application:stop(App) || App <- Apps ++ [emqx, mnesia]]. + +%% backward compatible +deps_path(App, RelativePath) -> app_path(App, RelativePath). + +app_path(App, RelativePath) -> + ok = ensure_app_loaded(App), + Lib = code:lib_dir(App), + safe_relative_path(filename:join([Lib, RelativePath])). + +assert_app_loaded(App) -> + case code:lib_dir(App) of + {error, bad_name} -> error({not_loaded, ?THIS_APP}); + _ -> ok + end. + +ensure_app_loaded(?THIS_APP) -> + ok = assert_app_loaded(?THIS_APP); +ensure_app_loaded(App) -> + case code:lib_dir(App) of + {error, bad_name} -> + ok = assert_app_loaded(?THIS_APP), + Dir0 = code:lib_dir(?THIS_APP), + LibRoot = upper_level(Dir0), + Dir = filename:join([LibRoot, atom_to_list(App), "ebin"]), + case code:add_pathz(Dir) of + true -> ok; + {error, bad_directory} -> error({bad_directory, Dir}) + end, + case application:load(App) of + ok -> ok; + {error, Reason} -> error({failed_to_load, App, Reason}) + end, + ok = assert_app_loaded(App); + _ -> + ok + end. + +upper_level(Dir) -> + Split = filename:split(Dir), + UpperReverse = tl(lists:reverse(Split)), + filename:join(lists:reverse(UpperReverse)). + +safe_relative_path(Path) -> + case filename:split(Path) of + ["/" | T] -> + T1 = do_safe_relative_path(filename:join(T)), + filename:join(["/", T1]); + _ -> + do_safe_relative_path(Path) + end. + +do_safe_relative_path(Path) -> + case safe_relative_path_2(Path) of + unsafe -> Path; + OK -> OK + end. + +-if(?OTP_RELEASE < 23). +safe_relative_path_2(Path) -> + filename:safe_relative_path(Path). +-else. +safe_relative_path_2(Path) -> + {ok, Cwd} = file:get_cwd(), + filelib:safe_relative_path(Path, Cwd). +-endif. + +-spec(reload(App :: atom(), SpecAppConfig :: special_config_handler()) -> ok). +reload(App, SpecAppConfigHandler) -> + application:stop(App), + start_app(App, SpecAppConfigHandler), + application:start(App). + +ensure_mnesia_stopped() -> + ekka_mnesia:ensure_stopped(), + ekka_mnesia:delete_schema(). + +%% Help function to wait for Fun to yield 'true'. +wait_for(Fn, Ln, F, Timeout) -> + {Pid, Mref} = erlang:spawn_monitor(fun() -> wait_loop(F, catch_call(F)) end), + wait_for_down(Fn, Ln, Timeout, Pid, Mref, false). + +change_emqx_opts(SslType) -> + change_emqx_opts(SslType, []). + +change_emqx_opts(SslType, MoreOpts) -> + {ok, Listeners} = application:get_env(emqx, listeners), + NewListeners = + lists:map(fun(Listener) -> + maybe_inject_listener_ssl_options(SslType, MoreOpts, Listener) + end, Listeners), + application:set_env(emqx, listeners, NewListeners). + +maybe_inject_listener_ssl_options(SslType, MoreOpts, {sll, Port, Opts}) -> + %% this clause is kept to be backward compatible + %% new config for listener is a map, old is a three-element tuple + {ssl, Port, inject_listener_ssl_options(SslType, Opts, MoreOpts)}; +maybe_inject_listener_ssl_options(SslType, MoreOpts, #{proto := ssl, opts := Opts} = Listener) -> + Listener#{opts := inject_listener_ssl_options(SslType, Opts, MoreOpts)}; +maybe_inject_listener_ssl_options(_SslType, _MoreOpts, Listener) -> + Listener. + +inject_listener_ssl_options(SslType, Opts, MoreOpts) -> + SslOpts = proplists:get_value(ssl_options, Opts), + Keyfile = app_path(emqx, filename:join(["etc", "certs", "key.pem"])), + Certfile = app_path(emqx, filename:join(["etc", "certs", "cert.pem"])), + TupleList1 = lists:keyreplace(keyfile, 1, SslOpts, {keyfile, Keyfile}), + TupleList2 = lists:keyreplace(certfile, 1, TupleList1, {certfile, Certfile}), + TupleList3 = + case SslType of + ssl_twoway -> + CAfile = app_path(emqx, proplists:get_value(cacertfile, ?MQTT_SSL_TWOWAY)), + MutSslList = lists:keyreplace(cacertfile, 1, ?MQTT_SSL_TWOWAY, {cacertfile, CAfile}), + lists:merge(TupleList2, MutSslList); + _ -> + lists:filter(fun ({cacertfile, _}) -> false; + ({verify, _}) -> false; + ({fail_if_no_peer_cert, _}) -> false; + (_) -> true + end, TupleList2) + end, + TupleList4 = emqx_misc:merge_opts(TupleList3, proplists:get_value(ssl_options, MoreOpts, [])), + NMoreOpts = emqx_misc:merge_opts(MoreOpts, [{ssl_options, TupleList4}]), + emqx_misc:merge_opts(Opts, NMoreOpts). + +flush() -> + flush([]). + +flush(Msgs) -> + receive + M -> flush([M|Msgs]) + after + 0 -> lists:reverse(Msgs) + end. + +client_ssl_twoway() -> + client_ssl_twoway(default). + +client_ssl_twoway(TLSVsn) -> + client_certs() ++ ciphers(TLSVsn). + +%% Paths prepended to cert filenames +client_certs() -> + [ { Key, app_path(emqx, FilePath) } || { Key, FilePath } <- ?MQTT_SSL_CLIENT_CERTS ]. + +client_ssl() -> + client_ssl(default). + +client_ssl(TLSVsn) -> + ciphers(TLSVsn) ++ [{reuse_sessions, true}]. + +ciphers(default) -> []; %% determined via config file defaults +ciphers('tlsv1.3') -> ?TLS_1_3_CIPHERS; +ciphers(_OlderTLSVsn) -> ?TLS_OLD_CIPHERS. + +wait_mqtt_payload(Payload) -> + receive + {publish, #{payload := Payload}} -> + ct:pal("OK - received msg: ~p~n", [Payload]) + after 1000 -> + ct:fail({timeout, Payload, {msg_box, flush()}}) + end. + +not_wait_mqtt_payload(Payload) -> + receive + {publish, #{payload := Payload}} -> + ct:fail({received, Payload}) + after 1000 -> + ct:pal("OK - msg ~p is not received", [Payload]) + end. + +wait_for_down(Fn, Ln, Timeout, Pid, Mref, Kill) -> + receive + {'DOWN', Mref, process, Pid, normal} -> + ok; + {'DOWN', Mref, process, Pid, {unexpected, Result}} -> + erlang:error({unexpected, Fn, Ln, Result}); + {'DOWN', Mref, process, Pid, {crashed, {C, E, S}}} -> + erlang:raise(C, {Fn, Ln, E}, S) + after + Timeout -> + case Kill of + true -> + erlang:demonitor(Mref, [flush]), + erlang:exit(Pid, kill), + erlang:error({Fn, Ln, timeout}); + false -> + Pid ! stop, + wait_for_down(Fn, Ln, Timeout, Pid, Mref, true) + end + end. + +wait_loop(_F, ok) -> exit(normal); +wait_loop(F, LastRes) -> + receive + stop -> erlang:exit(LastRes) + after + 100 -> + Res = catch_call(F), + wait_loop(F, Res) + end. + +catch_call(F) -> + try + case F() of + true -> ok; + Other -> {unexpected, Other} + end + catch + C : E : S -> + {crashed, {C, E, S}} + end. + +force_set_config_file_paths(emqx, Paths) -> + application:set_env(emqx, config_files, Paths); +force_set_config_file_paths(_, _) -> + ok. diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index 50d575c0e..88585492b 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -20,15 +20,15 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_fill_default_values(_) -> Conf = #{ diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 5784ad6aa..987b39d77 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -23,7 +23,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- %% CT callbacks diff --git a/apps/emqx/test/emqx_ctl_SUITE.erl b/apps/emqx/test/emqx_ctl_SUITE.erl index 2907ff787..6d8455e7d 100644 --- a/apps/emqx/test/emqx_ctl_SUITE.erl +++ b/apps/emqx/test/emqx_ctl_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = emqx_logger:set_log_level(emergency), @@ -66,13 +66,13 @@ t_run_commands(_) -> t_print(_) -> ok = emqx_ctl:print("help"), - ok = emqx_ctl:print("~s", [help]), - ok = emqx_ctl:print("~s", [<<"~!@#$%^&*()">>]), + ok = emqx_ctl:print("~ts", [help]), + ok = emqx_ctl:print("~ts", [<<"~!@#$%^&*()">>]), % - check the output of the usage mock_print(), ?assertEqual("help", emqx_ctl:print("help")), - ?assertEqual("help", emqx_ctl:print("~s", [help])), - ?assertEqual("~!@#$%^&*()", emqx_ctl:print("~s", [<<"~!@#$%^&*()">>])), + ?assertEqual("help", emqx_ctl:print("~ts", [help])), + ?assertEqual("~!@#$%^&*()", emqx_ctl:print("~ts", [<<"~!@#$%^&*()">>])), unmock_print(). t_usage(_) -> diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl index a8e783c49..88a4d8ee0 100644 --- a/apps/emqx/test/emqx_flapping_SUITE.erl +++ b/apps/emqx/test/emqx_flapping_SUITE.erl @@ -21,11 +21,11 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), emqx_config:put_zone_conf(default, [flapping_detect], #{max_count => 3, window_time => 100, % 0.1s @@ -34,7 +34,7 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]), + emqx_common_test_helpers:stop_apps([]), ekka_mnesia:delete_schema(), %% Clean emqx_banned table ok. diff --git a/apps/emqx/test/emqx_gc_SUITE.erl b/apps/emqx/test/emqx_gc_SUITE.erl index 6504234f9..00c790bb9 100644 --- a/apps/emqx/test/emqx_gc_SUITE.erl +++ b/apps/emqx/test/emqx_gc_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_init(_) -> GC1 = emqx_gc:init(#{count => 10, bytes => 0}), diff --git a/apps/emqx/test/emqx_guid_SUITE.erl b/apps/emqx/test/emqx_guid_SUITE.erl index b72062067..670321f20 100644 --- a/apps/emqx/test/emqx_guid_SUITE.erl +++ b/apps/emqx/test/emqx_guid_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_guid_gen(_) -> Guid1 = emqx_guid:gen(), diff --git a/apps/emqx/test/emqx_hooks_SUITE.erl b/apps/emqx/test/emqx_hooks_SUITE.erl index be8814ca4..c79c6f2ad 100644 --- a/apps/emqx/test/emqx_hooks_SUITE.erl +++ b/apps/emqx/test/emqx_hooks_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). % t_lookup(_) -> % error('TODO'). diff --git a/apps/emqx/test/emqx_inflight_SUITE.erl b/apps/emqx/test/emqx_inflight_SUITE.erl index 6e5d8f5cc..248b50ce3 100644 --- a/apps/emqx/test/emqx_inflight_SUITE.erl +++ b/apps/emqx/test/emqx_inflight_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_contain(_) -> Inflight = emqx_inflight:insert(k, v, emqx_inflight:new()), diff --git a/apps/emqx/test/emqx_json_SUITE.erl b/apps/emqx/test/emqx_json_SUITE.erl index 8f579f7be..b29edf41a 100644 --- a/apps/emqx/test/emqx_json_SUITE.erl +++ b/apps/emqx/test/emqx_json_SUITE.erl @@ -69,7 +69,7 @@ %m #{<<"foo">> => [{}]} NOT SUPPORT %%-------------------------------------------------------------------- -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_decode_encode(_) -> null = decode(encode(null)), diff --git a/apps/emqx/test/emqx_keepalive_SUITE.erl b/apps/emqx/test/emqx_keepalive_SUITE.erl index d8a0c1316..8371e0030 100644 --- a/apps/emqx/test/emqx_keepalive_SUITE.erl +++ b/apps/emqx/test/emqx_keepalive_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_check(_) -> Keepalive = emqx_keepalive:init(60), diff --git a/apps/emqx/test/emqx_limiter_SUITE.erl b/apps/emqx/test/emqx_limiter_SUITE.erl index b22840edc..a937f67ee 100644 --- a/apps/emqx/test/emqx_limiter_SUITE.erl +++ b/apps/emqx/test/emqx_limiter_SUITE.erl @@ -25,7 +25,7 @@ %% Setups %%-------------------------------------------------------------------- -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_testcase(_, Cfg) -> _ = esockd_limiter:start_link(), diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index a3bfb2d47..649fc0065 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -23,7 +23,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> NewConfig = generate_config(), diff --git a/apps/emqx/test/emqx_logger_SUITE.erl b/apps/emqx/test/emqx_logger_SUITE.erl index d09222b6b..46e2df60e 100644 --- a/apps/emqx/test/emqx_logger_SUITE.erl +++ b/apps/emqx/test/emqx_logger_SUITE.erl @@ -25,7 +25,7 @@ -define(a, "a"). -define(SUPPORTED_LEVELS, [emergency, alert, critical, error, warning, notice, info, debug]). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_testcase(_TestCase, Config) -> Config. diff --git a/apps/emqx/test/emqx_message_SUITE.erl b/apps/emqx/test/emqx_message_SUITE.erl index 7c8435a8d..0070c7d9e 100644 --- a/apps/emqx/test/emqx_message_SUITE.erl +++ b/apps/emqx/test/emqx_message_SUITE.erl @@ -23,7 +23,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). suite() -> [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}]. diff --git a/apps/emqx/test/emqx_metrics_SUITE.erl b/apps/emqx/test/emqx_metrics_SUITE.erl index 751a26780..f47e028bc 100644 --- a/apps/emqx/test/emqx_metrics_SUITE.erl +++ b/apps/emqx/test/emqx_metrics_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_new(_) -> with_metrics_server( diff --git a/apps/emqx/test/emqx_misc_SUITE.erl b/apps/emqx/test/emqx_misc_SUITE.erl index 2ab5ee3e0..f821e9cd2 100644 --- a/apps/emqx/test/emqx_misc_SUITE.erl +++ b/apps/emqx/test/emqx_misc_SUITE.erl @@ -28,7 +28,7 @@ {nodelay, true} ]). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_merge_opts(_) -> Opts = emqx_misc:merge_opts(?SOCKOPTS, [raw, diff --git a/apps/emqx/test/emqx_mountpoint_SUITE.erl b/apps/emqx/test/emqx_mountpoint_SUITE.erl index cb62db051..e8b4c0e5c 100644 --- a/apps/emqx/test/emqx_mountpoint_SUITE.erl +++ b/apps/emqx/test/emqx_mountpoint_SUITE.erl @@ -28,7 +28,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_mount(_) -> Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), diff --git a/apps/emqx/test/emqx_mqtt_SUITE.erl b/apps/emqx/test/emqx_mqtt_SUITE.erl index 42a4e5780..3d149ee08 100644 --- a/apps/emqx/test/emqx_mqtt_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_SUITE.erl @@ -29,15 +29,15 @@ send_pend ]). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). init_per_testcase(TestCase, Config) -> case erlang:function_exported(?MODULE, TestCase, 2) of diff --git a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl index f8b5a7ab6..51be4895e 100644 --- a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_check_pub(_) -> OldConf = emqx:get_config([zones]), diff --git a/apps/emqx/test/emqx_mqtt_props_SUITE.erl b/apps/emqx/test/emqx_mqtt_props_SUITE.erl index b4dcd2f18..d2dca041f 100644 --- a/apps/emqx/test/emqx_mqtt_props_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_props_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_id(_) -> foreach_prop( diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index 8f82d83bb..caa1e7216 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -39,19 +39,19 @@ all() -> ]. groups() -> - TCs = emqx_ct:all(?MODULE), + TCs = emqx_common_test_helpers:all(?MODULE), [ {tcp, [], TCs} , {quic, [], TCs} ]. init_per_group(tcp, Config) -> - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([]), [ {port, 1883}, {conn_fun, connect} | Config]; init_per_group(quic, Config) -> - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([]), [ {port, 14567}, {conn_fun, quic_connect} | Config]; init_per_group(_, Config) -> - emqx_ct_helpers:stop_apps([]), + emqx_common_test_helpers:stop_apps([]), Config. end_per_group(_Group, _Config) -> @@ -59,12 +59,12 @@ end_per_group(_Group, _Config) -> init_per_suite(Config) -> %% Start Apps - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). init_per_testcase(TestCase, Config) -> case erlang:function_exported(?MODULE, TestCase, 2) of diff --git a/apps/emqx/test/emqx_mqueue_SUITE.erl b/apps/emqx/test/emqx_mqueue_SUITE.erl index 5b94ecbbf..3f559c4b2 100644 --- a/apps/emqx/test/emqx_mqueue_SUITE.erl +++ b/apps/emqx/test/emqx_mqueue_SUITE.erl @@ -27,7 +27,7 @@ -define(Q, emqx_mqueue). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_info(_) -> Q = ?Q:init(#{max_len => 5, store_qos0 => true}), diff --git a/apps/emqx/test/emqx_olp_SUITE.erl b/apps/emqx/test/emqx_olp_SUITE.erl new file mode 100644 index 000000000..348c4c1b5 --- /dev/null +++ b/apps/emqx/test/emqx_olp_SUITE.erl @@ -0,0 +1,118 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_olp_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("lc/include/lc.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([]). + +init_per_testcase(_, Config) -> + emqx_olp:enable(), + case wait_for(fun() -> lc_sup:whereis_runq_flagman() end, 10) of + true -> ok; + false -> + ct:fail("runq_flagman is not up") + end, + ok = load_ctl:put_config(#{ ?RUNQ_MON_F0 => true + , ?RUNQ_MON_F1 => 5 + , ?RUNQ_MON_F2 => 1 + , ?RUNQ_MON_T1 => 200 + , ?RUNQ_MON_T2 => 50 + , ?RUNQ_MON_C1 => 2 + , ?RUNQ_MON_F5 => -1 + }), + Config. + +%% Test that olp could be enabled/disabled globally +t_disable_enable(_Config) -> + Old = load_ctl:whereis_runq_flagman(), + ok = emqx_olp:disable(), + ?assert(not is_process_alive(Old)), + {ok, Pid} = emqx_olp:enable(), + timer:sleep(1000), + ?assert(is_process_alive(Pid)). + +%% Test that overload detection works +t_is_overloaded(_Config) -> + P = burst_runq(), + timer:sleep(3000), + ?assert(emqx_olp:is_overloaded()), + exit(P, kill), + timer:sleep(3000), + ?assert(not emqx_olp:is_overloaded()). + +%% Test that new conn is rejected when olp is enabled +t_overloaded_conn(_Config) -> + process_flag(trap_exit, true), + ?assert(erlang:is_process_alive(load_ctl:whereis_runq_flagman())), + emqx_config:put([overload_protection, enable], true), + P = burst_runq(), + timer:sleep(1000), + ?assert(emqx_olp:is_overloaded()), + true = emqx:is_running(node()), + {ok, C} = emqtt:start_link([{host, "localhost"}, {clientid, "myclient"}]), + ?assertNotMatch({ok, _Pid}, emqtt:connect(C)), + exit(P, kill). + +%% Test that new conn is rejected when olp is enabled +t_overload_cooldown_conn(Config) -> + t_overloaded_conn(Config), + timer:sleep(1000), + ?assert(not emqx_olp:is_overloaded()), + {ok, C} = emqtt:start_link([{host, "localhost"}, {clientid, "myclient"}]), + ?assertMatch({ok, _Pid}, emqtt:connect(C)), + emqtt:stop(C). + +-spec burst_runq() -> ParentToKill :: pid(). +burst_runq() -> + NProc = erlang:system_info(schedulers_online), + spawn(?MODULE, worker_parent, [NProc * 10, {?MODULE, busy_loop, []}]). + +%% internal helpers +worker_parent(N, {M, F, A}) -> + lists:foreach(fun(_) -> + proc_lib:spawn_link(fun() -> apply(M, F, A) end) + end, lists:seq(1, N)), + receive stop -> ok end. + +busy_loop() -> + erlang:yield(), + busy_loop(). + +wait_for(_Fun, 0) -> + false; +wait_for(Fun, Retry) -> + case is_pid(Fun()) of + true -> + true; + false -> + timer:sleep(10), + wait_for(Fun, Retry - 1) + end. diff --git a/apps/emqx/test/emqx_os_mon_SUITE.erl b/apps/emqx/test/emqx_os_mon_SUITE.erl index 674d005d3..97aa07a97 100644 --- a/apps/emqx/test/emqx_os_mon_SUITE.erl +++ b/apps/emqx/test/emqx_os_mon_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_config:put([sysmon, os], #{ diff --git a/apps/emqx/test/emqx_packet_SUITE.erl b/apps/emqx/test/emqx_packet_SUITE.erl index ae7b16a8a..bcb345cc0 100644 --- a/apps/emqx/test/emqx_packet_SUITE.erl +++ b/apps/emqx/test/emqx_packet_SUITE.erl @@ -40,7 +40,7 @@ {?AUTH, 'AUTH', ?AUTH_PACKET()} ]). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_type(_) -> lists:foreach(fun({Type, _Name, Packet}) -> @@ -290,24 +290,24 @@ t_will_msg(_) -> ?assertEqual(<<"topic">>, Msg2#message.topic). t_format(_) -> - io:format("~s", [emqx_packet:format(#mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK, retain = true, dup = 0}, variable = undefined})]), - io:format("~s", [emqx_packet:format(#mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK}, variable = 1, payload = <<"payload">>})]), - io:format("~s", [emqx_packet:format(?CONNECT_PACKET(#mqtt_packet_connect{will_flag = true, + io:format("~ts", [emqx_packet:format(#mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK, retain = true, dup = 0}, variable = undefined})]), + io:format("~ts", [emqx_packet:format(#mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK}, variable = 1, payload = <<"payload">>})]), + io:format("~ts", [emqx_packet:format(?CONNECT_PACKET(#mqtt_packet_connect{will_flag = true, will_retain = true, will_qos = ?QOS_2, will_topic = <<"topic">>, will_payload = <<"payload">>}))]), - io:format("~s", [emqx_packet:format(?CONNECT_PACKET(#mqtt_packet_connect{password = password}))]), - io:format("~s", [emqx_packet:format(?CONNACK_PACKET(?CONNACK_SERVER))]), - io:format("~s", [emqx_packet:format(?PUBLISH_PACKET(?QOS_1, 1))]), - io:format("~s", [emqx_packet:format(?PUBLISH_PACKET(?QOS_2, <<"topic">>, 10, <<"payload">>))]), - io:format("~s", [emqx_packet:format(?PUBACK_PACKET(?PUBACK, 98))]), - io:format("~s", [emqx_packet:format(?PUBREL_PACKET(99))]), - io:format("~s", [emqx_packet:format(?SUBSCRIBE_PACKET(15, [{<<"topic">>, ?QOS_0}, {<<"topic1">>, ?QOS_1}]))]), - io:format("~s", [emqx_packet:format(?SUBACK_PACKET(40, [?QOS_0, ?QOS_1]))]), - io:format("~s", [emqx_packet:format(?UNSUBSCRIBE_PACKET(89, [<<"t">>, <<"t2">>]))]), - io:format("~s", [emqx_packet:format(?UNSUBACK_PACKET(90))]), - io:format("~s", [emqx_packet:format(?DISCONNECT_PACKET(128))]). + io:format("~ts", [emqx_packet:format(?CONNECT_PACKET(#mqtt_packet_connect{password = password}))]), + io:format("~ts", [emqx_packet:format(?CONNACK_PACKET(?CONNACK_SERVER))]), + io:format("~ts", [emqx_packet:format(?PUBLISH_PACKET(?QOS_1, 1))]), + io:format("~ts", [emqx_packet:format(?PUBLISH_PACKET(?QOS_2, <<"topic">>, 10, <<"payload">>))]), + io:format("~ts", [emqx_packet:format(?PUBACK_PACKET(?PUBACK, 98))]), + io:format("~ts", [emqx_packet:format(?PUBREL_PACKET(99))]), + io:format("~ts", [emqx_packet:format(?SUBSCRIBE_PACKET(15, [{<<"topic">>, ?QOS_0}, {<<"topic1">>, ?QOS_1}]))]), + io:format("~ts", [emqx_packet:format(?SUBACK_PACKET(40, [?QOS_0, ?QOS_1]))]), + io:format("~ts", [emqx_packet:format(?UNSUBSCRIBE_PACKET(89, [<<"t">>, <<"t2">>]))]), + io:format("~ts", [emqx_packet:format(?UNSUBACK_PACKET(90))]), + io:format("~ts", [emqx_packet:format(?DISCONNECT_PACKET(128))]). t_parse_empty_publish(_) -> %% 52: 0011(type=PUBLISH) 0100 (QoS=2) diff --git a/apps/emqx/test/emqx_pd_SUITE.erl b/apps/emqx/test/emqx_pd_SUITE.erl index ecd5b9e81..5aae09953 100644 --- a/apps/emqx/test/emqx_pd_SUITE.erl +++ b/apps/emqx/test/emqx_pd_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_update_counter(_) -> ?assertEqual(undefined, emqx_pd:inc_counter(bytes, 1)), diff --git a/apps/emqx/test/emqx_plugins_SUITE.erl b/apps/emqx/test/emqx_plugins_SUITE.erl index 7a68ab8dd..3aba5a997 100644 --- a/apps/emqx/test/emqx_plugins_SUITE.erl +++ b/apps/emqx/test/emqx_plugins_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> @@ -31,23 +31,23 @@ init_per_suite(Config) -> DataPath = proplists:get_value(data_dir, Config), AppPath = filename:join([DataPath, "emqx_mini_plugin"]), HoconPath = filename:join([DataPath, "emqx_hocon_plugin"]), - Cmd = lists:flatten(io_lib:format("cd ~s && make", [AppPath])), - CmdPath = lists:flatten(io_lib:format("cd ~s && make", [HoconPath])), + Cmd = lists:flatten(io_lib:format("cd ~ts && make", [AppPath])), + CmdPath = lists:flatten(io_lib:format("cd ~ts && make", [HoconPath])), - ct:pal("Executing ~s~n", [Cmd]), - ct:pal("~n ~s~n", [os:cmd(Cmd)]), + ct:pal("Executing ~ts~n", [Cmd]), + ct:pal("~n ~ts~n", [os:cmd(Cmd)]), - ct:pal("Executing ~s~n", [CmdPath]), - ct:pal("~n ~s~n", [os:cmd(CmdPath)]), + ct:pal("Executing ~ts~n", [CmdPath]), + ct:pal("~n ~ts~n", [os:cmd(CmdPath)]), - emqx_ct_helpers:boot_modules([]), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules([]), + emqx_common_test_helpers:start_apps([]), emqx_config:put([plugins, expand_plugins_dir], DataPath), ?assertEqual(ok, emqx_plugins:load()), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_load(_) -> ?assertEqual(ok, emqx_plugins:load()), diff --git a/apps/emqx/test/emqx_pmon_SUITE.erl b/apps/emqx/test/emqx_pmon_SUITE.erl index 382f885a9..2c6600f8e 100644 --- a/apps/emqx/test/emqx_pmon_SUITE.erl +++ b/apps/emqx/test/emqx_pmon_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). % t_new(_) -> % error('TODO'). diff --git a/apps/emqx/test/emqx_pqueue_SUITE.erl b/apps/emqx/test/emqx_pqueue_SUITE.erl index 4930f547d..a51c21473 100644 --- a/apps/emqx/test/emqx_pqueue_SUITE.erl +++ b/apps/emqx/test/emqx_pqueue_SUITE.erl @@ -24,7 +24,7 @@ -define(PQ, emqx_pqueue). -define(SUITE, ?MODULE). -all() -> emqx_ct:all(?SUITE). +all() -> emqx_common_test_helpers:all(?SUITE). t_is_queue(_) -> Q = ?PQ:new(), diff --git a/apps/emqx/test/emqx_reason_codes_SUITE.erl b/apps/emqx/test/emqx_reason_codes_SUITE.erl index fe261899f..1c02df197 100644 --- a/apps/emqx/test/emqx_reason_codes_SUITE.erl +++ b/apps/emqx/test/emqx_reason_codes_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_frame_error(_) -> ?assertEqual(?RC_PACKET_TOO_LARGE, emqx_reason_codes:frame_error(frame_too_large)), diff --git a/apps/emqx/test/emqx_request_handler.erl b/apps/emqx/test/emqx_request_handler.erl index 62bcf4260..a9b612312 100644 --- a/apps/emqx/test/emqx_request_handler.erl +++ b/apps/emqx/test/emqx_request_handler.erl @@ -65,7 +65,7 @@ handle_msg(ReqMsg, RequestHandler, Parent) -> props = RspProps, payload = RspPayload }, - emqx_logger:debug("~p sending response msg to topic ~s with~n" + emqx_logger:debug("~p sending response msg to topic ~ts with~n" "corr-data=~p~npayload=~p", [?MODULE, RspTopic, CorrData, RspPayload]), ok = send_response(RspMsg); diff --git a/apps/emqx/test/emqx_request_responser_SUITE.erl b/apps/emqx/test/emqx_request_responser_SUITE.erl index 91d50d280..6b5066904 100644 --- a/apps/emqx/test/emqx_request_responser_SUITE.erl +++ b/apps/emqx/test/emqx_request_responser_SUITE.erl @@ -22,12 +22,12 @@ -include_lib("common_test/include/ct.hrl"). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). all() -> [request_response]. diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index d16659844..aeb60fc2c 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -24,15 +24,15 @@ -define(R, emqx_router). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules([router]), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules([router]), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). init_per_testcase(_TestCase, Config) -> clear_tables(), diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl index 553f88b61..460b01543 100644 --- a/apps/emqx/test/emqx_router_helper_SUITE.erl +++ b/apps/emqx/test/emqx_router_helper_SUITE.erl @@ -23,14 +23,14 @@ -define(ROUTER_HELPER, emqx_router_helper). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_monitor(_) -> ok = emqx_router_helper:monitor({undefined, node()}), diff --git a/apps/emqx/test/emqx_sequence_SUITE.erl b/apps/emqx/test/emqx_sequence_SUITE.erl index 4a622f5c8..10c11b7b7 100644 --- a/apps/emqx/test/emqx_sequence_SUITE.erl +++ b/apps/emqx/test/emqx_sequence_SUITE.erl @@ -27,7 +27,7 @@ , reclaim/2 ]). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). % t_currval(_) -> % error('TODO'). diff --git a/apps/emqx/test/emqx_session_SUITE.erl b/apps/emqx/test/emqx_session_SUITE.erl index f52dacc14..70ae5310e 100644 --- a/apps/emqx/test/emqx_session_SUITE.erl +++ b/apps/emqx/test/emqx_session_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- %% CT callbacks diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 17ed1026b..d9b50d90a 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -26,21 +26,21 @@ -define(SUITE, ?MODULE). -define(wait(For, Timeout), - emqx_ct_helpers:wait_for( + emqx_common_test_helpers:wait_for( ?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). -define(ack, shared_sub_ack). -define(no_ack, no_ack). -all() -> emqx_ct:all(?SUITE). +all() -> emqx_common_test_helpers:all(?SUITE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_is_ack_required(_) -> ?assertEqual(false, emqx_shared_sub:is_ack_required(#message{headers = #{}})). diff --git a/apps/emqx/test/emqx_stats_SUITE.erl b/apps/emqx/test/emqx_stats_SUITE.erl index b37f0975b..ef53529e2 100644 --- a/apps/emqx/test/emqx_stats_SUITE.erl +++ b/apps/emqx/test/emqx_stats_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_cast_useless_msg(_)-> emqx_stats:setstat('notExis', 1), diff --git a/apps/emqx/test/emqx_sup_SUITE.erl b/apps/emqx/test/emqx_sup_SUITE.erl index 51672e1b6..185e1d752 100644 --- a/apps/emqx/test/emqx_sup_SUITE.erl +++ b/apps/emqx/test/emqx_sup_SUITE.erl @@ -21,15 +21,15 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_child(_) -> ?assertMatch({error, _}, emqx_sup:start_child(undef, worker)), diff --git a/apps/emqx/test/emqx_sys_SUITE.erl b/apps/emqx/test/emqx_sys_SUITE.erl index 354e1a8a2..eb8112772 100644 --- a/apps/emqx/test/emqx_sys_SUITE.erl +++ b/apps/emqx/test/emqx_sys_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> application:load(emqx), diff --git a/apps/emqx/test/emqx_sys_mon_SUITE.erl b/apps/emqx/test/emqx_sys_mon_SUITE.erl index 53770f7e2..1997312c7 100644 --- a/apps/emqx/test/emqx_sys_mon_SUITE.erl +++ b/apps/emqx/test/emqx_sys_mon_SUITE.erl @@ -42,11 +42,11 @@ fmt("long_schedule warning: port = ~p", [?FAKE_PORT]), ?FAKE_INFO} ]). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_testcase(t_sys_mon, Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([], + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([], fun(emqx) -> application:set_env(emqx, sysmon, [{busy_dist_port,true}, {busy_port,false}, @@ -58,8 +58,8 @@ init_per_testcase(t_sys_mon, Config) -> end), Config; init_per_testcase(t_sys_mon2, Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([], + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([], fun(emqx) -> application:set_env(emqx, sysmon, [{busy_dist_port,false}, {busy_port,true}, @@ -72,12 +72,12 @@ init_per_testcase(t_sys_mon2, Config) -> end), Config; init_per_testcase(_, Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_testcase(_, _Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_procinfo(_) -> ok = meck:new(emqx_vm, [passthrough, no_history]), diff --git a/apps/emqx/test/emqx_tables_SUITE.erl b/apps/emqx/test/emqx_tables_SUITE.erl index 17468cf17..f1068dfef 100644 --- a/apps/emqx/test/emqx_tables_SUITE.erl +++ b/apps/emqx/test/emqx_tables_SUITE.erl @@ -23,7 +23,7 @@ -define(TAB, ?MODULE). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_new(_) -> ok = emqx_tables:new(?TAB), diff --git a/apps/emqx/test/emqx_takeover_SUITE.erl b/apps/emqx/test/emqx_takeover_SUITE.erl index ebc942e7d..70561d9cc 100644 --- a/apps/emqx/test/emqx_takeover_SUITE.erl +++ b/apps/emqx/test/emqx_takeover_SUITE.erl @@ -29,14 +29,14 @@ %%-------------------------------------------------------------------- %% Inital funcs -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]), + emqx_common_test_helpers:stop_apps([]), ok. %%-------------------------------------------------------------------- %% Testcases diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl index e8262a8ec..76b95d9bc 100644 --- a/apps/emqx/test/emqx_topic_SUITE.erl +++ b/apps/emqx/test/emqx_topic_SUITE.erl @@ -36,7 +36,7 @@ -define(N, 100000). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_wildcard(_) -> true = wildcard(<<"a/b/#">>), @@ -214,5 +214,5 @@ bench(Case, Fun, Args) -> [fun(_) -> apply(Fun, Args) end, lists:seq(1, ?N) ]), - ct:pal("Time consumed by ~s: ~.3f(us)~nCall ~s per second: ~w", + ct:pal("Time consumed by ~ts: ~.3f(us)~nCall ~ts per second: ~w", [Case, Time/?N, Case, (?N * 1000000) div Time]). diff --git a/apps/emqx/test/emqx_tracer_SUITE.erl b/apps/emqx/test/emqx_tracer_SUITE.erl index f5af65440..f6f4c7a5b 100644 --- a/apps/emqx/test/emqx_tracer_SUITE.erl +++ b/apps/emqx/test/emqx_tracer_SUITE.erl @@ -26,12 +26,12 @@ all() -> [t_trace_clientid, t_trace_topic]. init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_trace_clientid(_Config) -> {ok, T} = emqtt:start_link([{host, "localhost"}, diff --git a/apps/emqx/test/emqx_trie_SUITE.erl b/apps/emqx/test/emqx_trie_SUITE.erl index 51106f529..e22e643b0 100644 --- a/apps/emqx/test/emqx_trie_SUITE.erl +++ b/apps/emqx/test/emqx_trie_SUITE.erl @@ -30,7 +30,7 @@ all() -> ]. groups() -> - Cases = emqx_ct:all(?MODULE), + Cases = emqx_common_test_helpers:all(?MODULE), [{compact, Cases}, {not_compact, Cases}]. init_per_group(compact, Config) -> diff --git a/apps/emqx/test/emqx_vm_SUITE.erl b/apps/emqx/test/emqx_vm_SUITE.erl index 51905010a..2c3edff6e 100644 --- a/apps/emqx/test/emqx_vm_SUITE.erl +++ b/apps/emqx/test/emqx_vm_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_load(_Config) -> ?assertMatch([{load1, _}, {load5, _}, {load15, _}], emqx_vm:loads()). @@ -73,9 +73,6 @@ t_get_memory(_Config) -> t_schedulers(_Config) -> emqx_vm:schedulers(). -t_get_process_group_leader_info(_Config) -> - emqx_vm:get_process_group_leader_info(self()). - t_get_process_limit(_Config) -> emqx_vm:get_process_limit(). diff --git a/apps/emqx/test/emqx_vm_mon_SUITE.erl b/apps/emqx/test/emqx_vm_mon_SUITE.erl index 5b39746a1..3eaa4de2e 100644 --- a/apps/emqx/test/emqx_vm_mon_SUITE.erl +++ b/apps/emqx/test/emqx_vm_mon_SUITE.erl @@ -21,11 +21,11 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_testcase(t_alarms, Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), emqx_config:put([sysmon, vm], #{ process_high_watermark => 0, process_low_watermark => 0, @@ -35,12 +35,12 @@ init_per_testcase(t_alarms, Config) -> {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_vm_mon), Config; init_per_testcase(_, Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), Config. end_per_testcase(_, _Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_alarms(_) -> timer:sleep(500), diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index b7484ba90..116605a96 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -35,7 +35,7 @@ -define(ws_conn, emqx_ws_connection). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- %% CT callbacks @@ -79,7 +79,7 @@ init_per_testcase(TestCase, Config) when Config; init_per_testcase(_, Config) -> - ok = emqx_ct_helpers:start_apps([]), + ok = emqx_common_test_helpers:start_apps([]), Config. end_per_testcase(TestCase, _Config) when @@ -98,7 +98,7 @@ end_per_testcase(TestCase, _Config) when ]); end_per_testcase(_, Config) -> - emqx_ct_helpers:stop_apps([]), + emqx_common_test_helpers:stop_apps([]), Config. %%-------------------------------------------------------------------- diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 827e08dab..0f89c68d8 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1991,13 +1991,13 @@ convert_certs(Config) -> serialize_error({not_found, {authenticator, ID}}) -> {404, #{code => <<"NOT_FOUND">>, message => list_to_binary( - io_lib:format("Authenticator '~s' does not exist", [ID]) + io_lib:format("Authenticator '~ts' does not exist", [ID]) )}}; serialize_error({not_found, {listener, ID}}) -> {404, #{code => <<"NOT_FOUND">>, message => list_to_binary( - io_lib:format("Listener '~s' does not exist", [ID]) + io_lib:format("Listener '~ts' does not exist", [ID]) )}}; serialize_error({not_found, {chain, ?GLOBAL}}) -> @@ -2007,13 +2007,13 @@ serialize_error({not_found, {chain, ?GLOBAL}}) -> serialize_error({not_found, {chain, Name}}) -> {400, #{code => <<"BAD_REQUEST">>, message => list_to_binary( - io_lib:format("No authentication has been create for listener '~s'", [Name]) + io_lib:format("No authentication has been create for listener '~ts'", [Name]) )}}; serialize_error({already_exists, {authenticator, ID}}) -> {409, #{code => <<"ALREADY_EXISTS">>, message => list_to_binary( - io_lib:format("Authenticator '~s' already exist", [ID]) + io_lib:format("Authenticator '~ts' already exist", [ID]) )}}; serialize_error(no_available_provider) -> diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index e0f37a50d..3daece601 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -77,7 +77,7 @@ mnesia(copy) -> %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn:scram:builtin-db". +namespace() -> "authn-scram-builtin_db". roots() -> [config]. @@ -148,7 +148,7 @@ add_user(#{user_id := UserID, case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> IsSuperuser = maps:get(is_superuser, UserInfo, false), - add_user(UserID, Password, IsSuperuser, State), + add_user(UserGroup, UserID, Password, IsSuperuser, State), {ok, #{user_id => UserID, is_superuser => IsSuperuser}}; [_] -> {error, already_exist} @@ -240,9 +240,9 @@ check_client_final_message(Bin, #{is_superuser := IsSuperuser} = Cache, #{algori {error, not_authorized} end. -add_user(UserID, Password, IsSuperuser, State) -> +add_user(UserGroup, UserID, Password, IsSuperuser, State) -> {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(Password, State), - UserInfo = #user_info{user_id = UserID, + UserInfo = #user_info{user_id = {UserGroup, UserID}, stored_key = StoredKey, server_key = ServerKey, salt = Salt, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 22f701220..0c5d3f7f7 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -40,7 +40,7 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn:password-based:http-server". +namespace() -> "authn-password_based-http_server". roots() -> [ {config, {union, [ hoconsc:ref(?MODULE, get) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 327e356f2..1b7c5b87d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -37,7 +37,7 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn:jwt". +namespace() -> "authn-jwt". roots() -> [ {config, {union, [ hoconsc:mk('hmac-based') diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index b69d613f8..c3b6badf7 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -84,7 +84,7 @@ mnesia(copy) -> %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn:password-based:builtin-db". +namespace() -> "authn-password_based-builtin_db". roots() -> [config]. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index e2459ffe8..7b986e6f9 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -39,7 +39,7 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn:password-based:mongodb". +namespace() -> "authn-password_based-mongodb". roots() -> [ {config, {union, [ hoconsc:mk(standalone) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 43a9bd252..b4c6dac08 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -39,7 +39,7 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn:password-based:mysql". +namespace() -> "authn-password_based-mysql". roots() -> [config]. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 99b83844b..5f1005e9f 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -40,7 +40,7 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn:password-based:postgresql". +namespace() -> "authn-password_based-postgresql". roots() -> [config]. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 9b8dbefbf..cb04b0274 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -39,7 +39,7 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn:password-based:redis". +namespace() -> "authn-password_based-redis". roots() -> [ {config, {union, [ hoconsc:mk(standalone) diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl index 74ec397cc..1ee419bb0 100644 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl @@ -19,4 +19,4 @@ -compile(export_all). -compile(nowarn_export_all). -all() -> emqx_ct:all(?MODULE). \ No newline at end of file +all() -> emqx_common_test_helpers:all(?MODULE). \ No newline at end of file diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 16c04771d..170449dcc 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -27,14 +27,14 @@ % -define(AUTH, emqx_authn). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). % init_per_suite(Config) -> -% emqx_ct_helpers:start_apps([emqx_authn]), +% emqx_common_test_helpers:start_apps([emqx_authn]), % Config. % end_per_suite(_) -> -% emqx_ct_helpers:stop_apps([emqx_authn]), +% emqx_common_test_helpers:stop_apps([emqx_authn]), % ok. % t_jwt_authenticator(_) -> diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index 959cf0323..a21d0e9ad 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -27,14 +27,14 @@ % -define(AUTH, emqx_authn). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). % init_per_suite(Config) -> -% emqx_ct_helpers:start_apps([emqx_authn]), +% emqx_common_test_helpers:start_apps([emqx_authn]), % Config. % end_per_suite(_) -> -% emqx_ct_helpers:stop_apps([emqx_authn]), +% emqx_common_test_helpers:stop_apps([emqx_authn]), % ok. % t_mnesia_authenticator(_) -> diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index d8ab36b32..44d4184de 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -73,10 +73,10 @@ move(Type, Position, Opts) -> update(Cmd, Sources) -> update(Cmd, Sources, #{}). -update({replace, Type}, Sources, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {{replace, type(Type)}, Sources}, Opts); -update({delete, Type}, Sources, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {{delete, type(Type)}, Sources}, Opts); +update({?CMD_REPLCAE, Type}, Sources, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {{?CMD_REPLCAE, type(Type)}, Sources}, Opts); +update({?CMD_DELETE, Type}, Sources, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {{?CMD_DELETE, type(Type)}, Sources}, Opts); update(Cmd, Sources, Opts) -> emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}, Opts). @@ -102,12 +102,12 @@ do_update({?CMD_APPEND, Sources}, Conf) when is_list(Sources), is_list(Conf) -> NConf = Conf ++ Sources, ok = check_dup_types(NConf), NConf; -do_update({{replace, Type}, Source}, Conf) when is_map(Source), is_list(Conf) -> +do_update({{?CMD_REPLCAE, Type}, Source}, Conf) when is_map(Source), is_list(Conf) -> {_Old, Front, Rear} = take(Type, Conf), NConf = Front ++ [Source | Rear], ok = check_dup_types(NConf), NConf; -do_update({{delete, Type}, _Source}, Conf) when is_list(Conf) -> +do_update({{?CMD_DELETE, Type}, _Source}, Conf) when is_list(Conf) -> {_Old, Front, Rear} = take(Type, Conf), NConf = Front ++ Rear, NConf; @@ -138,14 +138,14 @@ do_post_update({?CMD_APPEND, Sources}, _NewSources) -> InitedSources = init_sources(check_sources(Sources)), emqx_hooks:put('client.authorize', {?MODULE, authorize, [lookup() ++ InitedSources]}, -1), ok = emqx_authz_cache:drain_cache(); -do_post_update({{replace, Type}, #{type := Type} = Source}, _NewSources) when is_map(Source) -> +do_post_update({{?CMD_REPLCAE, Type}, Source}, _NewSources) when is_map(Source) -> OldInitedSources = lookup(), {OldSource, Front, Rear} = take(Type, OldInitedSources), ok = ensure_resource_deleted(OldSource), InitedSources = init_sources(check_sources([Source])), ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Front ++ InitedSources ++ Rear]}, -1), ok = emqx_authz_cache:drain_cache(); -do_post_update({{delete, Type}, _Source}, _NewSources) -> +do_post_update({{?CMD_DELETE, Type}, _Source}, _NewSources) -> OldInitedSources = lookup(), {OldSource, Front, Rear} = take(Type, OldInitedSources), ok = ensure_resource_deleted(OldSource), @@ -202,13 +202,13 @@ init_source(#{type := file, {ok, Terms} -> [emqx_authz_rule:compile(Term) || Term <- Terms]; {error, eacces} -> - ?LOG(alert, "Insufficient permissions to read the ~s file", [Path]), + ?LOG(alert, "Insufficient permissions to read the ~ts file", [Path]), error(eaccess); {error, enoent} -> - ?LOG(alert, "The ~s file does not exist", [Path]), + ?LOG(alert, "The ~ts file does not exist", [Path]), error(enoent); {error, Reason} -> - ?LOG(alert, "Failed to read ~s: ~p", [Path, Reason]), + ?LOG(alert, "Failed to read ~ts: ~p", [Path, Reason]), error(Reason) end, Source#{annotations => #{rules => Rules}}; @@ -315,7 +315,7 @@ find_action_in_hooks() -> Action. gen_id(Type) -> - iolist_to_binary([io_lib:format("~s_~s",[?APP, Type])]). + iolist_to_binary([io_lib:format("~ts_~ts",[?APP, Type])]). create_resource(#{type := DB, annotations := #{id := ResourceID}} = Source) -> diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index e0d4667cb..784519d54 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -379,7 +379,7 @@ source(get, #{bindings := #{type := Type}}) -> end; source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable}}) -> {ok, Filename} = write_file(maps:get(path, emqx_authz:lookup(file), ""), Rules), - case emqx_authz:update({?CMD_REPLCAE, file}, #{<<"type">> => file, <<"enable">> => Enable, <<"path">> => Filename}) of + case emqx_authz:update({?CMD_REPLCAE, <<"file">>}, #{<<"type">> => <<"file">>, <<"enable">> => Enable, <<"path">> => Filename}) of {ok, _} -> {204}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 50c2ec3ff..711863628 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -177,7 +177,7 @@ query() -> connector_fields(DB) -> connector_fields(DB, config). connector_fields(DB, Fields) -> - Mod0 = io_lib:format("~s_~s",[emqx_connector, DB]), + Mod0 = io_lib:format("~ts_~ts",[emqx_connector, DB]), Mod = try list_to_existing_atom(Mod0) catch diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 16bf39d49..d91de68a0 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -25,7 +25,7 @@ -define(CONF_DEFAULT, <<"authorization: {sources: []}">>). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). groups() -> []. @@ -44,14 +44,14 @@ init_per_suite(Config) -> meck:expect(emqx_resource, remove, fun(_) -> ok end ), ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), - ok = emqx_ct_helpers:start_apps([emqx_authz]), + ok = emqx_common_test_helpers:start_apps([emqx_authz]), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Config. end_per_suite(_Config) -> {ok, _} = emqx_authz:update(?CMD_REPLCAE, []), - emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), + emqx_common_test_helpers:stop_apps([emqx_authz, emqx_resource]), meck:unload(emqx_resource), meck:unload(emqx_schema), ok. @@ -111,7 +111,7 @@ init_per_testcase(_, Config) -> }). -define(SOURCE6, #{<<"type">> => <<"file">>, <<"enable">> => true, - <<"path">> => emqx_ct_helpers:deps_path(emqx_authz, "etc/acl.conf") + <<"path">> => emqx_common_test_helpers:deps_path(emqx_authz, "etc/acl.conf") }). diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index 1ea942c10..4f0881bfb 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -84,7 +84,7 @@ all() -> []. %% Todo: Waiting for @terry-xiaoyu to fix the config_not_found error - % emqx_ct:all(?MODULE). + % emqx_common_test_helpers:all(?MODULE). groups() -> []. @@ -99,7 +99,7 @@ init_per_suite(Config) -> ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), - ok = emqx_ct_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1), + ok = emqx_common_test_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), @@ -107,7 +107,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> {ok, _} = emqx_authz:update(replace, []), - emqx_ct_helpers:stop_apps([emqx_authz, emqx_dashboard]), + emqx_common_test_helpers:stop_apps([emqx_authz, emqx_dashboard]), meck:unload(emqx_schema), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl index 1db9fff2b..573b98f18 100644 --- a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl @@ -38,20 +38,20 @@ -define(BASE_PATH, "api"). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). groups() -> []. init_per_suite(Config) -> - ok = emqx_ct_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1), + ok = emqx_common_test_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_resource, emqx_authz, emqx_dashboard]), + emqx_common_test_helpers:stop_apps([emqx_resource, emqx_authz, emqx_dashboard]), ok. set_special_configs(emqx_dashboard) -> diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 86b347b98..72e16a0ec 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -97,7 +97,7 @@ all() -> []. %% Todo: Waiting for @terry-xiaoyu to fix the config_not_found error - % emqx_ct:all(?MODULE). + % emqx_common_test_helpers:all(?MODULE). groups() -> []. @@ -118,7 +118,7 @@ init_per_suite(Config) -> ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), - ok = emqx_ct_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1), + ok = emqx_common_test_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), @@ -126,7 +126,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> {ok, _} = emqx_authz:update(replace, []), - emqx_ct_helpers:stop_apps([emqx_resource, emqx_authz, emqx_dashboard]), + emqx_common_test_helpers:stop_apps([emqx_resource, emqx_authz, emqx_dashboard]), meck:unload(emqx_resource), meck:unload(emqx_schema), ok. @@ -154,7 +154,7 @@ init_per_testcase(t_api, Config) -> meck:new(emqx, [non_strict, passthrough, no_history, no_link]), meck:expect(emqx, get_config, fun([node, data_dir]) -> - % emqx_ct_helpers:deps_path(emqx_authz, "test"); + % emqx_common_test_helpers:deps_path(emqx_authz, "test"); {data_dir, Data} = lists:keyfind(data_dir, 1, Config), Data; (C) -> meck:passthrough([C]) diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index 17763d993..c0e66751a 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -24,7 +24,7 @@ -define(CONF_DEFAULT, <<"authorization: {sources: []}">>). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). groups() -> []. @@ -42,7 +42,7 @@ init_per_suite(Config) -> meck:expect(emqx_resource, remove, fun(_) -> ok end ), ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), - ok = emqx_ct_helpers:start_apps([emqx_authz]), + ok = emqx_common_test_helpers:start_apps([emqx_authz]), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), @@ -58,7 +58,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> {ok, _} = emqx_authz:update(replace, []), - emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), + emqx_common_test_helpers:stop_apps([emqx_authz, emqx_resource]), meck:unload(emqx_resource), meck:unload(emqx_schema), ok. diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl index 8b221d3e7..947f46a82 100644 --- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -25,7 +25,7 @@ -define(CONF_DEFAULT, <<"authorization: {sources: []}">>). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). groups() -> []. @@ -39,7 +39,7 @@ init_per_suite(Config) -> end), ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), - ok = emqx_ct_helpers:start_apps([emqx_authz]), + ok = emqx_common_test_helpers:start_apps([emqx_authz]), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), @@ -49,7 +49,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> {ok, _} = emqx_authz:update(replace, []), - emqx_ct_helpers:stop_apps([emqx_authz]), + emqx_common_test_helpers:stop_apps([emqx_authz]), meck:unload(emqx_schema), ok. diff --git a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl index 6925194bd..357d8a9ed 100644 --- a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl @@ -25,7 +25,7 @@ -define(CONF_DEFAULT, <<"authorization: {sources: []}">>). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). groups() -> []. @@ -43,7 +43,7 @@ init_per_suite(Config) -> meck:expect(emqx_resource, remove, fun(_) -> ok end ), ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), - ok = emqx_ct_helpers:start_apps([emqx_authz]), + ok = emqx_common_test_helpers:start_apps([emqx_authz]), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"mongodb">>, @@ -60,7 +60,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> {ok, _} = emqx_authz:update(replace, []), - emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), + emqx_common_test_helpers:stop_apps([emqx_authz, emqx_resource]), meck:unload(emqx_resource), meck:unload(emqx_schema), ok. diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index 32e52e7c0..c85422122 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -25,7 +25,7 @@ -define(CONF_DEFAULT, <<"authorization: {sources: []}">>). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). groups() -> []. @@ -43,7 +43,7 @@ init_per_suite(Config) -> meck:expect(emqx_resource, remove, fun(_) -> ok end ), ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), - ok = emqx_ct_helpers:start_apps([emqx_authz]), + ok = emqx_common_test_helpers:start_apps([emqx_authz]), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), @@ -62,7 +62,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> {ok, _} = emqx_authz:update(replace, []), - emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), + emqx_common_test_helpers:stop_apps([emqx_authz, emqx_resource]), meck:unload(emqx_resource), meck:unload(emqx_schema), ok. diff --git a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl index 9526adcda..2b9d4c62e 100644 --- a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl @@ -25,7 +25,7 @@ -define(CONF_DEFAULT, <<"authorization: {sources: []}">>). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). groups() -> []. @@ -43,7 +43,7 @@ init_per_suite(Config) -> meck:expect(emqx_resource, remove, fun(_) -> ok end ), ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), - ok = emqx_ct_helpers:start_apps([emqx_authz]), + ok = emqx_common_test_helpers:start_apps([emqx_authz]), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), @@ -62,7 +62,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> {ok, _} = emqx_authz:update(replace, []), - emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), + emqx_common_test_helpers:stop_apps([emqx_authz, emqx_resource]), meck:unload(emqx_resource), meck:unload(emqx_schema), ok. diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index 9949e8b51..b1657d558 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -24,7 +24,7 @@ -define(CONF_DEFAULT, <<"authorization: {sources: []}">>). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). groups() -> []. @@ -42,7 +42,7 @@ init_per_suite(Config) -> meck:expect(emqx_resource, remove, fun(_) -> ok end ), ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), - ok = emqx_ct_helpers:start_apps([emqx_authz]), + ok = emqx_common_test_helpers:start_apps([emqx_authz]), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), @@ -60,7 +60,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> {ok, _} = emqx_authz:update(replace, []), - emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), + emqx_common_test_helpers:stop_apps([emqx_authz, emqx_resource]), meck:unload(emqx_resource), meck:unload(emqx_schema), ok. diff --git a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl index 3c7e314cd..ec8ca929a 100644 --- a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl @@ -29,14 +29,14 @@ -define(SOURCE5, {allow, {'or', [{username, {re, "^test"}}, {clientid, {re, "test?"}}]}, publish, ["%u", "%c"]}). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_ct_helpers:start_apps([emqx_authz]), + ok = emqx_common_test_helpers:start_apps([emqx_authz]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_authz]), + emqx_common_test_helpers:stop_apps([emqx_authz]), ok. t_compile(_) -> diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl index 5b781455d..ba4b058ed 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl @@ -34,10 +34,12 @@ fields("auto_subscribe") -> fields("topic") -> [ {topic, sc(binary(), #{})} - , {qos, sc(typerefl:union([0, 1, 2]), #{default => 0})} - , {rh, sc(typerefl:union([0, 1, 2]), #{default => 0})} - , {rap, sc(typerefl:union([0, 1]), #{default => 0})} - , {nl, sc(typerefl:union([0, 1]), #{default => 0})} + , {qos, sc(hoconsc:union([typerefl:integer(0), typerefl:integer(1), typerefl:integer(2)]), + #{default => 0})} + , {rh, sc(hoconsc:union([typerefl:integer(0), typerefl:integer(1), typerefl:integer(2)]), + #{default => 0})} + , {rap, sc(hoconsc:union([typerefl:integer(0), typerefl:integer(1)]), #{default => 0})} + , {nl, sc(hoconsc:union([typerefl:integer(0), typerefl:integer(1)]), #{default => 0})} ]. %%-------------------------------------------------------------------- diff --git a/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl b/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl index 11865153a..8e541eaec 100644 --- a/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl +++ b/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl @@ -25,5 +25,5 @@ do_init(Config = #{topics := _Topics}) -> Options = emqx_auto_subscribe_internal:init(Config), {emqx_auto_subscribe_internal, Options}; -do_init(_) -> - erlang:error("only support in EMQ X Enterprise"). +do_init(_Config) -> + erlang:error(not_supported). diff --git a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl new file mode 100644 index 000000000..58f64be90 --- /dev/null +++ b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl @@ -0,0 +1,162 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_auto_subscribe_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-define(APP, emqx_auto_subscribe). + +-define(TOPIC_C, <<"/c/${clientid}">>). +-define(TOPIC_U, <<"/u/${username}">>). +-define(TOPIC_H, <<"/h/${host}">>). +-define(TOPIC_P, <<"/p/${port}">>). +-define(TOPIC_A, <<"/client/${clientid}/username/${username}/host/${host}/port/${port}">>). +-define(TOPIC_S, <<"/topic/simple">>). + +-define(TOPICS, [?TOPIC_C, ?TOPIC_U, ?TOPIC_H, ?TOPIC_P, ?TOPIC_A, ?TOPIC_S]). + +-define(ENSURE_TOPICS , [<<"/c/auto_sub_c">> + , <<"/u/auto_sub_u">> + , ?TOPIC_S]). + +-define(CLIENT_ID, <<"auto_sub_c">>). +-define(CLIENT_USERNAME, <<"auto_sub_u">>). + +all() -> + [t_auto_subscribe, t_update]. + +init_per_suite(Config) -> + ekka_mnesia:start(), + application:stop(?APP), + + meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_schema, fields, fun("auto_subscribe") -> + meck:passthrough(["auto_subscribe"]) ++ + emqx_auto_subscribe_schema:fields("auto_subscribe"); + (F) -> meck:passthrough([F]) + end), + + meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), + meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end), + meck:expect(emqx_resource, remove, fun(_) -> ok end ), + + application:load(emqx_dashboard), + application:load(?APP), + ok = emqx_config:init_load(emqx_auto_subscribe_schema, + <<"auto_subscribe { + topics = [ + { + topic = \"/c/${clientid}\" + }, + { + topic = \"/u/${username}\" + }, + { + topic = \"/h/${host}\" + }, + { + topic = \"/p/${port}\" + }, + { + topic = \"/client/${clientid}/username/${username}/host/${host}/port/${port}\" + }, + { + topic = \"/topic/simple\" + qos = 1 + rh = 0 + rap = 0 + nl = 0 + } + ] + }">>), + emqx_common_test_helpers:start_apps([emqx_dashboard], fun set_special_configs/1), + emqx_common_test_helpers:start_apps([?APP]), + Config. + +set_special_configs(emqx_dashboard) -> + Config = #{ + default_username => <<"admin">>, + default_password => <<"public">>, + listeners => [#{ + protocol => http, + port => 18083 + }] + }, + emqx_config:put([emqx_dashboard], Config), + ok; +set_special_configs(_) -> + ok. + +topic_config(T) -> + #{ + topic => T, + qos => 0, + rh => 0, + rap => 0, + nl => 0 + }. + +end_per_suite(_) -> + application:unload(emqx_management), + application:unload(?APP), + meck:unload(emqx_resource), + meck:unload(emqx_schema), + emqx_common_test_helpers:stop_apps([emqx_dashboard, ?APP]). + +t_auto_subscribe(_) -> + {ok, Client} = emqtt:start_link(#{username => ?CLIENT_USERNAME, clientid => ?CLIENT_ID}), + {ok, _} = emqtt:connect(Client), + timer:sleep(100), + ?assertEqual(check_subs(length(?TOPICS)), ok), + emqtt:disconnect(Client), + ok. + +t_update(_) -> + Path = emqx_mgmt_api_test_util:api_path(["mqtt", "auto_subscribe"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Body = [#{topic => ?TOPIC_S}], + {ok, Response} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, Body), + ResponseMap = emqx_json:decode(Response, [return_maps]), + ?assertEqual(1, erlang:length(ResponseMap)), + + {ok, Client} = emqtt:start_link(#{username => ?CLIENT_USERNAME, clientid => ?CLIENT_ID}), + {ok, _} = emqtt:connect(Client), + timer:sleep(100), + ?assertEqual(check_subs(ets:tab2list(emqx_suboption), [?TOPIC_S]), ok), + emqtt:disconnect(Client), + + {ok, GETResponse} = emqx_mgmt_api_test_util:request_api(get, Path), + GETResponseMap = emqx_json:decode(GETResponse, [return_maps]), + ?assertEqual(1, erlang:length(GETResponseMap)), + ok. + + +check_subs(Count) -> + Subs = ets:tab2list(emqx_suboption), + ?assert(length(Subs) >= Count), + check_subs((Subs), ?ENSURE_TOPICS). + +check_subs([], []) -> + ok; +check_subs([{{_, Topic}, #{subid := ?CLIENT_ID}} | Subs], List) -> + check_subs(Subs, lists:delete(Topic, List)); +check_subs([_ | Subs], List) -> + check_subs(Subs, List). + diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl index 8cb325e20..2f3601646 100644 --- a/apps/emqx_bridge/src/emqx_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -27,6 +27,7 @@ start(_StartType, _StartArgs) -> {ok, Sup}. stop(_State) -> + emqx_config_handler:remove_handler(emqx_bridge:config_key_path()), ok = emqx_bridge:unload_hook(), ok. diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 92c7c6d64..7c71e09b3 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -79,11 +79,7 @@ fields(config) -> ] ++ emqx_connector_schema_lib:ssl_fields(). method() -> - hoconsc:union([ typerefl:atom(post) - , typerefl:atom(put) - , typerefl:atom(get) - , typerefl:atom(delete) - ]). + hoconsc:enum([post, put, get, delete]). validations() -> [ {check_ssl_opts, fun check_ssl_opts/1} ]. diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index f1f5ab133..d4d5621e7 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -47,17 +47,10 @@ start_listeners() -> type => apiKey, name => "authorization", in => header}}}}, - %% TODO: make it permanent when it's ready to release - Dispatch = - case os:getenv("_EMQX_ENABLE_DASHBOARD") of - V when V =:= "true" orelse V =:= "1" -> - [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, - {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, - {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}} - ]; - _ -> - [] - end, + Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, + {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, + {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}} + ], BaseMinirest = #{ base_path => ?BASE_PATH, modules => minirest_api:find_api_modules(apps()), @@ -69,13 +62,13 @@ start_listeners() -> [begin Minirest = maps:put(protocol, Protocol, BaseMinirest), {ok, _} = minirest:start(Name, RanchOptions, Minirest), - ?ULOG("Start listener ~s on ~p successfully.~n", [Name, Port]) + ?ULOG("Start listener ~ts on ~p successfully.~n", [Name, Port]) end || {Name, Protocol, Port, RanchOptions} <- listeners()]. stop_listeners() -> [begin ok = minirest:stop(Name), - ?ULOG("Stop listener ~s on ~p successfully.~n", [Name, Port]) + ?ULOG("Stop listener ~ts on ~p successfully.~n", [Name, Port]) end || {Name, _, Port, _} <- listeners()]. %%-------------------------------------------------------------------- diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 75b3ab201..1c8aeaf83 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -75,7 +75,7 @@ translate_req(Request, #{module := Module, path := Path, method := Method}) -> catch throw:Error -> {_, [{validation_error, ValidErr}]} = Error, #{path := Key, reason := Reason} = ValidErr, - {400, 'BAD_REQUEST', iolist_to_binary(io_lib:format("~s : ~p", [Key, Reason]))} + {400, 'BAD_REQUEST', iolist_to_binary(io_lib:format("~ts : ~p", [Key, Reason]))} end. namespace() -> "public". @@ -103,7 +103,7 @@ error_codes(Codes = [_ | _], MsgExample) -> ]. support_check_schema(#{check_schema := true}) -> ?DEFAULT_FILTER; -support_check_schema(#{check_schema := Func})when is_function(Func, 2) -> #{filter => Func}; +support_check_schema(#{check_schema := Func}) when is_function(Func, 2) -> #{filter => Func}; support_check_schema(_) -> #{filter => undefined}. parse_spec_ref(Module, Path) -> @@ -191,11 +191,11 @@ parameters(Params, Module) -> lists:foldl(fun(Param, {Acc, RefsAcc}) -> case Param of ?REF(StructName) -> - {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(Module, StructName)} |Acc], - [{Module, StructName, parameter}|RefsAcc]}; + {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(Module, StructName)} | Acc], + [{Module, StructName, parameter} | RefsAcc]}; ?R_REF(RModule, StructName) -> - {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(RModule, StructName)} |Acc], - [{RModule, StructName, parameter}|RefsAcc]}; + {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(RModule, StructName)} | Acc], + [{RModule, StructName, parameter} | RefsAcc]}; {Name, Type} -> In = hocon_schema:field_schema(Type, in), In =:= undefined andalso throw({error, <<"missing in:path/query field in parameters">>}), @@ -300,10 +300,13 @@ components([{Module, Field, parameter} | Refs], SpecAcc, SubRefsAcc) -> NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Param}, components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc). +%% Semantic error at components.schemas.xxx:xx:xx +%% Component names can only contain the characters A-Z a-z 0-9 - . _ +%% So replace ':' by '-'. namespace(Module) -> case hocon_schema:namespace(Module) of undefined -> Module; - NameSpace -> NameSpace + NameSpace -> re:replace(to_bin(NameSpace), ":","-",[global]) end. hocon_schema_to_spec(?R_REF(Module, StructName), _LocalModule) -> @@ -312,13 +315,20 @@ hocon_schema_to_spec(?R_REF(Module, StructName), _LocalModule) -> hocon_schema_to_spec(?REF(StructName), LocalModule) -> {#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(LocalModule, StructName)}, [{LocalModule, StructName}]}; -hocon_schema_to_spec(Type, _LocalModule) when ?IS_TYPEREFL(Type) -> - {typename_to_spec(typerefl:name(Type)), []}; +hocon_schema_to_spec(Type, LocalModule) when ?IS_TYPEREFL(Type) -> + {typename_to_spec(typerefl:name(Type), LocalModule), []}; hocon_schema_to_spec(?ARRAY(Item), LocalModule) -> {Schema, Refs} = hocon_schema_to_spec(Item, LocalModule), {#{type => array, items => Schema}, Refs}; +hocon_schema_to_spec(?LAZY(Item), LocalModule) -> + hocon_schema_to_spec(Item, LocalModule); hocon_schema_to_spec(?ENUM(Items), _LocalModule) -> {#{type => string, enum => Items}, []}; +hocon_schema_to_spec(?MAP(Name, Type), LocalModule) -> + {Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule), + {#{<<"type">> => object, + <<"properties">> => #{<<"$", (to_bin(Name))/binary>> => Schema}}, + SubRefs}; hocon_schema_to_spec(?UNION(Types), LocalModule) -> {OneOf, Refs} = lists:foldl(fun(Type, {Acc, RefsAcc}) -> {Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule), @@ -328,38 +338,102 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) -> hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> {#{type => string, enum => [Atom]}, []}. -typename_to_spec("boolean()") -> #{type => boolean, example => true}; -typename_to_spec("binary()") -> #{type => string, example =><<"binary example">>}; -typename_to_spec("float()") -> #{type =>number, example =>3.14159}; -typename_to_spec("integer()") -> #{type =>integer, example =>100}; -typename_to_spec("number()") -> #{type =>number, example =>42}; -typename_to_spec("string()") -> #{type =>string, example =><<"string example">>}; -typename_to_spec("atom()") -> #{type =>string, example =>atom}; -typename_to_spec("duration()") -> #{type =>string, example =><<"12m">>}; -typename_to_spec("duration_s()") -> #{type =>string, example =><<"1h">>}; -typename_to_spec("duration_ms()") -> #{type =>string, example =><<"32s">>}; -typename_to_spec("percent()") -> #{type =>number, example =><<"12%">>}; -typename_to_spec("file()") -> #{type =>string, example =><<"/path/to/file">>}; -typename_to_spec("ip_port()") -> #{type => string, example =><<"127.0.0.1:80">>}; -typename_to_spec(Name) -> +%% todo: Find a way to fetch enum value from user_id_type(). +typename_to_spec("user_id_type()", _Mod) -> #{type => string, enum => [clientid, username]}; +typename_to_spec("term()", _Mod) -> #{type => string, example => "term"}; +typename_to_spec("boolean()", _Mod) -> #{type => boolean, example => true}; +typename_to_spec("binary()", _Mod) -> #{type => string, example => <<"binary-example">>}; +typename_to_spec("float()", _Mod) -> #{type => number, example => 3.14159}; +typename_to_spec("integer()", _Mod) -> #{type => integer, example => 100}; +typename_to_spec("non_neg_integer()", _Mod) -> #{type => integer, minimum => 1, example => 100}; +typename_to_spec("number()", _Mod) -> #{type => number, example => 42}; +typename_to_spec("string()", _Mod) -> #{type => string, example => <<"string-example">>}; +typename_to_spec("atom()", _Mod) -> #{type => string, example => atom}; +typename_to_spec("duration()", _Mod) -> #{type => string, example => <<"12m">>}; +typename_to_spec("duration_s()", _Mod) -> #{type => string, example => <<"1h">>}; +typename_to_spec("duration_ms()", _Mod) -> #{type => string, example => <<"32s">>}; +typename_to_spec("percent()", _Mod) -> #{type => number, example => <<"12%">>}; +typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>}; +typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>}; +typename_to_spec("url()", _Mod) -> #{type => string, example => <<"http://127.0.0.1">>}; +typename_to_spec("server()", Mod) -> typename_to_spec("ip_port()", Mod); +typename_to_spec("connect_timeout()", Mod) -> typename_to_spec("timeout()", Mod); +typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, example => infinity}, + #{type => integer, example => 100}], example => infinity}; +typename_to_spec("bytesize()", _Mod) -> #{type => string, example => <<"32MB">>}; +typename_to_spec("wordsize()", _Mod) -> #{type => string, example => <<"1024KB">>}; +typename_to_spec("map()", _Mod) -> #{type => string, example => <<>>}; +typename_to_spec("comma_separated_list()", _Mod) -> #{type => string, example => <<"item1,item2">>}; +typename_to_spec("comma_separated_atoms()", _Mod) -> #{type => string, example => <<"item1,item2">>}; +typename_to_spec("pool_type()", _Mod) -> #{type => string, enum => [random, hash], example => hash}; +typename_to_spec("log_level()", _Mod) -> + #{type => string, enum => [debug, info, notice, warning, error, critical, alert, emergency, all]}; +typename_to_spec("rate()", _Mod) -> + #{type => string, example => <<"10M/s">>}; +typename_to_spec("bucket_rate()", _Mod) -> + #{type => string, example => <<"10M/s, 100M">>}; +typename_to_spec(Name, Mod) -> + Spec = range(Name), + Spec1 = remote_module_type(Spec, Name, Mod), + Spec2 = typerefl_array(Spec1, Name, Mod), + Spec3 = integer(Spec2, Name), + Spec3 =:= nomatch andalso + throw({error, #{msg => <<"Unsupport Type">>, type => Name, module => Mod}}), + Spec3. + +range(Name) -> case string:split(Name, "..") of - [MinStr, MaxStr] -> %% 1..10 - {Min, []} = string:to_integer(MinStr), - {Max, []} = string:to_integer(MaxStr), - #{type => integer, example => Min, minimum => Min, maximum => Max}; - _ -> %% Module:Type(). - case string:split(Name, ":") of - [_Module, Type] -> typename_to_spec(Type); - _ -> throw({error, #{msg => <<"Unsupport Type">>, type => Name}}) - end + [MinStr, MaxStr] -> %% 1..10 1..inf -inf..10 + Schema = #{type => integer}, + Schema1 = add_integer_prop(Schema, minimum, MinStr), + add_integer_prop(Schema1, maximum, MaxStr); + _ -> nomatch end. -to_bin(List) when is_list(List) -> list_to_binary(List); +%% Module:Type +remote_module_type(nomatch, Name, Mod) -> + case string:split(Name, ":") of + [_Module, Type] -> typename_to_spec(Type, Mod); + _ -> nomatch + end; +remote_module_type(Spec, _Name, _Mod) -> Spec. + +%% [string()] or [integer()] or [xxx]. +typerefl_array(nomatch, Name, Mod) -> + case string:trim(Name, leading, "[") of + Name -> nomatch; + Name1 -> + case string:trim(Name1, trailing, "]") of + Name1 -> notmatch; + Name2 -> + Schema = typename_to_spec(Name2, Mod), + #{type => array, items => Schema} + end + end; +typerefl_array(Spec, _Name, _Mod) -> Spec. + +%% integer(1) +integer(nomatch, Name) -> + case string:to_integer(Name) of + {Int, []} -> #{type => integer, enum => [Int], example => Int, default => Int}; + _ -> nomatch + end; +integer(Spec, _Name) -> Spec. + +add_integer_prop(Schema, Key, Value) -> + case string:to_integer(Value) of + {error, no_integer} -> Schema; + {Int, []}when Key =:= minimum -> Schema#{Key => Int, example => Int}; + {Int, []} -> Schema#{Key => Int} + end. + +to_bin([Atom | _] = List) when is_atom(Atom) -> iolist_to_binary(io_lib:format("~p", [List])); +to_bin(List) when is_list(List) -> unicode:characters_to_binary(List); to_bin(B) when is_boolean(B) -> B; to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); to_bin(X) -> X. -parse_object(PropList = [_|_], Module) when is_list(PropList) -> +parse_object(PropList = [_ | _], Module) when is_list(PropList) -> {Props, Required, Refs} = lists:foldl(fun({Name, Hocon}, {Acc, RequiredAcc, RefsAcc}) -> NameBin = to_bin(Name), diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index da51a2418..b59a5f0b3 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -41,15 +41,15 @@ all() -> %% TODO: V5 API -%% emqx_ct:all(?MODULE). +%% emqx_common_test_helpers:all(?MODULE). []. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard],fun set_special_configs/1), + emqx_common_test_helpers:start_apps([emqx_management, emqx_dashboard],fun set_special_configs/1), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management]), + emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_management]), ekka_mnesia:ensure_stopped(). set_special_configs(emqx_management) -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index cea0a915d..9c9958880 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -112,15 +112,15 @@ t_without_in(_Config) -> t_require(_Config) -> ExpectSpec = [#{ in => query,name => userid, required => false, - schema => #{example => <<"binary example">>, type => string}}], + schema => #{example => <<"binary-example">>, type => string}}], validate("/required/false", ExpectSpec), ok. t_nullable(_Config) -> NullableFalse = [#{in => query,name => userid, required => true, - schema => #{example => <<"binary example">>, type => string}}], + schema => #{example => <<"binary-example">>, type => string}}], NullableTrue = [#{in => query,name => userid, - schema => #{example => <<"binary example">>, type => string, + schema => #{example => <<"binary-example">>, type => string, nullable => true}}], validate("/nullable/false", NullableFalse), validate("/nullable/true", NullableTrue), diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 84cbfe5fb..7aa986d1d 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -121,7 +121,7 @@ t_nest_ref(_Config) -> #{<<"emqx_swagger_requestBody_SUITE.good_ref">> => #{<<"properties">> => [ {<<"webhook-host">>, #{default => <<"127.0.0.1:80">>, example => <<"127.0.0.1:80">>,type => string}}, {<<"log_dir">>, #{example => <<"var/log/emqx">>,type => string}}, - {<<"tag">>, #{description => <<"tag">>, example => <<"binary example">>,type => string}}], + {<<"tag">>, #{description => <<"tag">>, example => <<"binary-example">>,type => string}}], <<"type">> => object}}]), {_, Components} = validate("/ref/nest/ref", Spec, Refs), ?assertEqual(ExpectComponents, Components), diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index fd6920549..a9969ba4b 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -13,7 +13,7 @@ -export([all/0, suite/0, groups/0]). -export([paths/0, api_spec/0, schema/1, fields/1]). -export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1, t_error/1, - t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1, + t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1, t_complicated_type/1, t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, t_ref_array_with_key/1, t_ref_array_without_key/1, t_api_spec/1]). @@ -21,7 +21,7 @@ all() -> [{group, spec}]. suite() -> [{timetrap, {minutes, 1}}]. groups() -> [ {spec, [parallel], [ - t_api_spec, t_simple_binary, t_object, t_nest_object, t_error, + t_api_spec, t_simple_binary, t_object, t_nest_object, t_error, t_complicated_type, t_raw_local_ref, t_raw_remote_ref, t_empty, t_hocon_schema_function, t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]} @@ -54,7 +54,7 @@ t_error(_Config) -> #{<<"application/json">> => #{<<"schema">> => #{<<"type">> => object, <<"properties">> => [ - {<<"code">>, #{enum => ['Bad1','Bad2'], type => string}}, + {<<"code">>, #{enum => ['Bad1', 'Bad2'], type => string}}, {<<"message">>, #{description => <<"Details description of the error.">>, example => <<"Bad request desc">>, type => string}}] }}}}, @@ -156,6 +156,38 @@ t_nest_ref(_Config) -> validate(Path, Object, ExpectRefs), ok. +t_complicated_type(_Config) -> + Path = "/ref/complicated_type", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{<<"properties">> => + [ + {<<"no_neg_integer">>, #{example => 100, minimum => 1, type => integer}}, + {<<"url">>, #{example => <<"http://127.0.0.1">>, type => string}}, + {<<"server">>, #{example => <<"127.0.0.1:80">>, type => string}}, + {<<"connect_timeout">>, #{example => infinity, <<"oneOf">> => [ + #{example => infinity, type => string}, + #{example => 100, type => integer}]}}, + {<<"pool_type">>, #{enum => [random, hash], example => hash, type => string}}, + {<<"timeout">>, #{example => infinity, + <<"oneOf">> => + [#{example => infinity, type => string}, #{example => 100, type => integer}]}}, + {<<"bytesize">>, #{example => <<"32MB">>, type => string}}, + {<<"wordsize">>, #{example => <<"1024KB">>, type => string}}, + {<<"maps">>, #{example => <<>>, type => string}}, + {<<"comma_separated_list">>, #{example => <<"item1,item2">>, type => string}}, + {<<"comma_separated_atoms">>, #{example => <<"item1,item2">>, type => string}}, + {<<"log_level">>, + #{enum => [debug, info, notice, warning, error, critical, alert, emergency, all], type => string}}, + {<<"fix_integer">>, #{default => 100, enum => [100], example => 100,type => integer}} + ], + <<"type">> => object}}}}, + {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), + ?assertEqual(test, OperationId), + Response = maps:get(responses, maps:get(post, Spec)), + ?assertEqual(Object, maps:get(<<"200">>, Response)), + ?assertEqual([], Refs), + ok. + + t_ref_array_with_key(_Config) -> Path = "/ref/array/with/key", Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ @@ -163,10 +195,10 @@ t_ref_array_with_key(_Config) -> {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, - {<<"assert">>, #{description => <<"money">>, example => 3.14159,type => number}}, - {<<"number_ex">>, #{description => <<"number example">>, example => 42,type => number}}, - {<<"percent_ex">>, #{description => <<"percent example">>, example => <<"12%">>,type => number}}, - {<<"duration_ms_ex">>, #{description => <<"duration ms example">>, example => <<"32s">>,type => string}}, + {<<"assert">>, #{description => <<"money">>, example => 3.14159, type => number}}, + {<<"number_ex">>, #{description => <<"number example">>, example => 42, type => number}}, + {<<"percent_ex">>, #{description => <<"percent example">>, example => <<"12%">>, type => number}}, + {<<"duration_ms_ex">>, #{description => <<"duration ms example">>, example => <<"32s">>, type => string}}, {<<"atom_ex">>, #{description => <<"atom ex">>, example => atom, type => string}}, {<<"array_refs">>, #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}, type => array}} ]} @@ -201,7 +233,7 @@ t_hocon_schema_function(_Config) -> }}, #{<<"emqx_swagger_remote_schema.ref3">> => #{<<"type">> => object, <<"properties">> => [ - {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}}, + {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>, type => string}}, {<<"version">>, #{description => <<"a good version">>, example => <<"1.0.0">>, type => string}}] }}, #{<<"emqx_swagger_remote_schema.root">> => #{required => [<<"default_password">>, <<"default_username">>], @@ -210,8 +242,8 @@ t_hocon_schema_function(_Config) -> [#{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref2">>}, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref1">>}]}, type => array}}, {<<"default_username">>, - #{default => <<"admin">>, example => <<"string example">>, type => string}}, - {<<"default_password">>, #{default => <<"public">>, example => <<"string example">>, type => string}}, + #{default => <<"admin">>, example => <<"string-example">>, type => string}}, + {<<"default_password">>, #{default => <<"public">>, example => <<"string-example">>, type => string}}, {<<"sample_interval">>, #{default => <<"10s">>, example => <<"1h">>, type => string}}, {<<"token_expired_time">>, #{default => <<"30m">>, example => <<"12m">>, type => string}}], <<"type">> => object}}], @@ -290,6 +322,27 @@ schema("/error") -> 400 => emqx_dashboard_swagger:error_codes(['Bad1', 'Bad2'], <<"Bad request desc">>), 404 => emqx_dashboard_swagger:error_codes(['Not-Found']) }} + }; +schema("/ref/complicated_type") -> + #{ + operationId => test, + post => #{responses => #{ + 200 => [ + {no_neg_integer, hoconsc:mk(non_neg_integer(), #{})}, + {url, hoconsc:mk(emqx_connector_http:url(), #{})}, + {server, hoconsc:mk(emqx_connector_redis:server(), #{})}, + {connect_timeout, hoconsc:mk(emqx_connector_http:connect_timeout(), #{})}, + {pool_type, hoconsc:mk(emqx_connector_http:pool_type(), #{})}, + {timeout, hoconsc:mk(timeout(), #{})}, + {bytesize, hoconsc:mk(emqx_schema:bytesize(), #{})}, + {wordsize, hoconsc:mk(emqx_schema:wordsize(), #{})}, + {maps, hoconsc:mk(map(), #{})}, + {comma_separated_list, hoconsc:mk(emqx_schema:comma_separated_list(), #{})}, + {comma_separated_atoms, hoconsc:mk(emqx_schema:comma_separated_atoms(), #{})}, + {log_level, hoconsc:mk(emqx_machine_schema:log_level(), #{})}, + {fix_integer, hoconsc:mk(typerefl:integer(100), #{})} + ] + }} }. validate(Path, ExpectObject, ExpectRefs) -> diff --git a/apps/emqx_exhook/src/emqx_exhook_cli.erl b/apps/emqx_exhook/src/emqx_exhook_cli.erl index 860499698..a96cdb6cc 100644 --- a/apps/emqx_exhook/src/emqx_exhook_cli.erl +++ b/apps/emqx_exhook/src/emqx_exhook_cli.erl @@ -23,7 +23,7 @@ cli(["server", "list"]) -> if_enabled(fun() -> ServerNames = emqx_exhook:list(), - [emqx_ctl:print("Server(~s)~n", [format(Name)]) || Name <- ServerNames] + [emqx_ctl:print("Server(~ts)~n", [format(Name)]) || Name <- ServerNames] end); cli(["server", "enable", Name]) -> @@ -78,7 +78,7 @@ format(Name) -> case emqx_exhook_mngr:server(Name) of undefined -> lists:flatten( - io_lib:format("name=~s, hooks=#{}, active=false", [Name])); + io_lib:format("name=~ts, hooks=#{}, active=false", [Name])); Server -> emqx_exhook_server:format(Server) end. diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl index 1a6e10bf0..eef1e67ec 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mngr.erl @@ -185,7 +185,7 @@ handle_info({timeout, _Ref, {reload, Name}}, State) -> {error, not_found} -> {noreply, NState}; {error, Reason} -> - ?LOG(warning, "Failed to reload exhook callback server \"~s\", " + ?LOG(warning, "Failed to reload exhook callback server \"~ts\", " "Reason: ~0p", [Name, Reason]), {noreply, ensure_reload_timer(NState)} end; @@ -231,7 +231,7 @@ do_load_server(Name, State0 = #state{ {ok, ServerState} -> save(Name, ServerState), ?LOG(info, "Load exhook callback server " - "\"~s\" successfully!", [Name]), + "\"~ts\" successfully!", [Name]), {ok, State#state{ running = maps:put(Name, Options, Running), waiting = maps:remove(Name, Waiting), diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 64d39eb52..9e988c6d8 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -40,13 +40,13 @@ roots() -> [exhook]. fields(exhook) -> [ {request_failed_action, - sc(union([deny, ignore]), + sc(hoconsc:enum([deny, ignore]), #{default => deny})} , {request_timeout, sc(duration(), #{default => "5s"})} , {auto_reconnect, - sc(union([false, duration()]), + sc(hoconsc:union([false, duration()]), #{ default => "60s" })} , {servers, diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index e776f8c62..84a143117 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -91,7 +91,7 @@ load(Name, Opts0, ReqOpts) -> {ok, HookSpecs} -> %% Reigster metrics Prefix = lists:flatten( - io_lib:format("exhook.~s.", [Name])), + io_lib:format("exhook.~ts.", [Name])), ensure_metrics(Prefix, HookSpecs), %% Ensure hooks ensure_hooks(HookSpecs), @@ -129,7 +129,7 @@ channel_opts(Opts = #{url := URL}) -> end. format_http_uri(Scheme, Host, Port) -> - lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])). + lists:flatten(io_lib:format("~ts://~ts:~w", [Scheme, Host, Port])). filter(Ls) -> [ E || E <- Ls, E /= undefined]. @@ -194,7 +194,7 @@ ensure_hooks(HookSpecs) -> lists:foreach(fun(Hookpoint) -> case lists:keyfind(Hookpoint, 1, ?ENABLED_HOOKS) of false -> - ?LOG(error, "Unknown name ~s to hook, skip it!", [Hookpoint]); + ?LOG(error, "Unknown name ~ts to hook, skip it!", [Hookpoint]); {Hookpoint, {M, F, A}} -> emqx_hooks:put(Hookpoint, {M, F, A}), ets:update_counter(?CNTER, Hookpoint, {2, 1}, {Hookpoint, 0}) @@ -217,7 +217,7 @@ may_unload_hooks(HookSpecs) -> format(#server{name = Name, hookspec = Hooks}) -> lists:flatten( - io_lib:format("name=~s, hooks=~0p, active=true", [Name, Hooks])). + io_lib:format("name=~ts, hooks=~0p, active=true", [Name, Hooks])). %%-------------------------------------------------------------------- %% APIs diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl index d2cc78b47..dd020ce85 100644 --- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl @@ -36,16 +36,16 @@ exhook: { %% Setups %%-------------------------------------------------------------------- -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Cfg) -> _ = emqx_exhook_demo_svr:start(), ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT), - emqx_ct_helpers:start_apps([emqx_exhook]), + emqx_common_test_helpers:start_apps([emqx_exhook]), Cfg. end_per_suite(_Cfg) -> - emqx_ct_helpers:stop_apps([emqx_exhook]), + emqx_common_test_helpers:stop_apps([emqx_exhook]), emqx_exhook_demo_svr:stop(). %%-------------------------------------------------------------------- diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index 74db72022..f755bf0ef 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -472,13 +472,13 @@ do_setup() -> logger:set_primary_config(#{level => warning}), _ = emqx_exhook_demo_svr:start(), ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT), - emqx_ct_helpers:start_apps([emqx_exhook]), + emqx_common_test_helpers:start_apps([emqx_exhook]), %% waiting first loaded event {'on_provider_loaded', _} = emqx_exhook_demo_svr:take(), ok. do_teardown(_) -> - emqx_ct_helpers:stop_apps([emqx_exhook]), + emqx_common_test_helpers:stop_apps([emqx_exhook]), %% waiting last unloaded event {'on_provider_unloaded', _} = emqx_exhook_demo_svr:take(), _ = emqx_exhook_demo_svr:stop(), diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index af34a1754..c998a1401 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -663,7 +663,7 @@ handle_incoming(Packet, State = #state{ }) -> Ctx = ChannMod:info(ctx, Channel), ok = inc_incoming_stats(Ctx, FrameMod, Packet), - ?LOG(debug, "RECV ~s", [FrameMod:format(Packet)]), + ?LOG(debug, "RECV ~ts", [FrameMod:format(Packet)]), with_channel(handle_in, [Packet], State). %%-------------------------------------------------------------------- @@ -715,12 +715,12 @@ serialize_and_inc_stats_fun(#state{ Ctx = ChannMod:info(ctx, Channel), fun(Packet) -> case FrameMod:serialize_pkt(Packet, Serialize) of - <<>> -> ?LOG(warning, "~s is discarded due to the frame is too large!", + <<>> -> ?LOG(warning, "~ts is discarded due to the frame is too large!", [FrameMod:format(Packet)]), ok = emqx_gateway_ctx:metrics_inc(Ctx, 'delivery.dropped.too_large'), ok = emqx_gateway_ctx:metrics_inc(Ctx, 'delivery.dropped'), <<>>; - Data -> ?LOG(debug, "SEND ~s", [FrameMod:format(Packet)]), + Data -> ?LOG(debug, "SEND ~ts", [FrameMod:format(Packet)]), ok = inc_outgoing_stats(Ctx, FrameMod, Packet), Data end diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index e84a67860..e17b72c6d 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -83,7 +83,7 @@ request_parameters() -> request_properties() -> properties([ {token, string, "message token, can be empty"} , {method, string, "request method type", ["get", "put", "post", "delete"]} - , {timeout, string, "timespan for response", "10s"} + , {timeout, string, "timespan for response"} , {content_type, string, "payload type", [<<"text/plain">>, <<"application/json">>, <<"application/octet-stream">>]} , {payload, string, "payload"}]). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index b6808c896..5b6642977 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -331,7 +331,7 @@ auth_connect(_Input, Channel = #channel{ctx = Ctx, {ok, NClientInfo} -> {ok, Channel#channel{clientinfo = NClientInfo}}; {error, Reason} -> - ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + ?LOG(warning, "Client ~ts (Username: '~ts') login failed for ~0p", [ClientId, Username, Reason]), {error, Reason} end. @@ -473,7 +473,7 @@ process_connection({open, Req}, Result, Channel, Iter) -> {ok, _Input, NChannel} -> process_connect(ensure_connected(NChannel), Req, Result, Iter); {error, ReasonCode, NChannel} -> - ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), + ErrMsg = io_lib:format("Login Failed: ~ts", [ReasonCode]), Payload = erlang:list_to_binary(lists:flatten(ErrMsg)), iter(Iter, reply({error, bad_request}, Payload, Req, Result), diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl index 5fd557def..6f576a2da 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl @@ -67,7 +67,7 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> on_gateway_load(Gateway#{config => Config}, Ctx) catch Class : Reason : Stk -> - logger:error("Failed to update ~s; " + logger:error("Failed to update ~ts; " "reason: {~0p, ~0p} stacktrace: ~0p", [GwName, Class, Reason, Stk]), {error, {Class, Reason}} @@ -89,11 +89,11 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Gateway ~s:~s:~s on ~s started.~n", + ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. @@ -118,10 +118,10 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", + ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index 41cc1fa26..58723cd22 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -77,25 +77,21 @@ apis() -> -define(format_fun, {?MODULE, format_channel_info}). clients(get, #{ bindings := #{name := Name0} - , query_string := Qs + , query_string := Params }) -> with_gateway(Name0, fun(GwName, _) -> TabName = emqx_gateway_cm:tabname(info, GwName), - case maps:get(<<"node">>, Qs, undefined) of + case maps:get(<<"node">>, Params, undefined) of undefined -> - Response = emqx_mgmt_api:cluster_query( - Qs, TabName, - ?CLIENT_QS_SCHEMA, ?query_fun - ), - {200, Response}; + Response = emqx_mgmt_api:cluster_query(Params, TabName, + ?CLIENT_QS_SCHEMA, ?query_fun), + emqx_mgmt_util:generate_response(Response); Node1 -> Node = binary_to_atom(Node1, utf8), - ParamsWithoutNode = maps:without([<<"node">>], Qs), - Response = emqx_mgmt_api:node_query( - Node, ParamsWithoutNode, - TabName, ?CLIENT_QS_SCHEMA, ?query_fun - ), - {200, Response} + ParamsWithoutNode = maps:without([<<"node">>], Params), + Response = emqx_mgmt_api:node_query(Node, ParamsWithoutNode, + TabName, ?CLIENT_QS_SCHEMA, ?query_fun), + emqx_mgmt_util:generate_response(Response) end end). @@ -109,7 +105,7 @@ clients_insta(get, #{ bindings := #{name := Name0, [ClientInfo] -> {200, ClientInfo}; [ClientInfo | _More] -> - ?LOG(warning, "More than one client info was returned on ~s", + ?LOG(warning, "More than one client info was returned on ~ts", [ClientId]), {200, ClientInfo}; [] -> diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 1f8d226e2..013bb35c9 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -51,10 +51,10 @@ gateway_type_searching() -> reg(Mod) -> try Mod:reg(), - ?LOG(info, "Register ~s gateway application successfully!", [Mod]) + ?LOG(info, "Register ~ts gateway application successfully!", [Mod]) catch Class : Reason : Stk -> - ?LOG(error, "Failed to register ~s gateway application: {~p, ~p}\n" + ?LOG(error, "Failed to register ~ts gateway application: {~p, ~p}\n" "Stacktrace: ~0p", [Mod, Class, Reason, Stk]) end. @@ -67,14 +67,14 @@ load_gateway_by_default([]) -> load_gateway_by_default([{Type, Confs}|More]) -> case emqx_gateway_registry:lookup(Type) of undefined -> - ?LOG(error, "Skip to load ~s gateway, because it is not registered", + ?LOG(error, "Skip to load ~ts gateway, because it is not registered", [Type]); _ -> case emqx_gateway:load(Type, Confs) of {ok, _} -> - ?LOG(debug, "Load ~s gateway successfully!", [Type]); + ?LOG(debug, "Load ~ts gateway successfully!", [Type]); {error, Reason} -> - ?LOG(error, "Failed to load ~s gateway: ~0p", [Type, Reason]) + ?LOG(error, "Failed to load ~ts gateway: ~0p", [Type, Reason]) end end, load_gateway_by_default(More). diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index fce969130..a441b384e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -53,7 +53,7 @@ gateway(["list"]) -> lists:foreach(fun(#{name := Name} = Gateway) -> %% TODO: More infos: listeners?, connected? Status = maps:get(status, Gateway, stopped), - print("Gateway(name=~s, status=~s)~n", [Name, Status]) + print("Gateway(name=~ts, status=~ts)~n", [Name, Status]) end, emqx_gateway:list()); gateway(["lookup", Name]) -> @@ -123,7 +123,7 @@ gateway(_) -> 'gateway-registry'(["list"]) -> lists:foreach( fun({Name, #{cbkmod := CbMod}}) -> - print("Registered Name: ~s, Callback Module: ~s~n", [Name, CbMod]) + print("Registered Name: ~ts, Callback Module: ~ts~n", [Name, CbMod]) end, emqx_gateway_registry:list()); @@ -229,10 +229,10 @@ print_record({client, {_, Infos, Stats}}) -> connected_at => ConnectedAt }, - print("Client(~s, username=~s, peername=~s, " - "clean_start=~s, keepalive=~w, " + print("Client(~ts, username=~ts, peername=~ts, " + "clean_start=~ts, keepalive=~w, " "subscriptions=~w, delivered_msgs=~w, " - "connected=~s, created_at=~w, connected_at=~w)~n", + "connected=~ts, created_at=~w, connected_at=~w)~n", [format(K, maps:get(K, Info)) || K <- InfoKeys]). print(S) -> emqx_ctl:print(S). @@ -243,7 +243,7 @@ format(_, undefined) -> format(peername, {IPAddr, Port}) -> IPStr = emqx_mgmt_util:ntoa(IPAddr), - io_lib:format("~s:~p", [IPStr, Port]); + io_lib:format("~ts:~p", [IPStr, Port]); format(_, Val) -> Val. diff --git a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl index 172193e28..bb928498c 100644 --- a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl +++ b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl @@ -103,7 +103,7 @@ init([Gateway, Ctx, _GwDscrptr]) -> }, case maps:get(enable, Config, true) of false -> - ?LOG(info, "Skipp to start ~s gateway due to disabled", [GwName]), + ?LOG(info, "Skipp to start ~ts gateway due to disabled", [GwName]), {ok, State}; true -> case cb_gateway_load(State) of @@ -266,13 +266,13 @@ do_create_authn_chain(ChainName, AuthConf) -> case emqx_authentication:create_authenticator(ChainName, AuthConf) of {ok, _} -> ok; {error, Reason} -> - ?LOG(error, "Failed to create authenticator chain ~s, " + ?LOG(error, "Failed to create authenticator chain ~ts, " "reason: ~p, config: ~p", [ChainName, Reason, AuthConf]), throw({badauth, Reason}) end; {error, Reason} -> - ?LOG(error, "Falied to create authn chain ~s, reason ~p", + ?LOG(error, "Falied to create authn chain ~ts, reason ~p", [ChainName, Reason]), throw({badauth, Reason}) end. @@ -293,7 +293,7 @@ do_deinit_authn(Names) -> ok -> ok; {error, {not_found, _}} -> ok; {error, Reason} -> - ?LOG(error, "Failed to clean authentication chain: ~s, " + ?LOG(error, "Failed to clean authentication chain: ~ts, " "reason: ~p", [ChainName, Reason]) end end, Names). @@ -388,7 +388,7 @@ cb_gateway_load(State = #state{name = GwName, end catch Class : Reason1 : Stk -> - ?LOG(error, "Failed to load ~s gateway (~0p, ~0p) " + ?LOG(error, "Failed to load ~ts gateway (~0p, ~0p) " "crashed: {~p, ~p}, stacktrace: ~0p", [GwName, Gateway, Ctx, Class, Reason1, Stk]), @@ -413,7 +413,7 @@ cb_gateway_update(Config, end catch Class : Reason1 : Stk -> - ?LOG(error, "Failed to update ~s gateway to config: ~0p crashed: " + ?LOG(error, "Failed to update ~ts gateway to config: ~0p crashed: " "{~p, ~p}, stacktrace: ~0p", [GwName, Config, Class, Reason1, Stk]), {error, {Class, Reason1, Stk}} diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index bb0bf9dbe..32f24b2dd 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -232,7 +232,7 @@ gateway_common_options() -> common_listener_opts() -> [ {enable, sc(boolean(), true)} - , {bind, sc(union(ip_port(), integer()))} + , {bind, sc(hoconsc:union([ip_port(), integer()]))} , {max_connections, sc(integer(), 1024)} , {max_conn_rate, sc(integer())} , {authentication, authentication()} diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 0f8ee99b0..91168eb70 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -112,9 +112,9 @@ apply(F, A2) when is_function(F), format_listenon(Port) when is_integer(Port) -> io_lib:format("0.0.0.0:~w", [Port]); format_listenon({Addr, Port}) when is_list(Addr) -> - io_lib:format("~s:~w", [Addr, Port]); + io_lib:format("~ts:~w", [Addr, Port]); format_listenon({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). + io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]). parse_listenon(Port) when is_integer(Port) -> Port; diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl index e1dacab7f..f8deff2fe 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl @@ -291,17 +291,17 @@ handle_call({auth, ClientInfo0, Password}, _From, SessFun ) of {ok, _Session} -> - ?LOG(debug, "Client ~s (Username: '~s') authorized successfully!", + ?LOG(debug, "Client ~ts (Username: '~ts') authorized successfully!", [ClientId, Username]), {reply, ok, [{event, connected}], ensure_connected(Channel1#channel{clientinfo = NClientInfo})}; {error, Reason} -> - ?LOG(warning, "Client ~s (Username: '~s') open session failed for ~0p", + ?LOG(warning, "Client ~ts (Username: '~ts') open session failed for ~0p", [ClientId, Username, Reason]), {reply, {error, ?RESP_PERMISSION_DENY, Reason}, Channel} end; {error, Reason} -> - ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + ?LOG(warning, "Client ~ts (Username: '~ts') login failed for ~0p", [ClientId, Username, Reason]), {reply, {error, ?RESP_PERMISSION_DENY, Reason}, Channel} end; diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl index 87170fff2..07207fb87 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl @@ -60,7 +60,7 @@ start_grpc_server(GwName, Options = #{bind := ListenOn}) -> [{ssl_options, SslOpts}] end, _ = grpc:start_server(GwName, ListenOn, Services, SvrOptions), - ?ULOG("Start ~s gRPC server on ~p successfully.~n", [GwName, ListenOn]). + ?ULOG("Start ~ts gRPC server on ~p successfully.~n", [GwName, ListenOn]). start_grpc_client_channel(_GwType, undefined) -> undefined; @@ -71,7 +71,7 @@ start_grpc_client_channel(GwName, Options = #{address := UriStr}) -> Port = maps:get(port, UriMap), SvrAddr = lists:flatten( io_lib:format( - "~s://~s:~w", [Scheme, Host, Port]) + "~ts://~ts:~w", [Scheme, Host, Port]) ), ClientOpts = case Scheme of "https" -> @@ -118,7 +118,7 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> on_gateway_load(Gateway#{config => Config}, Ctx) catch Class : Reason : Stk -> - logger:error("Failed to update ~s; " + logger:error("Failed to update ~ts; " "reason: {~0p, ~0p} stacktrace: ~0p", [GwName, Class, Reason, Stk]), {error, {Class, Reason}} @@ -143,11 +143,11 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Gateway ~s:~s:~s on ~s started.~n", + ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. @@ -198,10 +198,10 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", + ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl index c1648a037..5ac9c8ae3 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -249,7 +249,7 @@ do_connect(Req, Result, Channel, Iter) -> iter(Iter, maps:merge(Result, NewResult), NChannel) end; {error, ReasonCode, NChannel} -> - ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), + ErrMsg = io_lib:format("Login Failed: ~ts", [ReasonCode]), Payload = erlang:list_to_binary(lists:flatten(ErrMsg)), iter(Iter, reply({error, bad_request}, Payload, Req, Result), @@ -320,7 +320,7 @@ auth_connect(_Input, Channel = #channel{ctx = Ctx, {ok, Channel#channel{clientinfo = NClientInfo, with_context = with_context(Ctx, ClientInfo)}}; {error, Reason} -> - ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + ?LOG(warning, "Client ~ts (Username: '~ts') login failed for ~0p", [ClientId, Username, Reason]), {error, Reason} end. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl index b5cce573f..13edc785b 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl @@ -68,7 +68,7 @@ on_gateway_update(NewGateway, OldGateway, GwState = #{ctx := Ctx}) -> on_gateway_load(NewGateway, Ctx) catch Class : Reason : Stk -> - logger:error("Failed to update ~s; " + logger:error("Failed to update ~ts; " "reason: {~0p, ~0p} stacktrace: ~0p", [GwName, Class, Reason, Stk]), {error, {Class, Reason}} @@ -90,11 +90,11 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Gateway ~s:~s:~s on ~s started.~n", + ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. @@ -130,10 +130,10 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", + ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl index 98aaf3d31..6d1da48ea 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl @@ -287,7 +287,7 @@ auth_connect(_Packet, Channel = #channel{ctx = Ctx, {ok, NClientInfo} -> {ok, Channel#channel{clientinfo = NClientInfo}}; {error, Reason} -> - ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + ?LOG(warning, "Client ~ts (Username: '~ts') login failed for ~0p", [ClientId, Username, Reason]), %% FIXME: ReasonCode? {error, Reason} @@ -860,7 +860,7 @@ run_client_subs_hook({TopicId, TopicName, QoS}, case run_hooks(Ctx, 'client.subscribe', [ClientInfo, #{}], TopicFilters) of [] -> - ?LOG(warning, "Skip to subscribe ~s, " + ?LOG(warning, "Skip to subscribe ~ts, " "due to 'client.subscribe' denied!", [TopicName]), {error, ?SN_EXCEED_LIMITATION}; [{NTopicName, NSubOpts}|_] -> @@ -879,7 +879,7 @@ do_subscribe({TopicId, TopicName, SubOpts}, {ok, {TopicId, NTopicName, NSubOpts}, Channel#channel{session = NSession}}; {error, ?RC_QUOTA_EXCEEDED} -> - ?LOG(warning, "Cannot subscribe ~s due to ~s.", + ?LOG(warning, "Cannot subscribe ~ts due to ~ts.", [TopicName, emqx_reason_codes:text(?RC_QUOTA_EXCEEDED)]), {error, ?SN_EXCEED_LIMITATION} end. diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl index 1b2025d1b..f88fa210c 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl @@ -292,10 +292,10 @@ message_type(Type) -> io_lib:format("Unknown Type ~p", [Type]). format(?SN_PUBLISH_MSG(Flags, TopicId, MsgId, Data)) -> - io_lib:format("mqtt_sn_message SN_PUBLISH, ~s, TopicId=~w, MsgId=~w, Payload=~w", + io_lib:format("mqtt_sn_message SN_PUBLISH, ~ts, TopicId=~w, MsgId=~w, Payload=~w", [format_flag(Flags), TopicId, MsgId, Data]); format(?SN_PUBACK_MSG(Flags, MsgId, ReturnCode)) -> - io_lib:format("mqtt_sn_message SN_PUBACK, ~s, MsgId=~w, ReturnCode=~w", + io_lib:format("mqtt_sn_message SN_PUBACK, ~ts, MsgId=~w, ReturnCode=~w", [format_flag(Flags), MsgId, ReturnCode]); format(?SN_PUBREC_MSG(?SN_PUBCOMP, MsgId)) -> io_lib:format("mqtt_sn_message SN_PUBCOMP, MsgId=~w", [MsgId]); @@ -304,13 +304,13 @@ format(?SN_PUBREC_MSG(?SN_PUBREC, MsgId)) -> format(?SN_PUBREC_MSG(?SN_PUBREL, MsgId)) -> io_lib:format("mqtt_sn_message SN_PUBREL, MsgId=~w", [MsgId]); format(?SN_SUBSCRIBE_MSG(Flags, Msgid, Topic)) -> - io_lib:format("mqtt_sn_message SN_SUBSCRIBE, ~s, MsgId=~w, TopicId=~w", + io_lib:format("mqtt_sn_message SN_SUBSCRIBE, ~ts, MsgId=~w, TopicId=~w", [format_flag(Flags), Msgid, Topic]); format(?SN_SUBACK_MSG(Flags, TopicId, MsgId, ReturnCode)) -> - io_lib:format("mqtt_sn_message SN_SUBACK, ~s, MsgId=~w, TopicId=~w, ReturnCode=~w", + io_lib:format("mqtt_sn_message SN_SUBACK, ~ts, MsgId=~w, TopicId=~w, ReturnCode=~w", [format_flag(Flags), MsgId, TopicId, ReturnCode]); format(?SN_UNSUBSCRIBE_MSG(Flags, Msgid, Topic)) -> - io_lib:format("mqtt_sn_message SN_UNSUBSCRIBE, ~s, MsgId=~w, TopicId=~w", + io_lib:format("mqtt_sn_message SN_UNSUBSCRIBE, ~ts, MsgId=~w, TopicId=~w", [format_flag(Flags), Msgid, Topic]); format(?SN_UNSUBACK_MSG(MsgId)) -> io_lib:format("mqtt_sn_message SN_UNSUBACK, MsgId=~w", [MsgId]); @@ -321,7 +321,7 @@ format(?SN_REGACK_MSG(TopicId, MsgId, ReturnCode)) -> io_lib:format("mqtt_sn_message SN_REGACK, TopicId=~w, MsgId=~w, ReturnCode=~w", [TopicId, MsgId, ReturnCode]); format(#mqtt_sn_message{type = Type, variable = Var}) -> - io_lib:format("mqtt_sn_message type=~s, Var=~w", [emqx_sn_frame:message_type(Type), Var]). + io_lib:format("mqtt_sn_message type=~ts, Var=~w", [emqx_sn_frame:message_type(Type), Var]). format_flag(#mqtt_sn_flags{dup = Dup, qos = QoS, retain = Retain, will = Will, clean_start = CleanStart, topic_id_type = TopicType}) -> io_lib:format("mqtt_sn_flags{dup=~p, qos=~p, retain=~p, will=~p, clean_session=~p, topic_id_type=~p}", diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl index f5660e0dc..192e7c340 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl @@ -86,7 +86,7 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> on_gateway_load(Gateway#{config => Config}, Ctx) catch Class : Reason : Stk -> - logger:error("Failed to update ~s; " + logger:error("Failed to update ~ts; " "reason: {~0p, ~0p} stacktrace: ~0p", [GwName, Class, Reason, Stk]), {error, {Class, Reason}} @@ -108,11 +108,11 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Gateway ~s:~s:~s on ~s started.~n", + ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. @@ -142,10 +142,10 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", + ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl index 4e045ac3c..91c2ea5fb 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl @@ -280,7 +280,7 @@ auth_connect(_Packet, Channel = #channel{ctx = Ctx, {ok, NClientInfo} -> {ok, Channel#channel{clientinfo = NClientInfo}}; {error, Reason} -> - ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + ?LOG(warning, "Client ~ts (Username: '~ts') login failed for ~0p", [ClientId, Username, Reason]), {error, Reason} end. @@ -352,7 +352,7 @@ handle_in(Packet = ?PACKET(?CMD_CONNECT), Channel) -> {ok, _NPacket, NChannel} -> process_connect(ensure_connected(NChannel)); {error, ReasonCode, NChannel} -> - ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), + ErrMsg = io_lib:format("Login Failed: ~ts", [ReasonCode]), handle_out(connerr, {[], undefined, ErrMsg}, NChannel) end; @@ -403,7 +403,7 @@ handle_in(?PACKET(?CMD_SUBSCRIBE, Headers), handle_out(receipt, receipt_id(Headers), NChannel1) end; {error, ErrMsg, NChannel} -> - ?LOG(error, "Failed to subscribe topic ~s, reason: ~s", + ?LOG(error, "Failed to subscribe topic ~ts, reason: ~ts", [Topic, ErrMsg]), handle_out(error, {receipt_id(Headers), ErrMsg}, NChannel) end; @@ -485,7 +485,7 @@ handle_in(?PACKET(?CMD_COMMIT, Headers), Channel) -> maybe_outgoing_receipt(receipt_id(Headers), Outgoings, Chann1); {error, Reason, Chann1} -> %% FIXME: atomic for transaction ?? - ErrMsg = io_lib:format("Execute transaction ~s falied: ~0p", + ErrMsg = io_lib:format("Execute transaction ~ts falied: ~0p", [TxId, Reason] ), handle_out(error, {receipt_id(Headers), ErrMsg}, Chann1) @@ -653,7 +653,7 @@ handle_call({subscribe, Topic, SubOpts}, _From, NChannel1 = NChannel#channel{subscriptions = NSubs}, reply(ok, NChannel1); {error, ErrMsg, NChannel} -> - ?LOG(error, "Failed to subscribe topic ~s, reason: ~s", + ?LOG(error, "Failed to subscribe topic ~ts, reason: ~ts", [Topic, ErrMsg]), reply({error, ErrMsg}, NChannel) end @@ -829,7 +829,7 @@ handle_deliver(Delivers, [Frame|Acc]; false -> ?LOG(error, "Dropped message ~0p due to not found " - "subscription id for ~s", + "subscription id for ~ts", [Message, emqx_message:topic(Message)]), metrics_inc('delivery.dropped', Channel), metrics_inc('delivery.dropped.no_subid', Channel), diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl index a93240207..b33878a97 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl @@ -71,7 +71,7 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> on_gateway_load(Gateway#{config => Config}, Ctx) catch Class : Reason : Stk -> - logger:error("Failed to update ~s; " + logger:error("Failed to update ~ts; " "reason: {~0p, ~0p} stacktrace: ~0p", [GwName, Class, Reason, Stk]), {error, {Class, Reason}} @@ -93,11 +93,11 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Gateway ~s:~s:~s on ~s started.~n", + ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. @@ -127,10 +127,10 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", + ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. diff --git a/apps/emqx_gateway/test/emqx_coap_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_SUITE.erl index f55fdf88c..db27de1fe 100644 --- a/apps/emqx_gateway/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_coap_SUITE.erl @@ -45,10 +45,10 @@ gateway.coap -define(MQTT_PREFIX, "coap://127.0.0.1/mqtt"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_gateway], fun set_special_cfg/1), + emqx_common_test_helpers:start_apps([emqx_gateway], fun set_special_cfg/1), Config. set_special_cfg(emqx_gateway) -> @@ -58,7 +58,7 @@ set_special_cfg(_) -> ok. end_per_suite(Config) -> - emqx_ct_helpers:stop_apps([emqx_gateway]), + emqx_common_test_helpers:stop_apps([emqx_gateway]), Config. %%-------------------------------------------------------------------- @@ -74,7 +74,7 @@ t_connection(_Config) -> %% heartbeat HeartURI = ?MQTT_PREFIX ++ "/connection?clientid=client1&token=" ++ Token, - ?LOGT("send heartbeat request:~s~n", [HeartURI]), + ?LOGT("send heartbeat request:~ts~n", [HeartURI]), {ok, changed, _} = er_coap_client:request(put, HeartURI), disconnection(Channel, Token), @@ -140,7 +140,7 @@ t_subscribe(_Config) -> URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ Token, Req = make_req(get, Payload, [{observe, 0}]), {ok, content, _} = do_request(Channel, URI, Req), - ?LOGT("observer topic:~s~n", [Topic]), + ?LOGT("observer topic:~ts~n", [Topic]), timer:sleep(100), [SubPid] = emqx:subscribers(Topic), @@ -172,7 +172,7 @@ t_un_subscribe(_Config) -> Req = make_req(get, Payload, [{observe, 0}]), {ok, content, _} = do_request(Channel, URI, Req), - ?LOGT("observer topic:~s~n", [Topic]), + ?LOGT("observer topic:~ts~n", [Topic]), timer:sleep(100), [SubPid] = emqx:subscribers(Topic), @@ -180,7 +180,7 @@ t_un_subscribe(_Config) -> UnReq = make_req(get, Payload, [{observe, 1}]), {ok, nocontent, _} = do_request(Channel, URI, UnReq), - ?LOGT("un observer topic:~s~n", [Topic]), + ?LOGT("un observer topic:~ts~n", [Topic]), timer:sleep(100), ?assertEqual([], emqx:subscribers(Topic)) end, @@ -197,7 +197,7 @@ t_observe_wildcard(_Config) -> URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ Token, Req = make_req(get, Payload, [{observe, 0}]), {ok, content, _} = do_request(Channel, URI, Req), - ?LOGT("observer topic:~s~n", [Topic]), + ?LOGT("observer topic:~ts~n", [Topic]), timer:sleep(100), [SubPid] = emqx:subscribers(Topic), @@ -244,7 +244,7 @@ do_request(Channel, URI, #coap_message{options = Opts} = Req) -> {_, _, Path, Query} = er_coap_client:resolve_uri(URI), Opts2 = [{uri_path, Path}, {uri_query, Query} | Opts], Req2 = Req#coap_message{options = Opts2}, - ?LOGT("send request:~s~nReq:~p~n", [URI, Req2]), + ?LOGT("send request:~ts~nReq:~p~n", [URI, Req2]), {ok, _} = er_coap_channel:send(Channel, Req2), with_response(Channel). diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl index 935a4ec53..1bcd1586a 100644 --- a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl @@ -49,7 +49,7 @@ gateway.coap { %%-------------------------------------------------------------------- all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), @@ -119,7 +119,7 @@ test_send_coap_request(UdpSock, Method, Content, Options, MsgId) -> ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, RequestBinary]), ok = gen_udp:send(UdpSock, IpAddr, Port, RequestBinary); {SchemeDiff, ChIdDiff, _, _} -> - error(lists:flatten(io_lib:format("scheme ~s or ChId ~s does not match with socket", [SchemeDiff, ChIdDiff]))) + error(lists:flatten(io_lib:format("scheme ~ts or ChId ~ts does not match with socket", [SchemeDiff, ChIdDiff]))) end. test_recv_coap_response(UdpSock) -> diff --git a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl b/apps/emqx_gateway/test/emqx_exproto_SUITE.erl index b91cd03b9..fc07fb720 100644 --- a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_exproto_SUITE.erl @@ -45,7 +45,7 @@ all() -> [{group, Name} || Name <- metrics()]. groups() -> - Cases = emqx_ct:all(?MODULE), + Cases = emqx_common_test_helpers:all(?MODULE), [{Name, Cases} || Name <- metrics()]. %% @private @@ -55,12 +55,12 @@ metrics() -> init_per_group(GrpName, Cfg) -> put(grpname, GrpName), Svrs = emqx_exproto_echo_svr:start(), - emqx_ct_helpers:start_apps([emqx_gateway], fun set_special_cfg/1), + emqx_common_test_helpers:start_apps([emqx_gateway], fun set_special_cfg/1), emqx_logger:set_log_level(debug), [{servers, Svrs}, {listener_type, GrpName} | Cfg]. end_per_group(_, Cfg) -> - emqx_ct_helpers:stop_apps([emqx_gateway]), + emqx_common_test_helpers:stop_apps([emqx_gateway]), emqx_exproto_echo_svr:stop(proplists:get_value(servers, Cfg)). set_special_cfg(emqx_gateway) -> @@ -448,7 +448,7 @@ client_ssl_opts() -> certs("client-key.pem", "client-cert.pem", "cacert.pem"). certs(Key, Cert, CACert) -> - CertsPath = emqx_ct_helpers:deps_path(emqx, "etc/certs"), + CertsPath = emqx_common_test_helpers:deps_path(emqx, "etc/certs"), #{keyfile => filename:join([ CertsPath, Key ]), certfile => filename:join([ CertsPath, Cert ]), cacertfile => filename:join([ CertsPath, CACert])}. diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index 295df5737..c752150ac 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -26,16 +26,16 @@ %%-------------------------------------------------------------------- all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> %% FIXME: Magic line. for saving gateway schema name for emqx_config emqx_config:init_load(emqx_gateway_schema, <<"gateway {}">>), - emqx_ct_helpers:start_apps([emqx_gateway]), + emqx_common_test_helpers:start_apps([emqx_gateway]), Conf. end_per_suite(_Conf) -> - emqx_ct_helpers:stop_apps([emqx_gateway]). + emqx_common_test_helpers:stop_apps([emqx_gateway]). init_per_testcase(_CaseName, Conf) -> _ = emqx_gateway_conf:unload_gateway(stomp), diff --git a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl index 56776957f..3ff63539e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl @@ -27,7 +27,7 @@ gateway: { } """>>). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- %% Setups @@ -35,11 +35,11 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), - emqx_ct_helpers:start_apps([emqx_gateway]), + emqx_common_test_helpers:start_apps([emqx_gateway]), Cfg. end_per_suite(_Cfg) -> - emqx_ct_helpers:stop_apps([emqx_gateway]), + emqx_common_test_helpers:stop_apps([emqx_gateway]), ok. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl index f36c1e816..8f9e6108f 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl @@ -150,12 +150,12 @@ groups() -> ]. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(Config) -> timer:sleep(300), - emqx_ct_helpers:stop_apps([]), + emqx_common_test_helpers:stop_apps([]), Config. init_per_testcase(_AllTestCase, Config) -> @@ -190,7 +190,7 @@ case01_register(Config) -> test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, [], MsgId), @@ -216,7 +216,7 @@ case01_register(Config) -> MsgId3 = 52, test_send_coap_request( UdpSock, delete, - sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), + sprintf("coap://127.0.0.1:~b~ts", [?PORT, join_path(Location, <<>>)]), #coap_content{payload = <<>>}, [], MsgId3), @@ -235,7 +235,7 @@ case01_register_additional_opts(Config) -> MsgId = 12, SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), - AddOpts = "ep=~s<=345&lwm2m=1&apn=psmA.eDRX0.ctnb&cust_opt=shawn&im=123&ct=1.4&mt=mdm9620&mv=1.2", + AddOpts = "ep=~ts<=345&lwm2m=1&apn=psmA.eDRX0.ctnb&cust_opt=shawn&im=123&ct=1.4&mt=mdm9620&mv=1.2", test_send_coap_request( UdpSock, post, sprintf("coap://127.0.0.1:~b/rd?" ++ AddOpts, [?PORT, Epn]), @@ -264,7 +264,7 @@ case01_register_additional_opts(Config) -> MsgId3 = 52, test_send_coap_request( UdpSock, delete, - sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), + sprintf("coap://127.0.0.1:~b~ts", [?PORT, join_path(Location, <<>>)]), #coap_content{payload = <<>>}, [], MsgId3), @@ -283,7 +283,7 @@ case01_register_incorrect_opts(Config) -> MsgId = 12, - AddOpts = "ep=~s<=345&lwm2m=1&incorrect_opt", + AddOpts = "ep=~ts<=345&lwm2m=1&incorrect_opt", test_send_coap_request( UdpSock, post, sprintf("coap://127.0.0.1:~b/rd?" ++ AddOpts, [?PORT, Epn]), @@ -310,7 +310,7 @@ case01_register_report(Config) -> test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, [], MsgId), @@ -345,7 +345,7 @@ case01_register_report(Config) -> MsgId3 = 52, test_send_coap_request( UdpSock, delete, - sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), + sprintf("coap://127.0.0.1:~b~ts", [?PORT, join_path(Location, <<>>)]), #coap_content{payload = <<>>}, [], MsgId3), @@ -369,7 +369,7 @@ case02_update_deregister(Config) -> test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, [], MsgId), @@ -398,7 +398,7 @@ case02_update_deregister(Config) -> MsgId2 = 27, test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b~s?lt=789", [?PORT, join_path(Location, <<>>)]), + sprintf("coap://127.0.0.1:~b~ts?lt=789", [?PORT, join_path(Location, <<>>)]), #coap_content{content_format = <<"text/plain">>, payload = <<", , , , , ">>}, [], MsgId2), @@ -424,7 +424,7 @@ case02_update_deregister(Config) -> MsgId3 = 52, test_send_coap_request( UdpSock, delete, - sprintf("coap://127.0.0.1:~b~s", [?PORT, join_path(Location, <<>>)]), + sprintf("coap://127.0.0.1:~b~ts", [?PORT, join_path(Location, <<>>)]), #coap_content{payload = <<>>}, [], MsgId3), @@ -445,7 +445,7 @@ case03_register_wrong_version(Config) -> SubTopic = list_to_binary("lwm2m/"++Epn++"/dn/#"), test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=8.3", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=8.3", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, [], MsgId), @@ -466,7 +466,7 @@ case04_register_and_lifetime_timeout(Config) -> test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=2&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=2&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, [], MsgId), @@ -509,7 +509,7 @@ case05_register_wrong_epn(Config) -> %% test_send_coap_request( UdpSock, %% post, -%% sprintf("coap://127.0.0.1:~b/rd?ep=~s&lwm2m=1", [?PORT, Epn]), +%% sprintf("coap://127.0.0.1:~b/rd?ep=~ts&lwm2m=1", [?PORT, Epn]), %% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, %% [], %% MsgId), @@ -532,7 +532,7 @@ case07_register_alternate_path_01(Config) -> test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], @@ -554,7 +554,7 @@ case07_register_alternate_path_02(Config) -> test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], @@ -576,7 +576,7 @@ case08_reregister(Config) -> test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], @@ -602,7 +602,7 @@ case08_reregister(Config) -> %% the same lwm2mc client registers to server again test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], @@ -621,7 +621,7 @@ case10_read(Config) -> %% step 1, device register ... test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], @@ -1605,7 +1605,7 @@ case60_observe(Config) -> %% test_send_coap_request( UdpSock, %% post, -%% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), +%% sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), %% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, %% [], %% MsgId1), @@ -1671,7 +1671,7 @@ case60_observe(Config) -> %% test_send_coap_request( UdpSock, %% post, -%% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), +%% sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), %% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, %% [], %% MsgId1), @@ -1717,10 +1717,10 @@ case60_observe(Config) -> %% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case90_psm_mode(Config) -> - server_cache_mode(Config, "ep=~s<=345&lwm2m=1&apn=psmA.eDRX0.ctnb"). + server_cache_mode(Config, "ep=~ts<=345&lwm2m=1&apn=psmA.eDRX0.ctnb"). case90_queue_mode(Config) -> - server_cache_mode(Config, "ep=~s<=345&lwm2m=1&b=UQ"). + server_cache_mode(Config, "ep=~ts<=345&lwm2m=1&b=UQ"). server_cache_mode(Config, RegOption) -> #{lwm2m := LwM2M} = Gateway = emqx:get_config([gateway]), @@ -1817,7 +1817,7 @@ device_update_1(UdpSock, Location) -> MsgId2 = 27, test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b~s?lt=789", [?PORT, join_path(Location, <<>>)]), + sprintf("coap://127.0.0.1:~b~ts?lt=789", [?PORT, join_path(Location, <<>>)]), #coap_content{payload = <<>>}, [], MsgId2), @@ -1845,7 +1845,7 @@ test_send_coap_request(UdpSock, Method, Uri, Content, Options, MsgId) -> ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, RequestBinary]), ok = gen_udp:send(UdpSock, IpAddr, Port, RequestBinary); {SchemeDiff, ChIdDiff, _, _} -> - error(lists:flatten(io_lib:format("scheme ~s or ChId ~s does not match with socket", [SchemeDiff, ChIdDiff]))) + error(lists:flatten(io_lib:format("scheme ~ts or ChId ~ts does not match with socket", [SchemeDiff, ChIdDiff]))) end. test_recv_coap_response(UdpSock) -> @@ -1916,7 +1916,7 @@ test_send_coap_notif(UdpSock, Host, Port, Content, ObSeq, Request) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic) -> test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = ObjectList}, [], MsgId1), diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl index a875aceb6..ed817dbd8 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl @@ -66,7 +66,7 @@ gateway.lwm2m { %%-------------------------------------------------------------------- all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), @@ -108,7 +108,7 @@ t_lookup_cmd_read(Config) -> %% step 1, device register ... test_send_coap_request( UdpSock, post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=600&lwm2m=1", [?PORT, Epn]), + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=600&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], @@ -192,9 +192,9 @@ t_lookup_cmd_discover(Config) -> send_request(ClientId, Path, Action) -> ApiPath = emqx_mgmt_api_test_util:api_path(["gateway/lwm2m", ClientId, "lookup_cmd"]), Auth = emqx_mgmt_api_test_util:auth_header_(), - Query = io_lib:format("path=~s&action=~s", [Path, Action]), + Query = io_lib:format("path=~ts&action=~ts", [Path, Action]), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, ApiPath, Query, Auth), - ?LOGT("rest api response:~s~n", [Response]), + ?LOGT("rest api response:~ts~n", [Response]), Response. no_received_request(ClientId, Path, Action) -> diff --git a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl index 7437c10b8..c02b60dd0 100644 --- a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl @@ -27,7 +27,7 @@ %%-------------------------------------------------------------------- all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). parse(D) -> {ok, P, _Rest, _State} = emqx_sn_frame:parse(D, #{}), diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl index 23fb691d9..aac2f0d35 100644 --- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl @@ -79,15 +79,15 @@ gateway.mqttsn { %%-------------------------------------------------------------------- all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), - emqx_ct_helpers:start_apps([emqx_gateway]), + emqx_common_test_helpers:start_apps([emqx_gateway]), Config. end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_gateway]). + emqx_common_test_helpers:stop_apps([emqx_gateway]). %%-------------------------------------------------------------------- %% Test cases diff --git a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl index 9aebfe791..e727c6c88 100644 --- a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl @@ -31,7 +31,7 @@ %%-------------------------------------------------------------------- all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> application:ensure_all_started(ekka), diff --git a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl index 9c3f1090f..56b1174a2 100644 --- a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl @@ -35,7 +35,7 @@ gateway.stomp { } ">>). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- %% Setups @@ -43,11 +43,11 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), - emqx_ct_helpers:start_apps([emqx_gateway]), + emqx_common_test_helpers:start_apps([emqx_gateway]), Cfg. end_per_suite(_Cfg) -> - emqx_ct_helpers:stop_apps([emqx_gateway]), + emqx_common_test_helpers:stop_apps([emqx_gateway]), ok. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl index cbced5f43..2af2493d2 100644 --- a/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- %% Test Cases diff --git a/apps/emqx_limiter/etc/emqx_limiter.conf b/apps/emqx_limiter/etc/emqx_limiter.conf new file mode 100644 index 000000000..44bbb1740 --- /dev/null +++ b/apps/emqx_limiter/etc/emqx_limiter.conf @@ -0,0 +1,50 @@ +##-------------------------------------------------------------------- +## Emq X Rate Limiter +##-------------------------------------------------------------------- +emqx_limiter { + bytes_in { + global = "100KB/10s" # token generation rate + zone.default = "100kB/10s" + zone.external = "20kB/10s" + bucket.tcp { + zone = default + aggregated = "100kB/10s,1Mb" + per_client = "100KB/10s,10Kb" + } + bucket.ssl { + zone = external + aggregated = "100kB/10s,1Mb" + per_client = "100KB/10s,10Kb" + } + } + + message_in { + global = "100/10s" + zone.default = "100/10s" + bucket.bucket1 { + zone = default + aggregated = "100/10s,1000" + per_client = "100/10s,100" + } + } + + connection { + global = "100/10s" + zone.default = "100/10s" + bucket.bucket1 { + zone = default + aggregated = "100/10s,1000" + per_client = "100/10s,100" + } + } + + message_routing { + global = "100/10s" + zone.default = "100/10s" + bucket.bucket1 { + zone = default + aggregated = "100/10s,100" + per_client = "100/10s,10" + } + } +} \ No newline at end of file diff --git a/apps/emqx_limiter/src/emqx_limiter.app.src b/apps/emqx_limiter/src/emqx_limiter.app.src new file mode 100644 index 000000000..70fe89e97 --- /dev/null +++ b/apps/emqx_limiter/src/emqx_limiter.app.src @@ -0,0 +1,15 @@ +%% -*- mode: erlang -*- +{application, emqx_limiter, + [{description, "EMQ X Hierachical Limiter"}, + {vsn, "1.0.0"}, % strict semver, bump manually! + {modules, []}, + {registered, [emqx_limiter_sup]}, + {applications, [kernel,stdlib,emqx]}, + {mod, {emqx_limiter_app,[]}}, + {env, []}, + {licenses, ["Apache-2.0"]}, + {maintainers, ["EMQ X Team "]}, + {links, [{"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx/emqx-retainer"} + ]} + ]}. diff --git a/apps/emqx_limiter/src/emqx_limiter_app.erl b/apps/emqx_limiter/src/emqx_limiter_app.erl new file mode 100644 index 000000000..2244a0e91 --- /dev/null +++ b/apps/emqx_limiter/src/emqx_limiter_app.erl @@ -0,0 +1,55 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_limiter_app). + +-behaviour(application). + +%% Application callbacks +-export([start/2, stop/1]). + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called whenever an application is started using +%% application:start/[1,2], and should start the processes of the +%% application. If the application is structured according to the OTP +%% design principles as a supervision tree, this means starting the +%% top supervisor of the tree. +%% @end +%%-------------------------------------------------------------------- +-spec start(StartType :: normal | + {takeover, Node :: node()} | + {failover, Node :: node()}, + StartArgs :: term()) -> + {ok, Pid :: pid()} | + {ok, Pid :: pid(), State :: term()} | + {error, Reason :: term()}. +start(_StartType, _StartArgs) -> + {ok, _} = Result = emqx_limiter_sup:start_link(), + Result. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called whenever an application has stopped. It +%% is intended to be the opposite of Module:start/2 and should do +%% any necessary cleaning up. The return value is ignored. +%% @end +%%-------------------------------------------------------------------- +-spec stop(State :: term()) -> any(). +stop(_State) -> + ok. diff --git a/apps/emqx_limiter/src/emqx_limiter_client.erl b/apps/emqx_limiter/src/emqx_limiter_client.erl new file mode 100644 index 000000000..eb7c768ff --- /dev/null +++ b/apps/emqx_limiter/src/emqx_limiter_client.erl @@ -0,0 +1,144 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_limiter_client). + +%% API +-export([create/5, make_ref/3, consume/2]). +-export_type([limiter/0]). + +%% tocket bucket algorithm +-record(limiter, { tokens :: non_neg_integer() + , rate :: float() + , capacity :: decimal() + , lasttime :: millisecond() + , ref :: ref_limiter() + }). + +-record(ref, { counter :: counters:counters_ref() + , index :: index() + , rate :: decimal() + , obtained :: non_neg_integer() + }). + +%% TODO +%% we should add a nop-limiter, when all the upper layers (global, zone, and buckets ) are infinity + +-type limiter() :: #limiter{}. +-type ref_limiter() :: #ref{}. +-type client() :: limiter() | ref_limiter(). +-type millisecond() :: non_neg_integer(). +-type pause_result(Client) :: {pause, millisecond(), Client}. +-type consume_result(Client) :: {ok, Client} + | pause_result(Client). +-type decimal() :: emqx_limiter_decimal:decimal(). +-type index() :: emqx_limiter_server:index(). + +-define(NOW, erlang:monotonic_time(millisecond)). +-define(MINIUMN_PAUSE, 100). + +-import(emqx_limiter_decimal, [sub/2]). +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec create(float(), + decimal(), + counters:counters_ref(), + index(), + decimal()) -> limiter(). +create(Rate, Capacity, Counter, Index, CounterRate) -> + #limiter{ tokens = Capacity + , rate = Rate + , capacity = Capacity + , lasttime = ?NOW + , ref = make_ref(Counter, Index, CounterRate) + }. + +-spec make_ref(counters:counters_ref(), index(), decimal()) -> ref_limiter(). +make_ref(Counter, Idx, Rate) -> + #ref{counter = Counter, index = Idx, rate = Rate, obtained = 0}. + +-spec consume(pos_integer(), Client) -> consume_result(Client) + when Client :: client(). +consume(Need, #limiter{tokens = Tokens, + capacity = Capacity} = Limiter) -> + if Need =< Tokens -> + try_consume_counter(Need, Limiter); + Need > Capacity -> + %% FIXME + %% The client should be able to send 4kb data if the rate is configured to be 2kb/s, it just needs 2s to complete. + throw("too big request"); %% FIXME how to deal this? + true -> + try_reset(Need, Limiter) + end; + +consume(Need, #ref{counter = Counter, + index = Index, + rate = Rate, + obtained = Obtained} = Ref) -> + Tokens = counters:get(Counter, Index), + if Tokens >= Need -> + counters:sub(Counter, Index, Need), + {ok, Ref#ref{obtained = Obtained + Need}}; + true -> + return_pause(Need - Tokens, Rate, Ref) + end. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +-spec try_consume_counter(pos_integer(), limiter()) -> consume_result(limiter()). +try_consume_counter(Need, + #limiter{tokens = Tokens, + ref = #ref{counter = Counter, + index = Index, + obtained = Obtained, + rate = CounterRate} = Ref} = Limiter) -> + CT = counters:get(Counter, Index), + if CT >= Need -> + counters:sub(Counter, Index, Need), + {ok, Limiter#limiter{tokens = sub(Tokens, Need), + ref = Ref#ref{obtained = Obtained + Need}}}; + true -> + return_pause(Need - CT, CounterRate, Limiter) + end. + +-spec try_reset(pos_integer(), limiter()) -> consume_result(limiter()). +try_reset(Need, + #limiter{tokens = Tokens, + rate = Rate, + lasttime = LastTime, + capacity = Capacity} = Limiter) -> + Now = ?NOW, + Inc = erlang:floor((Now - LastTime) * Rate / emqx_limiter_schema:minimum_period()), + Tokens2 = erlang:min(Tokens + Inc, Capacity), + if Need > Tokens2 -> + return_pause(Need, Rate, Limiter); + true -> + Limiter2 = Limiter#limiter{tokens = Tokens2, + lasttime = Now}, + try_consume_counter(Need, Limiter2) + end. + +-spec return_pause(pos_integer(), decimal(), Client) -> pause_result(Client) + when Client :: client(). +return_pause(_, infinity, Limiter) -> + %% workaround when emqx_limiter_server's rate is infinity + {pause, ?MINIUMN_PAUSE, Limiter}; + +return_pause(Diff, Rate, Limiter) -> + Pause = erlang:round(Diff * emqx_limiter_schema:minimum_period() / Rate), + {pause, erlang:max(Pause, ?MINIUMN_PAUSE), Limiter}. diff --git a/apps/emqx_limiter/src/emqx_limiter_decimal.erl b/apps/emqx_limiter/src/emqx_limiter_decimal.erl new file mode 100644 index 000000000..26ae611e8 --- /dev/null +++ b/apps/emqx_limiter/src/emqx_limiter_decimal.erl @@ -0,0 +1,79 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% a simple decimal module for rate-related calculations + +-module(emqx_limiter_decimal). + +%% API +-export([ add/2, sub/2, mul/2 + , add_to_counter/3, put_to_counter/3]). +-export_type([decimal/0, zero_or_float/0]). + +-type decimal() :: infinity | number(). +-type zero_or_float() :: 0 | float(). + +%%-------------------------------------------------------------------- +%%% API +%%-------------------------------------------------------------------- +-spec add(decimal(), decimal()) -> decimal(). +add(A, B) when A =:= infinity + orelse B =:= infinity -> + infinity; + +add(A, B) -> + A + B. + +-spec sub(decimal(), decimal()) -> decimal(). +sub(A, B) when A =:= infinity + orelse B =:= infinity -> + infinity; + +sub(A, B) -> + A - B. + +-spec mul(decimal(), decimal()) -> decimal(). +mul(A, B) when A =:= infinity + orelse B =:= infinity -> + infinity; + +mul(A, B) -> + A * B. + +-spec add_to_counter(counters:counters_ref(), pos_integer(), decimal()) -> + {zero_or_float(), zero_or_float()}. +add_to_counter(_, _, infinity) -> + {0, 0}; +add_to_counter(Counter, Index, Val) when is_float(Val) -> + IntPart = erlang:floor(Val), + if IntPart > 0 -> + counters:add(Counter, Index, IntPart); + true -> + ok + end, + {IntPart, Val - IntPart}; +add_to_counter(Counter, Index, Val) -> + counters:add(Counter, Index, Val), + {Val, 0}. + +-spec put_to_counter(counters:counters_ref(), pos_integer(), decimal()) -> ok. +put_to_counter(_, _, infinity) -> + ok; +put_to_counter(Counter, Index, Val) when is_float(Val) -> + IntPart = erlang:floor(Val), + counters:put(Counter, Index, IntPart); +put_to_counter(Counter, Index, Val) -> + counters:put(Counter, Index, Val). diff --git a/apps/emqx_limiter/src/emqx_limiter_manager.erl b/apps/emqx_limiter/src/emqx_limiter_manager.erl new file mode 100644 index 000000000..471098242 --- /dev/null +++ b/apps/emqx_limiter/src/emqx_limiter_manager.erl @@ -0,0 +1,229 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_limiter_manager). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). + +%% API +-export([ start_link/0, start_server/1, find_counter/1 + , find_counter/3, insert_counter/4, insert_counter/6 + , make_path/3, restart_server/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3, format_status/2]). + +-type path() :: list(atom()). +-type limiter_type() :: emqx_limiter_schema:limiter_type(). +-type zone_name() :: emqx_limiter_schema:zone_name(). +-type bucket_name() :: emqx_limiter_schema:bucket_name(). + +%% counter record in ets table +-record(element, {path :: path(), + counter :: counters:counters_ref(), + index :: index(), + rate :: rate() + }). + + +-type index() :: emqx_limiter_server:index(). +-type rate() :: emqx_limiter_decimal:decimal(). + +-define(TAB, emqx_limiter_counters). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec start_server(limiter_type()) -> _. +start_server(Type) -> + emqx_limiter_server_sup:start(Type). + +-spec restart_server(limiter_type()) -> _. +restart_server(Type) -> + emqx_limiter_server_sup:restart(Type). + +-spec find_counter(limiter_type(), zone_name(), bucket_name()) -> + {ok, counters:counters_ref(), index(), rate()} | undefined. +find_counter(Type, Zone, BucketId) -> + find_counter(make_path(Type, Zone, BucketId)). + +-spec find_counter(path()) -> + {ok, counters:counters_ref(), index(), rate()} | undefined. +find_counter(Path) -> + case ets:lookup(?TAB, Path) of + [#element{counter = Counter, index = Index, rate = Rate}] -> + {ok, Counter, Index, Rate}; + _ -> + undefined + end. + +-spec insert_counter(limiter_type(), + zone_name(), + bucket_name(), + counters:counters_ref(), + index(), + rate()) -> boolean(). +insert_counter(Type, Zone, BucketId, Counter, Index, Rate) -> + insert_counter(make_path(Type, Zone, BucketId), + Counter, + Index, + Rate). + +-spec insert_counter(path(), + counters:counters_ref(), + index(), + rate()) -> boolean(). +insert_counter(Path, Counter, Index, Rate) -> + ets:insert(?TAB, + #element{path = Path, + counter = Counter, + index = Index, + rate = Rate}). + +-spec make_path(limiter_type(), zone_name(), bucket_name()) -> path(). +make_path(Type, Name, BucketId) -> + [Type, Name, BucketId]. + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% @end +%%-------------------------------------------------------------------- +-spec start_link() -> {ok, Pid :: pid()} | + {error, Error :: {already_started, pid()}} | + {error, Error :: term()} | + ignore. +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the server +%% @end +%%-------------------------------------------------------------------- +-spec init(Args :: term()) -> {ok, State :: term()} | + {ok, State :: term(), Timeout :: timeout()} | + {ok, State :: term(), hibernate} | + {stop, Reason :: term()} | + ignore. +init([]) -> + _ = ets:new(?TAB, [ set, public, named_table, {keypos, #element.path} + , {write_concurrency, true}, {read_concurrency, true} + , {heir, erlang:whereis(emqx_limiter_sup), none} + ]), + {ok, #{}}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_call(Request :: term(), From :: {pid(), term()}, State :: term()) -> + {reply, Reply :: term(), NewState :: term()} | + {reply, Reply :: term(), NewState :: term(), Timeout :: timeout()} | + {reply, Reply :: term(), NewState :: term(), hibernate} | + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: term(), Reply :: term(), NewState :: term()} | + {stop, Reason :: term(), NewState :: term()}. +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignore, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_cast(Request :: term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: term(), NewState :: term()}. +handle_cast(Req, State) -> + ?LOG(error, "Unexpected cast: ~p", [Req]), + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_info(Info :: timeout() | term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: normal | term(), NewState :: term()}. +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% @end +%%-------------------------------------------------------------------- +-spec terminate(Reason :: normal | shutdown | {shutdown, term()} | term(), + State :: term()) -> any(). +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% @end +%%-------------------------------------------------------------------- +-spec code_change(OldVsn :: term() | {down, term()}, + State :: term(), + Extra :: term()) -> {ok, NewState :: term()} | + {error, Reason :: term()}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called for changing the form and appearance +%% of gen_server status when it is returned from sys:get_status/1,2 +%% or when it appears in termination error logs. +%% @end +%%-------------------------------------------------------------------- +-spec format_status(Opt :: normal | terminate, + Status :: list()) -> Status :: term(). +format_status(_Opt, Status) -> + Status. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- diff --git a/apps/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx_limiter/src/emqx_limiter_schema.erl new file mode 100644 index 000000000..0e2977025 --- /dev/null +++ b/apps/emqx_limiter/src/emqx_limiter_schema.erl @@ -0,0 +1,140 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_limiter_schema). + +-include_lib("typerefl/include/types.hrl"). + +-export([ roots/0, fields/1, to_rate/1 + , to_bucket_rate/1, minimum_period/0]). + +-define(KILOBYTE, 1024). + +-type limiter_type() :: bytes_in + | message_in + | connection + | message_routing. + +-type bucket_name() :: atom(). +-type zone_name() :: atom(). +-type rate() :: infinity | float(). +-type bucket_rate() :: list(infinity | number()). + +-typerefl_from_string({rate/0, ?MODULE, to_rate}). +-typerefl_from_string({bucket_rate/0, ?MODULE, to_bucket_rate}). + +-reflect_type([ rate/0 + , bucket_rate/0 + ]). + +-export_type([limiter_type/0, bucket_name/0, zone_name/0]). + +-import(emqx_schema, [sc/2, map/2]). + +roots() -> [emqx_limiter]. + +fields(emqx_limiter) -> + [ {bytes_in, sc(ref(limiter), #{})} + , {message_in, sc(ref(limiter), #{})} + , {connection, sc(ref(limiter), #{})} + , {message_routing, sc(ref(limiter), #{})} + ]; + +fields(limiter) -> + [ {global, sc(rate(), #{})} + , {zone, sc(map("zone name", rate()), #{})} + , {bucket, sc(map("bucket id", ref(bucket)), + #{desc => "Token Buckets"})} + ]; + +fields(bucket) -> + [ {zone, sc(atom(), #{desc => "the zone which the bucket in"})} + , {aggregated, sc(bucket_rate(), #{})} + , {per_client, sc(bucket_rate(), #{})} + ]. + +%% minimum period is 100ms +minimum_period() -> + 100. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +ref(Field) -> hoconsc:ref(?MODULE, Field). + +to_rate(Str) -> + Tokens = [string:trim(T) || T <- string:tokens(Str, "/")], + case Tokens of + ["infinity"] -> + {ok, infinity}; + [Quota, Interval] -> + {ok, Val} = to_quota(Quota), + case emqx_schema:to_duration_ms(Interval) of + {ok, Ms} when Ms > 0 -> + {ok, Val * minimum_period() / Ms}; + _ -> + {error, Str} + end; + _ -> + {error, Str} + end. + +to_bucket_rate(Str) -> + Tokens = [string:trim(T) || T <- string:tokens(Str, "/,")], + case Tokens of + [Rate, Capa] -> + {ok, infinity} = to_quota(Rate), + {ok, CapaVal} = to_quota(Capa), + if CapaVal =/= infinity -> + {ok, [infinity, CapaVal]}; + true -> + {error, Str} + end; + [Quota, Interval, Capacity] -> + {ok, Val} = to_quota(Quota), + case emqx_schema:to_duration_ms(Interval) of + {ok, Ms} when Ms > 0 -> + {ok, CapaVal} = to_quota(Capacity), + {ok, [Val * minimum_period() / Ms, CapaVal]}; + _ -> + {error, Str} + end; + _ -> + {error, Str} + end. + + +to_quota(Str) -> + {ok, MP} = re:compile("^\s*(?:(?:([1-9][0-9]*)([a-zA-z]*))|infinity)\s*$"), + Result = re:run(Str, MP, [{capture, all_but_first, list}]), + case Result of + {match, [Quota, Unit]} -> + Val = erlang:list_to_integer(Quota), + Unit2 = string:to_lower(Unit), + {ok, apply_unit(Unit2, Val)}; + {match, [Quota]} -> + {ok, erlang:list_to_integer(Quota)}; + {match, []} -> + {ok, infinity}; + _ -> + {error, Str} + end. + +apply_unit("", Val) -> Val; +apply_unit("kb", Val) -> Val * ?KILOBYTE; +apply_unit("mb", Val) -> Val * ?KILOBYTE * ?KILOBYTE; +apply_unit("gb", Val) -> Val * ?KILOBYTE * ?KILOBYTE * ?KILOBYTE; +apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). diff --git a/apps/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx_limiter/src/emqx_limiter_server.erl new file mode 100644 index 000000000..8a712db2e --- /dev/null +++ b/apps/emqx_limiter/src/emqx_limiter_server.erl @@ -0,0 +1,426 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% A hierachical token bucket algorithm +%% Note: this is not the linux HTB algorithm(http://luxik.cdi.cz/~devik/qos/htb/manual/theory.htm) +%% Algorithm: +%% 1. the root node periodically generates tokens and then distributes them +%% just like the oscillation of water waves +%% 2. the leaf node has a counter, which is the place where the token is actually held. +%% 3. other nodes only play the role of transmission, and the rate of the node is like a valve, +%% limiting the oscillation transmitted from the parent node + +-module(emqx_limiter_server). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3, format_status/2]). + +-export([ start_link/1, connect/2, info/2 + , name/1]). + +-record(root, { rate :: rate() %% number of tokens generated per period + , period :: pos_integer() %% token generation interval(second) + , childs :: list(node_id()) %% node children + , consumed :: non_neg_integer() + }). + +-record(zone, { id :: pos_integer() + , name :: zone_name() + , rate :: rate() + , obtained :: non_neg_integer() %% number of tokens obtained + , childs :: list(node_id()) + }). + +-record(bucket, { id :: pos_integer() + , name :: bucket_name() + , rate :: rate() + , obtained :: non_neg_integer() + , correction :: emqx_limiter_decimal:zero_or_float() %% token correction value + , capacity :: capacity() + , counter :: counters:counters_ref() + , index :: index() + }). + +-record(state, { root :: undefined | root() + , counter :: undefined | counters:counters_ref() %% current counter to alloc + , index :: index() + , zones :: #{zone_name() => node_id()} + , nodes :: nodes() + , type :: limiter_type() + }). + +%% maybe use maps is better, but record is fastter +-define(FIELD_OBTAINED, #zone.obtained). +-define(GET_FIELD(F, Node), element(F, Node)). +-define(CALL(Type, Msg), gen_server:call(name(Type), {?FUNCTION_NAME, Msg})). + +-type node_id() :: pos_integer(). +-type root() :: #root{}. +-type zone() :: #zone{}. +-type bucket() :: #bucket{}. +-type node_data() :: zone() | bucket(). +-type nodes() :: #{node_id() => node_data()}. +-type zone_name() :: emqx_limiter_schema:zone_name(). +-type limiter_type() :: emqx_limiter_schema:limiter_type(). +-type bucket_name() :: emqx_limiter_schema:bucket_name(). +-type rate() :: decimal(). +-type flow() :: decimal(). +-type capacity() :: decimal(). +-type decimal() :: emqx_limiter_decimal:decimal(). +-type state() :: #state{}. +-type index() :: pos_integer(). + +-export_type([index/0]). +-import(emqx_limiter_decimal, [add/2, sub/2, mul/2, add_to_counter/3, put_to_counter/3]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec connect(limiter_type(), bucket_name()) -> emqx_limiter_client:client(). +connect(Type, Bucket) -> + #{zone := Zone, + aggregated := [Aggr, Capacity], + per_client := [Client, ClientCapa]} = emqx:get_config([emqx_limiter, Type, bucket, Bucket]), + case emqx_limiter_manager:find_counter(Type, Zone, Bucket) of + {ok, Counter, Idx, Rate} -> + if Client =/= infinity andalso (Client < Aggr orelse ClientCapa < Capacity) -> + emqx_limiter_client:create(Client, ClientCapa, Counter, Idx, Rate); + true -> + emqx_limiter_client:make_ref(Counter, Idx, Rate) + end; + _ -> + ?LOG(error, "can't find the bucket:~p which type is:~p~n", [Bucket, Type]), + throw("invalid bucket") + end. + +-spec info(limiter_type(), atom()) -> term(). +info(Type, Info) -> + ?CALL(Type, Info). + +-spec name(limiter_type()) -> atom(). +name(Type) -> + erlang:list_to_atom(io_lib:format("~s_~s", [?MODULE, Type])). + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% @end +%%-------------------------------------------------------------------- +-spec start_link(limiter_type()) -> _. +start_link(Type) -> + gen_server:start_link({local, name(Type)}, ?MODULE, [Type], []). + +%%-------------------------------------------------------------------- +%%% gen_server callbacks +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the server +%% @end +%%-------------------------------------------------------------------- +-spec init(Args :: term()) -> {ok, State :: term()} | + {ok, State :: term(), Timeout :: timeout()} | + {ok, State :: term(), hibernate} | + {stop, Reason :: term()} | + ignore. +init([Type]) -> + State = #state{zones = #{}, + nodes = #{}, + type = Type, + index = 1}, + State2 = init_tree(Type, State), + oscillate(State2#state.root#root.period), + {ok, State2}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_call(Request :: term(), From :: {pid(), term()}, State :: term()) -> + {reply, Reply :: term(), NewState :: term()} | + {reply, Reply :: term(), NewState :: term(), Timeout :: timeout()} | + {reply, Reply :: term(), NewState :: term(), hibernate} | + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: term(), Reply :: term(), NewState :: term()} | + {stop, Reason :: term(), NewState :: term()}. +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_cast(Request :: term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: term(), NewState :: term()}. +handle_cast(Req, State) -> + ?LOG(error, "Unexpected cast: ~p", [Req]), + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% @end +%%-------------------------------------------------------------------- +-spec handle_info(Info :: timeout() | term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), Timeout :: timeout()} | + {noreply, NewState :: term(), hibernate} | + {stop, Reason :: normal | term(), NewState :: term()}. +handle_info(oscillate, State) -> + {noreply, oscillation(State)}; + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% @end +%%-------------------------------------------------------------------- +-spec terminate(Reason :: normal | shutdown | {shutdown, term()} | term(), + State :: term()) -> any(). +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% @end +%%-------------------------------------------------------------------- +-spec code_change(OldVsn :: term() | {down, term()}, + State :: term(), + Extra :: term()) -> {ok, NewState :: term()} | + {error, Reason :: term()}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called for changing the form and appearance +%% of gen_server status when it is returned from sys:get_status/1,2 +%% or when it appears in termination error logs. +%% @end +%%-------------------------------------------------------------------- +-spec format_status(Opt :: normal | terminate, + Status :: list()) -> Status :: term(). +format_status(_Opt, Status) -> + Status. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +oscillate(Interval) -> + erlang:send_after(Interval, self(), ?FUNCTION_NAME). + +%% @doc generate tokens, and then spread to leaf nodes +-spec oscillation(state()) -> state(). +oscillation(#state{root = #root{rate = Flow, + period = Interval, + childs = ChildIds, + consumed = Consumed} = Root, + nodes = Nodes} = State) -> + oscillate(Interval), + Childs = get_orderd_childs(ChildIds, Nodes), + {Alloced, Nodes2} = transverse(Childs, Flow, 0, Nodes), + State#state{nodes = Nodes2, + root = Root#root{consumed = Consumed + Alloced}}. + +%% @doc horizontal spread +-spec transverse(list(node_data()), + flow(), + non_neg_integer(), + nodes()) -> {non_neg_integer(), nodes()}. +transverse([H | T], InFlow, Alloced, Nodes) when InFlow > 0 -> + {NodeAlloced, Nodes2} = longitudinal(H, InFlow, Nodes), + InFlow2 = sub(InFlow, NodeAlloced), + Alloced2 = Alloced + NodeAlloced, + transverse(T, InFlow2, Alloced2, Nodes2); + +transverse(_, _, Alloced, Nodes) -> + {Alloced, Nodes}. + +%% @doc vertical spread +-spec longitudinal(node_data(), flow(), nodes()) -> + {non_neg_integer(), nodes()}. +longitudinal(#zone{id = Id, + rate = Rate, + obtained = Obtained, + childs = ChildIds} = Node, InFlow, Nodes) -> + Flow = erlang:min(InFlow, Rate), + + if Flow > 0 -> + Childs = get_orderd_childs(ChildIds, Nodes), + {Alloced, Nodes2} = transverse(Childs, Flow, 0, Nodes), + if Alloced > 0 -> + {Alloced, + Nodes2#{Id => Node#zone{obtained = Obtained + Alloced}}}; + true -> + %% childs are empty or all counter childs are full + {0, Nodes} + end; + true -> + {0, Nodes} + end; + +longitudinal(#bucket{id = Id, + rate = Rate, + capacity = Capacity, + correction = Correction, + counter = Counter, + index = Index, + obtained = Obtained} = Node, InFlow, Nodes) -> + Flow = add(erlang:min(InFlow, Rate), Correction), + + Tokens = counters:get(Counter, Index), + %% toknes's value mayb be a negative value(stolen from the future) + Avaiable = erlang:min(if Tokens < 0 -> + add(Capacity, Tokens); + true -> + sub(Capacity, Tokens) + end, Flow), + FixAvaiable = erlang:min(Capacity, Avaiable), + if FixAvaiable > 0 -> + {Alloced, Decimal} = add_to_counter(Counter, Index, FixAvaiable), + + {Alloced, + Nodes#{Id => Node#bucket{obtained = Obtained + Alloced, + correction = Decimal}}}; + true -> + {0, Nodes} + end. + +-spec get_orderd_childs(list(node_id()), nodes()) -> list(node_data()). +get_orderd_childs(Ids, Nodes) -> + Childs = [maps:get(Id, Nodes) || Id <- Ids], + + %% sort by obtained, avoid node goes hungry + lists:sort(fun(A, B) -> + ?GET_FIELD(?FIELD_OBTAINED, A) < ?GET_FIELD(?FIELD_OBTAINED, B) + end, + Childs). + +-spec init_tree(emqx_limiter_schema:limiter_type(), state()) -> state(). +init_tree(Type, State) -> + #{global := Global, + zone := Zone, + bucket := Bucket} = emqx:get_config([emqx_limiter, Type]), + {Factor, Root} = make_root(Global, Zone), + State2 = State#state{root = Root}, + {NodeId, State3} = make_zone(maps:to_list(Zone), Factor, 1, State2), + State4 = State3#state{counter = counters:new(maps:size(Bucket), + [write_concurrency])}, + make_bucket(maps:to_list(Bucket), Factor, NodeId, State4). + +-spec make_root(decimal(), hocon:config()) -> {number(), root()}. +make_root(Rate, Zone) -> + ZoneNum = maps:size(Zone), + Childs = lists:seq(1, ZoneNum), + MiniPeriod = emqx_limiter_schema:minimum_period(), + if Rate >= 1 -> + {1, #root{rate = Rate, + period = MiniPeriod, + childs = Childs, + consumed = 0}}; + true -> + Factor = 1 / Rate, + {Factor, #root{rate = 1, + period = erlang:floor(Factor * MiniPeriod), + childs = Childs, + consumed = 0}} + end. + +make_zone([{Name, Rate} | T], Factor, NodeId, State) -> + #state{zones = Zones, nodes = Nodes} = State, + Zone = #zone{id = NodeId, + name = Name, + rate = mul(Rate, Factor), + obtained = 0, + childs = []}, + State2 = State#state{zones = Zones#{Name => NodeId}, + nodes = Nodes#{NodeId => Zone}}, + make_zone(T, Factor, NodeId + 1, State2); + +make_zone([], _, NodeId, State2) -> + {NodeId, State2}. + +make_bucket([{Name, Conf} | T], Factor, NodeId, State) -> + #{zone := ZoneName, + aggregated := [Rate, Capacity]} = Conf, + {Counter, Idx, State2} = alloc_counter(ZoneName, Name, Rate, State), + Node = #bucket{ id = NodeId + , name = Name + , rate = mul(Rate, Factor) + , obtained = 0 + , correction = 0 + , capacity = Capacity + , counter = Counter + , index = Idx}, + State3 = add_zone_child(NodeId, Node, ZoneName, State2), + make_bucket(T, Factor, NodeId + 1, State3); + +make_bucket([], _, _, State) -> + State. + +-spec alloc_counter(zone_name(), bucket_name(), rate(), state()) -> + {counters:counters_ref(), pos_integer(), state()}. +alloc_counter(Zone, Bucket, Rate, + #state{type = Type, counter = Counter, index = Index} = State) -> + Path = emqx_limiter_manager:make_path(Type, Zone, Bucket), + case emqx_limiter_manager:find_counter(Path) of + undefined -> + init_counter(Path, Counter, Index, + Rate, State#state{index = Index + 1}); + {ok, ECounter, EIndex, _} -> + init_counter(Path, ECounter, EIndex, Rate, State) + end. + +init_counter(Path, Counter, Index, Rate, State) -> + _ = put_to_counter(Counter, Index, 0), + emqx_limiter_manager:insert_counter(Path, Counter, Index, Rate), + {Counter, Index, State}. + +-spec add_zone_child(node_id(), bucket(), zone_name(), state()) -> state(). +add_zone_child(NodeId, Bucket, Name, #state{zones = Zones, nodes = Nodes} = State) -> + ZoneId = maps:get(Name, Zones), + #zone{childs = Childs} = Zone = maps:get(ZoneId, Nodes), + Nodes2 = Nodes#{ZoneId => Zone#zone{childs = [NodeId | Childs]}, + NodeId => Bucket}, + State#state{nodes = Nodes2}. diff --git a/apps/emqx_limiter/src/emqx_limiter_server_sup.erl b/apps/emqx_limiter/src/emqx_limiter_server_sup.erl new file mode 100644 index 000000000..56a2dd2dc --- /dev/null +++ b/apps/emqx_limiter/src/emqx_limiter_server_sup.erl @@ -0,0 +1,94 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_limiter_server_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0, start/1, restart/1]). + +%% Supervisor callbacks +-export([init/1]). + +%%--================================================================== +%% API functions +%%--================================================================== + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the supervisor +%% @end +%%-------------------------------------------------------------------- +-spec start_link() -> {ok, Pid :: pid()} | + {error, {already_started, Pid :: pid()}} | + {error, {shutdown, term()}} | + {error, term()} | + ignore. +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +-spec start(emqx_limiter_schema:limiter_type()) -> _. +start(Type) -> + Spec = make_child(Type), + supervisor:start_child(?MODULE, Spec). + +-spec restart(emqx_limiter_schema:limiter_type()) -> _. +restart(Type) -> + Id = emqx_limiter_server:name(Type), + _ = supervisor:terminate_child(?MODULE, Id), + supervisor:restart_child(?MODULE, Id). + +%%--================================================================== +%% Supervisor callbacks +%%--================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Whenever a supervisor is started using supervisor:start_link/[2,3], +%% this function is called by the new process to find out about +%% restart strategy, maximum restart intensity, and child +%% specifications. +%% @end +%%-------------------------------------------------------------------- +-spec init(Args :: term()) -> + {ok, {SupFlags :: supervisor:sup_flags(), + [ChildSpec :: supervisor:child_spec()]}} | + ignore. +init([]) -> + SupFlags = #{strategy => one_for_one, + intensity => 10, + period => 3600}, + + {ok, {SupFlags, childs()}}. + +%%--================================================================== +%% Internal functions +%%--================================================================== +make_child(Type) -> + Id = emqx_limiter_server:name(Type), + #{id => Id, + start => {emqx_limiter_server, start_link, [Type]}, + restart => transient, + shutdown => 5000, + type => worker, + modules => [emqx_limiter_server]}. + +childs() -> + Conf = emqx:get_config([emqx_limiter]), + Types = maps:keys(Conf), + [make_child(Type) || Type <- Types]. diff --git a/apps/emqx_limiter/src/emqx_limiter_sup.erl b/apps/emqx_limiter/src/emqx_limiter_sup.erl new file mode 100644 index 000000000..957f053af --- /dev/null +++ b/apps/emqx_limiter/src/emqx_limiter_sup.erl @@ -0,0 +1,76 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_limiter_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). + +%% Supervisor callbacks +-export([init/1]). + +%%-------------------------------------------------------------------- +%% API functions +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the supervisor +%% @end +%%-------------------------------------------------------------------- +-spec start_link() -> {ok, Pid :: pid()} | + {error, {already_started, Pid :: pid()}} | + {error, {shutdown, term()}} | + {error, term()} | + ignore. +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%-------------------------------------------------------------------- +%% Supervisor callbacks +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Whenever a supervisor is started using supervisor:start_link/[2,3], +%% this function is called by the new process to find out about +%% restart strategy, maximum restart intensity, and child +%% specifications. +%% @end +%%-------------------------------------------------------------------- +-spec init(Args :: term()) -> + {ok, {SupFlags :: supervisor:sup_flags(), + [ChildSpec :: supervisor:child_spec()]}} | + ignore. +init([]) -> + SupFlags = #{strategy => one_for_one, + intensity => 10, + period => 3600}, + + Childs = [ make_child(emqx_limiter_manager, worker) + , make_child(emqx_limiter_server_sup, supervisor)], + + {ok, {SupFlags, Childs}}. + +make_child(Mod, Type) -> + #{id => Mod, + start => {Mod, start_link, []}, + restart => transient, + type => Type, + modules => [Mod]}. diff --git a/apps/emqx_limiter/test/emqx_limiter_SUITE.erl b/apps/emqx_limiter/test/emqx_limiter_SUITE.erl new file mode 100644 index 000000000..499103f6d --- /dev/null +++ b/apps/emqx_limiter/test/emqx_limiter_SUITE.erl @@ -0,0 +1,272 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_limiter_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(APP, emqx_limiter). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(BASE_CONF, <<""" +emqx_limiter { + bytes_in {global = \"100KB/10s\" + zone.default = \"100kB/10s\" + zone.external = \"20kB/10s\" + bucket.tcp {zone = default + aggregated = \"100kB/10s,1Mb\" + per_client = \"100KB/10s,10Kb\"} + bucket.ssl {zone = external + aggregated = \"100kB/10s,1Mb\" + per_client = \"100KB/10s,10Kb\"} + } + + message_in {global = \"100/10s\" + zone.default = \"100/10s\" + bucket.bucket1 {zone = default + aggregated = \"100/10s,1000\" + per_client = \"100/10s,100\"} + } + + connection {global = \"100/10s\" + zone.default = \"100/10s\" + bucket.bucket1 {zone = default + aggregated = \"100/10s,100\" + per_client = \"100/10s,10\" + } + } + + message_routing {global = \"100/10s\" + zone.default = \"100/10s\" + bucket.bucket1 {zone = default + aggregated = \"100/10s,100\" + per_client = \"100/10s,10\" + } + } +}""">>). + +-define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). + +-record(client_options, { interval :: non_neg_integer() + , per_cost :: non_neg_integer() + , type :: atom() + , bucket :: atom() + , lifetime :: non_neg_integer() + , rates :: list(tuple()) + }). + +-record(client_state, { client :: emqx_limiter_client:limiter() + , pid :: pid() + , got :: non_neg_integer() + , options :: #client_options{}}). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_config:init_load(emqx_limiter_schema, ?BASE_CONF), + emqx_common_test_helpers:start_apps([?APP]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([?APP]). + +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Test Cases +%%-------------------------------------------------------------------- +t_un_overload(_) -> + Conf = emqx:get_config([emqx_limiter]), + Conn = #{global => to_rate("infinity"), + zone => #{z1 => to_rate("1000/1s"), + z2 => to_rate("1000/1s")}, + bucket => #{b1 => #{zone => z1, + aggregated => to_bucket_rate("100/1s, 500"), + per_client => to_bucket_rate("10/1s, 50")}, + b2 => #{zone => z2, + aggregated => to_bucket_rate("500/1s, 500"), + per_client => to_bucket_rate("100/1s, infinity") + }}}, + Conf2 = Conf#{connection => Conn}, + emqx_config:put([emqx_limiter], Conf2), + {ok, _} = emqx_limiter_manager:restart_server(connection), + + timer:sleep(200), + + B1C = #client_options{interval = 100, + per_cost = 1, + type = connection, + bucket = b1, + lifetime = timer:seconds(3), + rates = [{fun erlang:'=<'/2, ["1000/1s", "100/1s"]}, + {fun erlang:'=:='/2, ["10/1s"]}]}, + + B2C = #client_options{interval = 100, + per_cost = 10, + type = connection, + bucket = b2, + lifetime = timer:seconds(3), + rates = [{fun erlang:'=<'/2, ["1000/1s", "500/1s"]}, + {fun erlang:'=:='/2, ["100/1s"]}]}, + + lists:foreach(fun(_) -> start_client(B1C) end, + lists:seq(1, 10)), + + + lists:foreach(fun(_) -> start_client(B2C) end, + lists:seq(1, 5)), + + ?assert(check_client_result(10 + 5)). + +t_infinity(_) -> + Conf = emqx:get_config([emqx_limiter]), + Conn = #{global => to_rate("infinity"), + zone => #{z1 => to_rate("1000/1s"), + z2 => to_rate("infinity")}, + bucket => #{b1 => #{zone => z1, + aggregated => to_bucket_rate("100/1s, infinity"), + per_client => to_bucket_rate("10/1s, 100")}, + b2 => #{zone => z2, + aggregated => to_bucket_rate("infinity, 600"), + per_client => to_bucket_rate("100/1s, infinity") + }}}, + Conf2 = Conf#{connection => Conn}, + emqx_config:put([emqx_limiter], Conf2), + {ok, _} = emqx_limiter_manager:restart_server(connection), + + timer:sleep(200), + + B1C = #client_options{interval = 100, + per_cost = 1, + type = connection, + bucket = b1, + lifetime = timer:seconds(3), + rates = [{fun erlang:'=<'/2, ["1000/1s", "100/1s"]}, + {fun erlang:'=:='/2, ["10/1s"]}]}, + + B2C = #client_options{interval = 100, + per_cost = 10, + type = connection, + bucket = b2, + lifetime = timer:seconds(3), + rates = [{fun erlang:'=:='/2, ["100/1s"]}]}, + + lists:foreach(fun(_) -> start_client(B1C) end, + lists:seq(1, 8)), + + lists:foreach(fun(_) -> start_client(B2C) end, + lists:seq(1, 4)), + + ?assert(check_client_result(8 + 4)). + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +start_client(Opts) -> + Pid = self(), + erlang:spawn(fun() -> enter_client(Opts, Pid) end). + +enter_client(#client_options{type = Type, + bucket = Bucket, + lifetime = Lifetime} = Opts, + Pid) -> + erlang:send_after(Lifetime, self(), stop), + erlang:send(self(), consume), + Client = emqx_limiter_server:connect(Type, Bucket), + client_loop(#client_state{client = Client, + pid = Pid, + got = 0, + options = Opts}). + +client_loop(#client_state{client = Client, + got = Got, + pid = Pid, + options = #client_options{interval = Interval, + per_cost = PerCost, + lifetime = Lifetime, + rates = Rates}} = State) -> + receive + consume -> + case emqx_limiter_client:consume(PerCost, Client) of + {ok, Client2} -> + erlang:send_after(Interval, self(), consume), + client_loop(State#client_state{client = Client2, + got = Got + PerCost}); + {pause, MS, Client2} -> + erlang:send_after(MS, self(), {resume, erlang:system_time(millisecond)}), + client_loop(State#client_state{client = Client2}) + end; + stop -> + Rate = Got * emqx_limiter_schema:minimum_period() / Lifetime, + ?LOGT("Got:~p, Rate is:~p Checks:~p~n", [Got, Rate, Rate]), + Check = check_rates(Rate, Rates), + erlang:send(Pid, {client, Check}); + {resume, Begin} -> + case emqx_limiter_client:consume(PerCost, Client) of + {ok, Client2} -> + Now = erlang:system_time(millisecond), + Diff = erlang:max(0, Interval - (Now - Begin)), + erlang:send_after(Diff, self(), consume), + client_loop(State#client_state{client = Client2, + got = Got + PerCost}); + {pause, MS, Client2} -> + erlang:send_after(MS, self(), {resume, Begin}), + client_loop(State#client_state{client = Client2}) + end + end. + +check_rates(Rate, [{Fun, Rates} | T]) -> + case lists:all(fun(E) -> Fun(Rate, to_rate(E)) end, Rates) of + true -> + check_rates(Rate, T); + false -> + false + end; +check_rates(_, _) -> + true. + +check_client_result(0) -> + true; + +check_client_result(N) -> + ?LOGT("check_client_result:~p~n", [N]), + receive + {client, true} -> + check_client_result(N - 1); + {client, false} -> + false; + Any -> + ?LOGT(">>>> other:~p~n", [Any]) + + after 3500 -> + ?LOGT(">>>> timeout~n", []), + false + end. + +to_rate(Str) -> + {ok, Rate} = emqx_limiter_schema:to_rate(Str), + Rate. + +to_bucket_rate(Str) -> + {ok, Result} = emqx_limiter_schema:to_bucket_rate(Str), + Result. diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 8fc3a14f4..83c47331e 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -37,7 +37,7 @@ post_boot() -> print_vsn() -> ok. -else. % TEST print_vsn() -> - ?ULOG("~s ~s is running now!~n", [emqx_app:get_description(), emqx_app:get_release()]). + ?ULOG("~ts ~ts is running now!~n", [emqx_app:get_description(), emqx_app:get_release()]). -endif. % TEST diff --git a/apps/emqx_machine/src/emqx_machine_schema.erl b/apps/emqx_machine/src/emqx_machine_schema.erl index 369d7b3c5..a124166e5 100644 --- a/apps/emqx_machine/src/emqx_machine_schema.erl +++ b/apps/emqx_machine/src/emqx_machine_schema.erl @@ -54,6 +54,7 @@ , emqx_rule_engine_schema , emqx_exhook_schema , emqx_psk_schema + , emqx_limiter_schema ]). namespace() -> undefined. diff --git a/apps/emqx_machine/test/emqx_global_gc_SUITE.erl b/apps/emqx_machine/test/emqx_global_gc_SUITE.erl index dcdeeae57..b55bc0611 100644 --- a/apps/emqx_machine/test/emqx_global_gc_SUITE.erl +++ b/apps/emqx_machine/test/emqx_global_gc_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_run_gc(_) -> ok = emqx_config:put([node, global_gc_interval], 1000), diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl index 1e90d867b..95ce23e11 100644 --- a/apps/emqx_machine/test/emqx_machine_SUITE.erl +++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl @@ -23,14 +23,14 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([]), + emqx_common_test_helpers:start_apps([]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_common_test_helpers:stop_apps([]). t_shutdown_reboot(_Config) -> emqx_machine_boot:stop_apps(normal), diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index ef8d7c70c..c7da2e752 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -120,7 +120,7 @@ node_query(Node, Params, Tab, QsSchema, QueryFun) -> Limit = b2i(limit(Params)), Page = b2i(page(Params)), Meta = #{page => Page, limit => Limit, count => 0}, - do_node_query(Node, Tab, Qs, QueryFun, Meta). + page_limit_check_query(Meta, {fun do_node_query/5, [Node, Tab, Qs, QueryFun, Meta]}). %% @private do_node_query(Node, Tab, Qs, QueryFun, Meta) -> @@ -169,7 +169,7 @@ cluster_query(Params, Tab, QsSchema, QueryFun) -> Page = b2i(page(Params)), Nodes = ekka_mnesia:running_nodes(), Meta = #{page => Page, limit => Limit, count => 0}, - do_cluster_query(Nodes, Tab, Qs, QueryFun, Meta). + page_limit_check_query(Meta, {fun do_cluster_query/5, [Nodes, Tab, Qs, QueryFun, Meta]}). %% @private do_cluster_query(Nodes, Tab, Qs, QueryFun, Meta) -> @@ -367,6 +367,15 @@ rows_sub_params(Len, _Meta = #{page := Page, limit := Limit, count := Count}) -> {_SubStart = 1, _NeedNowNum = Len} end. +page_limit_check_query(Meta, {F, A}) -> + case Meta of + #{page := Page, limit := Limit} + when Page < 1; Limit < 1 -> + {error, page_limit_invalid}; + _ -> + erlang:apply(F, A) + end. + %%-------------------------------------------------------------------- %% Types %%-------------------------------------------------------------------- diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl index 3641f82b0..c4e49a616 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl @@ -79,7 +79,7 @@ alarms(get, #{query_string := Qs}) -> <<"false">> -> ?DEACTIVATED_ALARM end, Response = emqx_mgmt_api:cluster_query(Qs, Table, [], {?MODULE, query}), - {200, Response}; + emqx_mgmt_util:generate_response(Response); alarms(delete, _Params) -> _ = emqx_mgmt:delete_all_deactivated_alarms(), diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 618976cb5..454f27466 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -264,7 +264,8 @@ clients_api() -> } ], responses => #{ - <<"200">> => emqx_mgmt_util:array_schema(client, <<"List clients 200 OK">>)}}}, + <<"200">> => emqx_mgmt_util:array_schema(client, <<"List clients 200 OK">>), + <<"400">> => emqx_mgmt_util:error_schema(<<"Invalid parameters">>, ['INVALID_PARAMETER'])}}}, {"/clients", Metadata, clients}. client_api() -> @@ -435,13 +436,13 @@ list(Params) -> undefined -> Response = emqx_mgmt_api:cluster_query(Params, Tab, QuerySchema, ?query_fun), - {200, Response}; + emqx_mgmt_util:generate_response(Response); Node1 -> Node = binary_to_atom(Node1, utf8), ParamsWithoutNode = maps:without([<<"node">>], Params), Response = emqx_mgmt_api:node_query(Node, ParamsWithoutNode, Tab, QuerySchema, ?query_fun), - {200, Response} + emqx_mgmt_util:generate_response(Response) end. lookup(#{clientid := ClientID}) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index a859f2002..de2774cae 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -16,97 +16,106 @@ -module(emqx_mgmt_api_configs). +-include_lib("hocon/include/hoconsc.hrl"). -behaviour(minirest_api). --import(emqx_mgmt_util, [ schema/1 - , schema/2 - , error_schema/2 - ]). - -export([api_spec/0]). +-export([paths/0, schema/1, fields/1]). --export([ config/3 - , config_reset/3 - ]). +-export([config/3, config_reset/3, configs/3, get_full_config/0]). -export([get_conf_schema/2, gen_schema/1]). --define(PARAM_CONF_PATH, [#{ - name => conf_path, - in => query, - description => <<"The config path separated by '.' character">>, - required => false, - schema => #{type => string, default => <<".">>} -}]). - --define(PREFIX, "/configs"). --define(PREFIX_RESET, "/configs_reset"). - --define(MAX_DEPTH, 1). - +-define(PREFIX, "/configs/"). +-define(PREFIX_RESET, "/configs_reset/"). -define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))). - --define(CORE_CONFS, [ - %% from emqx_machine_schema - log, rpc, - %% from emqx_schema - zones, mqtt, flapping_detect, force_shutdown, force_gc, conn_congestion, rate_limit, quota, - broker, alarm, sysmon, - %% from other apps - emqx_dashboard, emqx_management]). +-define(EXCLUDES, [listeners, node, cluster, gateway, rule_engine]). api_spec() -> - {config_apis() ++ [config_reset_api()], []}. + emqx_dashboard_swagger:spec(?MODULE). -config_apis() -> - [config_api(ConfPath, Schema) || {ConfPath, Schema} <- - get_conf_schema(emqx:get_config([]), ?MAX_DEPTH), is_core_conf(ConfPath)]. +paths() -> + ["/configs", "/configs_reset/:rootname"] ++ + lists:map(fun({Name, _Type}) -> ?PREFIX ++ to_list(Name) end, config_list(?EXCLUDES)). -config_api(ConfPath, Schema) -> - Path = path_join(ConfPath), - Descr = fun(Str) -> - list_to_binary([Str, " ", path_join(ConfPath, ".")]) - end, - Metadata = #{ +schema("/configs") -> + #{ + operationId => configs, get => #{ - description => Descr("Get configs for"), + tags => [conf], + description => <<"Get all the configurations of the specified node, including hot and non-hot updatable items.">>, + parameters => [ + {node, hoconsc:mk(typerefl:atom(), + #{in => query, required => false, example => <<"emqx@127.0.0.1">>, + desc => <<"Node's name: If you do not fill in the fields, this node will be used by default.">>})}], responses => #{ - <<"200">> => schema(Schema, <<"Get configs successfully">>), - <<"404">> => emqx_mgmt_util:error_schema(<<"Config not found">>, ['NOT_FOUND']) - } - }, - put => #{ - description => Descr("Update configs for"), - 'requestBody' => schema(Schema), - responses => #{ - <<"200">> => schema(Schema, <<"Update configs successfully">>), - <<"400">> => error_schema(<<"Update configs failed">>, ['UPDATE_FAILED']) + 200 => config_list([]) } } - }, - {?PREFIX ++ "/" ++ Path, Metadata, config}. - -config_reset_api() -> - Metadata = #{ + }; +schema("/configs_reset/:rootname") -> + Paths = lists:map(fun({Path, _}) -> Path end, config_list(?EXCLUDES)), + #{ + operationId => config_reset, post => #{ - tags => [configs], + tags => [conf], description => <<"Reset the config entry specified by the query string parameter `conf_path`.
- For a config entry that has default value, this resets it to the default value; - For a config entry that has no default value, an error 400 will be returned">>, - parameters => ?PARAM_CONF_PATH, + %% We only return "200" rather than the new configs that has been changed, as + %% the schema of the changed configs is depends on the request parameter + %% `conf_path`, it cannot be defined here. + parameters => [ + {rootname, hoconsc:mk(hoconsc:enum(Paths), #{in => path, example => <<"authorization">>})}, + {conf_path, hoconsc:mk(typerefl:binary(), + #{in => query, required => false, example => <<"cache.enable">>, + desc => <<"The config path separated by '.' character">>})}], responses => #{ - %% We only return "200" rather than the new configs that has been changed, as - %% the schema of the changed configs is depends on the request parameter - %% `conf_path`, it cannot be defined here. - <<"200">> => schema(<<"Reset configs successfully">>), - <<"400">> => error_schema(<<"It's not able to reset the config">>, ['INVALID_OPERATION']) + 200 => <<"Rest config successfully">>, + 400 => emqx_dashboard_swagger:error_codes(['NO_DEFAULT_VALUE', 'REST_FAILED']) } } - }, - {?PREFIX_RESET, Metadata, config_reset}. + }; +schema(Path) -> + {Root, Schema} = find_schema(Path), + #{ + operationId => config, + get => #{ + tags => [conf], + description => iolist_to_binary([<<"Get the sub-configurations under *">>, Root, <<"*">>]), + responses => #{ + 200 => Schema, + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"config not found">>) + } + }, + put => #{ + tags => [conf], + description => iolist_to_binary([<<"Update the sub-configurations under *">>, Root, <<"*">>]), + requestBody => Schema, + responses => #{ + 200 => Schema, + 400 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED']) + } + } + }. + +find_schema(Path) -> + [_, _Prefix, Root | _] = string:split(Path, "/", all), + Configs = config_list(?EXCLUDES), + case lists:keyfind(Root, 1, Configs) of + {Root, Schema} -> {Root, Schema}; + false -> + RootAtom = list_to_existing_atom(Root), + {Root, element(2, lists:keyfind(RootAtom, 1, Configs))} + end. + +%% we load all configs from emqx_machine_schema, some of them are defined as local ref +%% we need redirect to emqx_machine_schema. +%% such as hoconsc:ref("node") to hoconsc:ref(emqx_machine_schema, "node") +fields(Field) -> emqx_machine_schema:fields(Field). %%%============================================================================================== -%% parameters trans +%% HTTP API Callbacks config(get, _Params, Req) -> Path = conf_path(Req), case emqx_map_lib:deep_find(Path, get_full_config()) of @@ -118,19 +127,33 @@ config(get, _Params, Req) -> config(put, #{body := Body}, Req) -> Path = conf_path(Req), - {ok, #{raw_config := RawConf}} = emqx:update_config(Path, Body, - #{rawconf_with_defaults => true}), - {200, emqx_map_lib:jsonable_map(RawConf)}. + case emqx:update_config(Path, Body, #{rawconf_with_defaults => true}) of + {ok, #{raw_config := RawConf}} -> + {200, emqx_map_lib:jsonable_map(RawConf)}; + {error, Reason} -> + {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Reason)}} + end. config_reset(post, _Params, Req) -> %% reset the config specified by the query string param 'conf_path' Path = conf_path_reset(Req) ++ conf_path_from_querystr(Req), case emqx:reset_config(Path, #{}) of {ok, _} -> {200}; + {error, no_default_value} -> + {400, #{code => 'NO_DEFAULT_VALUE', message => <<"No Default Value.">>}}; {error, Reason} -> - {400, ?ERR_MSG(Reason)} + {400, #{code => 'REST_FAILED', message => ?ERR_MSG(Reason)}} end. +configs(get, Params, _Req) -> + Node = maps:get(node, Params, node()), + Res = rpc:call(Node, ?MODULE, get_full_config, [[]]), + {200, Res}. + +conf_path_reset(Req) -> + <<"/api/v5", ?PREFIX_RESET, Path/binary>> = cowboy_req:path(Req), + string:lexemes(Path, "/ "). + get_full_config() -> emqx_map_lib:jsonable_map( emqx_config:fill_defaults(emqx:get_raw_config([]))). @@ -141,14 +164,17 @@ conf_path_from_querystr(Req) -> Path -> string:lexemes(Path, ". ") end. +config_list(Exclude) -> + Roots = emqx_machine_schema:roots(), + lists:foldl(fun(Key, Acc) -> lists:delete(Key, Acc) end, Roots, Exclude). + +to_list(L) when is_list(L) -> L; +to_list(Atom) when is_atom(Atom) -> atom_to_list(Atom). + conf_path(Req) -> <<"/api/v5", ?PREFIX, Path/binary>> = cowboy_req:path(Req), string:lexemes(Path, "/ "). -conf_path_reset(Req) -> - <<"/api/v5", ?PREFIX_RESET, Path/binary>> = cowboy_req:path(Req), - string:lexemes(Path, "/ "). - get_conf_schema(Conf, MaxDepth) -> get_conf_schema([], maps:to_list(Conf), [], MaxDepth). @@ -158,11 +184,11 @@ get_conf_schema(BasePath, [{Key, Conf} | Confs], Result, MaxDepth) -> Path = BasePath ++ [Key], Depth = length(Path), Result1 = case is_map(Conf) of - true when Depth < MaxDepth -> - get_conf_schema(Path, maps:to_list(Conf), Result, MaxDepth); - true when Depth >= MaxDepth -> Result; - false -> Result - end, + true when Depth < MaxDepth -> + get_conf_schema(Path, maps:to_list(Conf), Result, MaxDepth); + true when Depth >= MaxDepth -> Result; + false -> Result + end, get_conf_schema(BasePath, Confs, [{Path, gen_schema(Conf)} | Result1], MaxDepth). %% TODO: generate from hocon schema @@ -181,7 +207,7 @@ gen_schema(Conf) when is_list(Conf) -> end; gen_schema(Conf) when is_map(Conf) -> #{type => object, properties => - maps:map(fun(_K, V) -> gen_schema(V) end, Conf)}; + maps:map(fun(_K, V) -> gen_schema(V) end, Conf)}; gen_schema(_Conf) -> %% the conf is not of JSON supported type, it may have been converted %% by the hocon schema @@ -189,17 +215,3 @@ gen_schema(_Conf) -> with_default_value(Type, Value) -> Type#{example => emqx_map_lib:binary_string(Value)}. - -path_join(Path) -> - path_join(Path, "/"). - -path_join([P], _Sp) -> str(P); -path_join([P | Path], Sp) -> - str(P) ++ Sp ++ path_join(Path, Sp). - -is_core_conf(Path) -> - lists:member(hd(Path), ?CORE_CONFS). - -str(S) when is_list(S) -> S; -str(S) when is_binary(S) -> binary_to_list(S); -str(S) when is_atom(S) -> atom_to_list(S). diff --git a/apps/emqx_management/src/emqx_mgmt_api_routes.erl b/apps/emqx_management/src/emqx_mgmt_api_routes.erl index e95679400..28ddb8ca2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_routes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_routes.erl @@ -37,6 +37,7 @@ , error_schema/2 , properties/1 , page_params/0 + , generate_response/1 ]). api_spec() -> @@ -54,7 +55,8 @@ routes_api() -> description => <<"EMQ X routes">>, parameters => [topic_param(query) , node_param()] ++ page_params(), responses => #{ - <<"200">> => object_array_schema(properties(), <<"List route info">>) + <<"200">> => object_array_schema(properties(), <<"List route info">>), + <<"400">> => error_schema(<<"Invalid parameters">>, ['INVALID_PARAMETER']) } } }, @@ -87,7 +89,7 @@ route(get, #{bindings := Bindings}) -> %% api apply list(Params) -> Response = emqx_mgmt_api:node_query(node(), Params, emqx_route, ?ROUTES_QS_SCHEMA, {?MODULE, query}), - {200, Response}. + generate_response(Response). lookup(#{topic := Topic}) -> case emqx_mgmt:lookup_routes(Topic) of diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl index d13bc5394..97871d35c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_status.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -41,7 +41,7 @@ running_status(get, _Params) -> false -> not_running; {value, _Val} -> running end, - Status = io_lib:format("Node ~s is ~s~nemqx is ~s", [node(), InternalStatus, AppStatus]), + Status = io_lib:format("Node ~ts is ~ts~nemqx is ~ts", [node(), InternalStatus, AppStatus]), Body = list_to_binary(Status), {200, #{<<"content-type">> => <<"text/plain">>}, Body}. diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 2be5773f3..81022f580 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -21,6 +21,7 @@ -include_lib("emqx/include/emqx.hrl"). -import(emqx_mgmt_util, [ page_schema/1 + , error_schema/2 , properties/1 , page_params/0 ]). @@ -53,7 +54,8 @@ subscriptions_api() -> description => <<"List subscriptions">>, parameters => parameters(), responses => #{ - <<"200">> => page_schema(subscription) + <<"200">> => page_schema(subscription), + <<"400">> => error_schema(<<"Invalid parameters">>, ['INVALID_PARAMETER']) } } }, @@ -114,11 +116,13 @@ list(Params) -> {Tab, QuerySchema} = ?SUBS_QS_SCHEMA, case maps:get(<<"node">>, Params, undefined) of undefined -> - {200, emqx_mgmt_api:cluster_query(Params, Tab, - QuerySchema, ?query_fun)}; + Response = emqx_mgmt_api:cluster_query(Params, Tab, + QuerySchema, ?query_fun), + emqx_mgmt_util:generate_response(Response); Node -> - {200, emqx_mgmt_api:node_query(binary_to_atom(Node, utf8), Params, - Tab, QuerySchema, ?query_fun)} + Response = emqx_mgmt_api:node_query(binary_to_atom(Node, utf8), Params, + Tab, QuerySchema, ?query_fun), + emqx_mgmt_util:generate_response(Response) end. format(Items) when is_list(Items) -> diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index abe64d886..d664828bf 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -21,7 +21,7 @@ -include("emqx_mgmt.hrl"). --define(PRINT_CMD(Cmd, Descr), io:format("~-48s# ~s~n", [Cmd, Descr])). +-define(PRINT_CMD(Cmd, Descr), io:format("~-48s# ~ts~n", [Cmd, Descr])). -export([load/0]). @@ -38,6 +38,7 @@ , trace/1 , log/1 , authz/1 + , olp/1 ]). -define(PROC_INFOKEYS, [status, @@ -65,7 +66,7 @@ is_cmd(Fun) -> status([]) -> {InternalStatus, _ProvidedStatus} = init:get_status(), - emqx_ctl:print("Node ~p ~s is ~p~n", [node(), emqx_app:get_release(), InternalStatus]); + emqx_ctl:print("Node ~p ~ts is ~p~n", [node(), emqx_app:get_release(), InternalStatus]); status(_) -> emqx_ctl:usage("status", "Show broker status"). @@ -74,7 +75,7 @@ status(_) -> broker([]) -> Funs = [sysdescr, version, datetime], - [emqx_ctl:print("~-10s: ~s~n", [Fun, emqx_sys:Fun()]) || Fun <- Funs], + [emqx_ctl:print("~-10s: ~ts~n", [Fun, emqx_sys:Fun()]) || Fun <- Funs], emqx_ctl:print("~-10s: ~p~n", [uptime, emqx_sys:uptime()]); broker(["stats"]) -> @@ -227,9 +228,9 @@ plugins(["list"]) -> plugins(["load", Name]) -> case emqx_plugins:load(list_to_atom(Name)) of ok -> - emqx_ctl:print("Plugin ~s loaded successfully.~n", [Name]); + emqx_ctl:print("Plugin ~ts loaded successfully.~n", [Name]); {error, Reason} -> - emqx_ctl:print("Load plugin ~s error: ~p.~n", [Name, Reason]) + emqx_ctl:print("Load plugin ~ts error: ~p.~n", [Name, Reason]) end; plugins(["unload", "emqx_management"])-> @@ -238,9 +239,9 @@ plugins(["unload", "emqx_management"])-> plugins(["unload", Name]) -> case emqx_plugins:unload(list_to_atom(Name)) of ok -> - emqx_ctl:print("Plugin ~s unloaded successfully.~n", [Name]); + emqx_ctl:print("Plugin ~ts unloaded successfully.~n", [Name]); {error, Reason} -> - emqx_ctl:print("Unload plugin ~s error: ~p.~n", [Name, Reason]) + emqx_ctl:print("Unload plugin ~ts error: ~p.~n", [Name, Reason]) end; plugins(["reload", Name]) -> @@ -248,13 +249,13 @@ plugins(["reload", Name]) -> PluginName -> case emqx_mgmt:reload_plugin(node(), PluginName) of ok -> - emqx_ctl:print("Plugin ~s reloaded successfully.~n", [Name]); + emqx_ctl:print("Plugin ~ts reloaded successfully.~n", [Name]); {error, Reason} -> - emqx_ctl:print("Reload plugin ~s error: ~p.~n", [Name, Reason]) + emqx_ctl:print("Reload plugin ~ts error: ~p.~n", [Name, Reason]) end catch error:badarg -> - emqx_ctl:print("Reload plugin ~s error: The plugin doesn't exist.~n", [Name]) + emqx_ctl:print("Reload plugin ~ts error: The plugin doesn't exist.~n", [Name]) end; plugins(_) -> @@ -274,7 +275,7 @@ vm(["all"]) -> [vm([Name]) || Name <- ["load", "memory", "process", "io", "ports"]]; vm(["load"]) -> - [emqx_ctl:print("cpu/~-20s: ~s~n", [L, V]) || {L, V} <- emqx_vm:loads()]; + [emqx_ctl:print("cpu/~-20s: ~ts~n", [L, V]) || {L, V} <- emqx_vm:loads()]; vm(["memory"]) -> [emqx_ctl:print("memory/~-17s: ~w~n", [Cat, Val]) || {Cat, Val} <- erlang:memory()]; @@ -311,42 +312,42 @@ mnesia(_) -> log(["set-level", Level]) -> case emqx_logger:set_log_level(list_to_atom(Level)) of - ok -> emqx_ctl:print("~s~n", [Level]); + ok -> emqx_ctl:print("~ts~n", [Level]); Error -> emqx_ctl:print("[error] set overall log level failed: ~p~n", [Error]) end; log(["primary-level"]) -> Level = emqx_logger:get_primary_log_level(), - emqx_ctl:print("~s~n", [Level]); + emqx_ctl:print("~ts~n", [Level]); log(["primary-level", Level]) -> _ = emqx_logger:set_primary_log_level(list_to_atom(Level)), - emqx_ctl:print("~s~n", [emqx_logger:get_primary_log_level()]); + emqx_ctl:print("~ts~n", [emqx_logger:get_primary_log_level()]); log(["handlers", "list"]) -> - _ = [emqx_ctl:print("LogHandler(id=~s, level=~s, destination=~s, status=~s)~n", [Id, Level, Dst, Status]) + _ = [emqx_ctl:print("LogHandler(id=~ts, level=~ts, destination=~ts, status=~ts)~n", [Id, Level, Dst, Status]) || #{id := Id, level := Level, dst := Dst, status := Status} <- emqx_logger:get_log_handlers()], ok; log(["handlers", "start", HandlerId]) -> case emqx_logger:start_log_handler(list_to_atom(HandlerId)) of - ok -> emqx_ctl:print("log handler ~s started~n", [HandlerId]); + ok -> emqx_ctl:print("log handler ~ts started~n", [HandlerId]); {error, Reason} -> - emqx_ctl:print("[error] failed to start log handler ~s: ~p~n", [HandlerId, Reason]) + emqx_ctl:print("[error] failed to start log handler ~ts: ~p~n", [HandlerId, Reason]) end; log(["handlers", "stop", HandlerId]) -> case emqx_logger:stop_log_handler(list_to_atom(HandlerId)) of - ok -> emqx_ctl:print("log handler ~s stopped~n", [HandlerId]); + ok -> emqx_ctl:print("log handler ~ts stopped~n", [HandlerId]); {error, Reason} -> - emqx_ctl:print("[error] failed to stop log handler ~s: ~p~n", [HandlerId, Reason]) + emqx_ctl:print("[error] failed to stop log handler ~ts: ~p~n", [HandlerId, Reason]) end; log(["handlers", "set-level", HandlerId, Level]) -> case emqx_logger:set_log_handler_level(list_to_atom(HandlerId), list_to_atom(Level)) of ok -> #{level := NewLevel} = emqx_logger:get_log_handler(list_to_atom(HandlerId)), - emqx_ctl:print("~s~n", [NewLevel]); + emqx_ctl:print("~ts~n", [NewLevel]); {error, Error} -> emqx_ctl:print("[error] ~p~n", [Error]) end; @@ -365,7 +366,7 @@ log(_) -> trace(["list"]) -> lists:foreach(fun({{Who, Name}, {Level, LogFile}}) -> - emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Who, Name, Level, LogFile]) + emqx_ctl:print("Trace(~ts=~ts, level=~ts, destination=~p)~n", [Who, Name, Level, LogFile]) end, emqx_tracer:lookup_traces()); trace(["stop", "client", ClientId]) -> @@ -396,17 +397,17 @@ trace(_) -> trace_on(Who, Name, Level, LogFile) -> case emqx_tracer:start_trace({Who, iolist_to_binary(Name)}, Level, LogFile) of ok -> - emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]); + emqx_ctl:print("trace ~ts ~ts successfully~n", [Who, Name]); {error, Error} -> - emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Who, Name, Error]) + emqx_ctl:print("[error] trace ~ts ~ts: ~p~n", [Who, Name, Error]) end. trace_off(Who, Name) -> case emqx_tracer:stop_trace({Who, iolist_to_binary(Name)}) of ok -> - emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]); + emqx_ctl:print("stop tracing ~ts ~ts successfully~n", [Who, Name]); {error, Error} -> - emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Name, Error]) + emqx_ctl:print("[error] stop tracing ~ts ~ts: ~p~n", [Who, Name, Error]) end. %%-------------------------------------------------------------------- @@ -432,32 +433,32 @@ listeners([]) -> {proxy_protocol, ProxyProtocol}, {running, Running} ] ++ CurrentConns ++ MaxConn, - emqx_ctl:print("~s~n", [ID]), + emqx_ctl:print("~ts~n", [ID]), lists:foreach(fun indent_print/1, Info) end, emqx_listeners:list()); listeners(["stop", ListenerId]) -> case emqx_listeners:stop_listener(list_to_atom(ListenerId)) of ok -> - emqx_ctl:print("Stop ~s listener successfully.~n", [ListenerId]); + emqx_ctl:print("Stop ~ts listener successfully.~n", [ListenerId]); {error, Error} -> - emqx_ctl:print("Failed to stop ~s listener: ~0p~n", [ListenerId, Error]) + emqx_ctl:print("Failed to stop ~ts listener: ~0p~n", [ListenerId, Error]) end; listeners(["start", ListenerId]) -> case emqx_listeners:start_listener(list_to_atom(ListenerId)) of ok -> - emqx_ctl:print("Started ~s listener successfully.~n", [ListenerId]); + emqx_ctl:print("Started ~ts listener successfully.~n", [ListenerId]); {error, Error} -> - emqx_ctl:print("Failed to start ~s listener: ~0p~n", [ListenerId, Error]) + emqx_ctl:print("Failed to start ~ts listener: ~0p~n", [ListenerId, Error]) end; listeners(["restart", ListenerId]) -> case emqx_listeners:restart_listener(list_to_atom(ListenerId)) of ok -> - emqx_ctl:print("Restarted ~s listener successfully.~n", [ListenerId]); + emqx_ctl:print("Restarted ~ts listener successfully.~n", [ListenerId]); {error, Error} -> - emqx_ctl:print("Failed to restart ~s listener: ~0p~n", [ListenerId, Error]) + emqx_ctl:print("Failed to restart ~ts listener: ~0p~n", [ListenerId, Error]) end; listeners(_) -> @@ -473,9 +474,9 @@ listeners(_) -> authz(["cache-clean", "node", Node]) -> case emqx_mgmt:clean_authz_cache_all(erlang:list_to_existing_atom(Node)) of ok -> - emqx_ctl:print("Authorization cache drain started on node ~s.~n", [Node]); + emqx_ctl:print("Authorization cache drain started on node ~ts.~n", [Node]); {error, Reason} -> - emqx_ctl:print("Authorization drain failed on node ~s: ~0p.~n", [Node, Reason]) + emqx_ctl:print("Authorization drain failed on node ~ts: ~0p.~n", [Node, Reason]) end; authz(["cache-clean", "all"]) -> @@ -495,6 +496,27 @@ authz(_) -> {"authz cache-clean ", "Clears authorization cache for given client"} ]). + +%%-------------------------------------------------------------------- +%% @doc OLP (Overload Protection related) +olp(["status"]) -> + S = case emqx_olp:is_overloaded() of + true -> "overloaded"; + false -> "not overloaded" + end, + emqx_ctl:print("~p is ~s ~n", [node(), S]); +olp(["disable"]) -> + Res = emqx_olp:disable(), + emqx_ctl:print("Disable overload protetion ~p : ~p ~n", [node(), Res]); +olp(["enable"]) -> + Res = emqx_olp:enable(), + emqx_ctl:print("Enable overload protection ~p : ~p ~n", [node(), Res]); +olp(_) -> + emqx_ctl:usage([{"olp status", "Return OLP status if system is overloaded"}, + {"olp enable", "Enable overload protection"}, + {"olp disable", "Disable overload protection"} + ]). + %%-------------------------------------------------------------------- %% Dump ETS %%-------------------------------------------------------------------- @@ -549,10 +571,10 @@ print({client, {ClientId, ChanPid}}) -> false -> [] end, Info1 = Info#{expiry_interval => maps:get(expiry_interval, Info) div 1000}, - emqx_ctl:print("Client(~s, username=~s, peername=~s, " - "clean_start=~s, keepalive=~w, session_expiry_interval=~w, " + emqx_ctl:print("Client(~ts, username=~ts, peername=~ts, " + "clean_start=~ts, keepalive=~w, session_expiry_interval=~w, " "subscriptions=~w, inflight=~w, awaiting_rel=~w, delivered_msgs=~w, enqueued_msgs=~w, dropped_msgs=~w, " - "connected=~s, created_at=~w, connected_at=~w" ++ + "connected=~ts, created_at=~w, connected_at=~w" ++ case maps:is_key(disconnected_at, Info1) of true -> ", disconnected_at=~w)~n"; false -> ")~n" @@ -560,23 +582,23 @@ print({client, {ClientId, ChanPid}}) -> [format(K, maps:get(K, Info1)) || K <- InfoKeys]); print({emqx_route, #route{topic = Topic, dest = {_, Node}}}) -> - emqx_ctl:print("~s -> ~s~n", [Topic, Node]); + emqx_ctl:print("~ts -> ~ts~n", [Topic, Node]); print({emqx_route, #route{topic = Topic, dest = Node}}) -> - emqx_ctl:print("~s -> ~s~n", [Topic, Node]); + emqx_ctl:print("~ts -> ~ts~n", [Topic, Node]); print(#plugin{name = Name, descr = Descr, active = Active}) -> - emqx_ctl:print("Plugin(~s, description=~s, active=~s)~n", + emqx_ctl:print("Plugin(~ts, description=~ts, active=~ts)~n", [Name, Descr, Active]); print({emqx_suboption, {{Pid, Topic}, Options}}) when is_pid(Pid) -> - emqx_ctl:print("~s -> ~s~n", [maps:get(subid, Options), Topic]). + emqx_ctl:print("~ts -> ~ts~n", [maps:get(subid, Options), Topic]). format(_, undefined) -> undefined; format(peername, {IPAddr, Port}) -> IPStr = emqx_mgmt_util:ntoa(IPAddr), - io_lib:format("~s:~p", [IPStr, Port]); + io_lib:format("~ts:~p", [IPStr, Port]); format(_, Val) -> Val. @@ -584,13 +606,13 @@ format(_, Val) -> bin(S) -> iolist_to_binary(S). indent_print({Key, {string, Val}}) -> - emqx_ctl:print(" ~-16s: ~s~n", [Key, Val]); + emqx_ctl:print(" ~-16s: ~ts~n", [Key, Val]); indent_print({Key, Val}) -> emqx_ctl:print(" ~-16s: ~w~n", [Key, Val]). format_listen_on(Port) when is_integer(Port) -> io_lib:format("0.0.0.0:~w", [Port]); format_listen_on({Addr, Port}) when is_list(Addr) -> - io_lib:format("~s:~w", [Addr, Port]); + io_lib:format("~ts:~w", [Addr, Port]); format_listen_on({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). + io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]). diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl index 48bef33ac..b276988bd 100644 --- a/apps/emqx_management/src/emqx_mgmt_util.erl +++ b/apps/emqx_management/src/emqx_mgmt_util.erl @@ -43,6 +43,7 @@ , batch_schema/1 ]). +-export([generate_response/1]). -export([urldecode/1]). @@ -80,7 +81,7 @@ kmg(Byte) when Byte > ?KB -> kmg(Byte) -> Byte. kmg(F, S) -> - iolist_to_binary(io_lib:format("~.2f~s", [F, S])). + iolist_to_binary(io_lib:format("~.2f~ts", [F, S])). ntoa({0,0,0,0,0,16#ffff,AB,CD}) -> inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}); @@ -258,3 +259,13 @@ bad_request() -> bad_request(Desc) -> object_schema(properties([{message, string}, {code, string}]), Desc). +%%%============================================================================================== +%% Response util + +generate_response(QueryResult) -> + case QueryResult of + {error, page_limit_invalid} -> + {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; + Response -> + {200, Response} + end. diff --git a/apps/emqx_management/test/emqx_mgmt_alarms_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_alarms_api_SUITE.erl index 9293d8fe0..81ea7ac5b 100644 --- a/apps/emqx_management/test/emqx_mgmt_alarms_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_alarms_api_SUITE.erl @@ -25,7 +25,7 @@ -define(DE_ACT_ALARM, test_de_act_alarm). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 34fa731c7..c023267a6 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -26,7 +26,7 @@ init_suite() -> init_suite(Apps) -> ekka_mnesia:start(), application:load(emqx_management), - emqx_ct_helpers:start_apps(Apps ++ [emqx_dashboard], fun set_special_configs/1). + emqx_common_test_helpers:start_apps(Apps ++ [emqx_dashboard], fun set_special_configs/1). end_suite() -> @@ -34,7 +34,7 @@ end_suite() -> end_suite(Apps) -> application:unload(emqx_management), - emqx_ct_helpers:stop_apps(Apps ++ [emqx_dashboard]). + emqx_common_test_helpers:stop_apps(Apps ++ [emqx_dashboard]). set_special_configs(emqx_dashboard) -> Config = #{ diff --git a/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl index 74838c91b..615445f66 100644 --- a/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl @@ -20,7 +20,7 @@ -include_lib("eunit/include/eunit.hrl"). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_management/test/emqx_mgmt_listeners_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_listeners_api_SUITE.erl index 0dd00b38e..5277961a9 100644 --- a/apps/emqx_management/test/emqx_mgmt_listeners_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_listeners_api_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_management/test/emqx_mgmt_metrics_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_metrics_api_SUITE.erl index 5e9464a1e..0df940a50 100644 --- a/apps/emqx_management/test/emqx_mgmt_metrics_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_metrics_api_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_management/test/emqx_mgmt_nodes_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_nodes_api_SUITE.erl index f4c17a163..8c5754dcc 100644 --- a/apps/emqx_management/test/emqx_mgmt_nodes_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_nodes_api_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_management/test/emqx_mgmt_publish_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_publish_api_SUITE.erl index 311a2cc97..a7bb41ee2 100644 --- a/apps/emqx_management/test/emqx_mgmt_publish_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_publish_api_SUITE.erl @@ -27,7 +27,7 @@ -define(TOPIC2, <<"api_topic2">>). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_management/test/emqx_mgmt_routes_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_routes_api_SUITE.erl index 6963e42cf..bd848cbc8 100644 --- a/apps/emqx_management/test/emqx_mgmt_routes_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_routes_api_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_management/test/emqx_mgmt_stats_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_stats_api_SUITE.erl index 11c66daa1..54174f15e 100644 --- a/apps/emqx_management/test/emqx_mgmt_stats_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_stats_api_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_management/test/emqx_mgmt_subscription_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_subscription_api_SUITE.erl index b344bcd11..94f20d2aa 100644 --- a/apps/emqx_management/test/emqx_mgmt_subscription_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_subscription_api_SUITE.erl @@ -30,7 +30,7 @@ -define(TOPIC_SORT, #{?TOPIC1 => 1, ?TOPIC2 => 2}). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index c989ecbed..cec75c9b5 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -28,14 +28,12 @@ namespace() -> modules. roots() -> ["delayed", - "recon", "telemetry", "event_message", array("rewrite"), array("topic_metrics")]. -fields(Name) when Name =:= "recon"; - Name =:= "telemetry" -> +fields("telemetry") -> [ {enable, hoconsc:mk(boolean(), #{default => false})} ]; diff --git a/apps/emqx_modules/src/emqx_telemetry_api.erl b/apps/emqx_modules/src/emqx_telemetry_api.erl index 93f938a5c..5d1cffdcd 100644 --- a/apps/emqx_modules/src/emqx_telemetry_api.erl +++ b/apps/emqx_modules/src/emqx_telemetry_api.erl @@ -136,7 +136,7 @@ data(get, _Request) -> % TelemetryData = get_telemetry_data(), % case emqx_json:safe_encode(TelemetryData, [pretty]) of % {ok, Bin} -> -% emqx_ctl:print("~s~n", [Bin]); +% emqx_ctl:print("~ts~n", [Bin]); % {error, _Reason} -> % emqx_ctl:print("Failed to get telemetry data") % end; diff --git a/apps/emqx_modules/test/emqx_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl index 3e386ea1d..3a070a1e8 100644 --- a/apps/emqx_modules/test/emqx_delayed_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl @@ -32,16 +32,16 @@ %%-------------------------------------------------------------------- all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ekka_mnesia:start(), ok = emqx_delayed:mnesia(boot), - emqx_ct_helpers:start_apps([emqx_modules]), + emqx_common_test_helpers:start_apps([emqx_modules]), Config. end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_modules]). + emqx_common_test_helpers:stop_apps([emqx_modules]). %%-------------------------------------------------------------------- %% Test cases diff --git a/apps/emqx_modules/test/emqx_event_message_SUITE.erl b/apps/emqx_modules/test/emqx_event_message_SUITE.erl index 526113bcc..f4454fb94 100644 --- a/apps/emqx_modules/test/emqx_event_message_SUITE.erl +++ b/apps/emqx_modules/test/emqx_event_message_SUITE.erl @@ -33,16 +33,16 @@ event_message: { \"$event/message_dropped\": true }""">>). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([emqx_modules]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([emqx_modules]), ok = emqx_config:init_load(emqx_modules_schema, ?EVENT_MESSAGE), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules]). + emqx_common_test_helpers:stop_apps([emqx_modules]). t_event_topic(_) -> ok = emqx_event_message:enable(), diff --git a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl index 012f1cb5b..e5bbb6182 100644 --- a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl @@ -38,15 +38,15 @@ rewrite: [ } ]""">>). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([emqx_modules]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([emqx_modules]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules]). + emqx_common_test_helpers:stop_apps([emqx_modules]). %% Test case for emqx_mod_write t_mod_rewrite(_Config) -> diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl index 62c524bdf..cf1ddff93 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -25,16 +25,16 @@ -import(proplists, [get_value/2]). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = ekka_mnesia:start(), ok = emqx_telemetry:mnesia(boot), - emqx_ct_helpers:start_apps([emqx_modules]), + emqx_common_test_helpers:start_apps([emqx_modules]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules]). + emqx_common_test_helpers:stop_apps([emqx_modules]). t_uuid(_) -> UUID = emqx_telemetry:generate_uuid(), diff --git a/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl index 246eb2ab3..a77a7aa79 100644 --- a/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl +++ b/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl @@ -25,16 +25,16 @@ topic_metrics: []""">>). -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:boot_modules(all), - emqx_ct_helpers:start_apps([emqx_modules]), + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([emqx_modules]), ok = emqx_config:init_load(emqx_modules_schema, ?TOPIC), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules]). + emqx_common_test_helpers:stop_apps([emqx_modules]). t_nonexistent_topic_metrics(_) -> emqx_topic_metrics:enable(), diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl index e51a4b6a6..c601f219f 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl @@ -101,7 +101,7 @@ do_save_file(FileName, Content, Dir) -> ok -> ensure_str(FullFilename); {error, Reason} -> - logger:error("failed_to_save_ssl_file ~s: ~0p", [FullFilename, Reason]), + logger:error("failed_to_save_ssl_file ~ts: ~0p", [FullFilename, Reason]), error({"failed_to_save_ssl_file", FullFilename, Reason}) end. diff --git a/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl index 56733147f..b2054e99e 100644 --- a/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl @@ -23,7 +23,7 @@ -define(PORT, 9876). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_http_connectivity(_) -> {ok, Socket} = gen_tcp:listen(?PORT, []), diff --git a/apps/emqx_psk/test/emqx_psk_SUITE.erl b/apps/emqx_psk/test/emqx_psk_SUITE.erl index fa24322a2..5794b8634 100644 --- a/apps/emqx_psk/test/emqx_psk_SUITE.erl +++ b/apps/emqx_psk/test/emqx_psk_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("eunit/include/eunit.hrl"). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> meck:new(emqx_config, [non_strict, passthrough, no_history, no_link]), @@ -35,12 +35,12 @@ init_per_suite(Config) -> ([psk, separator], _) -> <<":">>; (KeyPath, Default) -> meck:passthrough([KeyPath, Default]) end), - emqx_ct_helpers:start_apps([emqx_psk]), + emqx_common_test_helpers:start_apps([emqx_psk]), Config. end_per_suite(_) -> meck:unload(emqx_config), - emqx_ct_helpers:stop_apps([emqx_psk]), + emqx_common_test_helpers:stop_apps([emqx_psk]), ok. t_psk_lookup(_) -> diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index f3587ff5a..dcd5255b5 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -172,7 +172,7 @@ do_create(InstId, ResourceType, Config) -> _ = do_health_check(InstId), {ok, force_lookup(InstId)}; {error, Reason} -> - logger:error("start ~s resource ~s failed: ~p", [ResourceType, InstId, Reason]), + logger:error("start ~ts resource ~ts failed: ~p", [ResourceType, InstId, Reason]), {error, Reason} end end. diff --git a/apps/emqx_resource/src/emqx_resource_validator.erl b/apps/emqx_resource/src/emqx_resource_validator.erl index ee8cb6067..1fe07a28f 100644 --- a/apps/emqx_resource/src/emqx_resource_validator.erl +++ b/apps/emqx_resource/src/emqx_resource_validator.erl @@ -56,7 +56,7 @@ len(string) -> fun string:length/1; len(_Type) -> fun(Val) -> Val end. err_limit({Type, {Op, Expected}, {got, Got}}) -> - io_lib:format("Expect the ~s value ~s ~p but got: ~p", [Type, Op, Expected, Got]). + io_lib:format("Expect the ~ts value ~ts ~p but got: ~p", [Type, Op, Expected, Got]). return(true, _) -> ok; return(false, Error) -> diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 147e61ab7..2bb62e645 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -288,7 +288,7 @@ store_retained(Context, #message{topic = Topic, payload = Payload} = Msg) -> Mod = get_backend_module(), Mod:store_retained(Context, Msg); _ -> - ?ERROR("Cannot retain message(topic=~s, payload_size=~p) for payload is too big!", + ?ERROR("Cannot retain message(topic=~ts, payload_size=~p) for payload is too big!", [Topic, iolist_size(Payload)]) end. @@ -441,17 +441,17 @@ get_backend_module() -> true -> Backend end, - erlang:list_to_existing_atom(io_lib:format("~s_~s", [?APP, ModName])). + erlang:list_to_existing_atom(io_lib:format("~ts_~ts", [?APP, ModName])). create_resource(Context, #{type := built_in_database} = Cfg) -> emqx_retainer_mnesia:create_resource(Cfg), Context; create_resource(Context, #{type := DB} = Config) -> - ResourceID = erlang:iolist_to_binary([io_lib:format("~s_~s", [?APP, DB])]), + ResourceID = erlang:iolist_to_binary([io_lib:format("~ts_~ts", [?APP, DB])]), case emqx_resource:create( ResourceID, - list_to_existing_atom(io_lib:format("~s_~s", [emqx_connector, DB])), + list_to_existing_atom(io_lib:format("~ts_~ts", [emqx_connector, DB])), Config) of {ok, already_created} -> Context#{resource_id => ResourceID}; diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl index 54d0f2c3a..0fc0a4615 100644 --- a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl +++ b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl @@ -89,7 +89,7 @@ store_retained(_, Msg =#message{topic = Topic}) -> write); [] -> ?LOG(error, - "Cannot retain message(topic=~s) for table is full!", + "Cannot retain message(topic=~ts) for table is full!", [Topic]), ok end diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 0348e065a..ccc647ddc 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). -define(BASE_CONF, <<""" emqx_retainer { @@ -50,11 +50,11 @@ emqx_retainer { init_per_suite(Config) -> ok = emqx_config:init_load(emqx_retainer_schema, ?BASE_CONF), - emqx_ct_helpers:start_apps([emqx_retainer]), + emqx_common_test_helpers:start_apps([emqx_retainer]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_retainer]). + emqx_common_test_helpers:stop_apps([emqx_retainer]). %%-------------------------------------------------------------------- %% Test Cases %%-------------------------------------------------------------------- diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index 5d379e8cd..90eb9f615 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -37,7 +37,7 @@ all() -> %% TODO: V5 API - %% emqx_ct:all(?MODULE). + %% emqx_common_test_helpers:all(?MODULE). []. groups() -> @@ -45,13 +45,13 @@ groups() -> init_per_suite(Config) -> application:stop(emqx_retainer), - emqx_ct_helpers:start_apps([emqx_retainer, emqx_management], fun set_special_configs/1), + emqx_common_test_helpers:start_apps([emqx_retainer, emqx_management], fun set_special_configs/1), create_default_app(), Config. end_per_suite(_Config) -> delete_default_app(), - emqx_ct_helpers:stop_apps([emqx_management, emqx_retainer]). + emqx_common_test_helpers:stop_apps([emqx_management, emqx_retainer]). init_per_testcase(_, Config) -> Config. diff --git a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl index f9f657483..eeead9bd8 100644 --- a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_testcase(_TestCase, Config) -> Config. diff --git a/apps/emqx_retainer/test/emqx_retainer_mqtt_v5_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_mqtt_v5_SUITE.erl index 0be5df732..3fa49f079 100644 --- a/apps/emqx_retainer/test/emqx_retainer_mqtt_v5_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_mqtt_v5_SUITE.erl @@ -39,19 +39,19 @@ emqx_retainer { } }""">>). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> ok = emqx_config:init_load(emqx_retainer_schema, ?BASE_CONF), %% Meck emqtt ok = meck:new(emqtt, [non_strict, passthrough, no_history, no_link]), %% Start Apps - emqx_ct_helpers:start_apps([emqx_retainer]), + emqx_common_test_helpers:start_apps([emqx_retainer]), Config. end_per_suite(_Config) -> ok = meck:unload(emqtt), - emqx_ct_helpers:stop_apps([emqx_retainer]). + emqx_common_test_helpers:stop_apps([emqx_retainer]). client_info(Key, Client) -> maps:get(Key, maps:from_list(emqtt:info(Client)), undefined). diff --git a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf index a4344cda8..aff89b77e 100644 --- a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf +++ b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf @@ -3,4 +3,19 @@ ##==================================================================== rule_engine { ignore_sys_message = true + #rules.my_republish_rule { + # description = "A simple rule that republishs MQTT messages from topic 't/1' to 't/2'" + # enable = true + # sql = "SELECT * FROM \"t/1\"" + # outputs = [ + # { + # function = republish + # args = { + # topic = "t/2" + # qos = "${qos}" + # payload = "${payload}" + # } + # } + # ] + #} } diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index c230aa8c3..b7ec37d8e 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -26,40 +26,37 @@ -type mf() :: {Module::atom(), Fun::atom()}. -type hook() :: atom() | 'any'. - -type topic() :: binary(). --type bridge_channel_id() :: binary(). + -type selected_data() :: map(). -type envs() :: map(). --type output_type() :: bridge | builtin | func. --type output_target() :: bridge_channel_id() | atom() | output_fun(). --type output_fun_args() :: map(). --type output() :: #{ - type := output_type(), - target := output_target(), - args => output_fun_args() -}. --type output_fun() :: fun((selected_data(), envs(), output_fun_args()) -> any()). --type rule_info() :: - #{ from := list(topic()) - , outputs := [output()] +-type builtin_output_func() :: republish | console. +-type builtin_output_module() :: emqx_rule_outputs. +-type bridge_channel_id() :: binary(). +-type output_fun_args() :: map(). + +-type output() :: #{ + mod := builtin_output_module() | module(), + func := builtin_output_func() | atom(), + args => output_fun_args() +} | bridge_channel_id(). + +-type rule() :: + #{ id := rule_id() , sql := binary() + , outputs := [output()] + , enabled := boolean() + , description => binary() + , created_at := integer() %% epoch in millisecond precision + , from := list(topic()) , is_foreach := boolean() , fields := list() , doeach := term() , incase := term() , conditions := tuple() - , enabled := boolean() - , description => binary() }. --record(rule, - { id :: rule_id() - , created_at :: integer() %% epoch in millisecond precision - , info :: rule_info() - }). - %% Arithmetic operators -define(is_arith(Op), (Op =:= '+' orelse Op =:= '-' orelse @@ -93,6 +90,4 @@ end()). %% Tables --define(RULE_TAB, emqx_rule). - --define(RULE_ENGINE_SHARD, emqx_rule_engine_shard). +-define(RULE_TAB, emqx_rule_engine). diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index c96e82ecb..448f63138 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -37,16 +37,7 @@ roots() -> fields("rule_creation") -> [ {"id", sc(binary(), #{desc => "The Id of the rule", nullable => false})} - , {"sql", sc(binary(), #{desc => "The SQL of the rule", nullable => false})} - , {"outputs", sc(hoconsc:array(hoconsc:union( - [ ref("bridge_output") - , ref("builtin_output") - ])), - #{desc => "The outputs of the rule", - default => []})} - , {"enable", sc(boolean(), #{desc => "Enable or disable the rule", default => true})} - , {"description", sc(binary(), #{desc => "The description of the rule", default => <<>>})} - ]; + ] ++ emqx_rule_engine_schema:fields("rules"); fields("rule_test") -> [ {"context", sc(hoconsc:union([ ref("ctx_pub") @@ -62,38 +53,6 @@ fields("rule_test") -> , {"sql", sc(binary(), #{desc => "The SQL of the rule for testing", nullable => false})} ]; -fields("bridge_output") -> - [ {type, bridge} - , {target, sc(binary(), #{desc => "The Channel ID of the bridge"})} - ]; - -fields("builtin_output") -> - [ {type, builtin} - , {target, sc(binary(), #{desc => "The Name of the built-on output"})} - , {args, sc(map(), #{desc => "The arguments of the built-in output", - default => #{}})} - ]; - -%% TODO: how to use this in "builtin_output".args ? -fields("republish_args") -> - [ {topic, sc(binary(), - #{desc => "The target topic of the re-published message." - " Template with with variables is allowed.", - nullable => false})} - , {qos, sc(binary(), - #{desc => "The qos of the re-published message." - " Template with with variables is allowed. Defaults to ${qos}.", - default => <<"${qos}">> })} - , {retain, sc(binary(), - #{desc => "The retain of the re-published message." - " Template with with variables is allowed. Defaults to ${retain}.", - default => <<"${retain}">> })} - , {payload, sc(binary(), - #{desc => "The payload of the re-published message." - " Template with with variables is allowed. Defaults to ${payload}.", - default => <<"${payload}">>})} - ]; - fields("ctx_pub") -> [ {"event_type", sc(message_publish, #{desc => "Event Type", nullable => false})} , {"id", sc(binary(), #{desc => "Message ID"})} diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index a93a7e14d..95b153191 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -3,7 +3,7 @@ [{description, "EMQ X Rule Engine"}, {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, - {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, + {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel,stdlib,rulesql,getopt]}, {mod, {emqx_rule_engine_app, []}}, {env, []}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 04d35931a..8a6d78ff6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -16,48 +16,201 @@ -module(emqx_rule_engine). +-behaviour(gen_server). +-behaviour(emqx_config_handler). + -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("stdlib/include/qlc.hrl"). --export([ create_rule/1 - , update_rule/1 - , delete_rule/1 +-export([start_link/0]). + +-export([ post_config_update/4 + , config_key_path/0 ]). --export_type([rule/0]). +%% Rule Management --type rule() :: #rule{}. +-export([ load_rules/0 + ]). --define(T_RETRY, 60000). +-export([ create_rule/1 + , insert_rule/1 + , update_rule/1 + , delete_rule/1 + , get_rule/1 + ]). + +-export([ get_rules/0 + , get_rules_for_topic/1 + , get_rules_with_same_event/1 + , get_rules_ordered_by_ts/0 + ]). + +%% exported for cluster_call +-export([ do_delete_rule/1 + , do_insert_rule/1 + ]). + +-export([ load_hooks_for_rule/1 + , unload_hooks_for_rule/1 + , add_metrics_for_rule/1 + , clear_metrics_for_rule/1 + ]). + +%% gen_server Callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-define(RULE_ENGINE, ?MODULE). + +-define(T_CALL, infinity). + +config_key_path() -> + [rule_engine, rules]. + +-spec(start_link() -> {ok, pid()} | ignore | {error, Reason :: term()}). +start_link() -> + gen_server:start_link({local, ?RULE_ENGINE}, ?MODULE, [], []). %%------------------------------------------------------------------------------ -%% APIs for rules and resources +%% The config handler for emqx_rule_engine %%------------------------------------------------------------------------------ +post_config_update(_Req, NewRules, OldRules, _AppEnvs) -> + #{added := Added, removed := Removed, changed := Updated} + = emqx_map_lib:diff_maps(NewRules, OldRules), + maps_foreach(fun({Id, {_Old, New}}) -> + {ok, _} = update_rule(New#{id => bin(Id)}) + end, Updated), + maps_foreach(fun({Id, _Rule}) -> + ok = delete_rule(bin(Id)) + end, Removed), + maps_foreach(fun({Id, Rule}) -> + {ok, _} = create_rule(Rule#{id => bin(Id)}) + end, Added), + {ok, get_rules()}. + +%%------------------------------------------------------------------------------ +%% APIs for rules +%%------------------------------------------------------------------------------ + +-spec load_rules() -> ok. +load_rules() -> + maps_foreach(fun({Id, Rule}) -> + {ok, _} = create_rule(Rule#{id => bin(Id)}) + end, emqx:get_config([rule_engine, rules], #{})). -spec create_rule(map()) -> {ok, rule()} | {error, term()}. -create_rule(Params = #{id := RuleId}) -> - case emqx_rule_registry:get_rule(RuleId) of +create_rule(Params = #{id := RuleId}) when is_binary(RuleId) -> + case get_rule(RuleId) of not_found -> do_create_rule(Params); {ok, _} -> {error, {already_exists, RuleId}} end. -spec update_rule(map()) -> {ok, rule()} | {error, term()}. -update_rule(Params = #{id := RuleId}) -> - case delete_rule(RuleId) of - ok -> do_create_rule(Params); - Error -> Error +update_rule(Params = #{id := RuleId}) when is_binary(RuleId) -> + ok = delete_rule(RuleId), + do_create_rule(Params). + +-spec(delete_rule(RuleId :: rule_id()) -> ok). +delete_rule(RuleId) when is_binary(RuleId) -> + gen_server:call(?RULE_ENGINE, {delete_rule, RuleId}, ?T_CALL). + +-spec(insert_rule(Rule :: rule()) -> ok). +insert_rule(Rule) -> + gen_server:call(?RULE_ENGINE, {insert_rule, Rule}, ?T_CALL). + +%%------------------------------------------------------------------------------ +%% Rule Management +%%------------------------------------------------------------------------------ + +-spec(get_rules() -> [rule()]). +get_rules() -> + get_all_records(?RULE_TAB). + +get_rules_ordered_by_ts() -> + lists:sort(fun(#{created_at := CreatedA}, #{created_at := CreatedB}) -> + CreatedA =< CreatedB + end, get_rules()). + +-spec(get_rules_for_topic(Topic :: binary()) -> [rule()]). +get_rules_for_topic(Topic) -> + [Rule || Rule = #{from := From} <- get_rules(), + emqx_plugin_libs_rule:can_topic_match_oneof(Topic, From)]. + +-spec(get_rules_with_same_event(Topic :: binary()) -> [rule()]). +get_rules_with_same_event(Topic) -> + EventName = emqx_rule_events:event_name(Topic), + [Rule || Rule = #{from := From} <- get_rules(), + lists:any(fun(T) -> is_of_event_name(EventName, T) end, From)]. + +is_of_event_name(EventName, Topic) -> + EventName =:= emqx_rule_events:event_name(Topic). + +-spec(get_rule(Id :: rule_id()) -> {ok, rule()} | not_found). +get_rule(Id) -> + case ets:lookup(?RULE_TAB, Id) of + [{Id, Rule}] -> {ok, Rule#{id => Id}}; + [] -> not_found end. --spec(delete_rule(RuleId :: rule_id()) -> ok | {error, term()}). -delete_rule(RuleId) -> - case emqx_rule_registry:get_rule(RuleId) of - {ok, Rule} -> - ok = emqx_rule_registry:remove_rule(Rule), - _ = emqx_plugin_libs_rule:cluster_call(emqx_rule_metrics, clear_rule_metrics, [RuleId]), - ok; - not_found -> - {error, not_found} - end. +load_hooks_for_rule(#{from := Topics}) -> + lists:foreach(fun emqx_rule_events:load/1, Topics). + +add_metrics_for_rule(#{id := Id}) -> + ok = emqx_rule_metrics:create_rule_metrics(Id). + +clear_metrics_for_rule(#{id := Id}) -> + ok = emqx_rule_metrics:clear_rule_metrics(Id). + +unload_hooks_for_rule(#{id := Id, from := Topics}) -> + lists:foreach(fun(Topic) -> + case get_rules_with_same_event(Topic) of + [#{id := Id0}] when Id0 == Id -> %% we are now deleting the last rule + emqx_rule_events:unload(Topic); + _ -> ok + end + end, Topics). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + _TableId = ets:new(?KV_TAB, [named_table, set, public, {write_concurrency, true}, + {read_concurrency, true}]), + {ok, #{}}. + +handle_call({insert_rule, Rule}, _From, State) -> + _ = emqx_plugin_libs_rule:cluster_call(?MODULE, do_insert_rule, [Rule]), + {reply, ok, State}; + +handle_call({delete_rule, Rule}, _From, State) -> + _ = emqx_plugin_libs_rule:cluster_call(?MODULE, do_delete_rule, [Rule]), + {reply, ok, State}; + +handle_call(Req, _From, State) -> + ?SLOG(error, #{msg => "unexpected_call", request => Req}), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?SLOG(error, #{msg => "unexpected_cast", request => Msg}), + {noreply, State}. + +handle_info(Info, State) -> + ?SLOG(error, #{msg => "unexpected_info", request => Info}), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. %%------------------------------------------------------------------------------ %% Internal Functions @@ -66,55 +219,56 @@ delete_rule(RuleId) -> do_create_rule(Params = #{id := RuleId, sql := Sql, outputs := Outputs}) -> case emqx_rule_sqlparser:parse(Sql) of {ok, Select} -> - Rule = #rule{ - id = RuleId, - created_at = erlang:system_time(millisecond), - info = #{ - enabled => maps:get(enabled, Params, true), - sql => Sql, - from => emqx_rule_sqlparser:select_from(Select), - outputs => parse_outputs(Outputs), - description => maps:get(description, Params, ""), - %% -- calculated fields: - is_foreach => emqx_rule_sqlparser:select_is_foreach(Select), - fields => emqx_rule_sqlparser:select_fields(Select), - doeach => emqx_rule_sqlparser:select_doeach(Select), - incase => emqx_rule_sqlparser:select_incase(Select), - conditions => emqx_rule_sqlparser:select_where(Select) - %% -- calculated fields end - } + Rule = #{ + id => RuleId, + created_at => erlang:system_time(millisecond), + enabled => maps:get(enabled, Params, true), + sql => Sql, + outputs => parse_outputs(Outputs), + description => maps:get(description, Params, ""), + %% -- calculated fields: + from => emqx_rule_sqlparser:select_from(Select), + is_foreach => emqx_rule_sqlparser:select_is_foreach(Select), + fields => emqx_rule_sqlparser:select_fields(Select), + doeach => emqx_rule_sqlparser:select_doeach(Select), + incase => emqx_rule_sqlparser:select_incase(Select), + conditions => emqx_rule_sqlparser:select_where(Select) + %% -- calculated fields end }, - ok = emqx_rule_registry:add_rule(Rule), - _ = emqx_plugin_libs_rule:cluster_call(emqx_rule_metrics, create_rule_metrics, [RuleId]), + ok = insert_rule(Rule), {ok, Rule}; {error, Reason} -> {error, Reason} end. +do_insert_rule(#{id := Id} = Rule) -> + ok = load_hooks_for_rule(Rule), + ok = add_metrics_for_rule(Rule), + true = ets:insert(?RULE_TAB, {Id, maps:remove(id, Rule)}), + ok. + +do_delete_rule(RuleId) -> + case get_rule(RuleId) of + {ok, Rule} -> + ok = unload_hooks_for_rule(Rule), + ok = clear_metrics_for_rule(Rule), + true = ets:delete(?RULE_TAB, RuleId), + ok; + not_found -> ok + end. + parse_outputs(Outputs) -> - [do_parse_outputs(Out) || Out <- Outputs]. + [do_parse_output(Out) || Out <- Outputs]. -do_parse_outputs(#{type := bridge, target := ChId}) -> - #{type => bridge, target => ChId}; -do_parse_outputs(#{type := builtin, target := Repub, args := Args}) - when Repub == republish; Repub == <<"republish">> -> - #{type => builtin, target => republish, args => pre_process_repub_args(Args)}; -do_parse_outputs(#{type := Type, target := Name} = Output) - when Type == func; Type == builtin -> - #{type => Type, target => Name, args => maps:get(args, Output, #{})}. +do_parse_output(Output) when is_map(Output) -> + emqx_rule_outputs:parse_output(Output); +do_parse_output(BridgeChannelId) when is_binary(BridgeChannelId) -> + BridgeChannelId. -pre_process_repub_args(#{<<"topic">> := Topic} = Args) -> - QoS = maps:get(<<"qos">>, Args, <<"${qos}">>), - Retain = maps:get(<<"retain">>, Args, <<"${retain}">>), - Payload = maps:get(<<"payload">>, Args, <<"${payload}">>), - #{topic => Topic, qos => QoS, payload => Payload, retain => Retain, - preprocessed_tmpl => #{ - topic => emqx_plugin_libs_rule:preproc_tmpl(Topic), - qos => preproc_vars(QoS), - retain => preproc_vars(Retain), - payload => emqx_plugin_libs_rule:preproc_tmpl(Payload) - }}. +get_all_records(Tab) -> + [Rule#{id => Id} || {Id, Rule} <- ets:tab2list(Tab)]. -preproc_vars(Data) when is_binary(Data) -> - emqx_plugin_libs_rule:preproc_tmpl(Data); -preproc_vars(Data) -> - Data. +maps_foreach(Fun, Map) -> + lists:foreach(Fun, maps:to_list(Map)). + +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(B) when is_binary(B) -> B. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index b097c7169..5e6b28b61 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -29,7 +29,7 @@ , rule_test/2 ]). --define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~s Not Found", [(ID)]))). +-define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))). -define(ERR_BADARGS(REASON), begin R0 = err_msg(REASON), @@ -141,21 +141,22 @@ put_req_schema() -> description => <<"The outputs of the rule">>, type => array, items => #{ - type => object, - properties => #{ - type => #{ + oneOf => [ + #{ type => string, - enum => [<<"bridge">>, <<"builtin">>], - example => <<"builtin">> + example => <<"channel_id_of_my_bridge">>, + description => <<"The channel id of an emqx bridge">> }, - target => #{ - type => string, - example => <<"console">> - }, - args => #{ - type => object + #{ + type => object, + properties => #{ + function => #{ + type => string, + example => <<"console">> + } + } } - } + ] } }, description => #{ @@ -241,16 +242,25 @@ list_events(#{}, _Params) -> {200, emqx_rule_events:event_info()}. crud_rules(get, _Params) -> - Records = emqx_rule_registry:get_rules_ordered_by_ts(), + Records = emqx_rule_engine:get_rules_ordered_by_ts(), {200, format_rule_resp(Records)}; -crud_rules(post, #{body := Params}) -> - ?CHECK_PARAMS(Params, rule_creation, case emqx_rule_engine:create_rule(CheckedParams) of - {ok, Rule} -> {201, format_rule_resp(Rule)}; - {error, Reason} -> - ?SLOG(error, #{msg => "create_rule_failed", reason => Reason}), - {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(Reason)}} - end). +crud_rules(post, #{body := #{<<"id">> := Id} = Params}) -> + ConfPath = emqx_rule_engine:config_key_path() ++ [Id], + case emqx_rule_engine:get_rule(Id) of + {ok, _Rule} -> + {400, #{code => 'BAD_ARGS', message => <<"rule id already exists">>}}; + not_found -> + case emqx:update_config(ConfPath, maps:remove(<<"id">>, Params), #{}) of + {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> + [Rule] = [R || R = #{id := Id0} <- AllRules, Id0 == Id], + {201, format_rule_resp(Rule)}; + {error, Reason} -> + ?SLOG(error, #{msg => "create_rule_failed", + id => Id, reason => Reason}), + {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(Reason)}} + end + end. rule_test(post, #{body := Params}) -> ?CHECK_PARAMS(Params, rule_test, case emqx_rule_sqltester:test(CheckedParams) of @@ -259,30 +269,33 @@ rule_test(post, #{body := Params}) -> end). crud_rules_by_id(get, #{bindings := #{id := Id}}) -> - case emqx_rule_registry:get_rule(Id) of + case emqx_rule_engine:get_rule(Id) of {ok, Rule} -> {200, format_rule_resp(Rule)}; not_found -> {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} end; -crud_rules_by_id(put, #{bindings := #{id := Id}, body := Params0}) -> - Params = maps:merge(Params0, #{id => Id}), - ?CHECK_PARAMS(Params, rule_creation, case emqx_rule_engine:update_rule(CheckedParams) of - {ok, Rule} -> {200, format_rule_resp(Rule)}; - {error, not_found} -> - {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}}; +crud_rules_by_id(put, #{bindings := #{id := Id}, body := Params}) -> + ConfPath = emqx_rule_engine:config_key_path() ++ [Id], + case emqx:update_config(ConfPath, maps:remove(<<"id">>, Params), #{}) of + {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> + [Rule] = [R || R = #{id := Id0} <- AllRules, Id0 == Id], + {200, format_rule_resp(Rule)}; {error, Reason} -> ?SLOG(error, #{msg => "update_rule_failed", - id => Id, - reason => Reason}), + id => Id, reason => Reason}), {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(Reason)}} - end); + end; crud_rules_by_id(delete, #{bindings := #{id := Id}}) -> - case emqx_rule_engine:delete_rule(Id) of - ok -> {200}; - {error, not_found} -> {200} + ConfPath = emqx_rule_engine:config_key_path() ++ [Id], + case emqx:remove_config(ConfPath, #{}) of + {ok, _} -> {200}; + {error, Reason} -> + ?SLOG(error, #{msg => "delete_rule_failed", + id => Id, reason => Reason}), + {500, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(Reason)}} end. %%------------------------------------------------------------------------------ @@ -295,13 +308,12 @@ err_msg(Msg) -> format_rule_resp(Rules) when is_list(Rules) -> [format_rule_resp(R) || R <- Rules]; -format_rule_resp(#rule{id = Id, created_at = CreatedAt, - info = #{ - from := Topics, - outputs := Output, - sql := SQL, - enabled := Enabled, - description := Descr}}) -> +format_rule_resp(#{ id := Id, created_at := CreatedAt, + from := Topics, + outputs := Output, + sql := SQL, + enabled := Enabled, + description := Descr}) -> #{id => Id, from => Topics, outputs => format_output(Output), @@ -318,12 +330,11 @@ format_datetime(Timestamp, Unit) -> format_output(Outputs) -> [do_format_output(Out) || Out <- Outputs]. -do_format_output(#{type := func}) -> - #{type => func, target => <<"internal_function">>}; -do_format_output(#{type := builtin, target := Name, args := Args}) -> - #{type => builtin, target => Name, args => maps:remove(preprocessed_tmpl, Args)}; -do_format_output(#{type := bridge, target := Name}) -> - #{type => bridge, target => Name}. +do_format_output(#{mod := Mod, func := Func, args := Args}) -> + #{function => list_to_binary(lists:concat([Mod,":",Func])), + args => maps:remove(preprocessed_tmpl, Args)}; +do_format_output(BridgeChannelId) when is_binary(BridgeChannelId) -> + BridgeChannelId. get_rule_metrics(Id) -> [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_rule_metrics, [Id])) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl index e3e959222..e9ca443ec 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl @@ -25,9 +25,13 @@ -export([stop/1]). start(_Type, _Args) -> - ok = ekka_rlog:wait_for_shards([?RULE_ENGINE_SHARD], infinity), + _ = ets:new(?RULE_TAB, [named_table, public, set, {read_concurrency, true}]), ok = emqx_rule_events:reload(), - emqx_rule_engine_sup:start_link(). + SupRet = emqx_rule_engine_sup:start_link(), + ok = emqx_rule_engine:load_rules(), + emqx_config_handler:add_handler(emqx_rule_engine:config_key_path(), emqx_rule_engine), + SupRet. stop(_State) -> + emqx_config_handler:remove_handler(emqx_rule_engine:config_key_path()), ok = emqx_rule_events:unload(). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 2614fb8b1..8daae99b5 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -24,9 +24,130 @@ , roots/0 , fields/1]). +-export([ validate_sql/1 + ]). + namespace() -> rule_engine. roots() -> ["rule_engine"]. fields("rule_engine") -> - [{ignore_sys_message, hoconsc:mk(boolean(), #{default => true})}]. + [ {ignore_sys_message, sc(boolean(), #{default => true, desc => +"When set to 'true' (default), rule-engine will ignore messages published to $SYS topics." + })} + , {rules, sc(hoconsc:map("id", ref("rules")), #{desc => "The rules", default => #{}})} + ]; + +fields("rules") -> + [ {"sql", sc(binary(), + #{ desc => """ +SQL query to transform the messages.
+Example: SELECT * FROM \"test/topic\" WHERE payload.x = 1
+""" + , nullable => false + , validator => fun ?MODULE:validate_sql/1})} + , {"outputs", sc(hoconsc:array(hoconsc:union( + [ binary() + , ref("builtin_output_republish") + , ref("builtin_output_console") + ])), + #{ desc => """ +A list of outputs of the rule.
+An output can be a string that refers to the channel Id of a emqx bridge, or a object +that refers to a function.
+There a some built-in functions like \"republish\" and \"console\", and we also support user +provided functions like \":\".
+The outputs in the list is executed one by one in order. +This means that if one of the output is executing slowly, all of the outputs comes after it will not +be executed until it returns.
+If one of the output crashed, all other outputs come after it will still be executed, in the +original order.
+If there's any error when running an output, there will be an error message, and the 'failure' +counter of the function output or the bridge channel will increase. +""" + , default => [] + })} + , {"enable", sc(boolean(), #{desc => "Enable or disable the rule", default => true})} + , {"description", sc(binary(), #{desc => "The description of the rule", default => <<>>})} + ]; + +fields("builtin_output_republish") -> + [ {function, sc(republish, #{desc => "Republish the message as a new MQTT message"})} + , {args, sc(ref("republish_args"), + #{ desc => """ +The arguments of the built-in 'republish' output.
+We can use variables in the args.
+ +The variables are selected by the rule. For exmaple, if the rule SQL is defined as following: + + SELECT clientid, qos, payload FROM \"t/1\" + +Then there are 3 variables available: clientid, qos and +payload. And if we've set the args to: + + { + topic = \"t/${clientid}\" + qos = \"${qos}\" + payload = \"msg: ${payload}\" + } + +When the rule is triggered by an MQTT message with payload = \"hello\", qos = 1, +clientid = \"steve\", the rule will republish a new MQTT message to topic \"t/steve\", +payload = \"msg: hello\", and qos = 1. +""" + , default => #{} + })} + ]; + +fields("builtin_output_console") -> + [ {function, sc(console, #{desc => "Print the outputs to the console"})} + %% we may support some args for the console output in the future + %, {args, sc(map(), #{desc => "The arguments of the built-in 'console' output", + % default => #{}})} + ]; + +fields("republish_args") -> + [ {topic, sc(binary(), + #{ desc =>""" +The target topic of message to be re-published.
+Template with variables is allowed, see description of the 'republish_args'. +""" + , nullable => false + })} + , {qos, sc(binary(), + #{ desc => """ +The qos of the message to be re-published. +Template with with variables is allowed, see description of the 'republish_args.
+Defaults to ${qos}. If variable ${qos} is not found from the selected result of the rule, +0 is used. +""" + , default => <<"${qos}">> + })} + , {retain, sc(binary(), + #{ desc => """ +The retain flag of the message to be re-published. +Template with with variables is allowed, see description of the 'republish_args.
+Defaults to ${retain}. If variable ${retain} is not found from the selected result +of the rule, false is used. +""" + , default => <<"${retain}">> + })} + , {payload, sc(binary(), + #{ desc => """ +The payload of the message to be re-published. +Template with with variables is allowed, see description of the 'republish_args.
. +Defaults to ${payload}. If variable ${payload} is not found from the selected result +of the rule, then the string \"undefined\" is used. +""" + , default => <<"${payload}">> + })} + ]. + +validate_sql(Sql) -> + case emqx_rule_sqlparser:parse(Sql) of + {ok, _Result} -> ok; + {error, Reason} -> {error, Reason} + end. + +sc(Type, Meta) -> hoconsc:mk(Type, Meta). +ref(Field) -> hoconsc:ref(?MODULE, Field). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl index 4fad54a4b..7fd44df82 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl @@ -28,12 +28,12 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - Registry = #{id => emqx_rule_registry, - start => {emqx_rule_registry, start_link, []}, + Registry = #{id => emqx_rule_engine, + start => {emqx_rule_engine, start_link, []}, restart => permanent, shutdown => 5000, type => worker, - modules => [emqx_rule_registry]}, + modules => [emqx_rule_engine]}, Metrics = #{id => emqx_rule_metrics, start => {emqx_rule_metrics, start_link, []}, restart => permanent, diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index d030917ef..614dc841b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -64,7 +64,9 @@ -endif. reload() -> - emqx_rule_registry:load_hooks_for_rule(emqx_rule_registry:get_rules()). + lists:foreach(fun(Rule) -> + ok = emqx_rule_engine:load_hooks_for_rule(Rule) + end, emqx_rule_engine:get_rules()). load(<<"$bridges/", _ChannelId/binary>> = BridgeTopic) -> emqx_hooks:put(BridgeTopic, {?MODULE, on_bridge_message_received, @@ -86,7 +88,7 @@ unload(Topic) -> %% Callbacks %%-------------------------------------------------------------------- on_bridge_message_received(Message, #{bridge_topic := BridgeTopic}) -> - case emqx_rule_registry:get_rules_for_topic(BridgeTopic) of + case emqx_rule_engine:get_rules_for_topic(BridgeTopic) of [] -> ok; Rules -> emqx_rule_runtime:apply_rules(Rules, Message) end. @@ -95,7 +97,7 @@ on_message_publish(Message = #message{topic = Topic}, _Env) -> case ignore_sys_message(Message) of true -> ok; false -> - case emqx_rule_registry:get_rules_for_topic(Topic) of + case emqx_rule_engine:get_rules_for_topic(Topic) of [] -> ok; Rules -> emqx_rule_runtime:apply_rules(Rules, eventmsg_publish(Message)) end @@ -310,7 +312,7 @@ with_basic_columns(EventName, Data) when is_map(Data) -> %%-------------------------------------------------------------------- apply_event(EventName, GenEventMsg, _Env) -> EventTopic = event_topic(EventName), - case emqx_rule_registry:get_rules_for_topic(EventTopic) of + case emqx_rule_engine:get_rules_for_topic(EventTopic) of [] -> ok; Rules -> emqx_rule_runtime:apply_rules(Rules, GenEventMsg()) end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index fd922db86..c94242a99 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -914,9 +914,9 @@ map_path(Key) -> function_literal(Fun, []) when is_atom(Fun) -> atom_to_list(Fun) ++ "()"; function_literal(Fun, [FArg | Args]) when is_atom(Fun), is_list(Args) -> - WithFirstArg = io_lib:format("~s(~0p", [atom_to_list(Fun), FArg]), + WithFirstArg = io_lib:format("~ts(~0p", [atom_to_list(Fun), FArg]), lists:foldl(fun(Arg, Literal) -> - io_lib:format("~s, ~0p", [Literal, Arg]) + io_lib:format("~ts, ~0p", [Literal, Arg]) end, WithFirstArg, Args) ++ ")"; function_literal(Fun, Args) -> {invalid_func, {Fun, Args}}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl index cd59d3fa5..61a520e81 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl @@ -16,16 +16,57 @@ %% Define the default actions. -module(emqx_rule_outputs). + +-include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). +%% APIs +-export([ parse_output/1 + ]). + +%% callbacks of emqx_rule_output +-export([ pre_process_output_args/2 + ]). + +%% output functions -export([ console/3 , republish/3 ]). +-optional_callbacks([ pre_process_output_args/2 + ]). + +-callback pre_process_output_args(FuncName :: atom(), output_fun_args()) -> output_fun_args(). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- +parse_output(#{function := OutputFunc} = Output) -> + {Mod, Func} = parse_output_func(OutputFunc), + #{mod => Mod, func => Func, + args => pre_process_args(Mod, Func, maps:get(args, Output, #{}))}. + +%%-------------------------------------------------------------------- +%% callbacks of emqx_rule_output +%%-------------------------------------------------------------------- +pre_process_output_args(republish, #{topic := Topic, qos := QoS, retain := Retain, + payload := Payload} = Args) -> + Args#{preprocessed_tmpl => #{ + topic => emqx_plugin_libs_rule:preproc_tmpl(Topic), + qos => preproc_vars(QoS), + retain => preproc_vars(Retain), + payload => emqx_plugin_libs_rule:preproc_tmpl(Payload) + }}; +pre_process_output_args(_, Args) -> + Args. + +%%-------------------------------------------------------------------- +%% output functions +%%-------------------------------------------------------------------- -spec console(map(), map(), map()) -> any(). console(Selected, #{metadata := #{rule_id := RuleId}} = Envs, _Args) -> - ?ULOG("[rule output] ~s~n" + ?ULOG("[rule output] ~ts~n" "\tOutput Data: ~p~n" "\tEnvs: ~p~n", [RuleId, Selected, Envs]). @@ -42,8 +83,8 @@ republish(Selected, #{flags := Flags, metadata := #{rule_id := RuleId}}, payload := PayloadTks}}) -> Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected), Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected), - QoS = replace_simple_var(QoSTks, Selected), - Retain = replace_simple_var(RetainTks, Selected), + QoS = replace_simple_var(QoSTks, Selected, 0), + Retain = replace_simple_var(RetainTks, Selected, false), ?SLOG(debug, #{msg => "republish", topic => Topic, payload => Payload}), safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload); @@ -56,11 +97,45 @@ republish(Selected, #{metadata := #{rule_id := RuleId}}, payload := PayloadTks}}) -> Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected), Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected), - QoS = replace_simple_var(QoSTks, Selected), - Retain = replace_simple_var(RetainTks, Selected), + QoS = replace_simple_var(QoSTks, Selected, 0), + Retain = replace_simple_var(RetainTks, Selected, false), ?SLOG(debug, #{msg => "republish", topic => Topic, payload => Payload}), safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload). +%%-------------------------------------------------------------------- +%% internal functions +%%-------------------------------------------------------------------- +parse_output_func(OutputFunc) -> + {Mod, Func} = get_output_mod_func(OutputFunc), + assert_function_supported(Mod, Func), + {Mod, Func}. + +get_output_mod_func(OutputFunc) when is_atom(OutputFunc) -> + {emqx_rule_outputs, OutputFunc}; +get_output_mod_func(OutputFunc) when is_binary(OutputFunc) -> + ToAtom = fun(Bin) -> + try binary_to_existing_atom(Bin) of Atom -> Atom + catch error:badarg -> error({unknown_output_function, OutputFunc}) + end + end, + case string:split(OutputFunc, ":", all) of + [Func1] -> {emqx_rule_outputs, ToAtom(Func1)}; + [Mod1, Func1] -> {ToAtom(Mod1), ToAtom(Func1)}; + _ -> error({invalid_output_function, OutputFunc}) + end. + +assert_function_supported(Mod, Func) -> + case erlang:function_exported(Mod, Func, 3) of + true -> ok; + false -> error({output_function_not_supported, Func}) + end. + +pre_process_args(Mod, Func, Args) -> + case erlang:function_exported(Mod, pre_process_output_args, 2) of + true -> Mod:pre_process_output_args(Func, Args); + false -> Args + end. + safe_publish(RuleId, Topic, QoS, Flags, Payload) -> Msg = #message{ id = emqx_guid:gen(), @@ -75,8 +150,16 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload) -> _ = emqx_broker:safe_publish(Msg), emqx_metrics:inc_msg(Msg). -replace_simple_var(Tokens, Data) when is_list(Tokens) -> +preproc_vars(Data) when is_binary(Data) -> + emqx_plugin_libs_rule:preproc_tmpl(Data); +preproc_vars(Data) -> + Data. + +replace_simple_var(Tokens, Data, Default) when is_list(Tokens) -> [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}), - Var; -replace_simple_var(Val, _Data) -> + case Var of + undefined -> Default; %% cannot find the variable from Data + _ -> Var + end; +replace_simple_var(Val, _Data, _Default) -> Val. diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl deleted file mode 100644 index aa2c97c76..000000000 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ /dev/null @@ -1,245 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_registry). - --behaviour(gen_server). - --include("rule_engine.hrl"). --include_lib("emqx/include/logger.hrl"). --include_lib("stdlib/include/qlc.hrl"). - --export([start_link/0]). - -%% Rule Management --export([ get_rules/0 - , get_rules_for_topic/1 - , get_rules_with_same_event/1 - , get_rules_ordered_by_ts/0 - , get_rule/1 - , add_rule/1 - , add_rules/1 - , remove_rule/1 - , remove_rules/1 - ]). - --export([ load_hooks_for_rule/1 - , unload_hooks_for_rule/1 - ]). - -%% for debug purposes --export([dump/0]). - -%% gen_server Callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - -%% Mnesia bootstrap --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). - --define(REGISTRY, ?MODULE). - --define(T_CALL, 10000). - -%%------------------------------------------------------------------------------ -%% Mnesia bootstrap -%%------------------------------------------------------------------------------ - -%% @doc Create or replicate tables. --spec(mnesia(boot | copy) -> ok). -mnesia(boot) -> - %% Optimize storage - StoreProps = [{ets, [{read_concurrency, true}]}], - %% Rule table - ok = ekka_mnesia:create_table(?RULE_TAB, [ - {rlog_shard, ?RULE_ENGINE_SHARD}, - {disc_copies, [node()]}, - {record_name, rule}, - {attributes, record_info(fields, rule)}, - {storage_properties, StoreProps}]); - -mnesia(copy) -> - ok = ekka_mnesia:copy_table(?RULE_TAB, disc_copies). - -dump() -> - ?ULOG("Rules: ~p~n", [ets:tab2list(?RULE_TAB)]). - -%%------------------------------------------------------------------------------ -%% Start the registry -%%------------------------------------------------------------------------------ - --spec(start_link() -> {ok, pid()} | ignore | {error, Reason :: term()}). -start_link() -> - gen_server:start_link({local, ?REGISTRY}, ?MODULE, [], []). - -%%------------------------------------------------------------------------------ -%% Rule Management -%%------------------------------------------------------------------------------ - --spec(get_rules() -> list(emqx_rule_engine:rule())). -get_rules() -> - get_all_records(?RULE_TAB). - -get_rules_ordered_by_ts() -> - F = fun() -> - Query = qlc:q([E || E <- mnesia:table(?RULE_TAB)]), - qlc:e(qlc:keysort(#rule.created_at, Query, [{order, ascending}])) - end, - {atomic, List} = ekka_mnesia:transaction(?RULE_ENGINE_SHARD, F), - List. - --spec(get_rules_for_topic(Topic :: binary()) -> list(emqx_rule_engine:rule())). -get_rules_for_topic(Topic) -> - [Rule || Rule = #rule{info = #{from := From}} <- get_rules(), - emqx_plugin_libs_rule:can_topic_match_oneof(Topic, From)]. - --spec(get_rules_with_same_event(Topic :: binary()) -> list(emqx_rule_engine:rule())). -get_rules_with_same_event(Topic) -> - EventName = emqx_rule_events:event_name(Topic), - [Rule || Rule = #rule{info = #{from := From}} <- get_rules(), - lists:any(fun(T) -> is_of_event_name(EventName, T) end, From)]. - -is_of_event_name(EventName, Topic) -> - EventName =:= emqx_rule_events:event_name(Topic). - --spec(get_rule(Id :: rule_id()) -> {ok, emqx_rule_engine:rule()} | not_found). -get_rule(Id) -> - case mnesia:dirty_read(?RULE_TAB, Id) of - [Rule] -> {ok, Rule}; - [] -> not_found - end. - --spec(add_rule(emqx_rule_engine:rule()) -> ok). -add_rule(Rule) when is_record(Rule, rule) -> - add_rules([Rule]). - --spec(add_rules(list(emqx_rule_engine:rule())) -> ok). -add_rules(Rules) -> - gen_server:call(?REGISTRY, {add_rules, Rules}, ?T_CALL). - --spec(remove_rule(emqx_rule_engine:rule() | rule_id()) -> ok). -remove_rule(RuleOrId) -> - remove_rules([RuleOrId]). - --spec(remove_rules(list(emqx_rule_engine:rule()) | list(rule_id())) -> ok). -remove_rules(Rules) -> - gen_server:call(?REGISTRY, {remove_rules, Rules}, ?T_CALL). - -%% @private - -insert_rules([]) -> ok; -insert_rules(Rules) -> - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, load_hooks_for_rule, [Rules]), - [mnesia:write(?RULE_TAB, Rule, write) ||Rule <- Rules]. - -%% @private -delete_rules([]) -> ok; -delete_rules(Rules = [R|_]) when is_binary(R) -> - RuleRecs = - lists:foldl(fun(RuleId, Acc) -> - case get_rule(RuleId) of - {ok, Rule} -> [Rule|Acc]; - not_found -> Acc - end - end, [], Rules), - delete_rules_unload_hooks(RuleRecs); -delete_rules(Rules = [Rule|_]) when is_record(Rule, rule) -> - delete_rules_unload_hooks(Rules). - -delete_rules_unload_hooks(Rules) -> - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, unload_hooks_for_rule, [Rules]), - [mnesia:delete_object(?RULE_TAB, Rule, write) ||Rule <- Rules]. - -load_hooks_for_rule(Rules) -> - lists:foreach(fun(#rule{info = #{from := Topics}}) -> - lists:foreach(fun emqx_rule_events:load/1, Topics) - end, Rules). - -unload_hooks_for_rule(Rules) -> - lists:foreach(fun(#rule{id = Id, info = #{from := Topics}}) -> - lists:foreach(fun(Topic) -> - case get_rules_with_same_event(Topic) of - [#rule{id = Id0}] when Id0 == Id -> %% we are now deleting the last rule - emqx_rule_events:unload(Topic); - _ -> ok - end - end, Topics) - end, Rules). - -%%------------------------------------------------------------------------------ -%% gen_server callbacks -%%------------------------------------------------------------------------------ - -init([]) -> - _TableId = ets:new(?KV_TAB, [named_table, set, public, {write_concurrency, true}, - {read_concurrency, true}]), - {ok, #{}}. - -handle_call({add_rules, Rules}, _From, State) -> - trans(fun insert_rules/1, [Rules]), - {reply, ok, State}; - -handle_call({remove_rules, Rules}, _From, State) -> - trans(fun delete_rules/1, [Rules]), - {reply, ok, State}; - -handle_call(Req, _From, State) -> - ?SLOG(error, #{msg => "unexpected_call", request => Req}), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - ?SLOG(error, #{msg => "unexpected_cast", request => Msg}), - {noreply, State}. - -handle_info(Info, State) -> - ?SLOG(error, #{msg => "unexpected_info", request => Info}), - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%------------------------------------------------------------------------------ -%% Private functions -%%------------------------------------------------------------------------------ - -get_all_records(Tab) -> - %mnesia:dirty_match_object(Tab, mnesia:table_info(Tab, wild_pattern)). - %% Wrapping ets to a transaction to avoid reading inconsistent - %% ( nest cluster_call transaction, no a r/o transaction) - %% data during shard bootstrap - {atomic, Ret} = - ekka_mnesia:transaction(?RULE_ENGINE_SHARD, - fun() -> - ets:tab2list(Tab) - end), - Ret. - -trans(Fun, Args) -> - case ekka_mnesia:transaction(?RULE_ENGINE_SHARD, Fun, Args) of - {atomic, Result} -> Result; - {aborted, Reason} -> error(Reason) - end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index aafc6cddc..dc162665c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -38,19 +38,19 @@ -type collection() :: {alias(), [term()]}. -define(ephemeral_alias(TYPE, NAME), - iolist_to_binary(io_lib:format("_v_~s_~p_~p", [TYPE, NAME, erlang:system_time()]))). + iolist_to_binary(io_lib:format("_v_~ts_~p_~p", [TYPE, NAME, erlang:system_time()]))). -define(ActionMaxRetry, 3). %%------------------------------------------------------------------------------ %% Apply rules %%------------------------------------------------------------------------------ --spec(apply_rules(list(emqx_rule_engine:rule()), input()) -> ok). +-spec(apply_rules(list(rule()), input()) -> ok). apply_rules([], _Input) -> ok; -apply_rules([#rule{info = #{enabled := false}}|More], Input) -> +apply_rules([#{enabled := false}|More], Input) -> apply_rules(More, Input); -apply_rules([Rule = #rule{id = RuleID}|More], Input) -> +apply_rules([Rule = #{id := RuleID}|More], Input) -> try apply_rule_discard_result(Rule, Input) catch %% ignore the errors if select or match failed @@ -80,18 +80,19 @@ apply_rule_discard_result(Rule, Input) -> _ = apply_rule(Rule, Input), ok. -apply_rule(Rule = #rule{id = RuleID}, Input) -> +apply_rule(Rule = #{id := RuleID}, Input) -> clear_rule_payload(), do_apply_rule(Rule, add_metadata(Input, #{rule_id => RuleID})). -do_apply_rule(#rule{id = RuleId, info = #{ +do_apply_rule(#{ + id := RuleId, is_foreach := true, fields := Fields, doeach := DoEach, incase := InCase, conditions := Conditions, outputs := Outputs - }}, Input) -> + }, Input) -> {Selected, Collection} = ?RAISE(select_and_collect(Fields, Input), {select_and_collect_error, {_EXCLASS_,_EXCPTION_,_ST_}}), ColumnsAndSelected = maps:merge(Input, Selected), @@ -105,12 +106,12 @@ do_apply_rule(#rule{id = RuleId, info = #{ {error, nomatch} end; -do_apply_rule(#rule{id = RuleId, info = #{ - is_foreach := false, - fields := Fields, - conditions := Conditions, - outputs := Outputs - }}, Input) -> +do_apply_rule(#{id := RuleId, + is_foreach := false, + fields := Fields, + conditions := Conditions, + outputs := Outputs + }, Input) -> Selected = ?RAISE(select_and_transform(Fields, Input), {select_and_transform_error, {_EXCLASS_,_EXCPTION_,_ST_}}), case ?RAISE(match_conditions(Conditions, maps:merge(Input, Selected)), @@ -246,27 +247,11 @@ handle_output(OutId, Selected, Envs) -> }) end. -do_handle_output(#{type := bridge, target := ChannelId}, Selected, _Envs) -> +do_handle_output(ChannelId, Selected, _Envs) when is_binary(ChannelId) -> ?SLOG(debug, #{msg => "output to bridge", channel_id => ChannelId}), emqx_bridge:send_message(ChannelId, Selected); -do_handle_output(#{type := func, target := Func} = Out, Selected, Envs) -> - erlang:apply(Func, [Selected, Envs, maps:get(args, Out, #{})]); -do_handle_output(#{type := builtin, target := Output} = Out, Selected, Envs) - when is_atom(Output) -> - handle_builtin_output(Output, Selected, Envs, maps:get(args, Out, #{})); -do_handle_output(#{type := builtin, target := Output} = Out, Selected, Envs) - when is_binary(Output) -> - try binary_to_existing_atom(Output) of - Func -> handle_builtin_output(Func, Selected, Envs, maps:get(args, Out, #{})) - catch - error:badarg -> error(not_found) - end. - -handle_builtin_output(Func, Selected, Envs, Args) -> - case erlang:function_exported(emqx_rule_outputs, Func, 3) of - true -> erlang:apply(emqx_rule_outputs, Func, [Selected, Envs, Args]); - false -> error(not_found) - end. +do_handle_output(#{mod := Mod, func := Func, args := Args}, Selected, Envs) -> + Mod:Func(Selected, Envs, Args). eval({path, [{key, <<"payload">>} | Path]}, #{payload := Payload}) -> nested_get({path, Path}, may_decode_payload(Payload)); diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index ec263b35a..a67e62355 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -42,20 +42,18 @@ test(#{sql := Sql, context := Context}) -> test_rule(Sql, Select, Context, EventTopics) -> RuleId = iolist_to_binary(["sql_tester:", emqx_misc:gen_id(16)]), ok = emqx_rule_metrics:create_rule_metrics(RuleId), - Rule = #rule{ - id = RuleId, - info = #{ - sql => Sql, - from => EventTopics, - outputs => [#{type => func, target => fun ?MODULE:get_selected_data/3, args => #{}}], - enabled => true, - is_foreach => emqx_rule_sqlparser:select_is_foreach(Select), - fields => emqx_rule_sqlparser:select_fields(Select), - doeach => emqx_rule_sqlparser:select_doeach(Select), - incase => emqx_rule_sqlparser:select_incase(Select), - conditions => emqx_rule_sqlparser:select_where(Select) - }, - created_at = erlang:system_time(millisecond) + Rule = #{ + id => RuleId, + sql => Sql, + from => EventTopics, + outputs => [#{mod => ?MODULE, func => get_selected_data, args => #{}}], + enabled => true, + is_foreach => emqx_rule_sqlparser:select_is_foreach(Select), + fields => emqx_rule_sqlparser:select_fields(Select), + doeach => emqx_rule_sqlparser:select_doeach(Select), + incase => emqx_rule_sqlparser:select_incase(Select), + conditions => emqx_rule_sqlparser:select_where(Select), + created_at => erlang:system_time(millisecond) }, FullContext = fill_default_values(hd(EventTopics), emqx_rule_maps:atom_key_map(Context)), try diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 45b30223a..56af13acc 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -30,7 +30,6 @@ all() -> [ {group, engine} - , {group, api} , {group, funcs} , {group, registry} , {group, runtime} @@ -45,9 +44,6 @@ groups() -> [{engine, [sequence], [t_create_rule ]}, - {api, [], - [t_crud_rule_api - ]}, {funcs, [], [t_kv_store ]}, @@ -108,13 +104,11 @@ groups() -> init_per_suite(Config) -> application:load(emqx_machine), - ok = ekka_mnesia:start(), - ok = emqx_rule_registry:mnesia(boot), - ok = emqx_ct_helpers:start_apps([emqx_rule_engine]), + ok = emqx_common_test_helpers:start_apps([emqx_rule_engine]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_rule_engine]), + emqx_common_test_helpers:stop_apps([emqx_rule_engine]), ok. on_resource_create(_id, _) -> #{}. @@ -155,12 +149,12 @@ init_per_testcase(t_events, Config) -> #{id => <<"rule:t_events">>, sql => SQL, outputs => [ - #{type => builtin, target => console}, - #{type => func, target => fun ?MODULE:output_record_triggered_events/3, + #{function => console}, + #{function => <<"emqx_rule_engine_SUITE:output_record_triggered_events">>, args => #{}} ], description => <<"to console and record triggered events">>}), - ?assertMatch(#rule{id = <<"rule:t_events">>}, Rule), + ?assertMatch(#{id := <<"rule:t_events">>}, Rule), [{hook_points_rules, Rule} | Config]; init_per_testcase(_TestCase, Config) -> emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), @@ -168,7 +162,7 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(t_events, Config) -> ets:delete(events_record_tab), - ok = emqx_rule_registry:remove_rule(?config(hook_points_rules, Config)); + ok = delete_rule(?config(hook_points_rules, Config)); end_per_testcase(_TestCase, _Config) -> ok. @@ -176,57 +170,15 @@ end_per_testcase(_TestCase, _Config) -> %% Test cases for rule engine %%------------------------------------------------------------------------------ t_create_rule(_Config) -> - {ok, #rule{id = Id}} = emqx_rule_engine:create_rule( + {ok, #{id := Id}} = emqx_rule_engine:create_rule( #{sql => <<"select * from \"t/a\"">>, id => <<"t_create_rule">>, - outputs => [#{type => builtin, target => console}], + outputs => [#{function => console}], description => <<"debug rule">>}), - ct:pal("======== emqx_rule_registry:get_rules :~p", [emqx_rule_registry:get_rules()]), - ?assertMatch({ok, #rule{id = Id, info = #{from := [<<"t/a">>]}}}, - emqx_rule_registry:get_rule(Id)), - emqx_rule_registry:remove_rule(Id), - ok. - -%%------------------------------------------------------------------------------ -%% Test cases for rule engine api -%%------------------------------------------------------------------------------ - -t_crud_rule_api(_Config) -> - RuleID = <<"my_rule">>, - Params0 = #{ - <<"description">> => <<"A simple rule">>, - <<"enable">> => true, - <<"id">> => RuleID, - <<"outputs">> => [#{<<"type">> => <<"builtin">>, <<"target">> => <<"console">>}], - <<"sql">> => <<"SELECT * from \"t/1\"">> - }, - {201, Rule} = emqx_rule_engine_api:crud_rules(post, #{body => Params0}), - - ?assertEqual(RuleID, maps:get(id, Rule)), - {200, Rules} = emqx_rule_engine_api:crud_rules(get, #{}), - ct:pal("RList : ~p", [Rules]), - ?assert(length(Rules) > 0), - - {200, Rule1} = emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}}), - ct:pal("RShow : ~p", [Rule1]), - ?assertEqual(Rule, Rule1), - - {200, Rule2} = emqx_rule_engine_api:crud_rules_by_id(put, #{ - bindings => #{id => RuleID}, - body => Params0#{<<"sql">> => <<"select * from \"t/b\"">>} - }), - - {200, Rule3} = emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}}), - %ct:pal("RShow : ~p", [Rule3]), - ?assertEqual(Rule3, Rule2), - ?assertEqual(<<"select * from \"t/b\"">>, maps:get(sql, Rule3)), - - ?assertMatch({200}, emqx_rule_engine_api:crud_rules_by_id(delete, - #{bindings => #{id => RuleID}})), - - %ct:pal("Show After Deleted: ~p", [NotFound]), - ?assertMatch({404, #{code := _, message := _Message}}, - emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}})), + ct:pal("======== emqx_rule_engine:get_rules :~p", [emqx_rule_engine:get_rules()]), + ?assertMatch({ok, #{id := Id, from := [<<"t/a">>]}}, + emqx_rule_engine:get_rule(Id)), + delete_rule(Id), ok. %%------------------------------------------------------------------------------ @@ -247,34 +199,27 @@ t_kv_store(_) -> t_add_get_remove_rule(_Config) -> RuleId0 = <<"rule-debug-0">>, - ok = emqx_rule_registry:add_rule(make_simple_rule(RuleId0)), - ?assertMatch({ok, #rule{id = RuleId0}}, emqx_rule_registry:get_rule(RuleId0)), - ok = emqx_rule_registry:remove_rule(RuleId0), - ?assertEqual(not_found, emqx_rule_registry:get_rule(RuleId0)), + ok = emqx_rule_engine:insert_rule(make_simple_rule(RuleId0)), + ?assertMatch({ok, #{id := RuleId0}}, emqx_rule_engine:get_rule(RuleId0)), + ok = delete_rule(RuleId0), + ?assertEqual(not_found, emqx_rule_engine:get_rule(RuleId0)), RuleId1 = <<"rule-debug-1">>, Rule1 = make_simple_rule(RuleId1), - ok = emqx_rule_registry:add_rule(Rule1), - ?assertMatch({ok, #rule{id = RuleId1}}, emqx_rule_registry:get_rule(RuleId1)), - ok = emqx_rule_registry:remove_rule(Rule1), - ?assertEqual(not_found, emqx_rule_registry:get_rule(RuleId1)), + ok = emqx_rule_engine:insert_rule(Rule1), + ?assertMatch({ok, #{id := RuleId1}}, emqx_rule_engine:get_rule(RuleId1)), + ok = delete_rule(Rule1), + ?assertEqual(not_found, emqx_rule_engine:get_rule(RuleId1)), ok. t_add_get_remove_rules(_Config) -> - emqx_rule_registry:remove_rules(emqx_rule_registry:get_rules()), - ok = emqx_rule_registry:add_rules( + delete_rules_by_ids(emqx_rule_engine:get_rules()), + ok = insert_rules( [make_simple_rule(<<"rule-debug-1">>), make_simple_rule(<<"rule-debug-2">>)]), - ?assertEqual(2, length(emqx_rule_registry:get_rules())), - ok = emqx_rule_registry:remove_rules([<<"rule-debug-1">>, <<"rule-debug-2">>]), - ?assertEqual([], emqx_rule_registry:get_rules()), - - Rule3 = make_simple_rule(<<"rule-debug-3">>), - Rule4 = make_simple_rule(<<"rule-debug-4">>), - ok = emqx_rule_registry:add_rules([Rule3, Rule4]), - ?assertEqual(2, length(emqx_rule_registry:get_rules())), - ok = emqx_rule_registry:remove_rules([Rule3, Rule4]), - ?assertEqual([], emqx_rule_registry:get_rules()), + ?assertEqual(2, length(emqx_rule_engine:get_rules())), + ok = delete_rules_by_ids([<<"rule-debug-1">>, <<"rule-debug-2">>]), + ?assertEqual([], emqx_rule_engine:get_rules()), ok. t_create_existing_rule(_Config) -> @@ -282,40 +227,40 @@ t_create_existing_rule(_Config) -> {ok, _} = emqx_rule_engine:create_rule( #{id => <<"an_existing_rule">>, sql => <<"select * from \"t/#\"">>, - outputs => [#{type => builtin, target => console}] + outputs => [#{function => console}] }), - {ok, #rule{info = #{sql := SQL}}} = emqx_rule_registry:get_rule(<<"an_existing_rule">>), + {ok, #{sql := SQL}} = emqx_rule_engine:get_rule(<<"an_existing_rule">>), ?assertEqual(<<"select * from \"t/#\"">>, SQL), - ok = emqx_rule_engine:delete_rule(<<"an_existing_rule">>), - ?assertEqual(not_found, emqx_rule_registry:get_rule(<<"an_existing_rule">>)), + ok = delete_rule(<<"an_existing_rule">>), + ?assertEqual(not_found, emqx_rule_engine:get_rule(<<"an_existing_rule">>)), ok. t_get_rules_for_topic(_Config) -> - Len0 = length(emqx_rule_registry:get_rules_for_topic(<<"simple/topic">>)), - ok = emqx_rule_registry:add_rules( + Len0 = length(emqx_rule_engine:get_rules_for_topic(<<"simple/topic">>)), + ok = insert_rules( [make_simple_rule(<<"rule-debug-1">>), make_simple_rule(<<"rule-debug-2">>)]), - ?assertEqual(Len0+2, length(emqx_rule_registry:get_rules_for_topic(<<"simple/topic">>))), - ok = emqx_rule_registry:remove_rules([<<"rule-debug-1">>, <<"rule-debug-2">>]), + ?assertEqual(Len0+2, length(emqx_rule_engine:get_rules_for_topic(<<"simple/topic">>))), + ok = delete_rules_by_ids([<<"rule-debug-1">>, <<"rule-debug-2">>]), ok. t_get_rules_ordered_by_ts(_Config) -> Now = fun() -> erlang:system_time(nanosecond) end, - ok = emqx_rule_registry:add_rules( + ok = insert_rules( [make_simple_rule_with_ts(<<"rule-debug-0">>, Now()), make_simple_rule_with_ts(<<"rule-debug-1">>, Now()), make_simple_rule_with_ts(<<"rule-debug-2">>, Now()) ]), ?assertMatch([ - #rule{id = <<"rule-debug-0">>}, - #rule{id = <<"rule-debug-1">>}, - #rule{id = <<"rule-debug-2">>} - ], emqx_rule_registry:get_rules_ordered_by_ts()). + #{id := <<"rule-debug-0">>}, + #{id := <<"rule-debug-1">>}, + #{id := <<"rule-debug-2">>} + ], emqx_rule_engine:get_rules_ordered_by_ts()). t_get_rules_for_topic_2(_Config) -> - Len0 = length(emqx_rule_registry:get_rules_for_topic(<<"simple/1">>)), - ok = emqx_rule_registry:add_rules( + Len0 = length(emqx_rule_engine:get_rules_for_topic(<<"simple/1">>)), + ok = insert_rules( [make_simple_rule(<<"rule-debug-1">>, <<"select * from \"simple/#\"">>, [<<"simple/#">>]), make_simple_rule(<<"rule-debug-2">>, <<"select * from \"simple/+\"">>, [<<"simple/+">>]), make_simple_rule(<<"rule-debug-3">>, <<"select * from \"simple/+/1\"">>, [<<"simple/+/1">>]), @@ -323,21 +268,21 @@ t_get_rules_for_topic_2(_Config) -> make_simple_rule(<<"rule-debug-5">>, <<"select * from \"simple/2,simple/+,simple/3\"">>, [<<"simple/2">>,<<"simple/+">>, <<"simple/3">>]), make_simple_rule(<<"rule-debug-6">>, <<"select * from \"simple/2,simple/3,simple/4\"">>, [<<"simple/2">>,<<"simple/3">>, <<"simple/4">>]) ]), - ?assertEqual(Len0+4, length(emqx_rule_registry:get_rules_for_topic(<<"simple/1">>))), - ok = emqx_rule_registry:remove_rules([<<"rule-debug-1">>, <<"rule-debug-2">>,<<"rule-debug-3">>, <<"rule-debug-4">>,<<"rule-debug-5">>, <<"rule-debug-6">>]), + ?assertEqual(Len0+4, length(emqx_rule_engine:get_rules_for_topic(<<"simple/1">>))), + ok = delete_rules_by_ids([<<"rule-debug-1">>, <<"rule-debug-2">>,<<"rule-debug-3">>, <<"rule-debug-4">>,<<"rule-debug-5">>, <<"rule-debug-6">>]), ok. t_get_rules_with_same_event(_Config) -> PubT = <<"simple/1">>, - PubN = length(emqx_rule_registry:get_rules_with_same_event(PubT)), - ?assertEqual([], emqx_rule_registry:get_rules_with_same_event(<<"$events/client_connected">>)), - ?assertEqual([], emqx_rule_registry:get_rules_with_same_event(<<"$events/client_disconnected">>)), - ?assertEqual([], emqx_rule_registry:get_rules_with_same_event(<<"$events/session_subscribed">>)), - ?assertEqual([], emqx_rule_registry:get_rules_with_same_event(<<"$events/session_unsubscribed">>)), - ?assertEqual([], emqx_rule_registry:get_rules_with_same_event(<<"$events/message_delivered">>)), - ?assertEqual([], emqx_rule_registry:get_rules_with_same_event(<<"$events/message_acked">>)), - ?assertEqual([], emqx_rule_registry:get_rules_with_same_event(<<"$events/message_dropped">>)), - ok = emqx_rule_registry:add_rules( + PubN = length(emqx_rule_engine:get_rules_with_same_event(PubT)), + ?assertEqual([], emqx_rule_engine:get_rules_with_same_event(<<"$events/client_connected">>)), + ?assertEqual([], emqx_rule_engine:get_rules_with_same_event(<<"$events/client_disconnected">>)), + ?assertEqual([], emqx_rule_engine:get_rules_with_same_event(<<"$events/session_subscribed">>)), + ?assertEqual([], emqx_rule_engine:get_rules_with_same_event(<<"$events/session_unsubscribed">>)), + ?assertEqual([], emqx_rule_engine:get_rules_with_same_event(<<"$events/message_delivered">>)), + ?assertEqual([], emqx_rule_engine:get_rules_with_same_event(<<"$events/message_acked">>)), + ?assertEqual([], emqx_rule_engine:get_rules_with_same_event(<<"$events/message_dropped">>)), + ok = insert_rules( [make_simple_rule(<<"r1">>, <<"select * from \"simple/#\"">>, [<<"simple/#">>]), make_simple_rule(<<"r2">>, <<"select * from \"abc/+\"">>, [<<"abc/+">>]), make_simple_rule(<<"r3">>, <<"select * from \"$events/client_connected\"">>, [<<"$events/client_connected">>]), @@ -349,15 +294,15 @@ t_get_rules_with_same_event(_Config) -> make_simple_rule(<<"r9">>, <<"select * from \"$events/message_dropped\"">>, [<<"$events/message_dropped">>]), make_simple_rule(<<"r10">>, <<"select * from \"t/1, $events/session_subscribed, $events/client_connected\"">>, [<<"t/1">>, <<"$events/session_subscribed">>, <<"$events/client_connected">>]) ]), - ?assertEqual(PubN + 3, length(emqx_rule_registry:get_rules_with_same_event(PubT))), - ?assertEqual(2, length(emqx_rule_registry:get_rules_with_same_event(<<"$events/client_connected">>))), - ?assertEqual(1, length(emqx_rule_registry:get_rules_with_same_event(<<"$events/client_disconnected">>))), - ?assertEqual(2, length(emqx_rule_registry:get_rules_with_same_event(<<"$events/session_subscribed">>))), - ?assertEqual(1, length(emqx_rule_registry:get_rules_with_same_event(<<"$events/session_unsubscribed">>))), - ?assertEqual(1, length(emqx_rule_registry:get_rules_with_same_event(<<"$events/message_delivered">>))), - ?assertEqual(1, length(emqx_rule_registry:get_rules_with_same_event(<<"$events/message_acked">>))), - ?assertEqual(1, length(emqx_rule_registry:get_rules_with_same_event(<<"$events/message_dropped">>))), - ok = emqx_rule_registry:remove_rules([<<"r1">>, <<"r2">>,<<"r3">>, <<"r4">>,<<"r5">>, <<"r6">>, <<"r7">>, <<"r8">>, <<"r9">>, <<"r10">>]), + ?assertEqual(PubN + 3, length(emqx_rule_engine:get_rules_with_same_event(PubT))), + ?assertEqual(2, length(emqx_rule_engine:get_rules_with_same_event(<<"$events/client_connected">>))), + ?assertEqual(1, length(emqx_rule_engine:get_rules_with_same_event(<<"$events/client_disconnected">>))), + ?assertEqual(2, length(emqx_rule_engine:get_rules_with_same_event(<<"$events/session_subscribed">>))), + ?assertEqual(1, length(emqx_rule_engine:get_rules_with_same_event(<<"$events/session_unsubscribed">>))), + ?assertEqual(1, length(emqx_rule_engine:get_rules_with_same_event(<<"$events/message_delivered">>))), + ?assertEqual(1, length(emqx_rule_engine:get_rules_with_same_event(<<"$events/message_acked">>))), + ?assertEqual(1, length(emqx_rule_engine:get_rules_with_same_event(<<"$events/message_dropped">>))), + ok = delete_rules_by_ids([<<"r1">>, <<"r2">>,<<"r3">>, <<"r4">>,<<"r5">>, <<"r6">>, <<"r7">>, <<"r8">>, <<"r9">>, <<"r10">>]), ok. %%------------------------------------------------------------------------------ @@ -453,7 +398,7 @@ t_match_atom_and_binary(_Config) -> end, emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). + delete_rule(TopicRule). t_sqlselect_0(_Config) -> %% Verify SELECT with and without 'AS' @@ -572,7 +517,7 @@ t_sqlselect_01(_Config) -> end, emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule1). + delete_rule(TopicRule1). t_sqlselect_02(_Config) -> SQL = "SELECT * " @@ -610,7 +555,7 @@ t_sqlselect_02(_Config) -> end, emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule1). + delete_rule(TopicRule1). t_sqlselect_1(_Config) -> SQL = "SELECT json_decode(payload) as p, payload " @@ -640,7 +585,7 @@ t_sqlselect_1(_Config) -> end, emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). + delete_rule(TopicRule). t_sqlselect_2(_Config) -> %% recursively republish to t2 @@ -666,7 +611,7 @@ t_sqlselect_2(_Config) -> received_nothing = Fun(), emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). + delete_rule(TopicRule). t_sqlselect_3(_Config) -> %% republish the client.connected msg @@ -698,7 +643,7 @@ t_sqlselect_3(_Config) -> end, emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). + delete_rule(TopicRule). t_sqlparse_event_1(_Config) -> Sql = "select topic as tp " @@ -1349,8 +1294,8 @@ t_sqlparse_nested_get(_Config) -> republish_output(Topic) -> republish_output(Topic, <<"${payload}">>). republish_output(Topic, Payload) -> - #{type => builtin, target => republish, - args => #{<<"payload">> => Payload, <<"topic">> => Topic, <<"qos">> => 0}}. + #{function => republish, + args => #{payload => Payload, topic => Topic, qos => 0, retain => false}}. make_simple_rule_with_ts(RuleId, Ts) when is_binary(RuleId) -> SQL = <<"select * from \"simple/topic\"">>, @@ -1366,18 +1311,16 @@ make_simple_rule(RuleId, SQL, Topics) when is_binary(RuleId) -> make_simple_rule(RuleId, SQL, Topics, erlang:system_time(millisecond)). make_simple_rule(RuleId, SQL, Topics, Ts) when is_binary(RuleId) -> - #rule{ - id = RuleId, - info = #{ - sql => SQL, - from => Topics, - fields => [<<"*">>], - is_foreach => false, - conditions => {}, - ouputs => [#{type => builtin, target => console}], - description => <<"simple rule">> - }, - created_at = Ts + #{ + id => RuleId, + sql => SQL, + from => Topics, + fields => [<<"*">>], + is_foreach => false, + conditions => {}, + outputs => [#{mod => emqx_rule_outputs, func => console, args => #{}}], + description => <<"simple rule">>, + created_at => Ts }. output_record_triggered_events(Data = #{event := EventName}, _Envs, _Args) -> @@ -1647,3 +1590,17 @@ deps_path(App, RelativePath) -> local_path(RelativePath) -> deps_path(emqx_rule_engine, RelativePath). +insert_rules(Rules) -> + lists:foreach(fun(Rule) -> + ok = emqx_rule_engine:insert_rule(Rule) + end, Rules). + +delete_rules_by_ids(Ids) -> + lists:foreach(fun(Id) -> + ok = emqx_rule_engine:delete_rule(Id) + end, Ids). + +delete_rule(#{id := Id}) -> + ok = emqx_rule_engine:delete_rule(Id); +delete_rule(Id) when is_binary(Id) -> + ok = emqx_rule_engine:delete_rule(Id). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl new file mode 100644 index 000000000..4b0f027f7 --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -0,0 +1,70 @@ +-module(emqx_rule_engine_api_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CONF_DEFAULT, <<"rule_engine {rules {}}">>). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_machine), + ok = emqx_config:init_load(emqx_rule_engine_schema, ?CONF_DEFAULT), + ok = emqx_common_test_helpers:start_apps([emqx_rule_engine]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_rule_engine]), + ok. + +init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + Config. + +end_per_testcase(_, _Config) -> + ok. + +t_crud_rule_api(_Config) -> + RuleID = <<"my_rule">>, + Params0 = #{ + <<"description">> => <<"A simple rule">>, + <<"enable">> => true, + <<"id">> => RuleID, + <<"outputs">> => [#{<<"function">> => <<"console">>}], + <<"sql">> => <<"SELECT * from \"t/1\"">> + }, + {201, Rule} = emqx_rule_engine_api:crud_rules(post, #{body => Params0}), + %% if we post again with the same params, it return with 400 "rule id already exists" + ?assertMatch({400, #{code := _, message := _Message}}, + emqx_rule_engine_api:crud_rules(post, #{body => Params0})), + + ?assertEqual(RuleID, maps:get(id, Rule)), + {200, Rules} = emqx_rule_engine_api:crud_rules(get, #{}), + ct:pal("RList : ~p", [Rules]), + ?assert(length(Rules) > 0), + + {200, Rule1} = emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}}), + ct:pal("RShow : ~p", [Rule1]), + ?assertEqual(Rule, Rule1), + + {200, Rule2} = emqx_rule_engine_api:crud_rules_by_id(put, #{ + bindings => #{id => RuleID}, + body => Params0#{<<"sql">> => <<"select * from \"t/b\"">>} + }), + + {200, Rule3} = emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}}), + %ct:pal("RShow : ~p", [Rule3]), + ?assertEqual(Rule3, Rule2), + ?assertEqual(<<"select * from \"t/b\"">>, maps:get(sql, Rule3)), + + ?assertMatch({200}, emqx_rule_engine_api:crud_rules_by_id(delete, + #{bindings => #{id => RuleID}})), + + %ct:pal("Show After Deleted: ~p", [NotFound]), + ?assertMatch({404, #{code := _, message := _Message}}, + emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}})), + ok. diff --git a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl index f586d6256..0226066b3 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl @@ -5,7 +5,7 @@ -include_lib("eunit/include/eunit.hrl"). -all() -> emqx_ct:all(?MODULE). +all() -> emqx_common_test_helpers:all(?MODULE). t_mod_hook_fun(_) -> Funcs = emqx_rule_events:module_info(exports), diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 32421df32..ae5524d36 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -423,9 +423,9 @@ t_concat(_) -> ?assertEqual(<<>>, apply_func(concat, [<<"">>, <<"">>])). t_sprintf(_) -> - ?assertEqual(<<"Hello Shawn!">>, apply_func(sprintf, [<<"Hello ~s!">>, <<"Shawn">>])), - ?assertEqual(<<"Name: ABC, Count: 2">>, apply_func(sprintf, [<<"Name: ~s, Count: ~p">>, <<"ABC">>, 2])), - ?assertEqual(<<"Name: ABC, Count: 2, Status: {ok,running}">>, apply_func(sprintf, [<<"Name: ~s, Count: ~p, Status: ~p">>, <<"ABC">>, 2, {ok, running}])). + ?assertEqual(<<"Hello Shawn!">>, apply_func(sprintf, [<<"Hello ~ts!">>, <<"Shawn">>])), + ?assertEqual(<<"Name: ABC, Count: 2">>, apply_func(sprintf, [<<"Name: ~ts, Count: ~p">>, <<"ABC">>, 2])), + ?assertEqual(<<"Name: ABC, Count: 2, Status: {ok,running}">>, apply_func(sprintf, [<<"Name: ~ts, Count: ~p, Status: ~p">>, <<"ABC">>, 2, {ok, running}])). t_pad(_) -> ?assertEqual(<<"abc ">>, apply_func(pad, [<<"abc">>, 5])), diff --git a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl index ff654ba94..e30966ab8 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl @@ -40,13 +40,13 @@ groups() -> ]. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx]), + emqx_common_test_helpers:start_apps([emqx]), {ok, _} = emqx_rule_metrics:start_link(), Config. end_per_suite(_Config) -> catch emqx_rule_metrics:stop(), - emqx_ct_helpers:stop_apps([emqx]), + emqx_common_test_helpers:stop_apps([emqx]), ok. init_per_testcase(_, Config) -> diff --git a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl index 9d06ee351..7b9736836 100644 --- a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl +++ b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl @@ -7,14 +7,14 @@ -include_lib("eunit/include/eunit.hrl"). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_statsd]), + emqx_common_test_helpers:start_apps([emqx_statsd]), Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_statsd]). + emqx_common_test_helpers:stop_apps([emqx_statsd]). all() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). t_statsd(_) -> {ok, Socket} = gen_udp:open(8125), diff --git a/rebar.config b/rebar.config index ca8dd3e22..89b8e3a9f 100644 --- a/rebar.config +++ b/rebar.config @@ -42,14 +42,15 @@ {post_hooks,[]}. {deps, - [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps + [ {lc, {git, "https://github.com/qzhuyan/lc.git", {tag, "0.1.1"}}} + , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.9"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.3"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.9"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} @@ -60,7 +61,7 @@ , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.5"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.6"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 6f56ce1d2..a3f32d0c4 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -114,21 +114,21 @@ plugins(HasElixir) -> %% emqx main project does not require port-compiler %% pin at root level for deterministic , {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}} - | [ rebar_mix || HasElixir ] + | [ {rebar_mix, "0.5.1"} || HasElixir ] ] %% test plugins are concatenated to default profile plugins %% otherwise rebar3 test profile runs are super slow ++ test_plugins(). test_plugins() -> - [ rebar3_proper, - {coveralls, {git, "https://github.com/emqx/coveralls-erl", {branch, "fix-git-info"}}} + [ {rebar3_proper, "0.12.1"} + , {coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}} ]. test_deps() -> [ {bbmustache, "1.10.0"} - , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "2.1.0"}}} - , meck + , {meck, "0.9.2"} + , {proper, "1.4.0"} ]. common_compile_opts() -> @@ -279,6 +279,7 @@ relx_apps(ReleaseType) -> , emqx_statsd , emqx_prometheus , emqx_psk + , emqx_limiter ] ++ [quicer || is_quicer_supported()] ++ [emqx_license || is_enterprise()]