diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index bb89dfe52..d954a6b1e 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -18,13 +18,13 @@ ]}. %% Deps here may duplicate with emqx.git root level rebar.config -%% but there not be any descrpancy. +%% but there may not be any discrepancy. %% This rebar.config is necessary because the app may be used as a %% `git_subdir` dependency in other projects. {deps, [ + {emqx_utils, {path, "../emqx_utils"}}, {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, - {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, @@ -36,7 +36,7 @@ {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} ]}. -{plugins, [{rebar3_proper, "0.12.1"}]}. +{plugins, [{rebar3_proper, "0.12.1"}, rebar3_path_deps]}. {extra_src_dirs, [{"etc", [recursive]}]}. {profiles, [ {test, [ diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 7014c8381..c4894c933 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -16,7 +16,6 @@ cowboy, sasl, os_mon, - jiffy, lc, hocon ]}, diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 6e4aa9922..ef870685a 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -164,29 +164,29 @@ run_hook(HookPoint, Args) -> run_fold_hook(HookPoint, Args, Acc) -> emqx_hooks:run_fold(HookPoint, Args, Acc). --spec get_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_config(emqx_utils_maps:config_key_path()) -> term(). get_config(KeyPath) -> emqx_config:get(KeyPath). --spec get_config(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_config(KeyPath, Default) -> emqx_config:get(KeyPath, Default). --spec get_raw_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw_config(emqx_utils_maps:config_key_path()) -> term(). get_raw_config(KeyPath) -> emqx_config:get_raw(KeyPath). --spec get_raw_config(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw_config(KeyPath, Default) -> emqx_config:get_raw(KeyPath, Default). --spec update_config(emqx_map_lib:config_key_path(), emqx_config:update_request()) -> +-spec update_config(emqx_utils_maps:config_key_path(), emqx_config:update_request()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_config(KeyPath, UpdateReq) -> update_config(KeyPath, UpdateReq, #{}). -spec update_config( - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -198,12 +198,12 @@ update_config([RootName | _] = KeyPath, UpdateReq, Opts) -> {{update, UpdateReq}, Opts} ). --spec remove_config(emqx_map_lib:config_key_path()) -> +-spec remove_config(emqx_utils_maps:config_key_path()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove_config(KeyPath) -> remove_config(KeyPath, #{}). --spec remove_config(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove_config([RootName | _] = KeyPath, Opts) -> emqx_config_handler:update_config( @@ -212,7 +212,7 @@ remove_config([RootName | _] = KeyPath, Opts) -> {remove, Opts} ). --spec reset_config(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset_config([RootName | _] = KeyPath, Opts) -> case emqx_config:get_default_value(KeyPath) of diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 84c40ef2a..6aa3cb95d 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -423,7 +423,7 @@ do_actions(deactivate, Alarm = #deactivated_alarm{name = Name}, [log | More]) -> do_actions(deactivate, Alarm, More); do_actions(Operation, Alarm, [publish | More]) -> Topic = topic(Operation), - {ok, Payload} = emqx_json:safe_encode(normalize(Alarm)), + {ok, Payload} = emqx_utils_json:safe_encode(normalize(Alarm)), Message = emqx_message:make( ?MODULE, 0, diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 5471b66fc..be3b35f57 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -277,9 +277,9 @@ atom(Bin) -> binary_to_existing_atom(Bin, utf8). certs_dir(ChainName, ConfigOrID) -> DirName = dir(ChainName, ConfigOrID), SubDir = iolist_to_binary(filename:join(["authn", DirName])), - emqx_misc:safe_filename(SubDir). + emqx_utils:safe_filename(SubDir). dir(ChainName, ID) when is_binary(ID) -> - emqx_misc:safe_filename(iolist_to_binary([to_bin(ChainName), "-", ID])); + emqx_utils:safe_filename(iolist_to_binary([to_bin(ChainName), "-", ID])); dir(ChainName, Config) when is_map(Config) -> dir(ChainName, authenticator_id(Config)). diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index 758c570da..a0ccd93d7 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -243,7 +243,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{expiry_timer := TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -254,10 +254,10 @@ code_change(_OldVsn, State, _Extra) -> -ifdef(TEST). ensure_expiry_timer(State) -> - State#{expiry_timer := emqx_misc:start_timer(10, expire)}. + State#{expiry_timer := emqx_utils:start_timer(10, expire)}. -else. ensure_expiry_timer(State) -> - State#{expiry_timer := emqx_misc:start_timer(timer:minutes(1), expire)}. + State#{expiry_timer := emqx_utils:start_timer(timer:minutes(1), expire)}. -endif. expire_banned_items(Now) -> diff --git a/apps/emqx/src/emqx_batch.erl b/apps/emqx/src/emqx_batch.erl index 2fe09942c..22e812975 100644 --- a/apps/emqx/src/emqx_batch.erl +++ b/apps/emqx/src/emqx_batch.erl @@ -85,7 +85,7 @@ commit(Batch = #batch{batch_q = Q, commit_fun = Commit}) -> reset(Batch). reset(Batch = #batch{linger_timer = TRef}) -> - _ = emqx_misc:cancel_timer(TRef), + _ = emqx_utils:cancel_timer(TRef), Batch#batch{batch_q = [], linger_timer = undefined}. -spec size(batch()) -> non_neg_integer(). diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index d56620123..5f7c4aaf5 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -71,7 +71,7 @@ code_change/3 ]). --import(emqx_tables, [lookup_value/2, lookup_value/3]). +-import(emqx_utils_ets, [lookup_value/2, lookup_value/3]). -ifdef(TEST). -compile(export_all). @@ -92,7 +92,7 @@ start_link(Pool, Id) -> ok = create_tabs(), gen_server:start_link( - {local, emqx_misc:proc_name(?BROKER, Id)}, + {local, emqx_utils:proc_name(?BROKER, Id)}, ?MODULE, [Pool, Id], [] @@ -107,15 +107,15 @@ create_tabs() -> TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}], %% SubOption: {Topic, SubPid} -> SubOption - ok = emqx_tables:new(?SUBOPTION, [ordered_set | TabOpts]), + ok = emqx_utils_ets:new(?SUBOPTION, [ordered_set | TabOpts]), %% Subscription: SubPid -> Topic1, Topic2, Topic3, ... %% duplicate_bag: o(1) insert - ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), + ok = emqx_utils_ets:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... %% bag: o(n) insert:( - ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). + ok = emqx_utils_ets:new(?SUBSCRIBER, [bag | TabOpts]). %%------------------------------------------------------------------------------ %% Subscribe API diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl index 91b4c4994..06f249678 100644 --- a/apps/emqx/src/emqx_broker_helper.erl +++ b/apps/emqx/src/emqx_broker_helper.erl @@ -73,11 +73,11 @@ register_sub(SubPid, SubId) when is_pid(SubPid) -> -spec lookup_subid(pid()) -> maybe(emqx_types:subid()). lookup_subid(SubPid) when is_pid(SubPid) -> - emqx_tables:lookup_value(?SUBMON, SubPid). + emqx_utils_ets:lookup_value(?SUBMON, SubPid). -spec lookup_subpid(emqx_types:subid()) -> maybe(pid()). lookup_subpid(SubId) -> - emqx_tables:lookup_value(?SUBID, SubId). + emqx_utils_ets:lookup_value(?SUBID, SubId). -spec get_sub_shard(pid(), emqx_types:topic()) -> non_neg_integer(). get_sub_shard(SubPid, Topic) -> @@ -105,15 +105,15 @@ reclaim_seq(Topic) -> init([]) -> %% Helper table - ok = emqx_tables:new(?HELPER, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?HELPER, [{read_concurrency, true}]), %% Shards: CPU * 32 true = ets:insert(?HELPER, {shards, emqx_vm:schedulers() * 32}), %% SubSeq: Topic -> SeqId ok = emqx_sequence:create(?SUBSEQ), %% SubId: SubId -> SubPid - ok = emqx_tables:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), %% SubMon: SubPid -> SubId - ok = emqx_tables:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), %% Stats timer ok = emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), {ok, #{pmon => emqx_pmon:new()}}. @@ -131,7 +131,7 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #{pmon := PMon}) -> - SubPids = [SubPid | emqx_misc:drain_down(?BATCH_SIZE)], + SubPids = [SubPid | emqx_utils:drain_down(?BATCH_SIZE)], ok = emqx_pool:async_submit( fun lists:foreach/2, [fun clean_down/1, SubPids] ), diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 29a59e482..745429536 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -61,7 +61,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [ run_fold/3, pipeline/3, @@ -622,7 +622,7 @@ process_connect( NChannel = Channel#channel{session = Session}, handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, ensure_connected(NChannel)); {ok, #{session := Session, present := true, pendings := Pendings}} -> - Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), + Pendings1 = lists:usort(lists:append(Pendings, emqx_utils:drain_deliver())), NChannel = Channel#channel{ session = Session, resuming = true, @@ -1203,7 +1203,7 @@ handle_call( ) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) - Delivers = emqx_misc:drain_deliver(), + Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), disconnect_and_shutdown(takenover, AllPendings, Channel); handle_call(list_authz_cache, Channel) -> @@ -1402,7 +1402,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> @@ -2045,7 +2045,7 @@ clear_keepalive(Channel = #channel{timers = Timers}) -> undefined -> Channel; TRef -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), Channel#channel{timers = maps:without([alive_timer], Timers)} end. %%-------------------------------------------------------------------- @@ -2241,7 +2241,7 @@ get_mqtt_conf(Zone, Key, Default) -> %%-------------------------------------------------------------------- set_field(Name, Value, Channel) -> - Pos = emqx_misc:index_of(Name, record_info(fields, channel)), + Pos = emqx_utils:index_of(Name, record_info(fields, channel)), setelement(Pos + 1, Channel, Value). get_mqueue(#channel{session = Session}) -> diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index f8c510482..0290b57d3 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -651,10 +651,10 @@ cast(Msg) -> gen_server:cast(?CM, Msg). init([]) -> TabOpts = [public, {write_concurrency, true}], - ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]), - ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]), - ok = emqx_tables:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]), - ok = emqx_tables:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_CONN_TAB, [bag | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]), ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0), State = #{chan_pmon => emqx_pmon:new()}, {ok, State}. @@ -672,7 +672,7 @@ handle_cast(Msg, State) -> handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> ?tp(emqx_cm_process_down, #{stale_pid => Pid, reason => _Reason}), - ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], + ChanPids = [Pid | emqx_utils:drain_down(?BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), lists:foreach(fun mark_channel_disconnected/1, ChanPids), ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]), diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index ab028cf7f..c94f25ead 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -144,7 +144,7 @@ -type app_envs() :: [proplists:property()]. %% @doc For the given path, get root value enclosed in a single-key map. --spec get_root(emqx_map_lib:config_key_path()) -> map(). +-spec get_root(emqx_utils_maps:config_key_path()) -> map(). get_root([RootName | _]) -> #{RootName => do_get(?CONF, [RootName], #{})}. @@ -155,14 +155,14 @@ get_root_raw([RootName | _]) -> %% @doc Get a config value for the given path. %% The path should at least include root config name. --spec get(emqx_map_lib:config_key_path()) -> term(). +-spec get(emqx_utils_maps:config_key_path()) -> term(). get(KeyPath) -> do_get(?CONF, KeyPath). --spec get(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get(emqx_utils_maps:config_key_path(), term()) -> term(). get(KeyPath, Default) -> do_get(?CONF, KeyPath, Default). --spec find(emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find(emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find([]) -> Ref = make_ref(), case do_get(?CONF, [], Ref) of @@ -172,12 +172,12 @@ find([]) -> find(KeyPath) -> atom_conf_path( KeyPath, - fun(AtomKeyPath) -> emqx_map_lib:deep_find(AtomKeyPath, get_root(KeyPath)) end, + fun(AtomKeyPath) -> emqx_utils_maps:deep_find(AtomKeyPath, get_root(KeyPath)) end, {return, {not_found, KeyPath}} ). --spec find_raw(emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find_raw(emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find_raw([]) -> Ref = make_ref(), case do_get_raw([], Ref) of @@ -185,9 +185,9 @@ find_raw([]) -> Res -> {ok, Res} end; find_raw(KeyPath) -> - emqx_map_lib:deep_find([bin(Key) || Key <- KeyPath], get_root_raw(KeyPath)). + emqx_utils_maps:deep_find([bin(Key) || Key <- KeyPath], get_root_raw(KeyPath)). --spec get_zone_conf(atom(), emqx_map_lib:config_key_path()) -> term(). +-spec get_zone_conf(atom(), emqx_utils_maps:config_key_path()) -> term(). get_zone_conf(Zone, KeyPath) -> case find(?ZONE_CONF_PATH(Zone, KeyPath)) of %% not found in zones, try to find the global config @@ -197,7 +197,7 @@ get_zone_conf(Zone, KeyPath) -> Value end. --spec get_zone_conf(atom(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_zone_conf(atom(), emqx_utils_maps:config_key_path(), term()) -> term(). get_zone_conf(Zone, KeyPath, Default) -> case find(?ZONE_CONF_PATH(Zone, KeyPath)) of %% not found in zones, try to find the global config @@ -207,24 +207,24 @@ get_zone_conf(Zone, KeyPath, Default) -> Value end. --spec put_zone_conf(atom(), emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_zone_conf(atom(), emqx_utils_maps:config_key_path(), term()) -> ok. put_zone_conf(Zone, KeyPath, Conf) -> ?MODULE:put(?ZONE_CONF_PATH(Zone, KeyPath), Conf). --spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> term(). +-spec get_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path()) -> term(). get_listener_conf(Type, Listener, KeyPath) -> ?MODULE:get(?LISTENER_CONF_PATH(Type, Listener, KeyPath)). --spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path(), term()) -> term(). get_listener_conf(Type, Listener, KeyPath, Default) -> ?MODULE:get(?LISTENER_CONF_PATH(Type, Listener, KeyPath), Default). --spec put_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path(), term()) -> ok. put_listener_conf(Type, Listener, KeyPath, Conf) -> ?MODULE:put(?LISTENER_CONF_PATH(Type, Listener, KeyPath), Conf). --spec find_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find_listener_conf(Type, Listener, KeyPath) -> find(?LISTENER_CONF_PATH(Type, Listener, KeyPath)). @@ -243,20 +243,20 @@ erase(RootName) -> persistent_term:erase(?PERSIS_KEY(?RAW_CONF, bin(RootName))), ok. --spec put(emqx_map_lib:config_key_path(), term()) -> ok. +-spec put(emqx_utils_maps:config_key_path(), term()) -> ok. put(KeyPath, Config) -> Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_put(Path, Map, Value) + emqx_utils_maps:deep_put(Path, Map, Value) end, do_put(?CONF, Putter, KeyPath, Config). %% Puts value into configuration even if path doesn't exist %% For paths of non-existing atoms use force_put(KeyPath, Config, unsafe) --spec force_put(emqx_map_lib:config_key_path(), term()) -> ok. +-spec force_put(emqx_utils_maps:config_key_path(), term()) -> ok. force_put(KeyPath, Config) -> force_put(KeyPath, Config, safe). --spec force_put(emqx_map_lib:config_key_path(), term(), safe | unsafe) -> ok. +-spec force_put(emqx_utils_maps:config_key_path(), term(), safe | unsafe) -> ok. force_put(KeyPath0, Config, Safety) -> KeyPath = case Safety of @@ -264,19 +264,19 @@ force_put(KeyPath0, Config, Safety) -> unsafe -> [unsafe_atom(Key) || Key <- KeyPath0] end, Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_force_put(Path, Map, Value) + emqx_utils_maps:deep_force_put(Path, Map, Value) end, do_put(?CONF, Putter, KeyPath, Config). --spec get_default_value(emqx_map_lib:config_key_path()) -> {ok, term()} | {error, term()}. +-spec get_default_value(emqx_utils_maps:config_key_path()) -> {ok, term()} | {error, term()}. get_default_value([RootName | _] = KeyPath) -> BinKeyPath = [bin(Key) || Key <- KeyPath], case find_raw([RootName]) of {ok, RawConf} -> - RawConf1 = emqx_map_lib:deep_remove(BinKeyPath, #{bin(RootName) => RawConf}), + RawConf1 = emqx_utils_maps:deep_remove(BinKeyPath, #{bin(RootName) => RawConf}), try fill_defaults(get_schema_mod(RootName), RawConf1, #{}) of FullConf -> - case emqx_map_lib:deep_find(BinKeyPath, FullConf) of + case emqx_utils_maps:deep_find(BinKeyPath, FullConf) of {not_found, _, _} -> {error, no_default_value}; {ok, Val} -> {ok, Val} end @@ -287,10 +287,10 @@ get_default_value([RootName | _] = KeyPath) -> {error, {rootname_not_found, RootName}} end. --spec get_raw(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path()) -> term(). get_raw(KeyPath) -> do_get_raw(KeyPath). --spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw(KeyPath, Default) -> do_get_raw(KeyPath, Default). -spec put_raw(map()) -> ok. @@ -303,10 +303,10 @@ put_raw(Config) -> hocon_maps:ensure_plain(Config) ). --spec put_raw(emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_raw(emqx_utils_maps:config_key_path(), term()) -> ok. put_raw(KeyPath, Config) -> Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_force_put(Path, Map, Value) + emqx_utils_maps:deep_force_put(Path, Map, Value) end, do_put(?RAW_CONF, Putter, KeyPath, Config). @@ -455,7 +455,7 @@ do_check_config(SchemaMod, RawConf, Opts0) -> Opts = maps:merge(Opts0, Opts1), {AppEnvs, CheckedConf} = hocon_tconf:map_translate(SchemaMod, RawConf, Opts), - {AppEnvs, emqx_map_lib:unsafe_atom_key_map(CheckedConf)}. + {AppEnvs, emqx_utils_maps:unsafe_atom_key_map(CheckedConf)}. fill_defaults(RawConf) -> fill_defaults(RawConf, #{}). @@ -709,11 +709,11 @@ do_put(Type, Putter, [RootName | KeyPath], DeepValue) -> do_deep_get(?CONF, KeyPath, Map, Default) -> atom_conf_path( KeyPath, - fun(AtomKeyPath) -> emqx_map_lib:deep_get(AtomKeyPath, Map, Default) end, + fun(AtomKeyPath) -> emqx_utils_maps:deep_get(AtomKeyPath, Map, Default) end, {return, Default} ); do_deep_get(?RAW_CONF, KeyPath, Map, Default) -> - emqx_map_lib:deep_get([bin(Key) || Key <- KeyPath], Map, Default). + emqx_utils_maps:deep_get([bin(Key) || Key <- KeyPath], Map, Default). do_deep_put(?CONF, Putter, KeyPath, Map, Value) -> atom_conf_path( diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index d5dc72acc..e664a7dd7 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -229,7 +229,7 @@ process_update_request([_], _Handlers, {remove, _Opts}) -> process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), BinKeyPath = bin_path(ConfKeyPath), - NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), + NewRawConf = emqx_utils_maps:deep_remove(BinKeyPath, OldRawConf), OverrideConf = remove_from_override_config(BinKeyPath, Opts), {ok, NewRawConf, OverrideConf, Opts}; process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) -> @@ -435,7 +435,7 @@ remove_from_override_config(_BinKeyPath, #{persistent := false}) -> undefined; remove_from_override_config(BinKeyPath, Opts) -> OldConf = emqx_config:read_override_conf(Opts), - emqx_map_lib:deep_remove(BinKeyPath, OldConf). + emqx_utils_maps:deep_remove(BinKeyPath, OldConf). %% apply new config on top of override config merge_to_override_config(_RawConf, #{persistent := false}) -> @@ -457,7 +457,7 @@ return_change_result(_ConfKeyPath, {remove, _Opts}) -> return_rawconf(ConfKeyPath, #{rawconf_with_defaults := true}) -> FullRawConf = emqx_config:fill_defaults(emqx_config:get_raw([])), - emqx_map_lib:deep_get(bin_path(ConfKeyPath), FullRawConf); + emqx_utils_maps:deep_get(bin_path(ConfKeyPath), FullRawConf); return_rawconf(ConfKeyPath, _) -> emqx_config:get_raw(ConfKeyPath). @@ -475,16 +475,16 @@ atom(Atom) when is_atom(Atom) -> -dialyzer({nowarn_function, do_remove_handler/2}). do_remove_handler(ConfKeyPath, Handlers) -> - NewHandlers = emqx_map_lib:deep_remove(ConfKeyPath ++ [?MOD], Handlers), + NewHandlers = emqx_utils_maps:deep_remove(ConfKeyPath ++ [?MOD], Handlers), remove_empty_leaf(ConfKeyPath, NewHandlers). remove_empty_leaf([], Handlers) -> Handlers; remove_empty_leaf(KeyPath, Handlers) -> - case emqx_map_lib:deep_find(KeyPath, Handlers) =:= {ok, #{}} of + case emqx_utils_maps:deep_find(KeyPath, Handlers) =:= {ok, #{}} of %% empty leaf true -> - Handlers1 = emqx_map_lib:deep_remove(KeyPath, Handlers), + Handlers1 = emqx_utils_maps:deep_remove(KeyPath, Handlers), SubKeyPath = lists:sublist(KeyPath, length(KeyPath) - 1), remove_empty_leaf(SubKeyPath, Handlers1); false -> @@ -501,7 +501,7 @@ assert_callback_function(Mod) -> end, ok. --spec schema(module(), emqx_map_lib:config_key_path()) -> hocon_schema:schema(). +-spec schema(module(), emqx_utils_maps:config_key_path()) -> hocon_schema:schema(). schema(SchemaModule, [RootKey | _]) -> Roots = hocon_schema:roots(SchemaModule), {Field, Translations} = diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index e5002cab4..8d47f033c 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -77,7 +77,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [start_timer/2] ). @@ -260,7 +260,7 @@ stats(#state{ {error, _} -> [] end, ChanStats = emqx_channel:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ChanStats, ProcStats]). %% @doc Set TCP keepalive socket options to override system defaults. @@ -392,7 +392,7 @@ run_loop( emqx_channel:info(zone, Channel), [force_shutdown] ), - emqx_misc:tune_heap_size(ShutdownPolicy), + emqx_utils:tune_heap_size(ShutdownPolicy), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); @@ -472,7 +472,7 @@ ensure_stats_timer(_Timeout, State) -> -compile({inline, [cancel_stats_timer/1]}). cancel_stats_timer(State = #state{stats_timer = TRef}) when is_reference(TRef) -> ?tp(debug, cancel_stats_timer, #{}), - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), State#state{stats_timer = undefined}; cancel_stats_timer(State) -> State. @@ -558,7 +558,7 @@ handle_msg( {incoming, Packet = ?CONNECT_PACKET(ConnPkt)}, State = #state{idle_timer = IdleTimer} ) -> - ok = emqx_misc:cancel_timer(IdleTimer), + ok = emqx_utils:cancel_timer(IdleTimer), Serialize = emqx_frame:serialize_opts(ConnPkt), NState = State#state{ serialize = Serialize, @@ -593,7 +593,7 @@ handle_msg( #state{listener = {Type, Listener}} = State ) -> ActiveN = get_active_n(Type, Listener), - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent handle_msg({inet_reply, _Sock, ok}, State = #state{listener = {Type, Listener}}) -> @@ -1073,7 +1073,7 @@ check_oom(State = #state{channel = Channel}) -> emqx_channel:info(zone, Channel), [force_shutdown] ), ?tp(debug, check_oom, #{policy => ShutdownPolicy}), - case emqx_misc:check_oom(ShutdownPolicy) of + case emqx_utils:check_oom(ShutdownPolicy) of {shutdown, Reason} -> %% triggers terminate/2 callback immediately erlang:exit({shutdown, Reason}); @@ -1200,7 +1200,7 @@ inc_counter(Key, Inc) -> %%-------------------------------------------------------------------- set_field(Name, Value, State) -> - Pos = emqx_misc:index_of(Name, record_info(fields, state)), + Pos = emqx_utils:index_of(Name, record_info(fields, state)), setelement(Pos + 1, State, Value). get_state(Pid) -> diff --git a/apps/emqx/src/emqx_crl_cache.erl b/apps/emqx/src/emqx_crl_cache.erl index 79e47a6dc..084313420 100644 --- a/apps/emqx/src/emqx_crl_cache.erl +++ b/apps/emqx/src/emqx_crl_cache.erl @@ -117,7 +117,7 @@ handle_call(Call, _From, State) -> handle_cast({evict, URL}, State0 = #state{refresh_timers = RefreshTimers0}) -> emqx_ssl_crl_cache:delete(URL), MTimer = maps:get(URL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = maps:without([URL], RefreshTimers0), State = State0#state{refresh_timers = RefreshTimers}, ?tp( @@ -223,9 +223,9 @@ ensure_timer(URL, State = #state{refresh_interval = Timeout}) -> ensure_timer(URL, State = #state{refresh_timers = RefreshTimers0}, Timeout) -> ?tp(crl_cache_ensure_timer, #{url => URL, timeout => Timeout}), MTimer = maps:get(URL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = RefreshTimers0#{ - URL => emqx_misc:start_timer( + URL => emqx_utils:start_timer( Timeout, {refresh, URL} ) @@ -297,7 +297,7 @@ handle_cache_overflow(State0) -> {_Time, OldestURL, InsertionTimes} = gb_trees:take_smallest(InsertionTimes0), emqx_ssl_crl_cache:delete(OldestURL), MTimer = maps:get(OldestURL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = maps:remove(OldestURL, RefreshTimers0), CachedURLs = sets:del_element(OldestURL, CachedURLs0), ?tp(debug, crl_cache_overflow, #{oldest_url => OldestURL}), diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 64e4ed6c3..46193fb16 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -99,7 +99,7 @@ now_diff(TS) -> erlang:system_time(millisecond) - TS. %%-------------------------------------------------------------------- init([]) -> - ok = emqx_tables:new(?FLAPPING_TAB, [ + ok = emqx_utils_ets:new(?FLAPPING_TAB, [ public, set, {keypos, #flapping.clientid}, @@ -184,7 +184,7 @@ code_change(_OldVsn, State, _Extra) -> start_timer(Zone) -> WindTime = maps:get(window_time, get_policy(Zone)), - emqx_misc:start_timer(WindTime, {garbage_collect, Zone}). + emqx_utils:start_timer(WindTime, {garbage_collect, Zone}). start_timers() -> lists:foreach( diff --git a/apps/emqx/src/emqx_guid.erl b/apps/emqx/src/emqx_guid.erl index fea4e70b0..d313723fb 100644 --- a/apps/emqx/src/emqx_guid.erl +++ b/apps/emqx/src/emqx_guid.erl @@ -145,10 +145,10 @@ npid() -> NPid. to_hexstr(I) when byte_size(I) =:= 16 -> - emqx_misc:bin_to_hexstr(I, upper). + emqx_utils:bin_to_hexstr(I, upper). from_hexstr(S) when byte_size(S) =:= 32 -> - emqx_misc:hexstr_to_bin(S). + emqx_utils:hexstr_to_bin(S). to_base62(<>) -> emqx_base62:encode(I). diff --git a/apps/emqx/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl index 1784d8ea3..0b8dc0941 100644 --- a/apps/emqx/src/emqx_hooks.erl +++ b/apps/emqx/src/emqx_hooks.erl @@ -229,7 +229,7 @@ lookup(HookPoint) -> %%-------------------------------------------------------------------- init([]) -> - ok = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), {ok, #{}}. handle_call({add, HookPoint, Callback = #callback{action = {M, F, _}}}, _From, State) -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl index 83bc2ec72..bbebd9460 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl @@ -375,7 +375,7 @@ return_pause(infinity, PauseType, Fun, Diff, Limiter) -> {PauseType, ?MINIMUM_PAUSE, make_retry_context(Fun, Diff), Limiter}; return_pause(Rate, PauseType, Fun, Diff, Limiter) -> Val = erlang:round(Diff * emqx_limiter_schema:default_period() / Rate), - Pause = emqx_misc:clamp(Val, ?MINIMUM_PAUSE, ?MAXIMUM_PAUSE), + Pause = emqx_utils:clamp(Val, ?MINIMUM_PAUSE, ?MAXIMUM_PAUSE), {PauseType, Pause, make_retry_context(Fun, Diff), Limiter}. -spec make_retry_context(undefined | retry_fun(Limiter), non_neg_integer()) -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 44663ceeb..f1daeaaeb 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -572,7 +572,7 @@ find_limiter_cfg(Type, #{rate := _} = Cfg) -> find_limiter_cfg(Type, Cfg) -> { maps:get(Type, Cfg, undefined), - find_client_cfg(Type, emqx_map_lib:deep_get([client, Type], Cfg, undefined)) + find_client_cfg(Type, emqx_utils_maps:deep_get([client, Type], Cfg, undefined)) }. find_client_cfg(Type, BucketCfg) -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 4e5843166..f82aebe7c 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -427,12 +427,12 @@ pre_config_update([listeners, _Type, _Name], {create, _NewConf}, _RawConf) -> pre_config_update([listeners, _Type, _Name], {update, _Request}, undefined) -> {error, not_found}; pre_config_update([listeners, Type, Name], {update, Request}, RawConf) -> - NewConfT = emqx_map_lib:deep_merge(RawConf, Request), + NewConfT = emqx_utils_maps:deep_merge(RawConf, Request), NewConf = ensure_override_limiter_conf(NewConfT, Request), CertsDir = certs_dir(Type, Name), {ok, convert_certs(CertsDir, NewConf)}; pre_config_update([listeners, _Type, _Name], {action, _Action, Updated}, RawConf) -> - NewConf = emqx_map_lib:deep_merge(RawConf, Updated), + NewConf = emqx_utils_maps:deep_merge(RawConf, Updated), {ok, NewConf}; pre_config_update(_Path, _Request, RawConf) -> {ok, RawConf}. @@ -500,7 +500,7 @@ esockd_opts(ListenerId, Type, Opts0) -> ws_opts(Type, ListenerName, Opts) -> WsPaths = [ - {emqx_map_lib:deep_get([websocket, mqtt_path], Opts, "/mqtt"), emqx_ws_connection, #{ + {emqx_utils_maps:deep_get([websocket, mqtt_path], Opts, "/mqtt"), emqx_ws_connection, #{ zone => zone(Opts), listener => {Type, ListenerName}, limiter => limiter(Opts), @@ -538,7 +538,7 @@ esockd_access_rules(StrRules) -> [A, CIDR] = string:tokens(S, " "), %% esockd rules only use words 'allow' and 'deny', both are existing %% comparison of strings may be better, but there is a loss of backward compatibility - case emqx_misc:safe_to_existing_atom(A) of + case emqx_utils:safe_to_existing_atom(A) of {ok, Action} -> [ { @@ -560,7 +560,7 @@ esockd_access_rules(StrRules) -> merge_default(Options) -> case lists:keytake(tcp_options, 1, Options) of {value, {tcp_options, TcpOpts}, Options1} -> - [{tcp_options, emqx_misc:merge_opts(?MQTT_SOCKOPTS, TcpOpts)} | Options1]; + [{tcp_options, emqx_utils:merge_opts(?MQTT_SOCKOPTS, TcpOpts)} | Options1]; false -> [{tcp_options, ?MQTT_SOCKOPTS} | Options] end. diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 22cf75153..0e72fd2b5 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -62,11 +62,11 @@ %% The JSON object is pretty-printed. %% NOTE: do not use this function for logging. best_effort_json(Input) -> - best_effort_json(Input, [space, {indent, 4}]). + best_effort_json(Input, [pretty, force_utf8]). best_effort_json(Input, Opts) -> Config = #{depth => unlimited, single_line => true}, JsonReady = best_effort_json_obj(Input, Config), - jsx:encode(JsonReady, Opts). + emqx_utils_json:encode(JsonReady, Opts). -spec format(logger:log_event(), config()) -> iodata(). format(#{level := Level, msg := Msg, meta := Meta} = Event, Config0) when is_map(Config0) -> @@ -92,7 +92,7 @@ format(Msg, Meta, Config) -> } end, Data = maps:without([report_cb], Data0), - jiffy:encode(json_obj(Data, Config)). + emqx_utils_json:encode(json_obj(Data, Config)). maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) -> case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of @@ -378,15 +378,15 @@ p_config() -> best_effort_json_test() -> ?assertEqual( - <<"{}">>, + <<"{\n \n}">>, emqx_logger_jsonfmt:best_effort_json([]) ), ?assertEqual( - <<"{\n \"key\": []\n}">>, + <<"{\n \"key\" : [\n \n ]\n}">>, emqx_logger_jsonfmt:best_effort_json(#{key => []}) ), ?assertEqual( - <<"[\n {\n \"key\": []\n }\n]">>, + <<"[\n {\n \"key\" : [\n \n ]\n }\n]">>, emqx_logger_jsonfmt:best_effort_json([#{key => []}]) ), ok. diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index c2e297623..21a114c0f 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -541,7 +541,7 @@ init([]) -> CRef = counters:new(?MAX_SIZE, [write_concurrency]), ok = persistent_term:put(?MODULE, CRef), % Create index mapping table - ok = emqx_tables:new(?TAB, [{keypos, 2}, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{keypos, 2}, {read_concurrency, true}]), Metrics = lists:append([ ?BYTES_METRICS, ?PACKET_METRICS, diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index 4e7ada044..3bb10ee5c 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -110,7 +110,7 @@ register_listener(ListenerID, Opts) -> -spec inject_sni_fun(emqx_listeners:listener_id(), map()) -> map(). inject_sni_fun(ListenerID, Conf0) -> SNIFun = emqx_const_v1:make_sni_fun(ListenerID), - Conf = emqx_map_lib:deep_merge(Conf0, #{ssl_options => #{sni_fun => SNIFun}}), + Conf = emqx_utils_maps:deep_merge(Conf0, #{ssl_options => #{sni_fun => SNIFun}}), ok = ?MODULE:register_listener(ListenerID, Conf), Conf. @@ -120,7 +120,7 @@ inject_sni_fun(ListenerID, Conf0) -> init(_Args) -> logger:set_process_metadata(#{domain => [emqx, ocsp, cache]}), - emqx_tables:new(?CACHE_TAB, [ + emqx_utils_ets:new(?CACHE_TAB, [ named_table, public, {heir, whereis(emqx_kernel_sup), none}, @@ -149,7 +149,7 @@ handle_call({register_listener, ListenerID, Conf}, _From, State0) -> msg => "registering_ocsp_cache", listener_id => ListenerID }), - RefreshInterval0 = emqx_map_lib:deep_get([ssl_options, ocsp, refresh_interval], Conf), + RefreshInterval0 = emqx_utils_maps:deep_get([ssl_options, ocsp, refresh_interval], Conf), RefreshInterval = max(RefreshInterval0, ?MIN_REFRESH_INTERVAL), State = State0#{{refresh_interval, ListenerID} => RefreshInterval}, %% we need to pass the config along because this might be called @@ -476,9 +476,9 @@ ensure_timer(ListenerID, State, Timeout) -> ensure_timer(ListenerID, {refresh, ListenerID}, State, Timeout). ensure_timer(ListenerID, Message, State, Timeout) -> - emqx_misc:cancel_timer(maps:get(?REFRESH_TIMER(ListenerID), State, undefined)), + emqx_utils:cancel_timer(maps:get(?REFRESH_TIMER(ListenerID), State, undefined)), State#{ - ?REFRESH_TIMER(ListenerID) => emqx_misc:start_timer( + ?REFRESH_TIMER(ListenerID) => emqx_utils:start_timer( Timeout, Message ) diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index c5ce35bf9..4810798eb 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -180,8 +180,8 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- cancel_outdated_timer(#{mem_time_ref := MemRef, cpu_time_ref := CpuRef}) -> - emqx_misc:cancel_timer(MemRef), - emqx_misc:cancel_timer(CpuRef), + emqx_utils:cancel_timer(MemRef), + emqx_utils:cancel_timer(CpuRef), ok. start_cpu_check_timer() -> @@ -204,7 +204,7 @@ start_mem_check_timer() -> end. start_timer(Interval, Msg) -> - emqx_misc:start_timer(Interval, Msg). + emqx_utils:start_timer(Interval, Msg). update_mem_alarm_status(HWM) when HWM > 1.0 orelse HWM < 0.0 -> ?SLOG(warning, #{msg => "discarded_out_of_range_mem_alarm_threshold", value => HWM}), diff --git a/apps/emqx/src/emqx_pool.erl b/apps/emqx/src/emqx_pool.erl index 1691a533a..1cb5f429c 100644 --- a/apps/emqx/src/emqx_pool.erl +++ b/apps/emqx/src/emqx_pool.erl @@ -57,7 +57,7 @@ -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 7c9cc61b0..42430af5d 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -98,7 +98,7 @@ mnesia(boot) -> -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ed20848d2..df5a91727 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2329,7 +2329,7 @@ mqtt_ssl_listener_ssl_options_validator(Conf) -> fun ocsp_outer_validator/1, fun crl_outer_validator/1 ], - case emqx_misc:pipeline(Checks, Conf, not_used) of + case emqx_utils:pipeline(Checks, Conf, not_used) of {ok, _, _} -> ok; {error, Reason, _NotUsed} -> @@ -2350,7 +2350,7 @@ ocsp_outer_validator(_Conf) -> ok. ocsp_inner_validator(#{enable_ocsp_stapling := _} = Conf) -> - ocsp_inner_validator(emqx_map_lib:binary_key_map(Conf)); + ocsp_inner_validator(emqx_utils_maps:binary_key_map(Conf)); ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := false} = _Conf) -> ok; ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := true} = Conf) -> @@ -2585,7 +2585,7 @@ to_url(Str) -> end. to_json_binary(Str) -> - case emqx_json:safe_decode(Str) of + case emqx_utils_json:safe_decode(Str) of {ok, _} -> {ok, iolist_to_binary(Str)}; Error -> diff --git a/apps/emqx/src/emqx_sequence.erl b/apps/emqx/src/emqx_sequence.erl index 60596324a..7acc87256 100644 --- a/apps/emqx/src/emqx_sequence.erl +++ b/apps/emqx/src/emqx_sequence.erl @@ -39,7 +39,7 @@ %% @doc Create a sequence. -spec create(name()) -> ok. create(Name) -> - emqx_tables:new(Name, [public, set, {write_concurrency, true}]). + emqx_utils_ets:new(Name, [public, set, {write_concurrency, true}]). %% @doc Next value of the sequence. -spec nextval(name(), key()) -> seqid(). diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index a13dfe491..8b15340e9 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -941,7 +941,7 @@ age(Now, Ts) -> Now - Ts. %%-------------------------------------------------------------------- set_field(Name, Value, Session) -> - Pos = emqx_misc:index_of(Name, record_info(fields, session)), + Pos = emqx_utils:index_of(Name, record_info(fields, session)), setelement(Pos + 1, Session, Value). get_mqueue(#session{mqueue = Q}) -> diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl index 0d4972e8c..0435ddca3 100644 --- a/apps/emqx/src/emqx_session_router.erl +++ b/apps/emqx/src/emqx_session_router.erl @@ -95,7 +95,7 @@ create_table(Tab, Storage) -> %%-------------------------------------------------------------------- create_init_tab() -> - emqx_tables:new(?SESSION_INIT_TAB, [ + emqx_utils_ets:new(?SESSION_INIT_TAB, [ public, {read_concurrency, true}, {write_concurrency, true} @@ -104,7 +104,7 @@ create_init_tab() -> -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] @@ -182,7 +182,7 @@ pending(SessionID, MarkerIDs) -> call(pick(SessionID), {pending, SessionID, MarkerIDs}). buffer(SessionID, STopic, Msg) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> ok; Worker -> emqx_session_router_worker:buffer(Worker, STopic, Msg) end. @@ -194,7 +194,7 @@ resume_begin(From, SessionID) when is_pid(From), is_binary(SessionID) -> -spec resume_end(pid(), binary()) -> {'ok', [emqx_types:message()]} | {'error', term()}. resume_end(From, SessionID) when is_pid(From), is_binary(SessionID) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> ?tp(ps_session_not_found, #{sid => SessionID}), {error, not_found}; @@ -249,7 +249,7 @@ handle_cast({delete_routes, SessionID, Subscriptions}, State) -> ok = lists:foreach(Fun, maps:to_list(Subscriptions)), {noreply, State}; handle_cast({resume_end, SessionID, Pid}, State) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> skip; P when P =:= Pid -> ets:delete(?SESSION_INIT_TAB, SessionID); P when is_pid(P) -> skip @@ -283,7 +283,7 @@ init_resume_worker(RemotePid, SessionID, #{pmon := Pmon} = State) -> error; {ok, Pid} -> Pmon1 = emqx_pmon:monitor(Pid, Pmon), - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> {ok, Pid, State#{pmon => Pmon1}}; {_, OldPid} -> diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 061f2a42f..d7dc8c5a6 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -399,9 +399,11 @@ init([]) -> ok = mria:wait_for_tables([?TAB]), {ok, _} = mnesia:subscribe({table, ?TAB, simple}), {atomic, PMon} = mria:transaction(?SHARED_SUB_SHARD, fun ?MODULE:init_monitors/0), - ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), - ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), - ok = emqx_tables:new(?SHARED_SUBS_ROUND_ROBIN_COUNTER, [public, set, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SHARED_SUBS, [protected, bag]), + ok = emqx_utils_ets:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?SHARED_SUBS_ROUND_ROBIN_COUNTER, [ + public, set, {write_concurrency, true} + ]), {ok, update_stats(#state{pmon = PMon})}. init_monitors() -> diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index ed901d9a9..ef9109e33 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -201,7 +201,7 @@ cast(Msg) -> gen_server:cast(?SERVER, Msg). %%-------------------------------------------------------------------- init(#{tick_ms := TickMs}) -> - ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [public, set, {write_concurrency, true}]), Stats = lists:append([ ?CONNECTION_STATS, ?CHANNEL_STATS, @@ -213,7 +213,7 @@ init(#{tick_ms := TickMs}) -> {ok, start_timer(#state{updates = [], tick_ms = TickMs}), hibernate}. start_timer(#state{tick_ms = Ms} = State) -> - State#state{timer = emqx_misc:start_timer(Ms, tick)}. + State#state{timer = emqx_utils:start_timer(Ms, tick)}. handle_call(stop, _From, State) -> {stop, normal, ok, State}; @@ -301,7 +301,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #state{timer = TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index a5f14e32a..509429796 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -62,7 +62,7 @@ -endif. -import(emqx_topic, [systop/1]). --import(emqx_misc, [start_timer/2]). +-import(emqx_utils, [start_timer/2]). -record(state, { heartbeat :: maybe(reference()), @@ -222,7 +222,7 @@ handle_info(Info, State) -> terminate(_Reason, #state{heartbeat = TRef1, ticker = TRef2}) -> _ = emqx_config_handler:remove_handler(?CONF_KEY_PATH), unload_event_hooks(sys_event_messages()), - lists:foreach(fun emqx_misc:cancel_timer/1, [TRef1, TRef2]). + lists:foreach(fun emqx_utils:cancel_timer/1, [TRef1, TRef2]). unload_event_hooks([]) -> ok; @@ -348,7 +348,7 @@ publish(Event, Payload) when Event == unsubscribed -> Topic = event_topic(Event, Payload), - safe_publish(Topic, emqx_json:encode(Payload)). + safe_publish(Topic, emqx_utils_json:encode(Payload)). metric_topic(Name) -> translate_topic("metrics/", Name). diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index 6ff68820e..f1190f586 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -77,7 +77,7 @@ init([]) -> {ok, start_timer(#{timer => undefined, events => []})}. start_timer(State) -> - State#{timer := emqx_misc:start_timer(timer:seconds(2), reset)}. + State#{timer := emqx_utils:start_timer(timer:seconds(2), reset)}. sysm_opts(VM) -> sysm_opts(maps:to_list(VM), []). @@ -204,7 +204,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), ok. code_change(_OldVsn, State, _Extra) -> diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 47797b326..d1c57bf0d 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -317,7 +317,9 @@ ensure_ssl_files(Dir, SSL, Opts) -> ensure_ssl_files(_Dir, SSL, [], _Opts) -> {ok, SSL}; ensure_ssl_files(Dir, SSL, [KeyPath | KeyPaths], Opts) -> - case ensure_ssl_file(Dir, KeyPath, SSL, emqx_map_lib:deep_get(KeyPath, SSL, undefined), Opts) of + case + ensure_ssl_file(Dir, KeyPath, SSL, emqx_utils_maps:deep_get(KeyPath, SSL, undefined), Opts) + of {ok, NewSSL} -> ensure_ssl_files(Dir, NewSSL, KeyPaths, Opts); {error, Reason} -> @@ -332,7 +334,7 @@ delete_ssl_files(Dir, NewOpts0, OldOpts0) -> {ok, OldOpts} = ensure_ssl_files(Dir, OldOpts0, #{dry_run => DryRun}), Get = fun (_KP, undefined) -> undefined; - (KP, Opts) -> emqx_map_lib:deep_get(KP, Opts, undefined) + (KP, Opts) -> emqx_utils_maps:deep_get(KP, Opts, undefined) end, lists:foreach( fun(KeyPath) -> delete_old_file(Get(KeyPath, NewOpts), Get(KeyPath, OldOpts)) end, @@ -372,7 +374,7 @@ do_ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, DryRun) -> true -> case save_pem_file(Dir, KeyPath, MaybePem, DryRun) of {ok, Path} -> - NewSSL = emqx_map_lib:deep_put(KeyPath, SSL, Path), + NewSSL = emqx_utils_maps:deep_put(KeyPath, SSL, Path), {ok, NewSSL}; {error, Reason} -> {error, Reason} @@ -482,9 +484,9 @@ is_valid_pem_file(Path) -> %% so they are forced to upload a cert file, or use an existing file path. -spec drop_invalid_certs(map()) -> map(). drop_invalid_certs(#{enable := False} = SSL) when ?IS_FALSE(False) -> - lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS_A); + lists:foldl(fun emqx_utils_maps:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS_A); drop_invalid_certs(#{<<"enable">> := False} = SSL) when ?IS_FALSE(False) -> - lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS); + lists:foldl(fun emqx_utils_maps:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS); drop_invalid_certs(#{enable := True} = SSL) when ?IS_TRUE(True) -> do_drop_invalid_certs(?SSL_FILE_OPT_PATHS_A, SSL); drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> @@ -493,7 +495,7 @@ drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> do_drop_invalid_certs([], SSL) -> SSL; do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> - case emqx_map_lib:deep_get(KeyPath, SSL, undefined) of + case emqx_utils_maps:deep_get(KeyPath, SSL, undefined) of undefined -> do_drop_invalid_certs(KeyPaths, SSL); PemOrPath -> @@ -501,7 +503,7 @@ do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> true -> do_drop_invalid_certs(KeyPaths, SSL); {error, _} -> - do_drop_invalid_certs(KeyPaths, emqx_map_lib:deep_remove(KeyPath, SSL)) + do_drop_invalid_certs(KeyPaths, emqx_utils_maps:deep_remove(KeyPath, SSL)) end end. @@ -586,7 +588,9 @@ ensure_ssl_file_key(_SSL, []) -> ok; ensure_ssl_file_key(SSL, RequiredKeyPaths) -> NotFoundRef = make_ref(), - Filter = fun(KeyPath) -> NotFoundRef =:= emqx_map_lib:deep_get(KeyPath, SSL, NotFoundRef) end, + Filter = fun(KeyPath) -> + NotFoundRef =:= emqx_utils_maps:deep_get(KeyPath, SSL, NotFoundRef) + end, case lists:filter(Filter, RequiredKeyPaths) of [] -> ok; Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index f14dc0c15..91194772f 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -272,7 +272,7 @@ handle_info({timeout, TRef, update_trace}, #{timer := TRef} = State) -> ?tp(update_trace_done, #{}), {noreply, State#{timer => NextTRef}}; handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), handle_info({timeout, TRef, update_trace}, State); handle_info(Info, State) -> ?SLOG(error, #{unexpected_info => Info}), @@ -280,7 +280,7 @@ handle_info(Info, State) -> terminate(_Reason, #{timer := TRef}) -> _ = mnesia:unsubscribe({table, ?TRACE, simple}), - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), stop_all_trace_handler(), update_trace_handler(), _ = file:del_dir_r(zip_dir()), @@ -302,7 +302,7 @@ update_trace(Traces) -> ok = stop_trace(NeedStop, Started), clean_stale_trace_files(), NextTime = find_closest_time(Traces, Now), - emqx_misc:start_timer(NextTime, update_trace). + emqx_utils:start_timer(NextTime, update_trace). stop_all_trace_handler() -> lists:foreach( diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index 231fd5e7b..528bc4d42 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -196,7 +196,7 @@ handler_id(Name, Type) -> do_handler_id(Name, Type) catch _:_ -> - Hash = emqx_misc:bin_to_hexstr(crypto:hash(md5, Name), lower), + Hash = emqx_utils:bin_to_hexstr(crypto:hash(md5, Name), lower), do_handler_id(Hash, Type) end. diff --git a/apps/emqx/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl index 1327a1bb0..d90d4139b 100644 --- a/apps/emqx/src/emqx_vm_mon.erl +++ b/apps/emqx/src/emqx_vm_mon.erl @@ -107,7 +107,7 @@ code_change(_OldVsn, State, _Extra) -> start_check_timer() -> Interval = emqx:get_config([sysmon, vm, process_check_interval]), - emqx_misc:start_timer(Interval, check). + emqx_utils:start_timer(Interval, check). usage(Percent) -> integer_to_list(floor(Percent * 100)) ++ "%". diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index ead609ed8..20962809f 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -52,7 +52,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [ maybe_apply/2, start_timer/2 @@ -172,7 +172,7 @@ stats(WsPid) when is_pid(WsPid) -> stats(#state{channel = Channel}) -> SockStats = emqx_pd:get_counters(?SOCK_STATS), ChanStats = emqx_channel:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ChanStats, ProcStats]). %% kick|discard|takeover @@ -340,7 +340,7 @@ tune_heap_size(Channel) -> ) of #{enable := false} -> ok; - ShutdownPolicy -> emqx_misc:tune_heap_size(ShutdownPolicy) + ShutdownPolicy -> emqx_utils:tune_heap_size(ShutdownPolicy) end. get_stats_enable(Zone) -> @@ -454,7 +454,7 @@ websocket_info( State = #state{listener = {Type, Listener}} ) -> ActiveN = get_active_n(Type, Listener), - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); websocket_info( {timeout, _, limit_timeout}, @@ -678,7 +678,7 @@ check_oom(State = #state{channel = Channel}) -> #{enable := false} -> State; #{enable := true} -> - case emqx_misc:check_oom(ShutdownPolicy) of + case emqx_utils:check_oom(ShutdownPolicy) of Shutdown = {shutdown, _Reason} -> postpone(Shutdown, State); _Other -> @@ -913,7 +913,7 @@ inc_qos_stats_key(_, _) -> undefined. %% Cancel idle timer cancel_idle_timer(State = #state{idle_timer = IdleTimer}) -> - ok = emqx_misc:cancel_timer(IdleTimer), + ok = emqx_utils:cancel_timer(IdleTimer), State#state{idle_timer = undefined}. %%-------------------------------------------------------------------- @@ -1046,7 +1046,7 @@ check_max_connection(Type, Listener) -> %%-------------------------------------------------------------------- set_field(Name, Value, State) -> - Pos = emqx_misc:index_of(Name, record_info(fields, state)), + Pos = emqx_utils:index_of(Name, record_info(fields, state)), setelement(Pos + 1, State, Value). %% ensure lowercase letters in headers diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 6e56c85ed..e9ddc61a8 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -482,7 +482,7 @@ copy_certs(_, _) -> load_config(SchemaModule, Config, Opts) -> ConfigBin = case is_map(Config) of - true -> jsx:encode(Config); + true -> emqx_utils_json:encode(Config); false -> Config end, ok = emqx_config:delete_override_conf_files(), @@ -1041,7 +1041,7 @@ switch_proxy(Switch, Name, ProxyHost, ProxyPort) -> off -> #{<<"enabled">> => false}; on -> #{<<"enabled">> => true} end, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, @@ -1061,7 +1061,7 @@ timeout_proxy(on, Name, ProxyHost, ProxyPort) -> <<"toxicity">> => 1.0, <<"attributes">> => #{<<"timeout">> => 0} }, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, @@ -1096,7 +1096,7 @@ latency_up_proxy(on, Name, ProxyHost, ProxyPort) -> <<"jitter">> => 3_000 } }, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl index 575bed5c3..e9064715d 100644 --- a/apps/emqx/test/emqx_common_test_http.erl +++ b/apps/emqx/test/emqx_common_test_http.erl @@ -54,7 +54,7 @@ request_api(Method, Url, QueryParams, Auth, Body, HttpOpts) -> [] -> {NewUrl, [Auth]}; _ -> - {NewUrl, [Auth], "application/json", emqx_json:encode(Body)} + {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)} end, do_request_api(Method, Request, HttpOpts). @@ -70,7 +70,7 @@ do_request_api(Method, Request, HttpOpts) -> end. get_http_data(ResponseBody) -> - emqx_json:decode(ResponseBody, [return_maps]). + emqx_utils_json:decode(ResponseBody, [return_maps]). auth_header(User, Pass) -> Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index fe8a5fed8..7befd7a16 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -57,5 +57,5 @@ t_fill_default_values(_) -> WithDefaults ), %% ensure JSON compatible - _ = emqx_json:encode(WithDefaults), + _ = emqx_utils_json:encode(WithDefaults), ok. diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index cc9e03168..21ed45119 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -496,16 +496,16 @@ t_get_conn_info(_) -> t_oom_shutdown(init, Config) -> ok = snabbkaffe:start_trace(), - ok = meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]), + ok = meck:new(emqx_utils, [non_strict, passthrough, no_history, no_link]), meck:expect( - emqx_misc, + emqx_utils, check_oom, fun(_) -> {shutdown, "fake_oom"} end ), Config; t_oom_shutdown('end', _Config) -> snabbkaffe:stop(), - meck:unload(emqx_misc), + meck:unload(emqx_utils), ok. t_oom_shutdown(_) -> diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl index 01f9c7172..dd3eb29e7 100644 --- a/apps/emqx/test/emqx_crl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -402,7 +402,7 @@ request(Method, Url, QueryParams, Body) -> Opts = #{return_all => true}, case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of {ok, {Reason, Headers, BodyR}} -> - {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}}; + {ok, {Reason, Headers, emqx_utils_json:decode(BodyR, [return_maps])}}; Error -> Error end. @@ -997,7 +997,7 @@ do_t_update_listener(Config) -> <<"enable_crl_check">> => true } }, - ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, CRLConfig), + ListenerData1 = emqx_utils_maps:deep_merge(ListenerData0, CRLConfig), {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1), ?assertMatch( #{ @@ -1040,7 +1040,7 @@ do_t_validations(_Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ListenerData1 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -1052,7 +1052,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} = - emqx_json:decode(ResRaw1, [return_maps]), + emqx_utils_json:decode(ResRaw1, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -1064,7 +1064,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw1, [return_maps]) + emqx_utils_json:decode(MsgRaw1, [return_maps]) ), ok. diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index 3c3fd0341..dff8ce5a7 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -143,7 +143,7 @@ init_per_testcase(t_ocsp_responder_error_responses, Config) -> } }, Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_map_lib:binary_key_map(Conf), + ConfBin = emqx_utils_maps:binary_key_map(Conf), hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), snabbkaffe:start_trace(), @@ -184,7 +184,7 @@ init_per_testcase(_TestCase, Config) -> } }, Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_map_lib:binary_key_map(Conf), + ConfBin = emqx_utils_maps:binary_key_map(Conf), hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), snabbkaffe:start_trace(), @@ -430,7 +430,7 @@ request(Method, Url, QueryParams, Body) -> Opts = #{return_all => true}, case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of {ok, {Reason, Headers, BodyR}} -> - {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}}; + {ok, {Reason, Headers, emqx_utils_json:decode(BodyR, [return_maps])}}; Error -> Error end. @@ -679,7 +679,7 @@ do_t_update_listener(Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ?assertEqual( undefined, - emqx_map_lib:deep_get([<<"ssl_options">>, <<"ocsp">>], ListenerData0, undefined) + emqx_utils_maps:deep_get([<<"ssl_options">>, <<"ocsp">>], ListenerData0, undefined) ), assert_no_http_get(), @@ -702,7 +702,7 @@ do_t_update_listener(Config) -> } } }, - ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, OCSPConfig), + ListenerData1 = emqx_utils_maps:deep_merge(ListenerData0, OCSPConfig), {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1), ?assertMatch( #{ @@ -722,14 +722,14 @@ do_t_update_listener(Config) -> %% location ?assertNotEqual( IssuerPemPath, - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [<<"ssl_options">>, <<"ocsp">>, <<"issuer_pem">>], ListenerData2 ) ), ?assertNotEqual( IssuerPem, - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [<<"ssl_options">>, <<"ocsp">>, <<"issuer_pem">>], ListenerData2 ) @@ -818,7 +818,7 @@ do_t_validations(_Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ListenerData1 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -827,7 +827,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} = - emqx_json:decode(ResRaw1, [return_maps]), + emqx_utils_json:decode(ResRaw1, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -839,11 +839,11 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw1, [return_maps]) + emqx_utils_json:decode(MsgRaw1, [return_maps]) ), ListenerData2 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -857,7 +857,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw2}} = update_listener_via_api(ListenerId, ListenerData2), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw2} = - emqx_json:decode(ResRaw2, [return_maps]), + emqx_utils_json:decode(ResRaw2, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -869,11 +869,11 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw2, [return_maps]) + emqx_utils_json:decode(MsgRaw2, [return_maps]) ), ListenerData3a = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -886,10 +886,12 @@ do_t_validations(_Config) -> } } ), - ListenerData3 = emqx_map_lib:deep_remove([<<"ssl_options">>, <<"certfile">>], ListenerData3a), + ListenerData3 = emqx_utils_maps:deep_remove( + [<<"ssl_options">>, <<"certfile">>], ListenerData3a + ), {error, {_, _, ResRaw3}} = update_listener_via_api(ListenerId, ListenerData3), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw3} = - emqx_json:decode(ResRaw3, [return_maps]), + emqx_utils_json:decode(ResRaw3, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -901,7 +903,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw3, [return_maps]) + emqx_utils_json:decode(MsgRaw3, [return_maps]) ), ok. diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index 298a33fe8..2db0acf82 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -119,7 +119,7 @@ t_has_routes(_) -> ?R:delete_route(<<"devices/+/messages">>). t_unexpected(_) -> - Router = emqx_misc:proc_name(?R, 1), + Router = emqx_utils:proc_name(?R, 1), ?assertEqual(ignored, gen_server:call(Router, bad_request)), ?assertEqual(ok, gen_server:cast(Router, bad_message)), Router ! bad_info. diff --git a/apps/emqx/test/emqx_tls_lib_tests.erl b/apps/emqx/test/emqx_tls_lib_tests.erl index 5510e4027..ad9598107 100644 --- a/apps/emqx/test/emqx_tls_lib_tests.erl +++ b/apps/emqx/test/emqx_tls_lib_tests.erl @@ -191,7 +191,7 @@ ssl_files_save_delete_test() -> FileKey = maps:get(<<"keyfile">>, SSL), ?assertMatch(<<"/tmp/ssl-test-dir/key-", _:16/binary>>, FileKey), ?assertEqual({ok, bin(test_key())}, file:read_file(FileKey)), - FileIssuerPem = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL), + FileIssuerPem = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL), ?assertMatch(<<"/tmp/ssl-test-dir/ocsp_issuer_pem-", _:16/binary>>, FileIssuerPem), ?assertEqual({ok, bin(test_key())}, file:read_file(FileIssuerPem)), %% no old file to delete @@ -251,8 +251,8 @@ ssl_file_replace_test() -> {ok, SSL3} = emqx_tls_lib:ensure_ssl_files(Dir, SSL1), File1 = maps:get(<<"keyfile">>, SSL2), File2 = maps:get(<<"keyfile">>, SSL3), - IssuerPem1 = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2), - IssuerPem2 = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL3), + IssuerPem1 = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2), + IssuerPem2 = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL3), ?assert(filelib:is_regular(File1)), ?assert(filelib:is_regular(File2)), ?assert(filelib:is_regular(IssuerPem1)), diff --git a/apps/emqx_authn/rebar.config b/apps/emqx_authn/rebar.config index 8fd9cea0f..5f0043c39 100644 --- a/apps/emqx_authn/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_connector, {path, "../emqx_connector"}} ]}. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index fc026a671..de856f163 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -929,7 +929,7 @@ aggregate_metrics([]) -> aggregate_metrics([HeadMetrics | AllMetrics]) -> ErrorLogger = fun(Reason) -> ?SLOG(info, #{msg => "bad_metrics_value", error => Reason}) end, Fun = fun(ElemMap, AccMap) -> - emqx_map_lib:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) + emqx_utils_maps:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) end, lists:foldl(Fun, HeadMetrics, AllMetrics). @@ -1069,7 +1069,7 @@ update_user(ChainName, AuthenticatorID, UserID, UserInfo0) -> true -> serialize_error({missing_parameter, password}); false -> - UserInfo = emqx_map_lib:safe_atom_key_map(UserInfo0), + UserInfo = emqx_utils_maps:safe_atom_key_map(UserInfo0), case emqx_authentication:update_user(ChainName, AuthenticatorID, UserID, UserInfo) of {ok, User} -> {200, User}; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 9be1d0a33..3c34d878e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -357,7 +357,7 @@ qs([{K, V} | More], Acc) -> qs(More, [["&", uri_encode(K), "=", uri_encode(V)] | Acc]). serialize_body(<<"application/json">>, Body) -> - emqx_json:encode(Body); + emqx_utils_json:encode(Body); serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> qs(maps:to_list(Body)). @@ -395,7 +395,7 @@ safely_parse_body(ContentType, Body) -> end. parse_body(<<"application/json", _/binary>>, Body) -> - {ok, emqx_json:decode(Body, [return_maps])}; + {ok, emqx_utils_json:decode(Body, [return_maps])}; parse_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> Flags = [<<"result">>, <<"is_superuser">>], RawMap = maps:from_list(cow_qs:parse_qs(Body)), diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl index 5ee923859..23d939f7d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl @@ -99,7 +99,7 @@ handle_info( State1; {StatusLine, Headers, Body} -> try - JWKS = jose_jwk:from(emqx_json:decode(Body, [return_maps])), + JWKS = jose_jwk:from(emqx_utils_json:decode(Body, [return_maps])), {_, JWKs} = JWKS#jose_jwk.keys, State1#{jwks := JWKs} catch diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 73ba6a2c1..a891a55e2 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -407,7 +407,7 @@ do_verify(_JWT, [], _VerifyClaims) -> do_verify(JWT, [JWK | More], VerifyClaims) -> try jose_jws:verify(JWK, JWT) of {true, Payload, _JWT} -> - Claims0 = emqx_json:decode(Payload, [return_maps]), + Claims0 = emqx_utils_json:decode(Payload, [return_maps]), Claims = try_convert_to_num(Claims0, [<<"exp">>, <<"iat">>, <<"nbf">>]), case verify_claims(Claims, VerifyClaims) of ok -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index 25a3a5976..20b604e70 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -332,7 +332,7 @@ run_fuzzy_filter( %% Example: data/user-credentials.json import_users_from_json(Bin, #{user_group := UserGroup}) -> - case emqx_json:safe_decode(Bin, [return_maps]) of + case emqx_utils_json:safe_decode(Bin, [return_maps]) of {ok, List} -> trans(fun ?MODULE:import/2, [UserGroup, List]); {error, Reason} -> diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 11e2c6773..c7f718dfc 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -29,7 +29,7 @@ -define(assertAuthenticatorsMatch(Guard, Path), (fun() -> {ok, 200, Response} = request(get, uri(Path)), - ?assertMatch(Guard, jiffy:decode(Response, [return_maps])) + ?assertMatch(Guard, emqx_utils_json:decode(Response, [return_maps])) end)() ). @@ -234,7 +234,7 @@ test_authenticator(PathPrefix) -> get, uri(PathPrefix ++ [?CONF_NS, "password_based:http", "status"]) ), - {ok, RList} = emqx_json:safe_decode(Res), + {ok, RList} = emqx_utils_json:safe_decode(Res), Snd = fun({_, Val}) -> Val end, LookupVal = fun LookupV(List, RestJson) -> case List of @@ -353,7 +353,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 0, <<"nomatch">> := 1 } - } = jiffy:decode(PageData0, [return_maps]); + } = emqx_utils_json:decode(PageData0, [return_maps]); ["listeners", 'tcp:default'] -> #{ <<"metrics">> := #{ @@ -361,7 +361,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 0, <<"nomatch">> := 1 } - } = jiffy:decode(PageData0, [return_maps]) + } = emqx_utils_json:decode(PageData0, [return_maps]) end, InvalidUsers = [ @@ -384,7 +384,7 @@ test_authenticator_users(PathPrefix) -> lists:foreach( fun(User) -> {ok, 201, UserData} = request(post, UsersUri, User), - CreatedUser = jiffy:decode(UserData, [return_maps]), + CreatedUser = emqx_utils_json:decode(UserData, [return_maps]), ?assertMatch(#{<<"user_id">> := _}, CreatedUser) end, ValidUsers @@ -411,7 +411,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 1, <<"nomatch">> := 1 } - } = jiffy:decode(PageData01, [return_maps]); + } = emqx_utils_json:decode(PageData01, [return_maps]); ["listeners", 'tcp:default'] -> #{ <<"metrics">> := #{ @@ -419,7 +419,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 1, <<"nomatch">> := 1 } - } = jiffy:decode(PageData01, [return_maps]) + } = emqx_utils_json:decode(PageData01, [return_maps]) end, {ok, 200, Page1Data} = request(get, UsersUri ++ "?page=1&limit=2"), @@ -433,7 +433,7 @@ test_authenticator_users(PathPrefix) -> <<"count">> := 3 } } = - jiffy:decode(Page1Data, [return_maps]), + emqx_utils_json:decode(Page1Data, [return_maps]), {ok, 200, Page2Data} = request(get, UsersUri ++ "?page=2&limit=2"), @@ -445,7 +445,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 2, <<"count">> := 3 } - } = jiffy:decode(Page2Data, [return_maps]), + } = emqx_utils_json:decode(Page2Data, [return_maps]), ?assertEqual(2, length(Page1Users)), ?assertEqual(1, length(Page2Users)), @@ -465,7 +465,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 3, <<"count">> := 1 } - } = jiffy:decode(Super1Data, [return_maps]), + } = emqx_utils_json:decode(Super1Data, [return_maps]), ?assertEqual( [<<"u2">>], @@ -482,7 +482,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 3, <<"count">> := 2 } - } = jiffy:decode(Super2Data, [return_maps]), + } = emqx_utils_json:decode(Super2Data, [return_maps]), ?assertEqual( [<<"u1">>, <<"u3">>], @@ -509,7 +509,7 @@ test_authenticator_user(PathPrefix) -> {ok, 200, UserData} = request(get, UsersUri ++ "/u1"), - FetchedUser = jiffy:decode(UserData, [return_maps]), + FetchedUser = emqx_utils_json:decode(UserData, [return_maps]), ?assertMatch(#{<<"user_id">> := <<"u1">>}, FetchedUser), ?assertNotMatch(#{<<"password">> := _}, FetchedUser), diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 9a3c7c833..851e80f6d 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -41,7 +41,7 @@ -define(SERVER_RESPONSE_JSON(Result), ?SERVER_RESPONSE_JSON(Result, false)). -define(SERVER_RESPONSE_JSON(Result, IsSuperuser), - jiffy:encode(#{ + emqx_utils_json:encode(#{ result => Result, is_superuser => IsSuperuser }) @@ -172,11 +172,11 @@ t_no_value_for_placeholder(_Config) -> #{ <<"cert_subject">> := <<"">>, <<"cert_common_name">> := <<"">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -444,7 +444,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req0 ), {ok, Req, State} @@ -459,7 +459,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => true}), + emqx_utils_json:encode(#{result => allow, is_superuser => true}), Req0 ), {ok, Req, State} @@ -512,11 +512,11 @@ samples() -> #{ <<"username">> := <<"plain">>, <<"password">> := <<"plain">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -539,7 +539,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -565,11 +565,11 @@ samples() -> <<"peerhost">> := <<"127.0.0.1">>, <<"cert_subject">> := <<"cert_subject_data">>, <<"cert_common_name">> := <<"cert_common_name_data">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} diff --git a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl index 7d51ff425..c4315b69f 100644 --- a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl @@ -168,7 +168,7 @@ cowboy_handler(Req0, State) -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req0 ), {ok, Req, State}. diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 7a51d2bbb..94c07ca96 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -467,7 +467,7 @@ jwks_handler(Req0, State) -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(JWKS), + emqx_utils_json:encode(JWKS), Req0 ), {ok, Req, State}. diff --git a/apps/emqx_authz/rebar.config b/apps/emqx_authz/rebar.config index da2fa7807..9fd61b060 100644 --- a/apps/emqx_authz/rebar.config +++ b/apps/emqx_authz/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info, nowarn_unused_import]}. {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_connector, {path, "../emqx_connector"}} ]}. diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 0c2dee340..2220e8f6e 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -403,7 +403,7 @@ aggregate_metrics([]) -> aggregate_metrics([HeadMetrics | AllMetrics]) -> ErrorLogger = fun(Reason) -> ?SLOG(info, #{msg => "bad_metrics_value", error => Reason}) end, Fun = fun(ElemMap, AccMap) -> - emqx_map_lib:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) + emqx_utils_maps:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) end, lists:foldl(Fun, HeadMetrics, AllMetrics). diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index 9aa2d506f..ede4a9582 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -47,7 +47,7 @@ create(#{path := Path} = Source) -> ?SLOG(alert, #{ msg => failed_to_read_acl_file, path => Path, - explain => emqx_misc:explain_posix(Reason) + explain => emqx_utils:explain_posix(Reason) }), throw(failed_to_read_acl_file); {error, Reason} -> diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 852a667c8..53378d9c2 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -227,7 +227,7 @@ encode_path(Path) -> lists:flatten(["/" ++ Part || Part <- lists:map(fun uri_encode/1, Parts)]). serialize_body(<<"application/json">>, Body) -> - jsx:encode(Body); + emqx_utils_json:encode(Body); serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> query_string(Body). diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 6630ed526..f03ae52a8 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -337,7 +337,7 @@ check_ssl_opts(Conf) -> (#{<<"url">> := Url} = Source) -> case emqx_authz_http:parse_url(Url) of {<<"https", _/binary>>, _, _} -> - case emqx_map_lib:deep_find([<<"ssl">>, <<"enable">>], Source) of + case emqx_utils_maps:deep_find([<<"ssl">>, <<"enable">>], Source) of {ok, true} -> true; {ok, false} -> throw({ssl_not_enable, Url}); _ -> throw({ssl_enable_not_found, Url}) diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index df77673a2..560141d0a 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -144,7 +144,7 @@ parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> end; parse_http_resp_body(<<"application/json", _/binary>>, Body) -> try - result(emqx_json:decode(Body, [return_maps])) + result(emqx_utils_json:decode(Body, [return_maps])) catch _:_ -> error end. diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index 45e6d7287..ab673b225 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -60,19 +60,19 @@ set_special_configs(emqx_authz) -> set_special_configs(_App) -> ok. -t_clean_cahce(_) -> +t_clean_cache(_) -> {ok, C} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), {ok, _} = emqtt:connect(C), {ok, _, _} = emqtt:subscribe(C, <<"a/b/c">>, 0), ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0), {ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(2, length(emqx_json:decode(Result3))), + ?assertEqual(2, length(emqx_utils_json:decode(Result3))), request(delete, uri(["authorization", "cache"])), {ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(0, length(emqx_json:decode(Result4))), + ?assertEqual(0, length(emqx_utils_json:decode(Result4))), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index 2aa2d9545..3775b9a1c 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -95,7 +95,7 @@ t_api(_) -> <<"page">> := 1, <<"hasnext">> := false } - } = jsx:decode(Request1), + } = emqx_utils_json:decode(Request1), ?assertEqual(3, length(Rules1)), {ok, 200, Request1_1} = @@ -119,7 +119,7 @@ t_api(_) -> <<"hasnext">> => false } }, - jsx:decode(Request1_1) + emqx_utils_json:decode(Request1_1) ), {ok, 200, Request2} = @@ -128,7 +128,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users", "user1"]), [] ), - #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = jsx:decode(Request2), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = emqx_utils_json:decode(Request2), {ok, 204, _} = request( @@ -142,7 +142,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users", "user1"]), [] ), - #{<<"username">> := <<"user1">>, <<"rules">> := Rules2} = jsx:decode(Request3), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules2} = emqx_utils_json:decode(Request3), ?assertEqual(0, length(Rules2)), {ok, 204, _} = @@ -202,8 +202,8 @@ t_api(_) -> <<"data">> := [#{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3}], <<"meta">> := #{<<"count">> := 1, <<"limit">> := 100, <<"page">> := 1} } = - jsx:decode(Request4), - #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = jsx:decode(Request5), + emqx_utils_json:decode(Request4), + #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = emqx_utils_json:decode(Request5), ?assertEqual(3, length(Rules3)), {ok, 204, _} = @@ -218,7 +218,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "clients", "client1"]), [] ), - #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules4} = jsx:decode(Request6), + #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules4} = emqx_utils_json:decode(Request6), ?assertEqual(0, length(Rules4)), {ok, 204, _} = @@ -252,7 +252,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "all"]), [] ), - #{<<"rules">> := Rules5} = jsx:decode(Request7), + #{<<"rules">> := Rules5} = emqx_utils_json:decode(Request7), ?assertEqual(3, length(Rules5)), {ok, 204, _} = @@ -267,7 +267,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "all"]), [] ), - #{<<"rules">> := Rules6} = jsx:decode(Request8), + #{<<"rules">> := Rules6} = emqx_utils_json:decode(Request8), ?assertEqual(0, length(Rules6)), {ok, 204, _} = @@ -285,7 +285,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users?page=2&limit=5"]), [] ), - #{<<"data">> := Data1} = jsx:decode(Request9), + #{<<"data">> := Data1} = emqx_utils_json:decode(Request9), ?assertEqual(5, length(Data1)), {ok, 204, _} = @@ -303,7 +303,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "clients?limit=5"]), [] ), - #{<<"data">> := Data2} = jsx:decode(Request10), + #{<<"data">> := Data2} = emqx_utils_json:decode(Request10), ?assertEqual(5, length(Data2)), {ok, 400, Msg1} = diff --git a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl index 41eba109e..e3412e169 100644 --- a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl @@ -76,7 +76,7 @@ t_api(_) -> {ok, 200, Result1} = request(put, uri(["authorization", "settings"]), Settings1), {ok, 200, Result1} = request(get, uri(["authorization", "settings"]), []), - ?assertEqual(Settings1, jsx:decode(Result1)), + ?assertEqual(Settings1, emqx_utils_json:decode(Result1)), Settings2 = #{ <<"no_match">> => <<"allow">>, @@ -90,7 +90,7 @@ t_api(_) -> {ok, 200, Result2} = request(put, uri(["authorization", "settings"]), Settings2), {ok, 200, Result2} = request(get, uri(["authorization", "settings"]), []), - ?assertEqual(Settings2, jsx:decode(Result2)), + ?assertEqual(Settings2, emqx_utils_json:decode(Result2)), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 411399d64..7a7dbb7e9 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -148,8 +148,8 @@ set_special_configs(_App) -> ok. init_per_testcase(t_api, Config) -> - meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_misc, gen_id, fun() -> "fake" end), + meck:new(emqx_utils, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_utils, gen_id, fun() -> "fake" end), meck:new(emqx, [non_strict, passthrough, no_history, no_link]), meck:expect( @@ -165,7 +165,7 @@ init_per_testcase(_, Config) -> Config. end_per_testcase(t_api, _Config) -> - meck:unload(emqx_misc), + meck:unload(emqx_utils), meck:unload(emqx), ok; end_per_testcase(_, _Config) -> @@ -182,7 +182,7 @@ t_api(_) -> {ok, 404, ErrResult} = request(get, uri(["authorization", "sources", "http"]), []), ?assertMatch( #{<<"code">> := <<"NOT_FOUND">>, <<"message">> := <<"Not found: http">>}, - emqx_json:decode(ErrResult, [return_maps]) + emqx_utils_json:decode(ErrResult, [return_maps]) ), [ @@ -215,7 +215,8 @@ t_api(_) -> ), {ok, 200, Result3} = request(get, uri(["authorization", "sources", "http"]), []), ?assertMatch( - #{<<"type">> := <<"http">>, <<"enable">> := false}, emqx_json:decode(Result3, [return_maps]) + #{<<"type">> := <<"http">>, <<"enable">> := false}, + emqx_utils_json:decode(Result3, [return_maps]) ), Keyfile = emqx_common_test_helpers:app_path( @@ -253,7 +254,7 @@ t_api(_) -> <<"total">> := 0, <<"nomatch">> := 0 } - } = emqx_json:decode(Status4, [return_maps]), + } = emqx_utils_json:decode(Status4, [return_maps]), ?assertMatch( #{ <<"type">> := <<"mongodb">>, @@ -265,7 +266,7 @@ t_api(_) -> <<"verify">> := <<"verify_none">> } }, - emqx_json:decode(Result4, [return_maps]) + emqx_utils_json:decode(Result4, [return_maps]) ), {ok, Cacert} = file:read_file(Cacertfile), @@ -297,7 +298,7 @@ t_api(_) -> <<"verify">> := <<"verify_none">> } }, - emqx_json:decode(Result5, [return_maps]) + emqx_utils_json:decode(Result5, [return_maps]) ), {ok, 200, Status5_1} = request(get, uri(["authorization", "sources", "mongodb", "status"]), []), @@ -308,7 +309,7 @@ t_api(_) -> <<"total">> := 0, <<"nomatch">> := 0 } - } = emqx_json:decode(Status5_1, [return_maps]), + } = emqx_utils_json:decode(Status5_1, [return_maps]), #{ ssl := #{ @@ -355,7 +356,7 @@ t_api(_) -> <<"code">> := <<"BAD_REQUEST">>, <<"message">> := <<"Type mismatch", _/binary>> }, - emqx_json:decode(TypeMismatch, [return_maps]) + emqx_utils_json:decode(TypeMismatch, [return_maps]) ), lists:foreach( @@ -443,7 +444,7 @@ t_api(_) -> <<"total">> := 1, <<"nomatch">> := 0 } - } = emqx_json:decode(Status5, [return_maps]) + } = emqx_utils_json:decode(Status5, [return_maps]) end ), @@ -469,7 +470,7 @@ t_api(_) -> <<"total">> := 2, <<"nomatch">> := 0 } - } = emqx_json:decode(Status6, [return_maps]) + } = emqx_utils_json:decode(Status6, [return_maps]) end ), @@ -495,7 +496,7 @@ t_api(_) -> <<"total">> := 3, <<"nomatch">> := 0 } - } = emqx_json:decode(Status7, [return_maps]) + } = emqx_utils_json:decode(Status7, [return_maps]) end ), ok. @@ -621,7 +622,7 @@ t_aggregate_metrics(_) -> ). get_sources(Result) -> - maps:get(<<"sources">>, emqx_json:decode(Result, [return_maps])). + maps:get(<<"sources">>, emqx_utils_json:decode(Result, [return_maps])). data_dir() -> emqx:data_dir(). diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index 0757113f3..9ff84b805 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -311,7 +311,7 @@ t_json_body(_Config) -> <<"topic">> := <<"t">>, <<"action">> := <<"publish">> }, - jiffy:decode(RawBody, [return_maps]) + emqx_utils_json:decode(RawBody, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, @@ -366,7 +366,7 @@ t_placeholder_and_body(_Config) -> <<"CN">> := ?PH_CERT_CN_NAME, <<"CS">> := ?PH_CERT_SUBJECT }, - jiffy:decode(PostVars, [return_maps]) + emqx_utils_json:decode(PostVars, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, @@ -418,7 +418,7 @@ t_no_value_for_placeholder(_Config) -> #{ <<"mountpoint">> := <<"[]">> }, - jiffy:decode(RawBody, [return_maps]) + emqx_utils_json:decode(RawBody, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, diff --git a/apps/emqx_auto_subscribe/rebar.config b/apps/emqx_auto_subscribe/rebar.config index 33e077f50..a19783033 100644 --- a/apps/emqx_auto_subscribe/rebar.config +++ b/apps/emqx_auto_subscribe/rebar.config @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {shell, [ {apps, [emqx_auto_subscribe]} diff --git a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl index 5c5a3ee79..9d8d47bf2 100644 --- a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl +++ b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl @@ -141,7 +141,7 @@ t_update(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Body = [#{topic => ?TOPIC_S}], {ok, Response} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, Body), - ResponseMap = emqx_json:decode(Response, [return_maps]), + ResponseMap = emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(1, erlang:length(ResponseMap)), BadBody1 = #{topic => ?TOPIC_S}, @@ -177,7 +177,7 @@ t_update(_) -> emqtt:disconnect(Client), {ok, GETResponse} = emqx_mgmt_api_test_util:request_api(get, Path), - GETResponseMap = emqx_json:decode(GETResponse, [return_maps]), + GETResponseMap = emqx_utils_json:decode(GETResponse, [return_maps]), ?assertEqual(1, erlang:length(GETResponseMap)), ok. diff --git a/apps/emqx_bridge/rebar.config b/apps/emqx_bridge/rebar.config index 79f2caf50..864c45e9a 100644 --- a/apps/emqx_bridge/rebar.config +++ b/apps/emqx_bridge/rebar.config @@ -1,7 +1,9 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../emqx"}} - , {emqx_resource, {path, "../../apps/emqx_resource"}} - ]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}} + ]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 2a20c5994..08b8222f2 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -207,7 +207,7 @@ send_message(BridgeId, Message) -> end. query_opts(Config) -> - case emqx_map_lib:deep_get([resource_opts, request_timeout], Config, false) of + case emqx_utils_maps:deep_get([resource_opts, request_timeout], Config, false) of Timeout when is_integer(Timeout) -> %% request_timeout is configured #{timeout => Timeout}; @@ -296,7 +296,7 @@ create(BridgeType, BridgeName, RawConf) -> brige_action => create, bridge_type => BridgeType, bridge_name => BridgeName, - bridge_raw_config => emqx_misc:redact(RawConf) + bridge_raw_config => emqx_utils:redact(RawConf) }), emqx_conf:update( emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], @@ -367,7 +367,7 @@ perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) -> perform_bridge_changes(Tasks, Result). diff_confs(NewConfs, OldConfs) -> - emqx_map_lib:diff_maps( + emqx_utils_maps:diff_maps( flatten_confs(NewConfs), flatten_confs(OldConfs) ). diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index b29cefacd..c87ea8ea6 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -20,7 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_api_lib.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). -import(hoconsc, [mk/2, array/1, enum/1]). @@ -220,7 +220,7 @@ info_example_basic(webhook) -> auto_restart_interval => 15000, query_mode => async, inflight_window => 100, - max_queue_bytes => 100 * 1024 * 1024 + max_buffer_bytes => 100 * 1024 * 1024 } }; info_example_basic(mqtt) -> @@ -245,7 +245,7 @@ mqtt_main_example() -> health_check_interval => <<"15s">>, auto_restart_interval => <<"60s">>, query_mode => sync, - max_queue_bytes => 100 * 1024 * 1024 + max_buffer_bytes => 100 * 1024 * 1024 }, ssl => #{ enable => false @@ -668,7 +668,7 @@ get_metrics_from_local_node(BridgeType, BridgeName) -> false -> ?BRIDGE_NOT_ENABLED; true -> - case emqx_misc:safe_to_existing_atom(Node, utf8) of + case emqx_utils:safe_to_existing_atom(Node, utf8) of {ok, TargetNode} -> call_operation(TargetNode, OperFunc, [ TargetNode, BridgeType, BridgeName @@ -835,7 +835,7 @@ format_resource_data(ResData) -> format_resource_data(error, undefined, Result) -> Result; format_resource_data(error, Error, Result) -> - Result#{status_reason => emqx_misc:readable_error_msg(Error)}; + Result#{status_reason => emqx_utils:readable_error_msg(Error)}; format_resource_data(K, V, Result) -> Result#{K => V}. @@ -1004,7 +1004,7 @@ supported_versions(get_metrics_from_all_nodes) -> [4]; supported_versions(_Call) -> [1, 2, 3, 4]. redact(Term) -> - emqx_misc:redact(Term). + emqx_utils:redact(Term). deobfuscate(NewConf, OldConf) -> maps:fold( @@ -1015,7 +1015,7 @@ deobfuscate(NewConf, OldConf) -> {ok, OldV} when is_map(V), is_map(OldV) -> Acc#{K => deobfuscate(V, OldV)}; {ok, OldV} -> - case emqx_misc:is_redacted(K, V) of + case emqx_utils:is_redacted(K, V) of true -> Acc#{K => OldV}; _ -> @@ -1028,6 +1028,6 @@ deobfuscate(NewConf, OldConf) -> ). map_to_json(M) -> - emqx_json:encode( - emqx_map_lib:jsonable_map(M, fun(K, V) -> {K, emqx_map_lib:binary_string(V)} end) + emqx_utils_json:encode( + emqx_utils_maps:jsonable_map(M, fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end) ). diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index b43cbe0ec..347f9d973 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -157,7 +157,7 @@ create(Type, Name, Conf, Opts0) -> msg => "create bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), Opts = override_start_after_created(Conf, Opts0), {ok, _Data} = emqx_resource:create_local( @@ -186,13 +186,13 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> %% without restarting the bridge. %% Opts = override_start_after_created(Conf, Opts0), - case emqx_map_lib:if_only_to_toggle_enable(OldConf, Conf) of + case emqx_utils_maps:if_only_to_toggle_enable(OldConf, Conf) of false -> ?SLOG(info, #{ msg => "update bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), case recreate(Type, Name, Conf, Opts) of {ok, _} -> @@ -202,7 +202,7 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> msg => "updating_a_non_existing_bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), create(Type, Name, Conf, Opts); {error, Reason} -> @@ -236,9 +236,9 @@ recreate(Type, Name, Conf, Opts) -> ). create_dry_run(Type, Conf0) -> - TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_misc:gen_id(8)]), - TmpPath = emqx_misc:safe_filename(TmpPath0), - Conf = emqx_map_lib:safe_atom_key_map(Conf0), + TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), + TmpPath = emqx_utils:safe_filename(TmpPath0), + Conf = emqx_utils_maps:safe_atom_key_map(Conf0), case emqx_connector_ssl:convert_certs(TmpPath, Conf) of {error, Reason} -> {error, Reason}; diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl index fe173fa89..595b75ecf 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl @@ -89,7 +89,7 @@ default_resource_opts() -> <<"inflight_window">> => 100, <<"auto_restart_interval">> => <<"60s">>, <<"health_check_interval">> => <<"15s">>, - <<"max_queue_bytes">> => <<"1GB">>, + <<"max_buffer_bytes">> => <<"1GB">>, <<"query_mode">> => <<"sync">>, %% there is only one underlying MQTT connection %% doesn't make a lot of sense to have a large pool diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 52000ba7f..e5def2d64 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -251,7 +251,7 @@ do_convert_webhook_config( case {MReqTRoot, MReqTResource} of {{ok, ReqTRoot}, {ok, ReqTResource}} -> {_Parsed, ReqTRaw} = max({ReqTRoot, ReqTRootRaw}, {ReqTResource, ReqTResourceRaw}), - Conf1 = emqx_map_lib:deep_merge( + Conf1 = emqx_utils_maps:deep_merge( Conf0, #{ <<"request_timeout">> => ReqTRaw, diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 4d5495ffe..3afe17080 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -975,7 +975,7 @@ t_with_redact_update(Config) -> ), %% update with redacted config - BridgeConf = emqx_misc:redact(Template), + BridgeConf = emqx_utils:redact(Template), BridgeID = emqx_bridge_resource:bridge_id(Type, Name), {ok, 200, _} = request(put, uri(["bridges", BridgeID]), BridgeConf, Config), ?assertEqual( @@ -1221,7 +1221,7 @@ t_inconsistent_webhook_request_timeouts(Config) -> URL1 = ?URL(Port, "path1"), Name = ?BRIDGE_NAME, BadBridgeParams = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ?HTTP_BRIDGE(URL1, Name), #{ <<"request_timeout">> => <<"1s">>, @@ -1304,4 +1304,4 @@ str(S) when is_list(S) -> S; str(S) when is_binary(S) -> binary_to_list(S). json(B) when is_binary(B) -> - emqx_json:decode(B, [return_maps]). + emqx_utils_json:decode(B, [return_maps]). diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index bd6af9323..bd5cda3f0 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -201,7 +201,7 @@ t_mqtt_conn_bridge_ingress(_) -> #{ <<"type">> := ?TYPE_MQTT, <<"name">> := ?BRIDGE_NAME_INGRESS - } = jsx:decode(Bridge), + } = emqx_utils_json:decode(Bridge), BridgeIDIngress = emqx_bridge_resource:bridge_id(?TYPE_MQTT, ?BRIDGE_NAME_INGRESS), @@ -270,7 +270,7 @@ t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> ?SERVER_CONF(<<"user1">>)#{ <<"type">> => ?TYPE_MQTT, <<"name">> => BridgeName, - <<"ingress">> => emqx_map_lib:deep_merge( + <<"ingress">> => emqx_utils_maps:deep_merge( ?INGRESS_CONF, #{<<"remote">> => #{<<"qos">> => 2}} ) @@ -313,7 +313,7 @@ t_mqtt_conn_bridge_ingress_no_payload_template(_) -> emqx:publish(emqx_message:make(RemoteTopic, Payload)), %% we should receive a message on the local broker, with specified topic Msg = assert_mqtt_msg_received(LocalTopic), - ?assertMatch(#{<<"payload">> := Payload}, jsx:decode(Msg#message.payload)), + ?assertMatch(#{<<"payload">> := Payload}, emqx_utils_json:decode(Msg#message.payload)), %% verify the metrics of the bridge ?assertMetrics( @@ -402,7 +402,7 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> Msg = assert_mqtt_msg_received(RemoteTopic), %% the MapMsg is all fields outputed by Rule-Engine. it's a binary coded json here. ?assertMatch(<>, Msg#message.from), - ?assertMatch(#{<<"payload">> := Payload}, jsx:decode(Msg#message.payload)), + ?assertMatch(#{<<"payload">> := Payload}, emqx_utils_json:decode(Msg#message.payload)), %% verify the metrics of the bridge ?retry( @@ -545,7 +545,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> <<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">> } ), - #{<<"id">> := RuleId} = jsx:decode(Rule), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule), %% we now test if the bridge works as expected @@ -562,7 +562,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> %% and also the rule should be matched, with matched + 1: {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), {ok, 200, Metrics} = request(get, uri(["rules", RuleId, "metrics"]), []), - ?assertMatch(#{<<"id">> := RuleId}, jsx:decode(Rule1)), + ?assertMatch(#{<<"id">> := RuleId}, emqx_utils_json:decode(Rule1)), ?assertMatch( #{ <<"metrics">> := #{ @@ -581,7 +581,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> <<"actions.failed.unknown">> := 0 } }, - jsx:decode(Metrics) + emqx_utils_json:decode(Metrics) ), %% we also check if the actions of the rule is triggered @@ -630,7 +630,7 @@ t_egress_mqtt_bridge_with_rules(_) -> <<"sql">> => <<"SELECT * from \"t/1\"">> } ), - #{<<"id">> := RuleId} = jsx:decode(Rule), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule), %% we now test if the bridge works as expected LocalTopic = <>, @@ -653,7 +653,7 @@ t_egress_mqtt_bridge_with_rules(_) -> timer:sleep(100), emqx:publish(emqx_message:make(RuleTopic, Payload2)), {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), - ?assertMatch(#{<<"id">> := RuleId, <<"name">> := _}, jsx:decode(Rule1)), + ?assertMatch(#{<<"id">> := RuleId, <<"name">> := _}, emqx_utils_json:decode(Rule1)), {ok, 200, Metrics} = request(get, uri(["rules", RuleId, "metrics"]), []), ?assertMatch( #{ @@ -673,7 +673,7 @@ t_egress_mqtt_bridge_with_rules(_) -> <<"actions.failed.unknown">> := 0 } }, - jsx:decode(Metrics) + emqx_utils_json:decode(Metrics) ), %% we should receive a message on the "remote" broker, with specified topic @@ -911,17 +911,17 @@ create_bridge(Config = #{<<"type">> := Type, <<"name">> := Name}) -> <<"type">> := Type, <<"name">> := Name }, - jsx:decode(Bridge) + emqx_utils_json:decode(Bridge) ), emqx_bridge_resource:bridge_id(Type, Name). request_bridge(BridgeID) -> {ok, 200, Bridge} = request(get, uri(["bridges", BridgeID]), []), - jsx:decode(Bridge). + emqx_utils_json:decode(Bridge). request_bridge_metrics(BridgeID) -> {ok, 200, BridgeMetrics} = request(get, uri(["bridges", BridgeID, "metrics"]), []), - jsx:decode(BridgeMetrics). + emqx_utils_json:decode(BridgeMetrics). request(Method, Url, Body) -> request(<<"connector_admin">>, Method, Url, Body). diff --git a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl index e222190d2..f08c87b6e 100644 --- a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl @@ -175,7 +175,7 @@ bridge_async_config(#{port := Port} = Config) -> " inflight_window = 100\n" " auto_restart_interval = \"60s\"\n" " health_check_interval = \"15s\"\n" - " max_queue_bytes = \"1GB\"\n" + " max_buffer_bytes = \"1GB\"\n" " query_mode = \"~s\"\n" " request_timeout = \"~s\"\n" " start_after_created = \"true\"\n" diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index 2dc43a130..fdfa3300c 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -156,7 +156,7 @@ on_start(InstanceId, Config) -> msg => "failed_to_start_kafka_consumer_client", instance_id => InstanceId, kafka_hosts => BootstrapHosts, - reason => emqx_misc:redact(Reason) + reason => emqx_utils:redact(Reason) }), throw(?CLIENT_DOWN_MESSAGE) end, @@ -344,7 +344,7 @@ start_consumer(Config, InstanceId, ClientID) -> msg => "failed_to_start_kafka_consumer", instance_id => InstanceId, kafka_hosts => emqx_bridge_kafka_impl:hosts(BootstrapHosts0), - reason => emqx_misc:redact(Reason2) + reason => emqx_utils:redact(Reason2) }), stop_client(ClientID), throw(failed_to_start_kafka_consumer) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index fb7cf524c..08fbf5e15 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -299,7 +299,7 @@ init_per_testcase(TestCase, Config) when common_init_per_testcase(TestCase, Config) end; init_per_testcase(t_cluster_group = TestCase, Config0) -> - Config = emqx_misc:merge_opts(Config0, [{num_partitions, 6}]), + Config = emqx_utils:merge_opts(Config0, [{num_partitions, 6}]), common_init_per_testcase(TestCase, Config); init_per_testcase(t_multiple_topic_mappings = TestCase, Config0) -> KafkaTopicBase = @@ -673,7 +673,7 @@ create_bridge(Config, Overrides) -> Type = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), emqx_bridge:create(Type, Name, KafkaConfig). delete_bridge(Config) -> @@ -696,7 +696,7 @@ create_bridge_api(Config, Overrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), @@ -705,7 +705,7 @@ create_bridge_api(Config, Overrides) -> Res = case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of {ok, {Status, Headers, Body0}} -> - {ok, {Status, Headers, emqx_json:decode(Body0, [return_maps])}}; + {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; Error -> Error end, @@ -719,7 +719,7 @@ update_bridge_api(Config, Overrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), BridgeId = emqx_bridge_resource:bridge_id(TypeBin, Name), Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId]), @@ -728,7 +728,7 @@ update_bridge_api(Config, Overrides) -> ct:pal("updating bridge (via http): ~p", [Params]), Res = case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, Params, Opts) of - {ok, {_Status, _Headers, Body0}} -> {ok, emqx_json:decode(Body0, [return_maps])}; + {ok, {_Status, _Headers, Body0}} -> {ok, emqx_utils_json:decode(Body0, [return_maps])}; Error -> Error end, ct:pal("bridge update result: ~p", [Res]), @@ -776,7 +776,7 @@ do_wait_for_expected_published_messages(Messages, Acc, _Timeout) when map_size(M do_wait_for_expected_published_messages(Messages0, Acc0, Timeout) -> receive {publish, Msg0 = #{payload := Payload}} -> - case emqx_json:safe_decode(Payload, [return_maps]) of + case emqx_utils_json:safe_decode(Payload, [return_maps]) of {error, _} -> ct:pal("unexpected message: ~p; discarding", [Msg0]), do_wait_for_expected_published_messages(Messages0, Acc0, Timeout); @@ -928,7 +928,7 @@ create_rule_and_action_http(Config) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), ct:pal("rule action params: ~p", [Params]), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -1188,7 +1188,7 @@ t_start_and_consume_ok(Config) -> <<"offset">> := OffsetReply, <<"headers">> := #{<<"hkey">> := <<"hvalue">>} }, - emqx_json:decode(PayloadBin, [return_maps]), + emqx_utils_json:decode(PayloadBin, [return_maps]), #{ offset_reply => OffsetReply, kafka_topic => KafkaTopic, @@ -1300,7 +1300,7 @@ t_multiple_topic_mappings(Config) -> %% as configured. Payloads = lists:sort([ - case emqx_json:safe_decode(P, [return_maps]) of + case emqx_utils_json:safe_decode(P, [return_maps]) of {ok, Decoded} -> Decoded; {error, _} -> P end @@ -1441,7 +1441,7 @@ do_t_failed_creation_then_fixed(Config) -> <<"offset">> := _, <<"headers">> := #{<<"hkey">> := <<"hvalue">>} }, - emqx_json:decode(PayloadBin, [return_maps]), + emqx_utils_json:decode(PayloadBin, [return_maps]), #{ kafka_topic => KafkaTopic, payload => Payload @@ -1543,7 +1543,7 @@ do_t_receive_after_recovery(Config) -> %% 2) publish messages while the consumer is down. %% we use `pmap' to avoid wolff sending the whole %% batch to a single partition. - emqx_misc:pmap(fun(Msg) -> publish(Config, [Msg]) end, Messages1), + emqx_utils:pmap(fun(Msg) -> publish(Config, [Msg]) end, Messages1), ok end), %% 3) restore and consume messages @@ -1636,7 +1636,7 @@ t_bridge_rule_action_source(Config) -> <<"headers">> := #{<<"hkey">> := <<"hvalue">>}, <<"topic">> := KafkaTopic }, - emqx_json:decode(RawPayload, [return_maps]) + emqx_utils_json:decode(RawPayload, [return_maps]) ), ?retry( _Interval = 200, @@ -1667,7 +1667,7 @@ t_cluster_group(Config) -> || {Name, Opts} <- Cluster ], on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1889,7 +1889,7 @@ t_cluster_node_down(Config) -> Cluster ), on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -2004,7 +2004,7 @@ t_begin_offset_earliest(Config) -> %% the consumers Published = receive_published(#{n => NumMessages}), Payloads = lists:map( - fun(#{payload := P}) -> emqx_json:decode(P, [return_maps]) end, + fun(#{payload := P}) -> emqx_utils_json:decode(P, [return_maps]) end, Published ), ?assert( diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index 6e3ddf5bb..a2111b1a8 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -320,7 +320,7 @@ kafka_bridge_rest_api_helper(Config) -> <<"sql">> => <<"SELECT * from \"kafka_bridge_topic/#\"">> } ), - #{<<"id">> := RuleId} = emqx_json:decode(Rule, [return_maps]), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule, [return_maps]), %% counters should be empty before ?assertEqual(0, emqx_resource_metrics:matched_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:success_get(ResourceId)), @@ -802,7 +802,7 @@ api_path(Parts) -> ?HOST ++ filename:join([?BASE_PATH | Parts]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. delete_all_bridges() -> diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl index fa352ce89..b16df854f 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl @@ -92,7 +92,7 @@ kafka_consumer_test() -> ), %% Bad: can't repeat kafka topics. - BadConf1 = emqx_map_lib:deep_put( + BadConf1 = emqx_utils_maps:deep_put( [<<"bridges">>, <<"kafka_consumer">>, <<"my_consumer">>, <<"topic_mapping">>], Conf1, [ @@ -121,7 +121,7 @@ kafka_consumer_test() -> ), %% Bad: there must be at least 1 mapping. - BadConf2 = emqx_map_lib:deep_put( + BadConf2 = emqx_utils_maps:deep_put( [<<"bridges">>, <<"kafka_consumer">>, <<"my_consumer">>, <<"topic_mapping">>], Conf1, [] diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 0382045d4..c82191bc3 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -501,15 +501,17 @@ log_and_alarm(IsSuccess, Res, #{kind := ?APPLY_KIND_INITIATE} = Meta) -> %% because nothing is committed case IsSuccess of true -> - ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_result", result => emqx_misc:redact(Res)}); + ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_result", result => emqx_utils:redact(Res)}); false -> - ?SLOG(warning, Meta#{msg => "cluster_rpc_apply_result", result => emqx_misc:redact(Res)}) + ?SLOG(warning, Meta#{ + msg => "cluster_rpc_apply_result", result => emqx_utils:redact(Res) + }) end; log_and_alarm(true, Res, Meta) -> - ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_ok", result => emqx_misc:redact(Res)}), + ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_ok", result => emqx_utils:redact(Res)}), do_alarm(deactivate, Res, Meta); log_and_alarm(false, Res, Meta) -> - ?SLOG(error, Meta#{msg => "cluster_rpc_apply_failed", result => emqx_misc:redact(Res)}), + ?SLOG(error, Meta#{msg => "cluster_rpc_apply_failed", result => emqx_utils:redact(Res)}), do_alarm(activate, Res, Meta). do_alarm(Fun, Res, #{tnx_id := Id} = Meta) -> diff --git a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl index bce866c2d..fe72cd65b 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl @@ -73,7 +73,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -82,7 +82,7 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- ensure_timer(State = #{cleanup_ms := Ms}) -> - State#{timer := emqx_misc:start_timer(Ms, del_stale_mfa)}. + State#{timer := emqx_utils:start_timer(Ms, del_stale_mfa)}. %% @doc Keep the latest completed 100 records for querying and troubleshooting. del_stale_mfa(MaxHistory) -> diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index d03cf9c27..1ecda913d 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -43,50 +43,50 @@ add_handler(ConfKeyPath, HandlerName) -> remove_handler(ConfKeyPath) -> emqx_config_handler:remove_handler(ConfKeyPath). --spec get(emqx_map_lib:config_key_path()) -> term(). +-spec get(emqx_utils_maps:config_key_path()) -> term(). get(KeyPath) -> emqx:get_config(KeyPath). --spec get(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get(emqx_utils_maps:config_key_path(), term()) -> term(). get(KeyPath, Default) -> emqx:get_config(KeyPath, Default). --spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw(KeyPath, Default) -> emqx_config:get_raw(KeyPath, Default). --spec get_raw(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path()) -> term(). get_raw(KeyPath) -> emqx_config:get_raw(KeyPath). %% @doc Returns all values in the cluster. --spec get_all(emqx_map_lib:config_key_path()) -> #{node() => term()}. +-spec get_all(emqx_utils_maps:config_key_path()) -> #{node() => term()}. get_all(KeyPath) -> {ResL, []} = emqx_conf_proto_v2:get_all(KeyPath), maps:from_list(ResL). %% @doc Returns the specified node's KeyPath, or exception if not found --spec get_by_node(node(), emqx_map_lib:config_key_path()) -> term(). +-spec get_by_node(node(), emqx_utils_maps:config_key_path()) -> term(). get_by_node(Node, KeyPath) when Node =:= node() -> emqx:get_config(KeyPath); get_by_node(Node, KeyPath) -> emqx_conf_proto_v2:get_config(Node, KeyPath). %% @doc Returns the specified node's KeyPath, or the default value if not found --spec get_by_node(node(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_by_node(node(), emqx_utils_maps:config_key_path(), term()) -> term(). get_by_node(Node, KeyPath, Default) when Node =:= node() -> emqx:get_config(KeyPath, Default); get_by_node(Node, KeyPath, Default) -> emqx_conf_proto_v2:get_config(Node, KeyPath, Default). %% @doc Returns the specified node's KeyPath, or config_not_found if key path not found --spec get_node_and_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_node_and_config(emqx_utils_maps:config_key_path()) -> term(). get_node_and_config(KeyPath) -> {node(), emqx:get_config(KeyPath, config_not_found)}. %% @doc Update all value of key path in cluster-override.conf or local-override.conf. -spec update( - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -97,7 +97,7 @@ update(KeyPath, UpdateReq, Opts) -> %% @doc Update the specified node's key path in local-override.conf. -spec update( node(), - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -108,13 +108,13 @@ update(Node, KeyPath, UpdateReq, Opts) -> emqx_conf_proto_v2:update(Node, KeyPath, UpdateReq, Opts). %% @doc remove all value of key path in cluster-override.conf or local-override.conf. --spec remove(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove(KeyPath, Opts) -> emqx_conf_proto_v2:remove_config(KeyPath, Opts). %% @doc remove the specified node's key path in local-override.conf. --spec remove(node(), emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove(node(), emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove(Node, KeyPath, Opts) when Node =:= node() -> emqx:remove_config(KeyPath, Opts#{override_to => local}); @@ -122,13 +122,13 @@ remove(Node, KeyPath, Opts) -> emqx_conf_proto_v2:remove_config(Node, KeyPath, Opts). %% @doc reset all value of key path in cluster-override.conf or local-override.conf. --spec reset(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset(KeyPath, Opts) -> emqx_conf_proto_v2:reset(KeyPath, Opts). %% @doc reset the specified node's key path in local-override.conf. --spec reset(node(), emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset(node(), emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset(Node, KeyPath, Opts) when Node =:= node() -> emqx:reset_config(KeyPath, Opts#{override_to => local}); @@ -166,7 +166,7 @@ gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> io:format(user, "===< Including fields from importance level: ~p~n", [IncludeImportance]), Opts = #{desc_file => I18nFile, lang => Lang, include_importance_up_from => IncludeImportance}, JsonMap = hocon_schema_json:gen(SchemaModule, Opts), - IoData = jsx:encode(JsonMap, [space, {indent, 4}]), + IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]), ok = file:write_file(SchemaJsonFile, IoData). gen_api_schema_json(Dir, I18nFile, Lang) -> @@ -268,13 +268,13 @@ do_gen_api_schema_json(File, SchemaMod, SchemaInfo) -> ApiSpec0 ), Components = lists:foldl(fun(M, Acc) -> maps:merge(M, Acc) end, #{}, Components0), - IoData = jsx:encode( + IoData = emqx_utils_json:encode( #{ info => SchemaInfo, paths => ApiSpec, components => #{schemas => Components} }, - [space, {indent, 4}] + [pretty, force_utf8] ), file:write_file(File, IoData). diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl index 84687e314..b66307a1b 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl @@ -38,22 +38,22 @@ -include_lib("emqx/include/bpapi.hrl"). --type update_config_key_path() :: [emqx_map_lib:config_key(), ...]. +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. introduced_in() -> "5.0.0". --spec get_config(node(), emqx_map_lib:config_key_path()) -> +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath) -> rpc:call(Node, emqx, get_config, [KeyPath]). --spec get_config(node(), emqx_map_lib:config_key_path(), _Default) -> +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath, Default) -> rpc:call(Node, emqx, get_config, [KeyPath, Default]). --spec get_all(emqx_map_lib:config_key_path()) -> emqx_rpc:multicall_result(). +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). get_all(KeyPath) -> rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl index dd8d2fedd..97446ee9f 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -44,19 +44,19 @@ introduced_in() -> -spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). sync_data_from_node(Node) -> rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). --type update_config_key_path() :: [emqx_map_lib:config_key(), ...]. +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. --spec get_config(node(), emqx_map_lib:config_key_path()) -> +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath) -> rpc:call(Node, emqx, get_config, [KeyPath]). --spec get_config(node(), emqx_map_lib:config_key_path(), _Default) -> +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath, Default) -> rpc:call(Node, emqx, get_config, [KeyPath, Default]). --spec get_all(emqx_map_lib:config_key_path()) -> emqx_rpc:multicall_result(). +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). get_all(KeyPath) -> rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 2ce6b00f8..03be87356 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -7,6 +7,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_resource, {path, "../emqx_resource"}}, {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}}, diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 401fc8812..ef2e11eb7 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -219,7 +219,7 @@ on_start( SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, Config)), {tls, SSLOpts} end, - NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + NTransportOpts = emqx_utils:ipv6_probe(TransportOpts), PoolOpts = [ {host, Host}, {port, Port}, @@ -425,7 +425,7 @@ do_get_status(PoolName, Timeout) -> Error end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of % we crash in case of non-empty lists since we don't know what to do in that case [_ | _] = Results -> case [E || {error, _} = E <- Results] of @@ -516,7 +516,7 @@ process_request( }. process_request_body(undefined, Msg) -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); process_request_body(BodyTks, Msg) -> emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg). @@ -603,7 +603,7 @@ is_sensitive_key(_) -> %% Function that will do a deep traversal of Data and remove sensitive %% information (i.e., passwords) redact(Data) -> - emqx_misc:redact(Data, fun is_sensitive_key/1). + emqx_utils:redact(Data, fun is_sensitive_key/1). %% because the body may contain some sensitive data %% and at the same time the redact function will not scan the binary data diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 82d622e09..ac2af301e 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -65,7 +65,7 @@ on_start( ?SLOG(info, #{ msg => "starting_ldap_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), Servers = emqx_schema:parse_servers(Servers0, ?LDAP_HOST_OPTIONS), SslOpts = diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index ead3e2e49..a5873bcf6 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -162,7 +162,7 @@ on_start( rs -> "starting_mongodb_replica_set_connector"; sharded -> "starting_mongodb_sharded_connector" end, - ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_misc:redact(Config)}), + ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_utils:redact(Config)}), NConfig = #{hosts := Hosts} = maybe_resolve_srv_and_txt_records(Config), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 8f2d06517..5b488825b 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -149,7 +149,7 @@ on_start(InstanceId, Conf) -> ?SLOG(info, #{ msg => "starting_mqtt_connector", connector => InstanceId, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), BasicConf = basic_config(Conf), BridgeConf = BasicConf#{ diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index c69cf5ca0..6600a5f77 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -102,7 +102,7 @@ on_start( ?SLOG(info, #{ msg => "starting_mysql_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 14cbbc80f..8796e00a5 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -95,7 +95,7 @@ on_start( ?SLOG(info, #{ msg => "starting_postgresql_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index c70e766af..4ef778e6b 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -123,7 +123,7 @@ on_start( ?SLOG(info, #{ msg => "starting_redis_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), ConfKey = case Type of diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl index 54dc0e022..e07d95d51 100644 --- a/apps/emqx_connector/src/emqx_connector_ssl.erl +++ b/apps/emqx_connector/src/emqx_connector_ssl.erl @@ -74,7 +74,7 @@ new_ssl_config(Config, _NewSSL) -> normalize_key_to_bin(undefined) -> undefined; normalize_key_to_bin(Map) when is_map(Map) -> - emqx_map_lib:binary_key_map(Map). + emqx_utils_maps:binary_key_map(Map). try_map_get(Key, Map, Default) when is_map(Map) -> maps:get(Key, Map, Default); diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index defbbaea2..df1114483 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -85,7 +85,7 @@ to_remote_msg(MapMsg, #{ qos = QoS, retain = Retain, topic = topic(Mountpoint, Topic), - props = emqx_misc:pub_props_to_packet(PubProps), + props = emqx_utils:pub_props_to_packet(PubProps), payload = Payload }; to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> @@ -112,7 +112,7 @@ to_broker_msg( Retain = replace_simple_var(RetainToken, MapMsg), PubProps = maps:get(pub_props, MapMsg, #{}), set_headers( - Props#{properties => emqx_misc:pub_props_to_packet(PubProps)}, + Props#{properties => emqx_utils:pub_props_to_packet(PubProps)}, emqx_message:set_flags( #{dup => Dup, retain => Retain}, emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload) @@ -123,7 +123,7 @@ process_payload(From, MapMsg) -> do_process_payload(maps:get(payload, From, undefined), MapMsg). do_process_payload(undefined, Msg) -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); do_process_payload(Tks, Msg) -> replace_vars_in_str(Tks, Msg). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 8e41a9f0f..880a99313 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -124,7 +124,7 @@ start_link(Name, BridgeOpts) -> {error, Reason} = Error -> ?SLOG(error, #{ msg => "client_start_failed", - config => emqx_misc:redact(BridgeOpts), + config => emqx_utils:redact(BridgeOpts), reason => Reason }), Error @@ -410,7 +410,7 @@ handle_disconnect(_Reason) -> ok. maybe_publish_local(Msg, Vars, Props) -> - case emqx_map_lib:deep_get([local, topic], Vars, undefined) of + case emqx_utils_maps:deep_get([local, topic], Vars, undefined) of %% local topic is not set, discard it undefined -> ok; diff --git a/apps/emqx_dashboard/rebar.config b/apps/emqx_dashboard/rebar.config index 9657d0bbf..440fde465 100644 --- a/apps/emqx_dashboard/rebar.config +++ b/apps/emqx_dashboard/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl index eac4f845f..01d96bdf0 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl @@ -119,7 +119,7 @@ remove_handler() -> pre_config_update(_Path, UpdateConf0, RawConf) -> UpdateConf = remove_sensitive_data(UpdateConf0), - NewConf = emqx_map_lib:deep_merge(RawConf, UpdateConf), + NewConf = emqx_utils_maps:deep_merge(RawConf, UpdateConf), ensure_ssl_cert(NewConf). -define(SENSITIVE_PASSWORD, <<"******">>). @@ -134,7 +134,7 @@ remove_sensitive_data(Conf0) -> end, case Conf1 of #{<<"listeners">> := #{<<"https">> := #{<<"password">> := ?SENSITIVE_PASSWORD}}} -> - emqx_map_lib:deep_remove([<<"listeners">>, <<"https">>, <<"password">>], Conf1); + emqx_utils_maps:deep_remove([<<"listeners">>, <<"https">>, <<"password">>], Conf1); _ -> Conf1 end. @@ -152,7 +152,7 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) -> ok. get_listener(Type, Conf) -> - emqx_map_lib:deep_get([listeners, Type], Conf, undefined). + emqx_utils_maps:deep_get([listeners, Type], Conf, undefined). diff_listeners(_, Listener, Listener) -> {#{}, #{}}; diff_listeners(Type, undefined, Start) -> {#{}, #{Type => Start}}; @@ -162,13 +162,14 @@ diff_listeners(Type, Stop, Start) -> {#{Type => Stop}, #{Type => Start}}. -define(DIR, <<"dashboard">>). ensure_ssl_cert(#{<<"listeners">> := #{<<"https">> := #{<<"enable">> := true}}} = Conf) -> - Https = emqx_map_lib:deep_get([<<"listeners">>, <<"https">>], Conf, undefined), + Https = emqx_utils_maps:deep_get([<<"listeners">>, <<"https">>], Conf, undefined), Opts = #{required_keys => [[<<"keyfile">>], [<<"certfile">>], [<<"cacertfile">>]]}, case emqx_tls_lib:ensure_ssl_files(?DIR, Https, Opts) of {ok, undefined} -> {error, <<"ssl_cert_not_found">>}; {ok, NewHttps} -> - {ok, emqx_map_lib:deep_merge(Conf, #{<<"listeners">> => #{<<"https">> => NewHttps}})}; + {ok, + emqx_utils_maps:deep_merge(Conf, #{<<"listeners">> => #{<<"https">> => NewHttps}})}; {error, Reason} -> ?SLOG(error, Reason#{msg => "bad_ssl_config"}), {error, Reason} diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index c2dd4a9dd..c0e162b62 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -122,7 +122,7 @@ fields(sampler_current) -> monitor(get, #{query_string := QS, bindings := Bindings}) -> Latest = maps:get(<<"latest">>, QS, infinity), RawNode = maps:get(node, Bindings, <<"all">>), - emqx_api_lib:with_node_or_cluster(RawNode, dashboard_samplers_fun(Latest)). + emqx_utils_api:with_node_or_cluster(RawNode, dashboard_samplers_fun(Latest)). dashboard_samplers_fun(Latest) -> fun(NodeOrCluster) -> @@ -133,10 +133,10 @@ dashboard_samplers_fun(Latest) -> end. monitor_current(get, #{bindings := []}) -> - emqx_api_lib:with_node_or_cluster(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); + emqx_utils_api:with_node_or_cluster(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); monitor_current(get, #{bindings := Bindings}) -> RawNode = maps:get(node, Bindings, <<"all">>), - emqx_api_lib:with_node_or_cluster(RawNode, fun current_rate/1). + emqx_utils_api:with_node_or_cluster(RawNode, fun current_rate/1). current_rate(Node) -> case emqx_dashboard_monitor:current_rate(Node) of diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index eb7f6c741..b2ad69997 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -474,7 +474,7 @@ maybe_add_summary_from_label(Spec, Hocon) -> get_i18n(Key, Struct, Default) -> {ok, #{cache := Cache, lang := Lang}} = emqx_dashboard:get_i18n(), Desc = hocon_schema:resolve_schema(Struct, Cache), - emqx_map_lib:deep_get([Key, Lang], Desc, Default). + emqx_utils_maps:deep_get([Key, Lang], Desc, Default). trans_label(Spec, Hocon, Default) -> Label = @@ -915,4 +915,4 @@ schema_converter(Options) -> maps:get(schema_converter, Options, fun hocon_schema_to_spec/2). hocon_error_msg(Reason) -> - emqx_misc:readable_error_msg(Reason). + emqx_utils:readable_error_msg(Reason). diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index e951a9a2a..1f14b02c0 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -142,7 +142,7 @@ t_swagger_json(_Config) -> %% with auth Auth = auth_header_(<<"admin">>, <<"public_www1">>), {ok, 200, Body1} = request_api(get, Url, Auth), - ?assert(jsx:is_json(Body1)), + ?assert(emqx_utils_json:is_json(Body1)), %% without auth {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, Body2}} = httpc:request(get, {Url, []}, [], [{body_format, binary}]), @@ -246,5 +246,5 @@ api_path(Parts) -> ?HOST ++ filename:join([?BASE_PATH | Parts]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index 8df130897..91c7729d3 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -82,7 +82,7 @@ request(Username, Method, Url, Body) -> -> {Url, [auth_header(Username)]}; _ -> - {Url, [auth_header(Username)], "application/json", jsx:encode(Body)} + {Url, [auth_header(Username)], "application/json", emqx_utils_json:encode(Body)} end, ct:pal("Method: ~p, Request: ~p", [Method, Request]), case httpc:request(Method, Request, [], [{body_format, binary}]) of diff --git a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl index 19d3f471e..c0a772d2d 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl @@ -100,7 +100,7 @@ request(Url) -> {ok, {{"HTTP/1.1", Code, _}, _, Return}} when Code >= 200 andalso Code =< 299 -> - {ok, emqx_json:decode(Return, [return_maps])}; + {ok, emqx_utils_json:decode(Return, [return_maps])}; {ok, {Reason, _, _}} -> {error, Reason} end. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index f35652f8e..a24fc2337 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -137,10 +137,10 @@ do_request_api(Method, Request) -> Code >= 200 andalso Code =< 299 -> ct:pal("Resp ~p ~p~n", [Code, Return]), - {ok, emqx_json:decode(Return, [return_maps])}; + {ok, emqx_utils_json:decode(Return, [return_maps])}; {ok, {{"HTTP/1.1", Code, _}, _, Return}} -> ct:pal("Resp ~p ~p~n", [Code, Return]), - {error, {Code, emqx_json:decode(Return, [return_maps])}}; + {error, {Code, emqx_utils_json:decode(Return, [return_maps])}}; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index fad539ed1..7abc601b4 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -5,7 +5,8 @@ ]}. {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. {grpc, [ diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl index aa5d1897f..9bfae9579 100644 --- a/apps/emqx_exhook/src/emqx_exhook_api.erl +++ b/apps/emqx_exhook/src/emqx_exhook_api.erl @@ -478,7 +478,7 @@ call_cluster(Fun) -> %%-------------------------------------------------------------------- %% Internal Funcs %%-------------------------------------------------------------------- -err_msg(Msg) -> emqx_misc:readable_error_msg(Msg). +err_msg(Msg) -> emqx_utils:readable_error_msg(Msg). get_raw_config() -> RawConfig = emqx:get_raw_config([exhook, servers], []), diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl index 77937a835..0647c80ea 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mgr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl @@ -507,11 +507,11 @@ sort_name_by_order(Names, Orders) -> lists:sort( fun (A, B) when is_binary(A) -> - emqx_map_lib:deep_get([A, order], Orders) < - emqx_map_lib:deep_get([B, order], Orders); + emqx_utils_maps:deep_get([A, order], Orders) < + emqx_utils_maps:deep_get([B, order], Orders); (#{name := A}, #{name := B}) -> - emqx_map_lib:deep_get([A, order], Orders) < - emqx_map_lib:deep_get([B, order], Orders) + emqx_utils_maps:deep_get([A, order], Orders) < + emqx_utils_maps:deep_get([B, order], Orders) end, Names ). diff --git a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl index 8a4fb7a44..c03b3f231 100644 --- a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl @@ -310,8 +310,8 @@ t_update(Cfg) -> ?assertMatch([], emqx_exhook_mgr:running()). decode_json(Data) -> - BinJosn = emqx_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJosn). + BinJosn = emqx_utils_json:decode(Data, [return_maps]), + emqx_utils_maps:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). @@ -332,7 +332,7 @@ request_api(Method, Url, QueryParams, Auth, Body) -> "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)}). do_request_api(Method, Request) -> case httpc:request(Method, Request, [], [{body_format, binary}]) of diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 7e5228a9e..2340a2dd8 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -1,5 +1,6 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 7f37061ac..4145a92a7 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -173,7 +173,7 @@ stats(#state{ end, ConnStats = emqx_pd:get_counters(?CONN_STATS), ChanStats = ChannMod:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ConnStats, ChanStats, ProcStats]). call(Pid, Req) -> @@ -297,7 +297,7 @@ init_state(WrappedSock, Peername, Options, FrameMod, ChannMod) -> StatsTimer = emqx_gateway_utils:stats_timer(Options), IdleTimeout = emqx_gateway_utils:idle_timeout(Options), OomPolicy = emqx_gateway_utils:oom_policy(Options), - IdleTimer = emqx_misc:start_timer(IdleTimeout, idle_timeout), + IdleTimer = emqx_utils:start_timer(IdleTimeout, idle_timeout), #state{ socket = WrappedSock, peername = Peername, @@ -327,7 +327,7 @@ run_loop( } ) -> emqx_logger:set_metadata_peername(esockd:format(Peername)), - _ = emqx_misc:tune_heap_size(OomPolicy), + _ = emqx_utils:tune_heap_size(OomPolicy), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); @@ -383,14 +383,14 @@ wakeup_from_hib(Parent, State) -> %% Ensure/cancel stats timer ensure_stats_timer(Timeout, State = #state{stats_timer = undefined}) -> - State#state{stats_timer = emqx_misc:start_timer(Timeout, emit_stats)}; + State#state{stats_timer = emqx_utils:start_timer(Timeout, emit_stats)}; ensure_stats_timer(_Timeout, State) -> State. cancel_stats_timer(State = #state{stats_timer = TRef}) when is_reference(TRef) -> - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), State#state{stats_timer = undefined}; cancel_stats_timer(State) -> State. @@ -471,7 +471,7 @@ handle_msg( State = #state{idle_timer = IdleTimer} ) -> IdleTimer /= undefined andalso - emqx_misc:cancel_timer(IdleTimer), + emqx_utils:cancel_timer(IdleTimer), NState = State#state{idle_timer = undefined}, handle_incoming(Packet, NState); handle_msg({outgoing, Data}, State) -> @@ -501,7 +501,7 @@ handle_msg( Deliver = {deliver, _Topic, _Msg}, State = #state{active_n = ActiveN} ) -> - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent %% TODO: Who will deliver this message? @@ -904,7 +904,7 @@ handle_info(Info, State) -> %% msg => "reach_rate_limit", %% pause => Time %% }), -%% TRef = emqx_misc:start_timer(Time, limit_timeout), +%% TRef = emqx_utils:start_timer(Time, limit_timeout), %% State#state{ %% sockstate = blocked, %% limiter = Limiter1, @@ -928,7 +928,7 @@ run_gc(Stats, State = #state{gc_state = GcSt}) -> end. check_oom(State = #state{oom_policy = OomPolicy}) -> - case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of + case ?ENABLED(OomPolicy) andalso emqx_utils:check_oom(OomPolicy) of {shutdown, Reason} -> %% triggers terminate/2 callback immediately erlang:exit({shutdown, Reason}); diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index e64e918b4..cd387e3bb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -115,7 +115,7 @@ clients(get, #{ fun ?MODULE:format_channel_info/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> QStringWithoutNode = maps:without([<<"node">>], QString), emqx_mgmt_api:node_query( diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 14b80a500..d90bf3689 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -96,7 +96,7 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> LName = binary_to_atom(maps:get(<<"name">>, LConf)), Path = [listeners, Type, LName], - case emqx_map_lib:deep_get(Path, RunningConf, undefined) of + case emqx_utils_maps:deep_get(Path, RunningConf, undefined) of undefined -> ListenerId = emqx_gateway_utils:listener_id( GwName, Type, LName diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index df808f295..fb4261065 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -74,7 +74,7 @@ gateway(["load", Name, Conf]) -> case emqx_gateway_conf:load_gateway( bin(Name), - emqx_json:decode(Conf, [return_maps]) + emqx_utils_json:decode(Conf, [return_maps]) ) of {ok, _} -> diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 71ec4bf59..837600811 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -766,9 +766,9 @@ init(Options) -> TabOpts = [public, {write_concurrency, true}], {ChanTab, ConnTab, InfoTab} = cmtabs(GwName), - ok = emqx_tables:new(ChanTab, [bag, {read_concurrency, true} | TabOpts]), - ok = emqx_tables:new(ConnTab, [bag | TabOpts]), - ok = emqx_tables:new(InfoTab, [ordered_set, compressed | TabOpts]), + ok = emqx_utils_ets:new(ChanTab, [bag, {read_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(ConnTab, [bag | TabOpts]), + ok = emqx_utils_ets:new(InfoTab, [ordered_set, compressed | TabOpts]), %% Start link cm-registry process %% XXX: Should I hang it under a higher level supervisor? @@ -802,7 +802,7 @@ handle_info( {'DOWN', _MRef, process, Pid, _Reason}, State = #state{gwname = GwName, chan_pmon = PMon} ) -> - ChanPids = [Pid | emqx_misc:drain_down(?DEFAULT_BATCH_SIZE)], + ChanPids = [Pid | emqx_utils:drain_down(?DEFAULT_BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), CmTabs = cmtabs(GwName), diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 07a4c1e2c..56a3e2068 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -106,7 +106,7 @@ unconvert_listeners(Ls) when is_list(Ls) -> {[Type, Name], Lis1} = maps_key_take([<<"type">>, <<"name">>], Lis), _ = vaildate_listener_name(Name), NLis1 = maps:without([<<"id">>, <<"running">>], Lis1), - emqx_map_lib:deep_merge(Acc, #{Type => #{Name => NLis1}}) + emqx_utils_maps:deep_merge(Acc, #{Type => #{Name => NLis1}}) end, #{}, Ls @@ -160,8 +160,8 @@ gateway(GwName0) -> RawConf = emqx_config:fill_defaults( emqx_config:get_root_raw(Path) ), - Confs = emqx_map_lib:jsonable_map( - emqx_map_lib:deep_get(Path, RawConf) + Confs = emqx_utils_maps:jsonable_map( + emqx_utils_maps:deep_get(Path, RawConf) ), LsConf = maps:get(<<"listeners">>, Confs, #{}), Confs#{<<"listeners">> => convert_listeners(GwName, LsConf)}. @@ -198,8 +198,8 @@ listeners(GwName0) -> RawConf = emqx_config:fill_defaults( emqx_config:get_root_raw([<<"gateway">>]) ), - Listeners = emqx_map_lib:jsonable_map( - emqx_map_lib:deep_get( + Listeners = emqx_utils_maps:jsonable_map( + emqx_utils_maps:deep_get( [<<"gateway">>, GwName, <<"listeners">>], RawConf ) ), @@ -213,12 +213,12 @@ listener(ListenerId) -> ), try Path = [<<"gateway">>, GwName, <<"listeners">>, Type, LName], - LConf = emqx_map_lib:deep_get(Path, RootConf), + LConf = emqx_utils_maps:deep_get(Path, RootConf), Running = emqx_gateway_utils:is_running( binary_to_existing_atom(ListenerId), LConf ), {ok, - emqx_map_lib:jsonable_map( + emqx_utils_maps:jsonable_map( LConf#{ id => ListenerId, type => Type, @@ -305,8 +305,8 @@ ret_ok_err({ok, _}) -> ok; ret_ok_err(Err) -> Err. ret_gw(GwName, {ok, #{raw_config := GwConf}}) -> - GwConf1 = emqx_map_lib:deep_get([bin(GwName)], GwConf), - LsConf = emqx_map_lib:deep_get( + GwConf1 = emqx_utils_maps:deep_get([bin(GwName)], GwConf), + LsConf = emqx_utils_maps:deep_get( [bin(GwName), <<"listeners">>], GwConf, #{} @@ -331,7 +331,7 @@ ret_gw(_GwName, Err) -> Err. ret_authn(GwName, {ok, #{raw_config := GwConf}}) -> - Authn = emqx_map_lib:deep_get( + Authn = emqx_utils_maps:deep_get( [bin(GwName), <<"authentication">>], GwConf ), @@ -340,7 +340,7 @@ ret_authn(_GwName, Err) -> Err. ret_authn(GwName, {LType, LName}, {ok, #{raw_config := GwConf}}) -> - Authn = emqx_map_lib:deep_get( + Authn = emqx_utils_maps:deep_get( [ bin(GwName), <<"listeners">>, @@ -355,7 +355,7 @@ ret_authn(_, _, Err) -> Err. ret_listener_or_err(GwName, {LType, LName}, {ok, #{raw_config := GwConf}}) -> - LConf = emqx_map_lib:deep_get( + LConf = emqx_utils_maps:deep_get( [bin(GwName), <<"listeners">>, bin(LType), bin(LName)], GwConf ), @@ -377,7 +377,7 @@ pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) -> case maps:get(GwName, RawConf, undefined) of undefined -> NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf), - {ok, emqx_map_lib:deep_put([GwName], RawConf, NConf)}; + {ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf)}; _ -> badres_gateway(already_exist, GwName) end; @@ -389,7 +389,7 @@ pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) -> Conf1 = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf), NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf1), NConf1 = maps:merge(GwRawConf, NConf), - {ok, emqx_map_lib:deep_put([GwName], RawConf, NConf1)} + {ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf1)} end; pre_config_update(_, {unload_gateway, GwName}, RawConf) -> _ = tune_gw_certs( @@ -400,7 +400,7 @@ pre_config_update(_, {unload_gateway, GwName}, RawConf) -> {ok, maps:remove(GwName, RawConf)}; pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined ) of @@ -408,7 +408,7 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> NConf = convert_certs(certs_dir(GwName), Conf), NListener = #{LType => #{LName => NConf}}, {ok, - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( RawConf, #{GwName => #{<<"listeners">> => NListener}} )}; @@ -417,7 +417,7 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined ) of @@ -425,7 +425,7 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) - badres_listener(not_found, GwName, LType, LName); OldConf -> NConf = convert_certs(certs_dir(GwName), Conf, OldConf), - NRawConf = emqx_map_lib:deep_put( + NRawConf = emqx_utils_maps:deep_put( [GwName, <<"listeners">>, LType, LName], RawConf, NConf @@ -434,22 +434,22 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) - end; pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) -> Path = [GwName, <<"listeners">>, LType, LName], - case emqx_map_lib:deep_get(Path, RawConf, undefined) of + case emqx_utils_maps:deep_get(Path, RawConf, undefined) of undefined -> {ok, RawConf}; OldConf -> clear_certs(certs_dir(GwName), OldConf), - {ok, emqx_map_lib:deep_remove(Path, RawConf)} + {ok, emqx_utils_maps:deep_remove(Path, RawConf)} end; pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, ?AUTHN_BIN], RawConf, undefined ) of undefined -> {ok, - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( RawConf, #{GwName => #{?AUTHN_BIN => Conf}} )}; @@ -458,7 +458,7 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> end; pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined @@ -477,25 +477,25 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> #{LType => #{LName => NListener}} } }, - {ok, emqx_map_lib:deep_merge(RawConf, NGateway)}; + {ok, emqx_utils_maps:deep_merge(RawConf, NGateway)}; _ -> badres_listener_authn(already_exist, GwName, LType, LName) end end; pre_config_update(_, {update_authn, GwName, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, ?AUTHN_BIN], RawConf, undefined ) of undefined -> badres_authn(not_found, GwName); _Authn -> - {ok, emqx_map_lib:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf)} + {ok, emqx_utils_maps:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf)} end; pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined @@ -514,7 +514,7 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> Listener ), {ok, - emqx_map_lib:deep_put( + emqx_utils_maps:deep_put( [GwName, <<"listeners">>, LType, LName], RawConf, NListener @@ -523,12 +523,12 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {remove_authn, GwName}, RawConf) -> {ok, - emqx_map_lib:deep_remove( + emqx_utils_maps:deep_remove( [GwName, ?AUTHN_BIN], RawConf )}; pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) -> Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN], - {ok, emqx_map_lib:deep_remove(Path, RawConf)}; + {ok, emqx_utils_maps:deep_remove(Path, RawConf)}; pre_config_update(_, UnknownReq, _RawConf) -> logger:error("Unknown configuration update request: ~0p", [UnknownReq]), {error, badreq}. diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index a0155a126..7aaaee9cb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -240,7 +240,7 @@ authn(GwName) -> ChainName = emqx_gateway_utils:global_chain(GwName), wrap_chain_name( ChainName, - emqx_map_lib:jsonable_map(emqx:get_raw_config(Path)) + emqx_utils_maps:jsonable_map(emqx:get_raw_config(Path)) ). -spec authn(gateway_name(), binary()) -> map(). @@ -250,7 +250,7 @@ authn(GwName, ListenerId) -> ChainName = emqx_gateway_utils:listener_chain(GwName, Type, Name), wrap_chain_name( ChainName, - emqx_map_lib:jsonable_map(emqx:get_raw_config(Path)) + emqx_utils_maps:jsonable_map(emqx:get_raw_config(Path)) ). wrap_chain_name(ChainName, Conf) -> @@ -404,7 +404,7 @@ return_http_error(Code, Msg) -> -spec reason2msg({atom(), map()} | any()) -> error | string(). reason2msg({badconf, #{key := Key, value := Value, reason := Reason}}) -> NValue = - case emqx_json:safe_encode(Value) of + case emqx_utils_json:safe_encode(Value) of {ok, Str} -> Str; {error, _} -> emqx_gateway_utils:stringfy(Value) end, @@ -495,7 +495,7 @@ reason2msg( reason2msg( {#{roots := [{gateway, _}]}, [_ | _]} = Error ) -> - Bin = emqx_misc:readable_error_msg(Error), + Bin = emqx_utils:readable_error_msg(Error), <<"Invalid configurations: ", Bin/binary>>; reason2msg(_) -> error. diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl index e94510387..0aa2ff210 100644 --- a/apps/emqx_gateway/src/emqx_gateway_metrics.erl +++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl @@ -89,7 +89,7 @@ tabname(GwName) -> init([GwName]) -> TabOpts = [public, {write_concurrency, true}], - ok = emqx_tables:new(tabname(GwName), [set | TabOpts]), + ok = emqx_utils_ets:new(tabname(GwName), [set | TabOpts]), {ok, #state{}}. handle_call(_Request, _From, State) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 9d80de00e..07185ef42 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -223,7 +223,7 @@ merge_default(Udp, Options) -> case lists:keytake(Key, 1, Options) of {value, {Key, TcpOpts}, Options1} -> [ - {Key, emqx_misc:merge_opts(Default, TcpOpts)} + {Key, emqx_utils:merge_opts(Default, TcpOpts)} | Options1 ]; false -> @@ -482,7 +482,7 @@ frame_options(Options) -> -spec init_gc_state(map()) -> emqx_gc:gc_state() | undefined. init_gc_state(Options) -> - emqx_misc:maybe_apply(fun emqx_gc:init/1, force_gc_policy(Options)). + emqx_utils:maybe_apply(fun emqx_gc:init/1, force_gc_policy(Options)). -spec force_gc_policy(map()) -> emqx_gc:opts() | undefined. force_gc_policy(Options) -> diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 10adcc428..fb648062a 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -164,7 +164,7 @@ t_gateway_stomp(_) -> {204, _} = request(put, "/gateways/stomp", GwConf), {200, ConfResp} = request(get, "/gateways/stomp"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{frame => #{max_headers => 10}}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{frame => #{max_headers => 10}}), {204, _} = request(put, "/gateways/stomp", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/stomp"), assert_confs(GwConf2, ConfResp2), @@ -186,7 +186,7 @@ t_gateway_mqttsn(_) -> {204, _} = request(put, "/gateways/mqttsn", GwConf), {200, ConfResp} = request(get, "/gateways/mqttsn"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{predefined => []}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{predefined => []}), {204, _} = request(put, "/gateways/mqttsn", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/mqttsn"), assert_confs(GwConf2, ConfResp2), @@ -206,7 +206,7 @@ t_gateway_coap(_) -> {204, _} = request(put, "/gateways/coap", GwConf), {200, ConfResp} = request(get, "/gateways/coap"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{heartbeat => <<"10s">>}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{heartbeat => <<"10s">>}), {204, _} = request(put, "/gateways/coap", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/coap"), assert_confs(GwConf2, ConfResp2), @@ -244,7 +244,7 @@ t_gateway_lwm2m(_) -> {204, _} = request(put, "/gateways/lwm2m", GwConf), {200, ConfResp} = request(get, "/gateways/lwm2m"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}), {204, _} = request(put, "/gateways/lwm2m", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/lwm2m"), assert_confs(GwConf2, ConfResp2), @@ -264,7 +264,7 @@ t_gateway_exproto(_) -> {204, _} = request(put, "/gateways/exproto", GwConf), {200, ConfResp} = request(get, "/gateways/exproto"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}), {204, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/exproto"), assert_confs(GwConf2, ConfResp2), @@ -293,7 +293,7 @@ t_gateway_exproto_with_ssl(_) -> {204, _} = request(put, "/gateways/exproto", GwConf), {200, ConfResp} = request(get, "/gateways/exproto"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{ + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{ server => #{ bind => <<"9200">>, ssl_options => SslCliOpts diff --git a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl index d75bf80eb..0ed66a38d 100644 --- a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl +++ b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl @@ -153,7 +153,7 @@ on_start_auth(authn_http) -> Handler = fun(Req0, State) -> ct:pal("Authn Req:~p~nState:~p~n", [Req0, State]), Headers = #{<<"content-type">> => <<"application/json">>}, - Response = jiffy:encode(#{result => allow, is_superuser => false}), + Response = emqx_utils_json:encode(#{result => allow, is_superuser => false}), case cowboy_req:match_qs([username, password], Req0) of #{ username := <<"admin">>, diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 1a4bab5f3..149e6acd6 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -266,7 +266,7 @@ t_case_exproto(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, 5000), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(Expect), C(Recv)) end ) @@ -282,7 +282,7 @@ t_case_exproto(_) -> disable_authn(GwName, Type, Name) -> RawCfg = emqx_conf:get_raw([gateway, GwName], #{}), - ListenerCfg = emqx_map_lib:deep_get( + ListenerCfg = emqx_utils_maps:deep_get( [<<"listeners">>, atom_to_binary(Type), atom_to_binary(Name)], RawCfg ), {ok, _} = emqx_gateway_conf:update_listener(GwName, {Type, Name}, ListenerCfg#{ diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index 9bbcf2711..c62e840df 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -165,7 +165,7 @@ t_case_lwm2m(_) -> Test("lwm2m", fun(SubTopic, Msg) -> ?assertEqual(true, lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics())), Payload = emqx_message:payload(Msg), - Cmd = emqx_json:decode(Payload, [return_maps]), + Cmd = emqx_utils_json:decode(Payload, [return_maps]), ?assertMatch(#{<<"msgType">> := <<"register">>, <<"data">> := _}, Cmd) end), @@ -350,7 +350,7 @@ t_case_exproto_publish(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, 5000), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(SvrMod:frame_connack(0)), C(Recv)), Send = fun() -> @@ -387,7 +387,7 @@ t_case_exproto_subscribe(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, WaitTime), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(SvrMod:frame_connack(0)), C(Recv)), SubBin = SvrMod:frame_subscribe(Topic, 0), diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index 33c307770..1e947e793 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -413,7 +413,7 @@ t_load_gateway_with_certs_content(_) -> ), {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), - SslConf = emqx_map_lib:deep_get( + SslConf = emqx_utils_maps:deep_get( [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], emqx:get_raw_config([gateway, stomp]) ), @@ -436,7 +436,7 @@ t_load_gateway_with_certs_content(_) -> % ), % {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), % assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), -% SslConf = emqx_map_lib:deep_get( +% SslConf = emqx_utils_maps:deep_get( % [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], % emqx:get_raw_config([gateway, stomp]) % ), @@ -471,7 +471,7 @@ t_add_listener_with_certs_content(_) -> emqx:get_raw_config([gateway, stomp]) ), - SslConf = emqx_map_lib:deep_get( + SslConf = emqx_utils_maps:deep_get( [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], emqx:get_raw_config([gateway, stomp]) ), diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index 7ec0c9538..bb378ef10 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -159,8 +159,8 @@ do_request(Mth, Req) -> <<>> -> #{}; _ -> - emqx_map_lib:unsafe_atom_key_map( - emqx_json:decode(Resp, [return_maps]) + emqx_utils_maps:unsafe_atom_key_map( + emqx_utils_json:decode(Resp, [return_maps]) ) end, {Code, NResp}; @@ -172,7 +172,7 @@ req(Path, Qs) -> {url(Path, Qs), auth([])}. req(Path, Qs, Body) -> - {url(Path, Qs), auth([]), "application/json", emqx_json:encode(Body)}. + {url(Path, Qs), auth([]), "application/json", emqx_utils_json:encode(Body)}. url(Path, []) -> lists:concat([?http_api_host, Path]); diff --git a/apps/emqx_gateway_coap/rebar.config b/apps/emqx_gateway_coap/rebar.config index c8675c3ba..3b070a72a 100644 --- a/apps/emqx_gateway_coap/rebar.config +++ b/apps/emqx_gateway_coap/rebar.config @@ -1,4 +1,6 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../../apps/emqx"}}, - {emqx_gateway, {path, "../../apps/emqx_gateway"}} - ]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../emqx_gateway"}} +]}. diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index 4cf362d9d..b90fd630d 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -111,7 +111,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_coap_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_coap_session:info/1, Session); info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> ClientId; info(ctx, #channel{ctx = Ctx}) -> @@ -366,7 +366,7 @@ ensure_timer(Name, Time, Msg, #channel{timers = Timers} = Channel) -> end. make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. ensure_keepalive_timer(Channel) -> @@ -710,7 +710,7 @@ process_connection( ) -> Queries = emqx_coap_message:get_option(uri_query, Req), case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun run_conn_hooks/2, diff --git a/apps/emqx_gateway_coap/src/emqx_coap_session.erl b/apps/emqx_gateway_coap/src/emqx_coap_session.erl index 688defcbb..5ae169675 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_session.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_session.erl @@ -81,7 +81,7 @@ %%%------------------------------------------------------------------- -spec new() -> session(). new() -> - _ = emqx_misc:rand_seed(), + _ = emqx_utils:rand_seed(), #session{ transport_manager = emqx_coap_tm:new(), observe_manager = emqx_coap_observe_res:new_manager(), diff --git a/apps/emqx_gateway_coap/src/emqx_coap_tm.erl b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl index 82f616b25..68a7ae237 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_tm.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl @@ -272,12 +272,12 @@ cancel_state_timer(#state_machine{timers = Timers} = Machine) -> undefined -> Machine; Ref -> - _ = emqx_misc:cancel_timer(Ref), + _ = emqx_utils:cancel_timer(Ref), Machine#state_machine{timers = maps:remove(state_timer, Timers)} end. process_timer(SeqId, {Type, Interval, Msg}, Timers) -> - Ref = emqx_misc:start_timer(Interval, {state_machine, {SeqId, Type, Msg}}), + Ref = emqx_utils:start_timer(Interval, {state_machine, {SeqId, Type, Msg}}), Timers#{Type => Ref}. -spec delete_machine(manager_key(), manager()) -> manager(). @@ -293,7 +293,7 @@ delete_machine(Id, Manager) -> } -> lists:foreach( fun({_, Ref}) -> - emqx_misc:cancel_timer(Ref) + emqx_utils:cancel_timer(Ref) end, maps:to_list(Timers) ), diff --git a/apps/emqx_gateway_coap/src/emqx_coap_transport.erl b/apps/emqx_gateway_coap/src/emqx_coap_transport.erl index 1948c969d..daea13ba8 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_transport.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_transport.erl @@ -119,7 +119,7 @@ idle(out, #coap_message{type = non} = Msg, _) -> timeouts => [{stop_timeout, ?NON_LIFETIME}] }); idle(out, Msg, Transport) -> - _ = emqx_misc:rand_seed(), + _ = emqx_utils:rand_seed(), Timeout = ?ACK_TIMEOUT + rand:uniform(?ACK_RANDOM_FACTOR), out(Msg, #{ next => wait_ack, diff --git a/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl index a0d6dbaaa..cec09a016 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl @@ -92,7 +92,7 @@ t_send_request_api(_) -> Req ), #{<<"token">> := RToken, <<"payload">> := RPayload} = - emqx_json:decode(Response, [return_maps]), + emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(Token, RToken), ?assertEqual(Payload, RPayload) end, diff --git a/apps/emqx_gateway_exproto/rebar.config b/apps/emqx_gateway_exproto/rebar.config index 928949c69..473fa9b67 100644 --- a/apps/emqx_gateway_exproto/rebar.config +++ b/apps/emqx_gateway_exproto/rebar.config @@ -1,7 +1,9 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../../apps/emqx"}}, - {emqx_gateway, {path, "../../apps/emqx_gateway"}} - ]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../emqx_gateway"}} +]}. {plugins, [ {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl index 7234e7a2f..3b2c8d73b 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl @@ -681,14 +681,14 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> ensure_timer(Name, remove_timer_ref(Name, Channel)). cancel_timer(Name, Channel = #channel{timers = Timers}) -> - emqx_misc:cancel_timer(maps:get(Name, Timers, undefined)), + emqx_utils:cancel_timer(maps:get(Name, Timers, undefined)), remove_timer_ref(Name, Channel). remove_timer_ref(Name, Channel = #channel{timers = Timers}) -> @@ -792,4 +792,4 @@ proto_name_to_protocol(ProtoName) when is_binary(ProtoName) -> binary_to_atom(ProtoName). anonymous_clientid() -> - iolist_to_binary(["exproto-", emqx_misc:gen_id()]). + iolist_to_binary(["exproto-", emqx_utils:gen_id()]). diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl index af15ef9d3..34883cdce 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl @@ -50,7 +50,7 @@ start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [] diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl index 7860f1220..ff105b931 100644 --- a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl @@ -149,7 +149,7 @@ start_grpc_server(GwName, Options = #{bind := ListenOn}) -> } }, SvrOptions = - case emqx_map_lib:deep_get([ssl, enable], Options, false) of + case emqx_utils_maps:deep_get([ssl, enable], Options, false) of false -> []; true -> @@ -201,7 +201,7 @@ start_grpc_client_channel(GwName, Options = #{address := Address}) -> }} ) end, - case emqx_map_lib:deep_get([ssl, enable], Options, false) of + case emqx_utils_maps:deep_get([ssl, enable], Options, false) of false -> SvrAddr = compose_http_uri(http, Host, Port), grpc_client_sup:create_channel_pool(GwName, SvrAddr, #{}); diff --git a/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl b/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl index b2e3ad4a7..e04990f5f 100644 --- a/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl +++ b/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl @@ -148,7 +148,7 @@ on_received_bytes(Stream, _Md) -> fun(Reqs) -> lists:foreach( fun(#{conn := Conn, bytes := Bytes}) -> - #{<<"type">> := Type} = Params = emqx_json:decode(Bytes, [return_maps]), + #{<<"type">> := Type} = Params = emqx_utils_json:decode(Bytes, [return_maps]), _ = handle_in(Conn, Type, Params) end, Reqs @@ -284,16 +284,16 @@ handle_out(Conn, ?TYPE_DISCONNECT) -> %% Frame frame_connect(ClientInfo, Password) -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ type => ?TYPE_CONNECT, clientinfo => ClientInfo, password => Password }). frame_connack(Code) -> - emqx_json:encode(#{type => ?TYPE_CONNACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_CONNACK, code => Code}). frame_publish(Topic, Qos, Payload) -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ type => ?TYPE_PUBLISH, topic => Topic, qos => Qos, @@ -301,19 +301,19 @@ frame_publish(Topic, Qos, Payload) -> }). frame_puback(Code) -> - emqx_json:encode(#{type => ?TYPE_PUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_PUBACK, code => Code}). frame_subscribe(Topic, Qos) -> - emqx_json:encode(#{type => ?TYPE_SUBSCRIBE, topic => Topic, qos => Qos}). + emqx_utils_json:encode(#{type => ?TYPE_SUBSCRIBE, topic => Topic, qos => Qos}). frame_suback(Code) -> - emqx_json:encode(#{type => ?TYPE_SUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_SUBACK, code => Code}). frame_unsubscribe(Topic) -> - emqx_json:encode(#{type => ?TYPE_UNSUBSCRIBE, topic => Topic}). + emqx_utils_json:encode(#{type => ?TYPE_UNSUBSCRIBE, topic => Topic}). frame_unsuback(Code) -> - emqx_json:encode(#{type => ?TYPE_UNSUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_UNSUBACK, code => Code}). frame_disconnect() -> - emqx_json:encode(#{type => ?TYPE_DISCONNECT}). + emqx_utils_json:encode(#{type => ?TYPE_DISCONNECT}). diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl index 80afadb8e..ca32d03db 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl @@ -227,7 +227,7 @@ to_operations(Obj, ObjDefinition) -> }. path_list(Path) -> - case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(Path, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId, ResInstId] -> [ObjId, ObjInsId, ResId, ResInstId]; [ObjId, ObjInsId, ResId] -> [ObjId, ObjInsId, ResId]; [ObjId, ObjInsId] -> [ObjId, ObjInsId]; diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl index a424375dd..77652744a 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl @@ -105,7 +105,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_lwm2m_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_lwm2m_session:info/1, Session); info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> ClientId; info(ctx, #channel{ctx = Ctx}) -> @@ -286,7 +286,7 @@ handle_call(discard, _From, Channel) -> % pendings = Pendings}) -> % ok = emqx_session:takeover(Session), % %% TODO: Should not drain deliver here (side effect) -% Delivers = emqx_misc:drain_deliver(), +% Delivers = emqx_utils:drain_deliver(), % AllPendings = lists:append(Delivers, Pendings), % shutdown_and_reply(takenover, AllPendings, Channel); @@ -390,7 +390,7 @@ set_peercert_infos(Peercert, ClientInfo) -> ClientInfo#{dn => DN, cn => CN}. make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. update_life_timer(#channel{session = Session, timers = Timers} = Channel) -> @@ -413,7 +413,7 @@ do_takeover(_DesireId, Msg, Channel) -> do_connect(Req, Result, Channel, Iter) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun check_lwm2m_version/2, fun enrich_conninfo/2, diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl index 53995c97c..8e4286343 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl @@ -335,7 +335,7 @@ remove_tmp_fields(Ref) -> -spec path_list(Path :: binary()) -> {[PathWord :: binary()], [Query :: binary()]}. path_list(Path) -> - case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(Path, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId, LastPart] -> {ResInstId, QueryList} = query_list(LastPart), {[ObjId, ObjInsId, ResId, ResInstId], QueryList}; @@ -389,7 +389,7 @@ observe_seq(Options) -> add_alternate_path_prefix(<<"/">>, PathList) -> PathList; add_alternate_path_prefix(AlternatePath, PathList) -> - [binary_util:trim(AlternatePath, $/) | PathList]. + [emqx_utils_binary:trim(AlternatePath, $/) | PathList]. extract_path(Ref = #{}) -> drop_query( diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl index 90a0306b7..8b9ba2491 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl @@ -97,7 +97,7 @@ tlv_single_resource(BaseName, Id, Value, ObjDefinition) -> [#{path => BaseName, value => Val}]. basename(OldBaseName, _ObjectId, ObjectInstanceId, ResourceId, 3) -> - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary, $/, ResId/binary>>; [ObjId, ObjInsId] -> @@ -113,13 +113,13 @@ basename(OldBaseName, _ObjectId, ObjectInstanceId, ResourceId, 3) -> >> end; basename(OldBaseName, _ObjectId, ObjectInstanceId, _ResourceId, 2) -> - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, _ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; [ObjId, ObjInsId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; [ObjId] -> <<$/, ObjId/binary, $/, (integer_to_binary(ObjectInstanceId))/binary>> end. % basename(OldBaseName, _ObjectId, _ObjectInstanceId, _ResourceId, 1) -> -% case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of +% case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of % [ObjId, _ObjInsId, _ResId] -> <<$/, ObjId/binary>>; % [ObjId, _ObjInsId] -> <<$/, ObjId/binary>>; % [ObjId] -> <<$/, ObjId/binary>> @@ -129,7 +129,7 @@ make_path(RelativePath, Id) -> <>. object_id(BaseName) -> - case binary:split(binary_util:trim(BaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(BaseName, $/), [<<$/>>], [global]) of [ObjId] -> binary_to_integer(ObjId); [ObjId, _] -> binary_to_integer(ObjId); [ObjId, _, _] -> binary_to_integer(ObjId); @@ -137,7 +137,7 @@ object_id(BaseName) -> end. object_resource_id(BaseName) -> - case binary:split(binary_util:trim(BaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(BaseName, $/), [<<$/>>], [global]) of [_ObjIdBin1] -> error({invalid_basename, BaseName}); [_ObjIdBin2, _] -> @@ -152,7 +152,7 @@ object_resource_id(BaseName) -> value(Value, ResourceId, ObjDefinition) -> case emqx_lwm2m_xml_object:get_resource_type(ResourceId, ObjDefinition) of "String" -> - % keep binary type since it is same as a string for jsx + % keep binary type since it is same as a string for emqx_utils_json Value; "Integer" -> Size = byte_size(Value) * 8, @@ -351,7 +351,7 @@ opaque_to_json(BaseName, Binary) -> [#{path => BaseName, value => base64:encode(Binary)}]. translate_json(JSONBin) -> - JSONTerm = emqx_json:decode(JSONBin, [return_maps]), + JSONTerm = emqx_utils_json:decode(JSONBin, [return_maps]), BaseName = maps:get(<<"bn">>, JSONTerm, <<>>), ElementList = maps:get(<<"e">>, JSONTerm, []), translate_element(BaseName, ElementList, []). @@ -371,8 +371,8 @@ translate_element(BaseName, [Element | ElementList], Acc) -> translate_element(BaseName, ElementList, NewAcc). full_path(BaseName, RelativePath) -> - Prefix = binary_util:rtrim(BaseName, $/), - Path = binary_util:ltrim(RelativePath, $/), + Prefix = emqx_utils_binary:rtrim(BaseName, $/), + Path = emqx_utils_binary:ltrim(RelativePath, $/), <>. get_element_value(#{<<"t">> := Value}) -> Value; diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl index 2752804dc..e267692a6 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl @@ -386,11 +386,11 @@ is_alternate_path(LinkAttrs) -> LinkAttrs ). -trim(Str) -> binary_util:trim(Str, $\s). +trim(Str) -> emqx_utils_binary:trim(Str, $\s). delink(Str) -> - Ltrim = binary_util:ltrim(Str, $<), - binary_util:rtrim(Ltrim, $>). + Ltrim = emqx_utils_binary:ltrim(Str, $<), + emqx_utils_binary:rtrim(Ltrim, $>). get_lifetime(#{<<"lt">> := LT}) -> case LT of @@ -737,7 +737,7 @@ proto_publish( Epn, Qos, MountedTopic, - emqx_json:encode(Payload), + emqx_utils_json:encode(Payload), #{}, Headers ), @@ -786,7 +786,7 @@ deliver_to_coap(AlternatePath, JsonData, MQTT, CacheMode, WithContext, Session) is_binary(JsonData) -> try - TermData = emqx_json:decode(JsonData, [return_maps]), + TermData = emqx_utils_json:decode(JsonData, [return_maps]), deliver_to_coap(AlternatePath, TermData, MQTT, CacheMode, WithContext, Session) catch ExClass:Error:ST -> diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl index 33ccec2c7..9f388b07c 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -402,7 +402,7 @@ case01_register_report(Config) -> timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -478,7 +478,7 @@ case02_update_deregister(Config) -> ?LOGT("Options got: ~p", [Opts]), Location = maps:get(location_path, Opts), - Register = emqx_json:encode( + Register = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -521,7 +521,7 @@ case02_update_deregister(Config) -> } = test_recv_coap_response(UdpSock), {ok, changed} = Method2, MsgId2 = RspId2, - Update = emqx_json:encode( + Update = emqx_utils_json:encode( #{ <<"msgType">> => <<"update">>, <<"data">> => #{ @@ -754,7 +754,7 @@ case08_reregister(Config) -> timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -871,7 +871,7 @@ case10_read(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -902,7 +902,7 @@ case10_read(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -957,7 +957,7 @@ case10_read_bad_request(Config) -> <<"path">> => <<"/3333/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -979,7 +979,7 @@ case10_read_bad_request(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode(#{ + ReadResult = emqx_utils_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"read">>, @@ -1015,7 +1015,7 @@ case10_read_separate_ack(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1032,7 +1032,7 @@ case10_read_separate_ack(Config) -> ?assertEqual(<<>>, Payload2), test_send_empty_ack(UdpSock, "127.0.0.1", ?PORT, Request2), - ReadResultACK = emqx_json:encode( + ReadResultACK = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1057,7 +1057,7 @@ case10_read_separate_ack(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1100,7 +1100,7 @@ case11_read_object_tlv(Config) -> <<"path">> => <<"/3/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1132,7 +1132,7 @@ case11_read_object_tlv(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1185,7 +1185,7 @@ case11_read_object_json(Config) -> <<"path">> => <<"/3/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1215,7 +1215,7 @@ case11_read_object_json(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1267,7 +1267,7 @@ case12_read_resource_opaque(Config) -> <<"path">> => <<"/3/0/8">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1293,7 +1293,7 @@ case12_read_resource_opaque(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1335,7 +1335,7 @@ case13_read_no_xml(Config) -> <<"msgType">> => <<"read">>, <<"data">> => #{<<"path">> => <<"/9723/0/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1360,7 +1360,7 @@ case13_read_no_xml(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1399,7 +1399,7 @@ case20_single_write(Config) -> <<"value">> => <<"12345">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1426,7 +1426,7 @@ case20_single_write(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1470,7 +1470,7 @@ case20_write(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1497,7 +1497,7 @@ case20_write(Config) -> ), timer:sleep(100), - WriteResult = emqx_json:encode( + WriteResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1547,7 +1547,7 @@ case21_write_object(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1574,7 +1574,7 @@ case21_write_object(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1618,7 +1618,7 @@ case22_write_error(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1639,7 +1639,7 @@ case22_write_error(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1677,7 +1677,7 @@ case_create_basic(Config) -> <<"basePath">> => <<"/5">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1703,7 +1703,7 @@ case_create_basic(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1738,7 +1738,7 @@ case_delete_basic(Config) -> <<"msgType">> => <<"delete">>, <<"data">> => #{<<"path">> => <<"/5/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1764,7 +1764,7 @@ case_delete_basic(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1804,7 +1804,7 @@ case30_execute(Config) -> <<"args">> => <<"2,7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1830,7 +1830,7 @@ case30_execute(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1868,7 +1868,7 @@ case31_execute_error(Config) -> <<"args">> => <<"2,7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1894,7 +1894,7 @@ case31_execute_error(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1931,7 +1931,7 @@ case40_discover(Config) -> <<"path">> => <<"/3/0/7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1961,7 +1961,7 @@ case40_discover(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2006,7 +2006,7 @@ case50_write_attribute(Config) -> <<"lt">> => <<"5">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(100), Request2 = test_recv_coap_request(UdpSock), @@ -2042,7 +2042,7 @@ case50_write_attribute(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2079,7 +2079,7 @@ case60_observe(Config) -> <<"msgType">> => <<"observe">>, <<"data">> => #{<<"path">> => <<"/3/0/10">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -2106,7 +2106,7 @@ case60_observe(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2141,7 +2141,7 @@ case60_observe(Config) -> timer:sleep(100), #coap_message{} = test_recv_coap_response(UdpSock), - ReadResult2 = emqx_json:encode( + ReadResult2 = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2173,7 +2173,7 @@ case60_observe(Config) -> <<"path">> => <<"/3/0/10">> } }, - CommandJson3 = emqx_json:encode(Command3), + CommandJson3 = emqx_utils_json:encode(Command3), test_mqtt_broker:publish(CommandTopic, CommandJson3, 0), timer:sleep(50), Request3 = test_recv_coap_request(UdpSock), @@ -2200,7 +2200,7 @@ case60_observe(Config) -> ), timer:sleep(100), - ReadResult3 = emqx_json:encode( + ReadResult3 = emqx_utils_json:encode( #{ <<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, @@ -2242,7 +2242,7 @@ case60_observe(Config) -> %% MsgId1), %% #coap_message{method = Method1} = test_recv_coap_response(UdpSock), %% ?assertEqual({ok,created}, Method1), -%% ReadResult = emqx_json:encode( +%% ReadResult = emqx_utils_json:encode( %% #{<<"msgType">> => <<"register">>, %% <<"data">> => #{ %% <<"alternatePath">> => <<"/">>, @@ -2268,7 +2268,7 @@ case60_observe(Config) -> %% <<"path">> => <<"/19/0/0">> %% } %% }, -%% CommandJson = emqx_json:encode(Command), +%% CommandJson = emqx_utils_json:encode(Command), %% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), %% timer:sleep(50), %% Request2 = test_recv_coap_request(UdpSock), @@ -2325,7 +2325,7 @@ case60_observe(Config) -> %% <<"value">> => base64:encode(<<12345:32>>) %% }}, %% -%% CommandJson = emqx_json:encode(Command), +%% CommandJson = emqx_utils_json:encode(Command), %% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), %% timer:sleep(50), %% Request2 = test_recv_coap_request(UdpSock), @@ -2342,7 +2342,7 @@ case60_observe(Config) -> %% {ok, changed}, #coap_content{}, Request2, true), %% timer:sleep(100), %% -%% ReadResult = emqx_json:encode( +%% ReadResult = emqx_utils_json:encode( %% #{<<"requestID">> => CmdId, %% <<"cacheID">> => CmdId, %% <<"data">> => #{ @@ -2502,7 +2502,7 @@ send_read_command_1(CmdId, _UdpSock) -> <<"msgType">> => <<"read">>, <<"data">> => #{<<"path">> => <<"/3/0/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50). @@ -2528,7 +2528,7 @@ verify_read_response_1(CmdId, UdpSock) -> true ), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl index b3c059b31..6fa46ebbc 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl @@ -131,7 +131,7 @@ t_lookup_read(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -178,7 +178,7 @@ t_lookup_discover(Config) -> <<"path">> => <<"/3/0/7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(200), @@ -350,10 +350,10 @@ no_received_request(ClientId, Path, Action) -> <<"codeMsg">> => <<"reply_not_received">>, <<"path">> => Path }, - ?assertEqual(NotReceived, emqx_json:decode(Response, [return_maps])). + ?assertEqual(NotReceived, emqx_utils_json:decode(Response, [return_maps])). normal_received_request(ClientId, Path, Action) -> Response = call_lookup_api(ClientId, Path, Action), - RCont = emqx_json:decode(Response, [return_maps]), + RCont = emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(list_to_binary(ClientId), maps:get(<<"clientid">>, RCont, undefined)), ?assertEqual(Path, maps:get(<<"path">>, RCont, undefined)), ?assertEqual(Action, maps:get(<<"action">>, RCont, undefined)), diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index c27c0ba3f..1ccc8b95a 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -218,7 +218,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_session:info/1, Session); info(will_msg, #channel{will_msg = WillMsg}) -> WillMsg; info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> @@ -282,7 +282,7 @@ enrich_clientinfo( feedvar(Override, Packet, ConnInfo, ClientInfo0), ClientInfo0 ), - {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + {ok, NPacket, NClientInfo} = emqx_utils:pipeline( [ fun maybe_assign_clientid/2, %% FIXME: CALL After authentication successfully @@ -414,7 +414,7 @@ process_connect( Channel#channel{session = Session} ); {ok, #{session := Session, present := true, pendings := Pendings}} -> - Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), + Pendings1 = lists:usort(lists:append(Pendings, emqx_utils:drain_deliver())), NChannel = Channel#channel{ session = Session, resuming = true, @@ -595,7 +595,7 @@ handle_in( Channel = #channel{conn_state = idle} ) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun run_conn_hooks/2, @@ -718,7 +718,7 @@ handle_in(PubPkt = ?SN_PUBLISH_MSG(_Flags, TopicId0, MsgId, _Data), Channel) -> Id end, case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun check_qos3_enable/2, fun preproc_pub_pkt/2, @@ -877,7 +877,7 @@ handle_in( end; handle_in(SubPkt = ?SN_SUBSCRIBE_MSG(_, MsgId, _), Channel) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun preproc_subs_type/2, fun check_subscribe_authz/2, @@ -911,7 +911,7 @@ handle_in( Channel ) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun preproc_unsub_type/2, fun run_client_unsub_hook/2, @@ -1823,7 +1823,7 @@ handle_call( ) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) - Delivers = emqx_misc:drain_deliver(), + Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), shutdown_and_reply(takenover, AllPendings, Channel); %handle_call(list_authz_cache, _From, Channel) -> @@ -2247,7 +2247,7 @@ ensure_register_timer(Channel) -> ensure_register_timer(RetryTimes, Channel = #channel{timers = Timers}) -> Msg = maps:get(register_timer, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(?REGISTER_TIMEOUT, {Msg, RetryTimes}), + TRef = emqx_utils:start_timer(?REGISTER_TIMEOUT, {Msg, RetryTimes}), Channel#channel{timers = Timers#{register_timer => TRef}}. cancel_timer(Name, Channel = #channel{timers = Timers}) -> @@ -2255,7 +2255,7 @@ cancel_timer(Name, Channel = #channel{timers = Timers}) -> undefined -> Channel; TRef -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), Channel#channel{timers = maps:without([Name], Timers)} end. @@ -2270,7 +2270,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> diff --git a/apps/emqx_gateway_stomp/rebar.config b/apps/emqx_gateway_stomp/rebar.config index c8675c3ba..cfeb0a195 100644 --- a/apps/emqx_gateway_stomp/rebar.config +++ b/apps/emqx_gateway_stomp/rebar.config @@ -1,4 +1,6 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../../apps/emqx"}}, - {emqx_gateway, {path, "../../apps/emqx_gateway"}} - ]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} +]}. diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index 13b70348a..316432dea 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -252,7 +252,7 @@ enrich_clientinfo( feedvar(Override, Packet, ConnInfo, ClientInfo0), ClientInfo0 ), - {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + {ok, NPacket, NClientInfo} = emqx_utils:pipeline( [ fun maybe_assign_clientid/2, fun parse_heartbeat/2, @@ -416,7 +416,7 @@ handle_in( {error, unexpected_connect, Channel}; handle_in(Packet = ?PACKET(?CMD_CONNECT), Channel) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun negotiate_version/2, @@ -474,7 +474,7 @@ handle_in( Topic = header(<<"destination">>, Headers), Ack = header(<<"ack">>, Headers, <<"auto">>), case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun parse_topic_filter/2, fun check_subscribed_status/2, @@ -796,7 +796,7 @@ handle_call( reply({error, no_subid}, Channel); SubId -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun parse_topic_filter/2, fun check_subscribed_status/2 @@ -869,7 +869,7 @@ handle_call(discard, _From, Channel) -> % pendings = Pendings}) -> % ok = emqx_session:takeover(Session), % %% TODO: Should not drain deliver here (side effect) -% Delivers = emqx_misc:drain_deliver(), +% Delivers = emqx_utils:drain_deliver(), % AllPendings = lists:append(Delivers, Pendings), % shutdown_and_reply(takenover, AllPendings, Channel); @@ -1289,7 +1289,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> diff --git a/apps/emqx_machine/rebar.config b/apps/emqx_machine/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_machine/rebar.config +++ b/apps/emqx_machine/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_machine/src/emqx_global_gc.erl b/apps/emqx_machine/src/emqx_global_gc.erl index f17ab2d16..121855e68 100644 --- a/apps/emqx_machine/src/emqx_global_gc.erl +++ b/apps/emqx_machine/src/emqx_global_gc.erl @@ -86,7 +86,7 @@ ensure_timer(State) -> disabled -> State; Interval when is_integer(Interval) -> - TRef = emqx_misc:start_timer(Interval, run), + TRef = emqx_utils:start_timer(Interval, run), State#{timer := TRef} end. diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 0bee30e35..6bd36aab5 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index 243c4bb8c..6872b150c 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -88,7 +88,7 @@ start_sysmon() -> end. node_status() -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ backend => mria_rlog:backend(), role => mria_rlog:role() }). diff --git a/apps/emqx_management/rebar.config b/apps/emqx_management/rebar.config index 73cbf471f..b2f5a40af 100644 --- a/apps/emqx_management/rebar.config +++ b/apps/emqx_management/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 0a5cc3afe..508cf7d07 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -165,7 +165,7 @@ banned(post, #{body := Body}) -> {ok, Banned} -> {200, format(Banned)}; {error, {already_exist, Old}} -> - OldBannedFormat = emqx_json:encode(format(Old)), + OldBannedFormat = emqx_utils_json:encode(format(Old)), {400, 'ALREADY_EXISTS', OldBannedFormat} end end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index cac3edaed..681c851bf 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -644,7 +644,7 @@ list_clients(QString) -> fun ?MODULE:format_channel_info/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> QStringWithoutNode = maps:without([<<"node">>], QString), emqx_mgmt_api:node_query( @@ -860,8 +860,8 @@ format_channel_info(ChannInfo = {_, _ClientInfo, _ClientStats}) -> format_channel_info(WhichNode, {_, ClientInfo0, ClientStats}) -> Node = maps:get(node, ClientInfo0, WhichNode), - ClientInfo1 = emqx_map_lib:deep_remove([conninfo, clientid], ClientInfo0), - ClientInfo2 = emqx_map_lib:deep_remove([conninfo, username], ClientInfo1), + ClientInfo1 = emqx_utils_maps:deep_remove([conninfo, clientid], ClientInfo0), + ClientInfo2 = emqx_utils_maps:deep_remove([conninfo, username], ClientInfo1), StatsMap = maps:without( [memory, next_pkt_id, total_heap_size], maps:from_list(ClientStats) @@ -958,4 +958,4 @@ format_authz_cache({{PubSub, Topic}, {AuthzResult, Timestamp}}) -> to_topic_info(Data) -> M = maps:with([<<"topic">>, <<"qos">>, <<"nl">>, <<"rap">>, <<"rh">>], Data), - emqx_map_lib:safe_atom_key_map(M). + emqx_utils_maps:safe_atom_key_map(M). diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index c126cfe19..d7f3ff321 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -390,7 +390,7 @@ crud_listeners_by_id(put, #{bindings := #{id := Id}, body := Body0}) -> undefined -> {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}}; PrevConf -> - MergeConfT = emqx_map_lib:deep_merge(PrevConf, Conf), + MergeConfT = emqx_utils_maps:deep_merge(PrevConf, Conf), MergeConf = emqx_listeners:ensure_override_limiter_conf(MergeConfT, Conf), case update(Path, MergeConf) of {ok, #{raw_config := _RawConf}} -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index a4173f5b0..ecf465f43 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -247,13 +247,13 @@ nodes(get, _Params) -> list_nodes(#{}). node(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun get_node/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun get_node/1)). node_metrics(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_metrics/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_metrics/1)). node_stats(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_stats/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_stats/1)). %%-------------------------------------------------------------------- %% api apply diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 409af4d95..1b69835f9 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -149,7 +149,7 @@ subscriptions(get, #{query_string := QString}) -> fun ?MODULE:format/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> emqx_mgmt_api:node_query( Node1, diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index a69837eb3..5df641add 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -498,7 +498,7 @@ download_trace_log(get, #{bindings := #{name := Name}, query_string := Query}) - %% We generate a session ID so that we name files %% with unique names. Then we won't cause %% overwrites for concurrent requests. - SessionId = emqx_misc:gen_id(), + SessionId = emqx_utils:gen_id(), ZipDir = filename:join([emqx_trace:zip_dir(), SessionId]), ok = file:make_dir(ZipDir), %% Write files to ZipDir and create an in-memory zip file diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 6f2a27414..12a7a6641 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -174,7 +174,7 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) -> desc = Desc, created_at = erlang:system_time(second), api_secret_hash = emqx_dashboard_admin:hash(ApiSecret), - api_key = list_to_binary(emqx_misc:gen_id(16)) + api_key = list_to_binary(emqx_utils:gen_id(16)) }, case create_app(App) of {ok, Res} -> @@ -213,7 +213,7 @@ do_force_create_app(App, ApiKey, NamePrefix) -> end. generate_unique_name(NamePrefix) -> - New = list_to_binary(NamePrefix ++ emqx_misc:gen_id(16)), + New = list_to_binary(NamePrefix ++ emqx_utils:gen_id(16)), case mnesia:read(?APP, New) of [] -> New; _ -> generate_unique_name(NamePrefix) @@ -246,7 +246,7 @@ init_bootstrap_file(File) -> {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), init_bootstrap_file(File, Dev, MP); {error, Reason0} -> - Reason = emqx_misc:explain_posix(Reason0), + Reason = emqx_utils:explain_posix(Reason0), ?SLOG( error, #{ diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 442d5c7de..253d527ac 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -356,7 +356,7 @@ mnesia(_) -> %% @doc Logger Command log(["set-level", Level]) -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> case emqx_logger:set_log_level(Level1) of ok -> emqx_ctl:print("~ts~n", [Level]); @@ -369,7 +369,7 @@ log(["primary-level"]) -> Level = emqx_logger:get_primary_log_level(), emqx_ctl:print("~ts~n", [Level]); log(["primary-level", Level]) -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> _ = emqx_logger:set_primary_log_level(Level1), ok; @@ -392,7 +392,7 @@ log(["handlers", "list"]) -> ], ok; log(["handlers", "start", HandlerId]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> case emqx_logger:start_log_handler(HandlerId1) of ok -> @@ -406,7 +406,7 @@ log(["handlers", "start", HandlerId]) -> emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(["handlers", "stop", HandlerId]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> case emqx_logger:stop_log_handler(HandlerId1) of ok -> @@ -420,9 +420,9 @@ log(["handlers", "stop", HandlerId]) -> emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(["handlers", "set-level", HandlerId, Level]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> case emqx_logger:set_log_handler_level(HandlerId1, Level1) of ok -> @@ -628,7 +628,7 @@ listeners([]) -> emqx_listeners:list() ); listeners(["stop", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:stop_listener(ListenerId1) of ok -> @@ -640,7 +640,7 @@ listeners(["stop", ListenerId]) -> emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(["start", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:start_listener(ListenerId1) of ok -> @@ -652,7 +652,7 @@ listeners(["start", ListenerId]) -> emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(["restart", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:restart_listener(ListenerId1) of ok -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl index 69ace16e8..dc7be7671 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl @@ -56,7 +56,7 @@ get_alarms(AssertCount, Activated) -> Qs = "activated=" ++ Activated, Headers = emqx_mgmt_api_test_util:auth_header_(), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, Qs, Headers), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, Data), Page = maps:get(<<"page">>, Meta), Limit = maps:get(<<"limit">>, Meta), diff --git a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl index 241a73dc4..1a396d795 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl @@ -228,7 +228,7 @@ list_app() -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key"]), case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. @@ -236,7 +236,7 @@ read_app(Name) -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key", Name]), case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -251,7 +251,7 @@ create_app(Name) -> enable => true }, case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -260,7 +260,7 @@ create_unexpired_app(Name, Params) -> Path = emqx_mgmt_api_test_util:api_path(["api_key"]), App = maps:merge(#{name => Name, desc => <<"Note"/utf8>>, enable => true}, Params), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -273,7 +273,7 @@ update_app(Name, Change) -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), UpdatePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]), case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of - {ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])}; + {ok, Update} -> {ok, emqx_utils_json:decode(Update, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl index c765f00bc..9f1b560f7 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl @@ -160,7 +160,7 @@ t_delete(_Config) -> list_banned() -> Path = emqx_mgmt_api_test_util:api_path(["banned"]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. @@ -168,7 +168,7 @@ create_banned(Banned) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["banned"]), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Banned) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 9f26f8542..6d7733b22 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -58,7 +58,7 @@ t_clients(_) -> %% get /clients ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), - ClientsResponse = emqx_json:decode(Clients, [return_maps]), + ClientsResponse = emqx_utils_json:decode(Clients, [return_maps]), ClientsMeta = maps:get(<<"meta">>, ClientsResponse), ClientsPage = maps:get(<<"page">>, ClientsMeta), ClientsLimit = maps:get(<<"limit">>, ClientsMeta), @@ -70,7 +70,7 @@ t_clients(_) -> %% get /clients/:clientid Client1Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1)]), {ok, Client1} = emqx_mgmt_api_test_util:request_api(get, Client1Path), - Client1Response = emqx_json:decode(Client1, [return_maps]), + Client1Response = emqx_utils_json:decode(Client1, [return_maps]), ?assertEqual(Username1, maps:get(<<"username">>, Client1Response)), ?assertEqual(ClientId1, maps:get(<<"clientid">>, Client1Response)), ?assertEqual(120, maps:get(<<"expiry_interval">>, Client1Response)), @@ -130,7 +130,7 @@ t_clients(_) -> "", AuthHeader ), - [SubscriptionsData] = emqx_json:decode(SubscriptionsRes, [return_maps]), + [SubscriptionsData] = emqx_utils_json:decode(SubscriptionsRes, [return_maps]), ?assertMatch( #{ <<"clientid">> := ClientId1, @@ -210,7 +210,7 @@ t_query_clients_with_time(_) -> GteParamRfc3339 ++ GteParamStamp ], DecodedResults = [ - emqx_json:decode(Response, [return_maps]) + emqx_utils_json:decode(Response, [return_maps]) || {ok, Response} <- RequestResults ], {LteResponseDecodeds, GteResponseDecodeds} = lists:split(4, DecodedResults), @@ -247,7 +247,7 @@ t_keepalive(_Config) -> {ok, C1} = emqtt:start_link(#{username => Username, clientid => ClientId}), {ok, _} = emqtt:connect(C1), {ok, NewClient} = emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body), - #{<<"keepalive">> := 11} = emqx_json:decode(NewClient, [return_maps]), + #{<<"keepalive">> := 11} = emqx_utils_json:decode(NewClient, [return_maps]), [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), #{conninfo := #{keepalive := Keepalive}} = emqx_connection:info(Pid), ?assertEqual(11, Keepalive), diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index fc1620fec..5fe9f881e 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -68,7 +68,7 @@ t_update(_Config) -> ), %% update failed - ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), + ErrorSysMon = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), ?assertMatch( {error, {"HTTP/1.1", 400, _}}, update_config(<<"sysmon">>, ErrorSysMon) @@ -83,7 +83,7 @@ t_update(_Config) -> assert_busy_port(true), %% reset no_default_value config - NewSysMon1 = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false), + NewSysMon1 = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false), {ok, #{}} = update_config(<<"sysmon">>, NewSysMon1), ?assertMatch({error, {"HTTP/1.1", 400, _}}, reset_config(<<"sysmon">>, "")), {ok, SysMon4} = get_config(<<"sysmon">>), @@ -99,27 +99,33 @@ t_log(_Config) -> {ok, Log} = get_config("log"), File = "log/emqx-test.log", %% update handler - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), {ok, #{}} = update_config(<<"log">>, Log2), {ok, Log3} = logger:get_handler_config(default), ?assertMatch(#{config := #{file := File}}, Log3), - ErrLog1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, 1), + ErrLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, 1), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog1)), - ErrLog2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enabfe">>], Log, true), + ErrLog2 = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"default">>, <<"enabfe">>], Log, true + ), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog2)), %% add new handler File1 = "log/emqx-test1.log", - Handler = emqx_map_lib:deep_get([<<"file_handlers">>, <<"default">>], Log2), - NewLog1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>], Log2, Handler), - NewLog2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>, <<"file">>], NewLog1, File1), + Handler = emqx_utils_maps:deep_get([<<"file_handlers">>, <<"default">>], Log2), + NewLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"new">>], Log2, Handler), + NewLog2 = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"new">>, <<"file">>], NewLog1, File1 + ), {ok, #{}} = update_config(<<"log">>, NewLog2), {ok, Log4} = logger:get_handler_config(new), ?assertMatch(#{config := #{file := File1}}, Log4), %% disable new handler - Disable = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>, <<"enable">>], NewLog2, false), + Disable = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"new">>, <<"enable">>], NewLog2, false + ), {ok, #{}} = update_config(<<"log">>, Disable), ?assertEqual({error, {not_found, new}}, logger:get_handler_config(new)), ok. @@ -130,27 +136,27 @@ t_global_zone(_Config) -> ?assertEqual(lists:usort(ZonesKeys), lists:usort(maps:keys(Zones))), ?assertEqual( emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed]), - emqx_map_lib:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones) + emqx_utils_maps:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones) ), - NewZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), + NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), {ok, #{}} = update_global_zone(NewZones), ?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])), %% Make sure the override config is updated, and remove the default value. ?assertMatch(#{<<"max_qos_allowed">> := 1}, read_conf(<<"mqtt">>)), - BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), + BadZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)), %% Remove max_qos_allowed from raw config, but we still get default value(2). Mqtt0 = emqx_conf:get_raw([<<"mqtt">>]), - ?assertEqual(1, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt0)), + ?assertEqual(1, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt0)), Mqtt1 = maps:remove(<<"max_qos_allowed">>, Mqtt0), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt1), Mqtt2 = emqx_conf:get_raw([<<"mqtt">>]), ?assertNot(maps:is_key(<<"max_qos_allowed">>, Mqtt2), Mqtt2), {ok, #{<<"mqtt">> := Mqtt3}} = get_global_zone(), %% the default value is 2 - ?assertEqual(2, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt3)), + ?assertEqual(2, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt3)), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0), DefaultZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2), @@ -261,7 +267,7 @@ t_configs_node(_) -> ?assertEqual(error, ExpType), ?assertMatch({{_, 404, _}, _, _}, ExpRes), {_, _, Body} = ExpRes, - ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_json:decode(Body, [return_maps])), + ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_utils_json:decode(Body, [return_maps])), ?assertMatch({error, {_, 500, _}}, get_configs("bad_node")). @@ -271,7 +277,7 @@ get_config(Name) -> Path = emqx_mgmt_api_test_util:api_path(["configs", Name]), case emqx_mgmt_api_test_util:request_api(get, Path) of {ok, Res} -> - {ok, emqx_json:decode(Res, [return_maps])}; + {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -290,8 +296,8 @@ get_configs(Node, Opts) -> end, URI = emqx_mgmt_api_test_util:api_path(Path), case emqx_mgmt_api_test_util:request_api(get, URI, [], [], [], Opts) of - {ok, {_, _, Res}} -> {ok, emqx_json:decode(Res, [return_maps])}; - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, {_, _, Res}} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -299,7 +305,7 @@ update_config(Name, Change) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), UpdatePath = emqx_mgmt_api_test_util:api_path(["configs", Name]), case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of - {ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])}; + {ok, Update} -> {ok, emqx_utils_json:decode(Update, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index 3238588e2..62f689a84 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -193,10 +193,10 @@ t_clear_certs(Config) when is_list(Config) -> }, %% create, make sure the cert files are created - NewConf = emqx_map_lib:deep_put( + NewConf = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"certfile">>], ConfTemp, cert_file("certfile") ), - NewConf2 = emqx_map_lib:deep_put( + NewConf2 = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"keyfile">>], NewConf, cert_file("keyfile") ), @@ -205,7 +205,7 @@ t_clear_certs(Config) when is_list(Config) -> ?assertMatch({ok, [_, _]}, ListResult1), %% update - UpdateConf = emqx_map_lib:deep_put( + UpdateConf = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"keyfile">>], NewConf2, cert_file("keyfile2") ), _ = request(put, NewPath, [], UpdateConf), @@ -385,7 +385,7 @@ action_listener(ID, Action, Running) -> request(Method, Url, QueryParams, Body) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body) of - {ok, Res} -> emqx_json:decode(Res, [return_maps]); + {ok, Res} -> emqx_utils_json:decode(Res, [return_maps]); Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl index 93cb69f0a..7ecfe9817 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl @@ -32,13 +32,13 @@ end_per_suite(_) -> t_metrics_api(_) -> {ok, MetricsResponse} = request_helper("metrics?aggregate=true"), - MetricsFromAPI = emqx_json:decode(MetricsResponse, [return_maps]), + MetricsFromAPI = emqx_utils_json:decode(MetricsResponse, [return_maps]), AggregateMetrics = emqx_mgmt:get_metrics(), match_helper(AggregateMetrics, MetricsFromAPI). t_single_node_metrics_api(_) -> {ok, MetricsResponse} = request_helper("metrics"), - [MetricsFromAPI] = emqx_json:decode(MetricsResponse, [return_maps]), + [MetricsFromAPI] = emqx_utils_json:decode(MetricsResponse, [return_maps]), LocalNodeMetrics = maps:from_list( emqx_mgmt:get_metrics(node()) ++ [{node, to_bin(node())}] ), diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 30313e555..1f14d075e 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -34,8 +34,8 @@ init_per_testcase(t_log_path, Config) -> emqx_config_logger:add_handler(), Log = emqx_conf:get_raw([log], #{}), File = "log/emqx-test.log", - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), {ok, #{}} = emqx_conf:update([log], Log2, #{rawconf_with_defaults => true}), Config; init_per_testcase(_, Config) -> @@ -43,7 +43,7 @@ init_per_testcase(_, Config) -> end_per_testcase(t_log_path, Config) -> Log = emqx_conf:get_raw([log], #{}), - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, false), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, false), {ok, #{}} = emqx_conf:update([log], Log1, #{rawconf_with_defaults => true}), emqx_config_logger:remove_handler(), Config; @@ -53,7 +53,7 @@ end_per_testcase(_, Config) -> t_nodes_api(_) -> NodesPath = emqx_mgmt_api_test_util:api_path(["nodes"]), {ok, Nodes} = emqx_mgmt_api_test_util:request_api(get, NodesPath), - NodesResponse = emqx_json:decode(Nodes, [return_maps]), + NodesResponse = emqx_utils_json:decode(Nodes, [return_maps]), LocalNodeInfo = hd(NodesResponse), Node = binary_to_atom(maps:get(<<"node">>, LocalNodeInfo), utf8), ?assertEqual(Node, node()), @@ -63,7 +63,7 @@ t_nodes_api(_) -> NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]), {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), NodeNameResponse = - binary_to_atom(maps:get(<<"node">>, emqx_json:decode(NodeInfo, [return_maps])), utf8), + binary_to_atom(maps:get(<<"node">>, emqx_utils_json:decode(NodeInfo, [return_maps])), utf8), ?assertEqual(node(), NodeNameResponse), BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode"]), @@ -75,7 +75,7 @@ t_nodes_api(_) -> t_log_path(_) -> NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]), {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), - #{<<"log_path">> := Path} = emqx_json:decode(NodeInfo, [return_maps]), + #{<<"log_path">> := Path} = emqx_utils_json:decode(NodeInfo, [return_maps]), ?assertEqual( <<"log">>, filename:basename(Path) @@ -85,7 +85,7 @@ t_node_stats_api(_) -> StatsPath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "stats"]), SystemStats = emqx_mgmt:get_stats(), {ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath), - Stats = emqx_json:decode(StatsResponse, [return_maps]), + Stats = emqx_utils_json:decode(StatsResponse, [return_maps]), Fun = fun(Key) -> ?assertEqual(maps:get(Key, SystemStats), maps:get(atom_to_binary(Key, utf8), Stats)) @@ -103,7 +103,7 @@ t_node_metrics_api(_) -> emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "metrics"]), SystemMetrics = emqx_mgmt:get_metrics(), {ok, MetricsResponse} = emqx_mgmt_api_test_util:request_api(get, MetricsPath), - Metrics = emqx_json:decode(MetricsResponse, [return_maps]), + Metrics = emqx_utils_json:decode(MetricsResponse, [return_maps]), Fun = fun(Key) -> ?assertEqual(maps:get(Key, SystemMetrics), maps:get(atom_to_binary(Key, utf8), Metrics)) diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 24e55494d..62fed8211 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -136,14 +136,14 @@ t_bad_plugin(Config) -> list_plugins() -> Path = emqx_mgmt_api_test_util:api_path(["plugins"]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. describe_plugins(Name) -> Path = emqx_mgmt_api_test_util:api_path(["plugins", Name]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -172,7 +172,7 @@ update_boot_order(Name, MoveBody) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["plugins", Name, "move"]), case emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, MoveBody) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -206,7 +206,7 @@ create_renamed_package(PackagePath, NewNameVsn) -> NewPackagePath. update_release_json(["release.json"], FileContent, NewName) -> - ContentMap = emqx_json:decode(FileContent, [return_maps]), - emqx_json:encode(ContentMap#{<<"name">> => NewName}); + ContentMap = emqx_utils_json:decode(FileContent, [return_maps]), + emqx_utils_json:encode(ContentMap#{<<"name">> => NewName}); update_release_json(_FileName, FileContent, _NewName) -> FileContent. diff --git a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl index 44b8069f5..303a73b41 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl @@ -352,4 +352,4 @@ receive_assert(Topic, Qos, Payload) -> end. decode_json(In) -> - emqx_json:decode(In, [return_maps]). + emqx_utils_json:decode(In, [return_maps]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl index 7236ac9e4..4099426b8 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl @@ -33,7 +33,7 @@ end_per_suite(_) -> t_stats_api(_) -> S = emqx_mgmt_api_test_util:api_path(["stats?aggregate=false"]), {ok, S1} = emqx_mgmt_api_test_util:request_api(get, S), - [Stats1] = emqx_json:decode(S1, [return_maps]), + [Stats1] = emqx_utils_json:decode(S1, [return_maps]), SystemStats1 = emqx_mgmt:get_stats(), Fun1 = fun(Key) -> @@ -43,7 +43,7 @@ t_stats_api(_) -> StatsPath = emqx_mgmt_api_test_util:api_path(["stats?aggregate=true"]), SystemStats = emqx_mgmt:get_stats(), {ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath), - Stats = emqx_json:decode(StatsResponse, [return_maps]), + Stats = emqx_utils_json:decode(StatsResponse, [return_maps]), ?assertEqual(erlang:length(maps:keys(SystemStats)), erlang:length(maps:keys(Stats))), Fun = fun(Key) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index b9e9fffd8..a23d70f2f 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -55,7 +55,7 @@ t_subscription_api(Config) -> {ok, _, _} = emqtt:subscribe(Client, ?TOPIC2), Path = emqx_mgmt_api_test_util:api_path(["subscriptions"]), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, Data), ?assertEqual(1, maps:get(<<"page">>, Meta)), ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), @@ -158,7 +158,7 @@ t_list_with_internal_subscription(_Config) -> request_json(Method, Query, Headers) when is_list(Query) -> Qs = uri_string:compose_query(Query), {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers), - emqx_json:decode(MatchRes, [return_maps]). + emqx_utils_json:decode(MatchRes, [return_maps]). path() -> emqx_mgmt_api_test_util:api_path(["subscriptions"]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 782f493fe..985b95d5b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -108,7 +108,8 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when end, do_request_api( Method, - {NewUrl, build_http_header(AuthOrHeaders), "application/json", emqx_json:encode(Body)}, + {NewUrl, build_http_header(AuthOrHeaders), "application/json", + emqx_utils_json:encode(Body)}, Opts ). @@ -156,7 +157,7 @@ api_path_without_base_path(Parts) -> %% %% Usage with RequestData: %% Payload = [{upload_type, <<"user_picture">>}], -%% PayloadContent = jsx:encode(Payload), +%% PayloadContent = emqx_utils_json:encode(Payload), %% RequestData = [ %% {<<"payload">>, PayloadContent} %% ] diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl index 0c2e684b4..659ae0d44 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl @@ -49,7 +49,7 @@ t_nodes_api(Config) -> %% list all Path = emqx_mgmt_api_test_util:api_path(["topics"]), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path), - RoutesData = emqx_json:decode(Response, [return_maps]), + RoutesData = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, RoutesData), ?assertEqual(1, maps:get(<<"page">>, Meta)), ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), @@ -68,7 +68,7 @@ t_nodes_api(Config) -> ]), Headers = emqx_mgmt_api_test_util:auth_header_(), {ok, MatchResponse} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers), - MatchData = emqx_json:decode(MatchResponse, [return_maps]), + MatchData = emqx_utils_json:decode(MatchResponse, [return_maps]), ?assertMatch( #{<<"count">> := 1, <<"page">> := 1, <<"limit">> := 100}, maps:get(<<"meta">>, MatchData) @@ -90,6 +90,6 @@ t_nodes_api(Config) -> [ #{<<"topic">> := Topic, <<"node">> := Node1}, #{<<"topic">> := Topic, <<"node">> := Node2} - ] = emqx_json:decode(RouteResponse, [return_maps]), + ] = emqx_utils_json:decode(RouteResponse, [return_maps]), ?assertEqual(lists:usort([Node, atom_to_binary(Slave)]), lists:usort([Node1, Node2])). diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 7922bbb40..0102eb56c 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -384,7 +384,7 @@ api_path(Path) -> emqx_mgmt_api_test_util:api_path([Path]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. load() -> diff --git a/apps/emqx_modules/rebar.config b/apps/emqx_modules/rebar.config index 9688d5043..ff542aed7 100644 --- a/apps/emqx_modules/rebar.config +++ b/apps/emqx_modules/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_conf, {path, "../emqx_conf"}} ]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 91cdbedd3..85313c181 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -328,7 +328,7 @@ handle_info(Info, State) -> terminate(_Reason, #{stats_timer := StatsTimer} = State) -> emqx_conf:remove_handler([delayed]), - emqx_misc:cancel_timer(StatsTimer), + emqx_utils:cancel_timer(StatsTimer), do_load_or_unload(false, State). code_change(_Vsn, State, _Extra) -> @@ -370,14 +370,14 @@ ensure_publish_timer({Ts, _Id}, State = #{publish_timer := undefined}) -> ensure_publish_timer({Ts, _Id}, State = #{publish_timer := TRef, publish_at := PubAt}) when Ts < PubAt -> - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), ensure_publish_timer(Ts, ?NOW, State); ensure_publish_timer(_Key, State) -> State. ensure_publish_timer(Ts, Now, State) -> Interval = max(1, Ts - Now), - TRef = emqx_misc:start_timer(Interval, do_publish), + TRef = emqx_utils:start_timer(Interval, do_publish), State#{publish_timer := TRef, publish_at := Now + Interval}. do_publish(Key, Now) -> @@ -418,7 +418,7 @@ do_load_or_unload(true, State) -> State; do_load_or_unload(false, #{publish_timer := PubTimer} = State) -> emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), - emqx_misc:cancel_timer(PubTimer), + emqx_utils:cancel_timer(PubTimer), ets:delete_all_objects(?TAB), State#{publish_timer := undefined, publish_at := 0}; do_load_or_unload(_, State) -> diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl index 6d5c772f0..3b27302df 100644 --- a/apps/emqx_modules/src/emqx_telemetry.erl +++ b/apps/emqx_modules/src/emqx_telemetry.erl @@ -161,7 +161,7 @@ handle_call(enable, _From, State) -> FirstReportTimeoutMS = timer:seconds(10), {reply, ok, ensure_report_timer(FirstReportTimeoutMS, State)}; handle_call(disable, _From, State = #state{timer = Timer}) -> - emqx_misc:cancel_timer(Timer), + emqx_utils:cancel_timer(Timer), {reply, ok, State#state{timer = undefined}}; handle_call(get_node_uuid, _From, State = #state{node_uuid = UUID}) -> {reply, {ok, UUID}, State}; @@ -205,7 +205,7 @@ ensure_report_timer(State = #state{report_interval = ReportInterval}) -> ensure_report_timer(ReportInterval, State). ensure_report_timer(ReportInterval, State) -> - State#state{timer = emqx_misc:start_timer(ReportInterval, time_to_report_telemetry_data)}. + State#state{timer = emqx_utils:start_timer(ReportInterval, time_to_report_telemetry_data)}. os_info() -> case erlang:system_info(os_type) of @@ -356,7 +356,7 @@ get_telemetry(State0 = #state{node_uuid = NodeUUID, cluster_uuid = ClusterUUID}) report_telemetry(State0 = #state{url = URL}) -> {State, Data} = get_telemetry(State0), - case emqx_json:safe_encode(Data) of + case emqx_utils_json:safe_encode(Data) of {ok, Bin} -> httpc_request(post, URL, [], Bin), ?tp(debug, telemetry_data_reported, #{}); diff --git a/apps/emqx_modules/src/emqx_telemetry_api.erl b/apps/emqx_modules/src/emqx_telemetry_api.erl index 798c3ad17..b7209d146 100644 --- a/apps/emqx_modules/src/emqx_telemetry_api.erl +++ b/apps/emqx_modules/src/emqx_telemetry_api.erl @@ -243,7 +243,7 @@ status(put, #{body := Body}) -> data(get, _Request) -> case emqx_modules_conf:is_telemetry_enabled() of true -> - {200, emqx_json:encode(get_telemetry_data())}; + {200, emqx_utils_json:encode(get_telemetry_data())}; false -> {404, #{ code => ?NOT_FOUND, diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl index dfc6b07ab..de09e568f 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics.erl @@ -201,7 +201,7 @@ reset() -> init([Opts]) -> erlang:process_flag(trap_exit, true), - ok = emqx_tables:new(?TAB, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{read_concurrency, true}]), erlang:send_after(timer:seconds(?TICKING_INTERVAL), self(), ticking), Fun = fun(#{topic := Topic}, CurrentSpeeds) -> diff --git a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl index ed3cd9292..4ae3dec88 100644 --- a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl @@ -229,8 +229,8 @@ t_large_payload(_) -> %%-------------------------------------------------------------------- decode_json(Data) -> - BinJson = emqx_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJson). + BinJson = emqx_utils_json:decode(Data, [return_maps]), + emqx_utils_maps:unsafe_atom_key_map(BinJson). clear_all_record() -> ets:delete_all_objects(emqx_delayed). diff --git a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl index ddb136f1e..68d12a2c1 100644 --- a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl @@ -75,7 +75,7 @@ t_mqtt_topic_rewrite(_) -> ?assertEqual( Rules, - emqx_json:decode(Result, [return_maps]) + emqx_utils_json:decode(Result, [return_maps]) ). t_mqtt_topic_rewrite_limit(_) -> diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl index a61781e13..bb5f39c1f 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -512,7 +512,7 @@ t_send_after_enable(_) -> ), receive {request, post, _URL, _Headers, Body} -> - {ok, Decoded} = emqx_json:safe_decode(Body, [return_maps]), + {ok, Decoded} = emqx_utils_json:safe_decode(Body, [return_maps]), ?assertMatch( #{ <<"uuid">> := _, diff --git a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl index 16f942bc0..ac6d12039 100644 --- a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl @@ -113,7 +113,7 @@ t_status(_) -> ?assertEqual( #{<<"enable">> => false}, - jsx:decode(Result0) + emqx_utils_json:decode(Result0) ), ?assertMatch( @@ -139,7 +139,7 @@ t_status(_) -> ?assertEqual( #{<<"enable">> => true}, - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -180,7 +180,7 @@ t_data(_) -> <<"uuid">> := _, <<"vm_specs">> := _ }, - jsx:decode(Result) + emqx_utils_json:decode(Result) ), {ok, 200, _} = diff --git a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl index ea85d1fe9..5d64f123d 100644 --- a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl @@ -74,7 +74,7 @@ t_mqtt_topic_metrics_collection(_) -> ?assertEqual( [], - jsx:decode(Result0) + emqx_utils_json:decode(Result0) ), {ok, 200, _} = request( @@ -95,7 +95,7 @@ t_mqtt_topic_metrics_collection(_) -> <<"metrics">> := #{} } ], - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -150,7 +150,7 @@ t_mqtt_topic_metrics(_) -> uri(["mqtt", "topic_metrics"]) ), - ?assertMatch([_], jsx:decode(Result0)), + ?assertMatch([_], emqx_utils_json:decode(Result0)), {ok, 200, Result1} = request( get, @@ -162,7 +162,7 @@ t_mqtt_topic_metrics(_) -> <<"topic">> := <<"topic/1/2">>, <<"metrics">> := #{} }, - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -288,7 +288,7 @@ t_node_aggregation(_) -> <<"topic">> := <<"topic/1/2">>, <<"metrics">> := #{<<"messages.dropped.count">> := 3} }, - jsx:decode(Result) + emqx_utils_json:decode(Result) ), meck:unload(emqx_topic_metrics_proto_v1). diff --git a/apps/emqx_plugin_libs/rebar.config b/apps/emqx_plugin_libs/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_plugin_libs/rebar.config +++ b/apps/emqx_plugin_libs/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_plugin_libs/src/emqx_placeholder.erl b/apps/emqx_plugin_libs/src/emqx_placeholder.erl index 1f93c1d3e..18ef9e8fb 100644 --- a/apps/emqx_plugin_libs/src/emqx_placeholder.erl +++ b/apps/emqx_plugin_libs/src/emqx_placeholder.erl @@ -240,7 +240,7 @@ sql_data(Bin) when is_binary(Bin) -> Bin; sql_data(Num) when is_number(Num) -> Num; sql_data(Bool) when is_boolean(Bool) -> Bool; sql_data(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -sql_data(Map) when is_map(Map) -> emqx_json:encode(Map). +sql_data(Map) when is_map(Map) -> emqx_utils_json:encode(Map). -spec bin(term()) -> binary(). bin(Val) -> emqx_plugin_libs_rule:bin(Val). diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl index 9b286f360..a3048e5dd 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl @@ -79,7 +79,7 @@ health_check_ecpool_workers(PoolName, CheckFunc, Timeout) -> false end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of [_ | _] = Status -> lists:all(fun(St) -> St =:= true end, Status); [] -> diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index d1a821895..8844fe586 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -225,7 +225,7 @@ tcp_connectivity(Host, Port) -> ) -> ok | {error, Reason :: term()}. tcp_connectivity(Host, Port, Timeout) -> - case gen_tcp:connect(Host, Port, emqx_misc:ipv6_probe([]), Timeout) of + case gen_tcp:connect(Host, Port, emqx_utils:ipv6_probe([]), Timeout) of {ok, Sock} -> gen_tcp:close(Sock), ok; @@ -236,11 +236,11 @@ tcp_connectivity(Host, Port, Timeout) -> str(Bin) when is_binary(Bin) -> binary_to_list(Bin); str(Num) when is_number(Num) -> number_to_list(Num); str(Atom) when is_atom(Atom) -> atom_to_list(Atom); -str(Map) when is_map(Map) -> binary_to_list(emqx_json:encode(Map)); +str(Map) when is_map(Map) -> binary_to_list(emqx_utils_json:encode(Map)); str(List) when is_list(List) -> case io_lib:printable_list(List) of true -> List; - false -> binary_to_list(emqx_json:encode(List)) + false -> binary_to_list(emqx_utils_json:encode(List)) end; str(Data) -> error({invalid_str, Data}). @@ -258,11 +258,11 @@ utf8_str(Str) -> bin(Bin) when is_binary(Bin) -> Bin; bin(Num) when is_number(Num) -> number_to_binary(Num); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -bin(Map) when is_map(Map) -> emqx_json:encode(Map); +bin(Map) when is_map(Map) -> emqx_utils_json:encode(Map); bin(List) when is_list(List) -> case io_lib:printable_list(List) of true -> list_to_binary(List); - false -> emqx_json:encode(List) + false -> emqx_utils_json:encode(List) end; bin(Data) -> error({invalid_bin, Data}). @@ -312,7 +312,7 @@ float2str(Float, Precision) when is_float(Float) and is_integer(Precision) -> float_to_binary(Float, [{decimals, Precision}, compact]). map(Bin) when is_binary(Bin) -> - case emqx_json:decode(Bin, [return_maps]) of + case emqx_utils_json:decode(Bin, [return_maps]) of Map = #{} -> Map; _ -> error({invalid_map, Bin}) end; diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 88b3d27a2..7b9a6cc48 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}} ]}. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 60d52f58b..05d9508b6 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -144,7 +144,7 @@ terminate(_Reason, _State) -> ok. ensure_timer(Interval) -> - emqx_misc:start_timer(Interval, ?TIMER_MSG). + emqx_utils:start_timer(Interval, ?TIMER_MSG). %%-------------------------------------------------------------------- %% prometheus callbacks diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index c4867d9fd..e29d46720 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -66,7 +66,7 @@ t_prometheus_api(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), - Conf = emqx_json:decode(Response, [return_maps]), + Conf = emqx_utils_json:decode(Response, [return_maps]), ?assertMatch( #{ <<"push_gateway_server">> := _, @@ -84,7 +84,7 @@ t_prometheus_api(_) -> NewConf = Conf#{<<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"disabled">>}, {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf), - Conf2 = emqx_json:decode(Response2, [return_maps]), + Conf2 = emqx_utils_json:decode(Response2, [return_maps]), ?assertMatch(NewConf, Conf2), ?assertEqual({ok, []}, application:get_env(prometheus, vm_statistics_collector_metrics)), ?assertEqual({ok, all}, application:get_env(prometheus, vm_memory_collector_metrics)), @@ -106,7 +106,7 @@ t_stats_api(_) -> Headers = [{"accept", "application/json"}, Auth], {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Headers), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), ?assertMatch(#{<<"client">> := _, <<"delivery">> := _}, Data), {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 7f88e1440..91572eac3 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -70,7 +70,7 @@ auto_restart_interval => pos_integer(), batch_size => pos_integer(), batch_time => pos_integer(), - max_queue_bytes => pos_integer(), + max_buffer_bytes => pos_integer(), query_mode => query_mode(), resume_interval => pos_integer(), inflight_window => pos_integer() @@ -85,11 +85,8 @@ -define(WORKER_POOL_SIZE, 16). --define(DEFAULT_QUEUE_SEG_SIZE, 10 * 1024 * 1024). --define(DEFAULT_QUEUE_SEG_SIZE_RAW, <<"10MB">>). - --define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024). --define(DEFAULT_QUEUE_SIZE_RAW, <<"100MB">>). +-define(DEFAULT_BUFFER_BYTES, 256 * 1024 * 1024). +-define(DEFAULT_BUFFER_BYTES_RAW, <<"256MB">>). -define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(15)). diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 0fa4c0bd8..e34cf5d0a 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -178,20 +178,7 @@ init({Id, Index, Opts}) -> process_flag(trap_exit, true), true = gproc_pool:connect_worker(Id, {Id, Index}), BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), - SegBytes0 = maps:get(queue_seg_bytes, Opts, ?DEFAULT_QUEUE_SEG_SIZE), - TotalBytes = maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE), - SegBytes = min(SegBytes0, TotalBytes), - QueueOpts = - #{ - dir => disk_queue_dir(Id, Index), - marshaller => fun ?MODULE:queue_item_marshaller/1, - max_total_bytes => TotalBytes, - %% we don't want to retain the queue after - %% resource restarts. - offload => {true, volatile}, - seg_bytes => SegBytes, - sizer => fun ?MODULE:estimate_size/1 - }, + QueueOpts = replayq_opts(Id, Index, Opts), Queue = replayq:open(QueueOpts), emqx_resource_metrics:queuing_set(Id, Index, queue_count(Queue)), emqx_resource_metrics:inflight_set(Id, Index, 0), @@ -214,7 +201,7 @@ init({Id, Index, Opts}) -> resume_interval => ResumeInterval, tref => undefined }, - ?tp(buffer_worker_init, #{id => Id, index => Index}), + ?tp(buffer_worker_init, #{id => Id, index => Index, queue_opts => QueueOpts}), {ok, running, Data}. running(enter, _, #{tref := _Tref} = Data) -> @@ -1535,7 +1522,7 @@ queue_count(Q) -> disk_queue_dir(Id, Index) -> QDir0 = binary_to_list(Id) ++ ":" ++ integer_to_list(Index), QDir = filename:join([emqx:data_dir(), "bufs", node(), QDir0]), - emqx_misc:safe_filename(QDir). + emqx_utils:safe_filename(QDir). clear_disk_queue_dir(Id, Index) -> ReplayQDir = disk_queue_dir(Id, Index), @@ -1679,6 +1666,32 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) -> end, BatchTime. +replayq_opts(Id, Index, Opts) -> + BufferMode = maps:get(buffer_mode, Opts, memory_only), + TotalBytes = maps:get(max_buffer_bytes, Opts, ?DEFAULT_BUFFER_BYTES), + case BufferMode of + memory_only -> + #{ + mem_only => true, + marshaller => fun ?MODULE:queue_item_marshaller/1, + max_total_bytes => TotalBytes, + sizer => fun ?MODULE:estimate_size/1 + }; + volatile_offload -> + SegBytes0 = maps:get(buffer_seg_bytes, Opts, TotalBytes), + SegBytes = min(SegBytes0, TotalBytes), + #{ + dir => disk_queue_dir(Id, Index), + marshaller => fun ?MODULE:queue_item_marshaller/1, + max_total_bytes => TotalBytes, + %% we don't want to retain the queue after + %% resource restarts. + offload => {true, volatile}, + seg_bytes => SegBytes, + sizer => fun ?MODULE:estimate_size/1 + } + end. + %% The request timeout should be greater than the resume interval, as %% it defines how often the buffer worker tries to unblock. If request %% timeout is <= resume interval and the buffer worker is ever diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 9fe0fbdf2..7ecf56c18 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -539,7 +539,7 @@ stop_resource(#data{state = ResState, id = ResId} = Data) -> Data#data{status = stopped}. make_test_id() -> - RandId = iolist_to_binary(emqx_misc:gen_id(16)), + RandId = iolist_to_binary(emqx_utils:gen_id(16)), <>. handle_manually_health_check(From, Data) -> @@ -613,7 +613,7 @@ maybe_alarm(_Status, ResId, Error) -> HrError = case Error of undefined -> <<"Unknown reason">>; - _Else -> emqx_misc:readable_error_msg(Error) + _Else -> emqx_utils:readable_error_msg(Error) end, emqx_alarm:activate( ResId, diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 647a40fed..3b4fb66e5 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -40,6 +40,7 @@ fields("resource_opts") -> ]; fields("creation_opts") -> [ + {buffer_mode, fun buffer_mode/1}, {worker_pool_size, fun worker_pool_size/1}, {health_check_interval, fun health_check_interval/1}, {resume_interval, fun resume_interval/1}, @@ -53,7 +54,8 @@ fields("creation_opts") -> {batch_size, fun batch_size/1}, {batch_time, fun batch_time/1}, {enable_queue, fun enable_queue/1}, - {max_queue_bytes, fun max_queue_bytes/1} + {max_buffer_bytes, fun max_buffer_bytes/1}, + {buffer_seg_bytes, fun buffer_seg_bytes/1} ]. resource_opts_meta() -> @@ -143,10 +145,24 @@ batch_time(default) -> ?DEFAULT_BATCH_TIME_RAW; batch_time(required) -> false; batch_time(_) -> undefined. -max_queue_bytes(type) -> emqx_schema:bytesize(); -max_queue_bytes(desc) -> ?DESC("max_queue_bytes"); -max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW; -max_queue_bytes(required) -> false; -max_queue_bytes(_) -> undefined. +max_buffer_bytes(type) -> emqx_schema:bytesize(); +max_buffer_bytes(aliases) -> [max_queue_bytes]; +max_buffer_bytes(desc) -> ?DESC("max_buffer_bytes"); +max_buffer_bytes(default) -> ?DEFAULT_BUFFER_BYTES_RAW; +max_buffer_bytes(required) -> false; +max_buffer_bytes(_) -> undefined. + +buffer_mode(type) -> enum([memory_only, volatile_offload]); +buffer_mode(desc) -> ?DESC("buffer_mode"); +buffer_mode(default) -> memory_only; +buffer_mode(required) -> false; +buffer_mode(importance) -> ?IMPORTANCE_HIDDEN; +buffer_mode(_) -> undefined. + +buffer_seg_bytes(type) -> emqx_schema:bytesize(); +buffer_seg_bytes(desc) -> ?DESC("buffer_seg_bytes"); +buffer_seg_bytes(required) -> false; +buffer_seg_bytes(importance) -> ?IMPORTANCE_HIDDEN; +buffer_seg_bytes(_) -> undefined. desc("creation_opts") -> ?DESC("creation_opts"). diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 5d8e85697..385b4cb91 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1314,7 +1314,8 @@ t_delete_and_re_create_with_same_name(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => NumBufferWorkers, - queue_seg_bytes => 100, + buffer_mode => volatile_offload, + buffer_seg_bytes => 100, resume_interval => 1_000 } ), @@ -1373,7 +1374,7 @@ t_delete_and_re_create_with_same_name(_Config) -> query_mode => async, batch_size => 1, worker_pool_size => 2, - queue_seg_bytes => 100, + buffer_seg_bytes => 100, resume_interval => 1_000 } ), @@ -1405,7 +1406,7 @@ t_always_overflow(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => 1, - max_queue_bytes => 1, + max_buffer_bytes => 1, resume_interval => 1_000 } ), @@ -2639,6 +2640,117 @@ t_call_mode_uncoupled_from_query_mode(_Config) -> end ). +%% The default mode is currently `memory_only'. +t_volatile_offload_mode(_Config) -> + MaxBufferBytes = 1_000, + DefaultOpts = #{ + max_buffer_bytes => MaxBufferBytes, + worker_pool_size => 1 + }, + ?check_trace( + begin + emqx_connector_demo:set_callback_mode(async_if_possible), + %% Create without any specified segment bytes; should + %% default to equal max bytes. + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{buffer_mode => volatile_offload} + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + %% Create with segment bytes < max bytes + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + buffer_mode => volatile_offload, + buffer_seg_bytes => MaxBufferBytes div 2 + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + %% Create with segment bytes = max bytes + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + buffer_mode => volatile_offload, + buffer_seg_bytes => MaxBufferBytes + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + %% Create with segment bytes > max bytes; should normalize + %% to max bytes. + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + buffer_mode => volatile_offload, + buffer_seg_bytes => 2 * MaxBufferBytes + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + ok + end, + fun(Trace) -> + HalfMaxBufferBytes = MaxBufferBytes div 2, + ?assertMatch( + [ + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + %% uses the specified value since it's smaller + %% than max bytes. + seg_bytes := HalfMaxBufferBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + } + ], + ?projection(queue_opts, ?of_kind(buffer_worker_init, Trace)) + ), + ok + end + ). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index 65de71fdd..a178e10a1 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 11013cdd3..7bfc8ee4e 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl index 454a65eb3..7b0a0dc2a 100644 --- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl +++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl @@ -91,7 +91,7 @@ worker() -> | ignore. start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] @@ -156,7 +156,7 @@ handle_cast({dispatch, Context, Pid, Topic}, #{limiter := Limiter} = State) -> {ok, Limiter2} = dispatch(Context, Pid, Topic, undefined, Limiter), {noreply, State#{limiter := Limiter2}}; handle_cast({refresh_limiter, Conf}, State) -> - BucketCfg = emqx_map_lib:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), + BucketCfg = emqx_utils_maps:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), {ok, Limiter} = emqx_limiter_server:connect(?APP, internal, BucketCfg), {noreply, State#{limiter := Limiter}}; handle_cast(Msg, State) -> diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index ba96887a2..61eee0510 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -72,7 +72,7 @@ t_config(_Config) -> ), UpdateConf = fun(Enable) -> - RawConf = emqx_json:decode(ConfJson, [return_maps]), + RawConf = emqx_utils_json:decode(ConfJson, [return_maps]), UpdateJson = RawConf#{<<"enable">> := Enable}, {ok, UpdateResJson} = request_api( put, @@ -81,7 +81,7 @@ t_config(_Config) -> auth_header_(), UpdateJson ), - UpdateRawConf = emqx_json:decode(UpdateResJson, [return_maps]), + UpdateRawConf = emqx_utils_json:decode(UpdateResJson, [return_maps]), ?assertEqual(Enable, maps:get(<<"enable">>, UpdateRawConf)) end, @@ -224,7 +224,7 @@ t_lookup_and_delete(_) -> t_change_storage_type(_Config) -> Path = api_path(["mqtt", "retainer"]), {ok, ConfJson} = request_api(get, Path), - RawConf = emqx_json:decode(ConfJson, [return_maps]), + RawConf = emqx_utils_json:decode(ConfJson, [return_maps]), %% pre-conditions ?assertMatch( #{ @@ -257,7 +257,7 @@ t_change_storage_type(_Config) -> #{data := Msgs0, meta := _} = decode_json(MsgsJson0), ?assertEqual(1, length(Msgs0)), - ChangedConf = emqx_map_lib:deep_merge( + ChangedConf = emqx_utils_maps:deep_merge( RawConf, #{ <<"backend">> => @@ -271,7 +271,7 @@ t_change_storage_type(_Config) -> auth_header_(), ChangedConf ), - UpdatedRawConf = emqx_json:decode(UpdateResJson, [return_maps]), + UpdatedRawConf = emqx_utils_json:decode(UpdateResJson, [return_maps]), ?assertMatch( #{ <<"backend">> := #{ @@ -311,8 +311,8 @@ t_change_storage_type(_Config) -> %% HTTP Request %%-------------------------------------------------------------------- decode_json(Data) -> - BinJson = emqx_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJson). + BinJson = emqx_utils_json:decode(Data, [return_maps]), + emqx_utils_maps:unsafe_atom_key_map(BinJson). %%-------------------------------------------------------------------- %% Internal funcs diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 110caa33d..07c53d3e3 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. {erl_opts, [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index c4a6e2e73..a9f24ddcd 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -175,7 +175,7 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) -> flags = Flags, headers = #{ republish_by => RuleId, - properties => emqx_misc:pub_props_to_packet(PubProps) + properties => emqx_utils:pub_props_to_packet(PubProps) }, topic = Topic, payload = Payload, @@ -213,7 +213,7 @@ replace_simple_var(Val, _Data, _Default) -> Val. format_msg([], Selected) -> - emqx_json:encode(Selected); + emqx_utils_json:encode(Selected); format_msg(Tokens, Selected) -> emqx_plugin_libs_rule:proc_tmpl(Tokens, Selected). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 44b49a75b..ada52c5aa 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -115,7 +115,7 @@ start_link() -> %%------------------------------------------------------------------------------ post_config_update(_, _Req, NewRules, OldRules, _AppEnvs) -> #{added := Added, removed := Removed, changed := Updated} = - emqx_map_lib:diff_maps(NewRules, OldRules), + emqx_utils_maps:diff_maps(NewRules, OldRules), maps_foreach( fun({Id, {_Old, New}}) -> {ok, _} = update_rule(New#{id => bin(Id)}) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index f640f8303..d66f2c1c9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -343,7 +343,7 @@ param_path_id() -> {200, Result} end; '/rules'(post, #{body := Params0}) -> - case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of + case maps:get(<<"id">>, Params0, list_to_binary(emqx_utils:gen_id(8))) of <<>> -> {400, #{code => 'BAD_REQUEST', message => <<"empty rule id is not allowed">>}}; Id -> @@ -459,16 +459,16 @@ param_path_id() -> %%------------------------------------------------------------------------------ err_msg({RuleError, {_E, Reason, _S}}) -> - emqx_misc:readable_error_msg(encode_nested_error(RuleError, Reason)); + emqx_utils:readable_error_msg(encode_nested_error(RuleError, Reason)); err_msg({Reason, _Details}) -> - emqx_misc:readable_error_msg(Reason); + emqx_utils:readable_error_msg(Reason); err_msg(Msg) -> - emqx_misc:readable_error_msg(Msg). + emqx_utils:readable_error_msg(Msg). encode_nested_error(RuleError, Reason) when is_tuple(Reason) -> encode_nested_error(RuleError, element(1, Reason)); encode_nested_error(RuleError, Reason) -> - case emqx_json:safe_encode([{RuleError, Reason}]) of + case emqx_utils_json:safe_encode([{RuleError, Reason}]) of {ok, Json} -> Json; _ -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 79e0406c1..81435d9ac 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -643,10 +643,10 @@ map(Data) -> emqx_plugin_libs_rule:map(Data). bin2hexstr(Bin) when is_binary(Bin) -> - emqx_misc:bin_to_hexstr(Bin, upper). + emqx_utils:bin_to_hexstr(Bin, upper). hexstr2bin(Str) when is_binary(Str) -> - emqx_misc:hexstr_to_bin(Str). + emqx_utils:hexstr_to_bin(Str). %%------------------------------------------------------------------------------ %% NULL Funcs @@ -944,7 +944,7 @@ sha256(S) when is_binary(S) -> hash(sha256, S). hash(Type, Data) -> - emqx_misc:bin_to_hexstr(crypto:hash(Type, Data), lower). + emqx_utils:bin_to_hexstr(crypto:hash(Type, Data), lower). %%------------------------------------------------------------------------------ %% gzip Funcs @@ -987,10 +987,10 @@ base64_decode(Data) when is_binary(Data) -> base64:decode(Data). json_encode(Data) -> - emqx_json:encode(Data). + emqx_utils_json:encode(Data). json_decode(Data) -> - emqx_json:decode(Data, [return_maps]). + emqx_utils_json:decode(Data, [return_maps]). term_encode(Term) -> erlang:term_to_binary(Term). diff --git a/apps/emqx_rule_engine/src/emqx_rule_maps.erl b/apps/emqx_rule_engine/src/emqx_rule_maps.erl index 13f99c88b..3dfffca46 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_maps.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_maps.erl @@ -86,7 +86,7 @@ general_map_put(Key, Val, Map, OrgData) -> ). general_find(KeyOrIndex, Data, OrgData, Handler) when is_binary(Data) -> - try emqx_json:decode(Data, [return_maps]) of + try emqx_utils_json:decode(Data, [return_maps]) of Json -> general_find(KeyOrIndex, Json, OrgData, Handler) catch _:_ -> Handler(not_found) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 51491df53..d7412d03c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -495,7 +495,7 @@ cache_payload(DecodedP) -> safe_decode_and_cache(MaybeJson) -> try - cache_payload(emqx_json:decode(MaybeJson, [return_maps])) + cache_payload(emqx_utils_json:decode(MaybeJson, [return_maps])) catch _:_ -> error({decode_json_failed, MaybeJson}) end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index e97c45b35..455efe389 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -48,7 +48,7 @@ test(#{sql := Sql, context := Context}) -> end. test_rule(Sql, Select, Context, EventTopics) -> - RuleId = iolist_to_binary(["sql_tester:", emqx_misc:gen_id(16)]), + RuleId = iolist_to_binary(["sql_tester:", emqx_utils:gen_id(16)]), ok = emqx_rule_engine:maybe_add_metrics_for_rule(RuleId), Rule = #{ id => RuleId, diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 93d7c7352..eb253e516 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -614,7 +614,9 @@ t_event_client_disconnected_normal(_Config) -> receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), - ?assertMatch(#{<<"reason">> := <<"normal">>}, emqx_json:decode(Payload, [return_maps])) + ?assertMatch( + #{<<"reason">> := <<"normal">>}, emqx_utils_json:decode(Payload, [return_maps]) + ) after 1000 -> ct:fail(wait_for_repub_disconnected_normal) end, @@ -651,7 +653,9 @@ t_event_client_disconnected_kicked(_Config) -> receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), - ?assertMatch(#{<<"reason">> := <<"kicked">>}, emqx_json:decode(Payload, [return_maps])) + ?assertMatch( + #{<<"reason">> := <<"kicked">>}, emqx_utils_json:decode(Payload, [return_maps]) + ) after 1000 -> ct:fail(wait_for_repub_disconnected_kicked) end, @@ -692,7 +696,7 @@ t_event_client_disconnected_discarded(_Config) -> {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), ?assertMatch( - #{<<"reason">> := <<"discarded">>}, emqx_json:decode(Payload, [return_maps]) + #{<<"reason">> := <<"discarded">>}, emqx_utils_json:decode(Payload, [return_maps]) ) after 1000 -> ct:fail(wait_for_repub_disconnected_discarded) @@ -737,7 +741,7 @@ t_event_client_disconnected_takenover(_Config) -> {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), ?assertMatch( - #{<<"reason">> := <<"takenover">>}, emqx_json:decode(Payload, [return_maps]) + #{<<"reason">> := <<"takenover">>}, emqx_utils_json:decode(Payload, [return_maps]) ) after 1000 -> ct:fail(wait_for_repub_disconnected_discarded) @@ -2800,7 +2804,7 @@ verify_event(EventName) -> [ begin %% verify fields can be formatted to JSON string - _ = emqx_json:encode(Fields), + _ = emqx_utils_json:encode(Fields), %% verify metadata fields verify_metadata_fields(EventName, Fields), %% verify available fields for each event name diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index e94806a7b..8d7546fca 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -46,13 +46,13 @@ end_per_suite(_Config) -> ok. init_per_testcase(t_crud_rule_api, Config) -> - meck:new(emqx_json, [passthrough]), + meck:new(emqx_utils_json, [passthrough]), init_per_testcase(common, Config); init_per_testcase(_, Config) -> Config. end_per_testcase(t_crud_rule_api, Config) -> - meck:unload(emqx_json), + meck:unload(emqx_utils_json), end_per_testcase(common, Config); end_per_testcase(_, _Config) -> {200, #{data := Rules}} = @@ -136,7 +136,7 @@ t_crud_rule_api(_Config) -> ), ?assertMatch( #{<<"select_and_transform_error">> := <<"decode_json_failed">>}, - emqx_json:decode(SelectAndTransformJsonError, [return_maps]) + emqx_utils_json:decode(SelectAndTransformJsonError, [return_maps]) ), {400, #{ code := 'BAD_REQUEST', @@ -150,7 +150,7 @@ t_crud_rule_api(_Config) -> ), ?assertMatch( #{<<"select_and_transform_error">> := <<"badarg">>}, - emqx_json:decode(SelectAndTransformBadArgError, [return_maps]) + emqx_utils_json:decode(SelectAndTransformBadArgError, [return_maps]) ), {400, #{ code := 'BAD_REQUEST', @@ -162,7 +162,7 @@ t_crud_rule_api(_Config) -> ) ), ?assertMatch({match, _}, re:run(BadSqlMessage, "syntax error")), - meck:expect(emqx_json, safe_encode, 1, {error, foo}), + meck:expect(emqx_utils_json, safe_encode, 1, {error, foo}), ?assertMatch( {400, #{ code := 'BAD_REQUEST', diff --git a/apps/emqx_slow_subs/rebar.config b/apps/emqx_slow_subs/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_slow_subs/rebar.config +++ b/apps/emqx_slow_subs/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl index 6b0721e3d..5196868c7 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl @@ -108,7 +108,7 @@ t_get_history(_) -> "page=1&limit=10", auth_header_() ), - #{<<"data">> := [First | _]} = emqx_json:decode(Data, [return_maps]), + #{<<"data">> := [First | _]} = emqx_utils_json:decode(Data, [return_maps]), ?assertMatch( #{ @@ -165,8 +165,8 @@ t_settting(_) -> ?assertEqual(Conf2#{stats_type := <<"internal">>}, GetReturn). decode_json(Data) -> - BinJosn = emqx_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJosn). + BinJosn = emqx_utils_json:decode(Data, [return_maps]), + emqx_utils_maps:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). @@ -187,7 +187,7 @@ request_api(Method, Url, QueryParams, Auth, Body) -> "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)}). do_request_api(Method, Request) -> ct:pal("Method: ~p, Request: ~p", [Method, Request]), diff --git a/apps/emqx_statsd/rebar.config b/apps/emqx_statsd/rebar.config index bb9a14272..a1383d920 100644 --- a/apps/emqx_statsd/rebar.config +++ b/apps/emqx_statsd/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {estatsd, {git, "https://github.com/emqx/estatsd", {tag, "0.1.0"}}} ]}. diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 9d40c6857..412e0b685 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_statsd, [ {description, "EMQX Statsd"}, - {vsn, "5.0.7"}, + {vsn, "5.0.8"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, [ diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl index c2e1819ac..c5a7fc1c8 100644 --- a/apps/emqx_statsd/src/emqx_statsd.erl +++ b/apps/emqx_statsd/src/emqx_statsd.erl @@ -144,7 +144,7 @@ flush_interval(_FlushInterval, SampleInterval) -> SampleInterval. ensure_timer(State = #{sample_time_interval := SampleTimeInterval}) -> - State#{timer => emqx_misc:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}. + State#{timer => emqx_utils:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}. check_multicall_result({Results, []}) -> case diff --git a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl index bcc710050..b5669e4b9 100644 --- a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl +++ b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl @@ -200,7 +200,7 @@ request(Method) -> request(Method, []). request(Method, Body) -> case request(Method, uri(["statsd"]), Body) of {ok, 200, Res} -> - {ok, emqx_json:decode(Res, [return_maps])}; + {ok, emqx_utils_json:decode(Res, [return_maps])}; {ok, _Status, _} -> error end. diff --git a/apps/emqx_utils/README.md b/apps/emqx_utils/README.md new file mode 100644 index 000000000..f8c386f3d --- /dev/null +++ b/apps/emqx_utils/README.md @@ -0,0 +1,24 @@ +# Erlang utility library for EMQX + +## Overview + +`emqx_utils` is a collection of utility functions for EMQX, organized into +several modules. It provides various functionalities to make it easier to work +with EMQX, such as binary manipulations, maps, JSON en- and decoding, ets table +handling, data conversions, and more. + +## Features + +- `emqx_utils`: unsorted helper functions, formerly known as `emqx_misc` - NEEDS WORK +- `emqx_utils_api`: collection of helper functions for API responses +- `emqx_utils_binary`: binary reverse, join, trim etc +- `emqx_utils_ets`: convenience functions for creating and looking up data in ets tables. +- `emqx_utils_json`: JSON encoding and decoding +- `emqx_utils_maps`: convenience functions for map lookup and manipulation like + deep_get etc. + +## Contributing + +Please see our [contributing guidelines](../../CONTRIBUTING.md) for information +on how to contribute to `emqx_utils`. We welcome bug reports, feature requests, +and pull requests. diff --git a/apps/emqx/include/emqx_api_lib.hrl b/apps/emqx_utils/include/emqx_utils_api.hrl similarity index 97% rename from apps/emqx/include/emqx_api_lib.hrl rename to apps/emqx_utils/include/emqx_utils_api.hrl index 549b0f94c..bfc8e0a53 100644 --- a/apps/emqx/include/emqx_api_lib.hrl +++ b/apps/emqx_utils/include/emqx_utils_api.hrl @@ -17,7 +17,7 @@ -ifndef(EMQX_API_LIB_HRL). -define(EMQX_API_LIB_HRL, true). --define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_misc:readable_error_msg(REASON)}). +-define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_utils:readable_error_msg(REASON)}). -define(OK(CONTENT), {200, CONTENT}). diff --git a/apps/emqx_utils/rebar.config b/apps/emqx_utils/rebar.config new file mode 100644 index 000000000..4c39cfe64 --- /dev/null +++ b/apps/emqx_utils/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang -*- + +{erl_opts, [ + debug_info +]}. + +{deps, [ + {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src new file mode 100644 index 000000000..eb6371411 --- /dev/null +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -0,0 +1,27 @@ +%% -*- mode: erlang -*- +{application, emqx_utils, [ + {description, "Miscellaneous utilities for EMQX apps"}, + % strict semver, bump manually! + {vsn, "5.0.0"}, + {modules, [ + emqx_utils, + emqx_utils_api, + emqx_utils_binary, + emqx_utils_ets, + emqx_utils_json, + emqx_utils_maps + ]}, + {registered, []}, + {applications, [ + kernel, + stdlib, + jiffy + ]}, + {env, []}, + {licenses, ["Apache-2.0"]}, + {maintainers, ["EMQX Team "]}, + {links, [ + {"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx/emqx"} + ]} +]}. diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx_utils/src/emqx_utils.erl similarity index 99% rename from apps/emqx/src/emqx_misc.erl rename to apps/emqx_utils/src/emqx_utils.erl index d0c7397c8..21dbd339d 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -14,14 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_misc). +-module(emqx_utils). -compile(inline). +%% [TODO] Cleanup so the instruction below is not necessary. -elvis([{elvis_style, god_modules, disable}]). --include("types.hrl"). --include("logger.hrl"). - -export([ merge_opts/2, maybe_apply/2, @@ -71,6 +69,8 @@ -export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]). +-type maybe(T) :: undefined | T. + -dialyzer({nowarn_function, [nolink_apply/2]}). -define(SHORT, 8). @@ -221,6 +221,7 @@ drain_down(Cnt, Acc) -> %% `ok': There is nothing out of the ordinary. %% `shutdown': Some numbers (message queue length hit the limit), %% hence shutdown for greater good (system stability). +%% [FIXME] cross-dependency on `emqx_types`. -spec check_oom(emqx_types:oom_policy()) -> ok | {shutdown, term()}. check_oom(Policy) -> check_oom(self(), Policy). @@ -279,6 +280,7 @@ proc_name(Mod, Id) -> list_to_atom(lists:concat([Mod, "_", Id])). %% Get Proc's Stats. +%% [FIXME] cross-dependency on `emqx_types`. -spec proc_stats() -> emqx_types:stats(). proc_stats() -> proc_stats(self()). diff --git a/apps/emqx/src/emqx_api_lib.erl b/apps/emqx_utils/src/emqx_utils_api.erl similarity index 95% rename from apps/emqx/src/emqx_api_lib.erl rename to apps/emqx_utils/src/emqx_utils_api.erl index f1d65f350..e6bd07272 100644 --- a/apps/emqx/src/emqx_api_lib.erl +++ b/apps/emqx_utils/src/emqx_utils_api.erl @@ -14,14 +14,14 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_api_lib). +-module(emqx_utils_api). -export([ with_node/2, with_node_or_cluster/2 ]). --include("emqx_api_lib.hrl"). +-include("emqx_utils_api.hrl"). -define(NODE_NOT_FOUND(NODE), ?NOT_FOUND(<<"Node not found: ", NODE/binary>>)). @@ -51,7 +51,7 @@ with_node_or_cluster(Node, Fun) -> -spec lookup_node(atom() | binary()) -> {ok, atom()} | not_found. lookup_node(BinNode) when is_binary(BinNode) -> - case emqx_misc:safe_to_existing_atom(BinNode, utf8) of + case emqx_utils:safe_to_existing_atom(BinNode, utf8) of {ok, Node} -> is_running_node(Node); _Error -> diff --git a/apps/emqx_gateway_lwm2m/src/binary_util.erl b/apps/emqx_utils/src/emqx_utils_binary.erl similarity index 97% rename from apps/emqx_gateway_lwm2m/src/binary_util.erl rename to apps/emqx_utils/src/emqx_utils_binary.erl index 68ac7a0d7..26976496d 100644 --- a/apps/emqx_gateway_lwm2m/src/binary_util.erl +++ b/apps/emqx_utils/src/emqx_utils_binary.erl @@ -1,4 +1,6 @@ %%-------------------------------------------------------------------- +%% Original file taken from https://github.com/arcusfelis/binary2 +%% Copyright (c) 2016 Michael Uvarov %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,9 +16,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(binary_util). - -%% copied from https://github.com/arcusfelis/binary2 +-module(emqx_utils_binary). %% Bytes -export([ diff --git a/apps/emqx/src/emqx_tables.erl b/apps/emqx_utils/src/emqx_utils_ets.erl similarity index 98% rename from apps/emqx/src/emqx_tables.erl rename to apps/emqx_utils/src/emqx_utils_ets.erl index ffdf7d891..099152675 100644 --- a/apps/emqx/src/emqx_tables.erl +++ b/apps/emqx_utils/src/emqx_utils_ets.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_tables). +-module(emqx_utils_ets). -export([ new/1, diff --git a/apps/emqx/src/emqx_json.erl b/apps/emqx_utils/src/emqx_utils_json.erl similarity index 91% rename from apps/emqx/src/emqx_json.erl rename to apps/emqx_utils/src/emqx_utils_json.erl index 7827b98c9..df7388c94 100644 --- a/apps/emqx/src/emqx_json.erl +++ b/apps/emqx_utils/src/emqx_utils_json.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_json). +-module(emqx_utils_json). -compile(inline). @@ -46,6 +46,10 @@ ]} ). +-export([is_json/1]). + +-compile({inline, [is_json/1]}). + -type encode_options() :: jiffy:encode_options(). -type decode_options() :: jiffy:decode_options(). @@ -79,7 +83,7 @@ safe_encode(Term, Opts) -> end. -spec decode(json_text()) -> json_term(). -decode(Json) -> decode(Json, []). +decode(Json) -> decode(Json, [return_maps]). -spec decode(json_text(), decode_options()) -> json_term(). decode(Json, Opts) -> @@ -100,6 +104,10 @@ safe_decode(Json, Opts) -> {error, Reason} end. +-spec is_json(json_text()) -> boolean(). +is_json(Json) -> + element(1, safe_decode(Json)) =:= ok. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -117,6 +125,8 @@ to_ejson([{_, _} | _] = L) -> {[{K, to_ejson(V)} || {K, V} <- L]}; to_ejson(L) when is_list(L) -> [to_ejson(E) || E <- L]; +to_ejson(M) when is_map(M) -> + maps:map(fun(_K, V) -> to_ejson(V) end, M); to_ejson(T) -> T. diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx_utils/src/emqx_utils_maps.erl similarity index 99% rename from apps/emqx/src/emqx_map_lib.erl rename to apps/emqx_utils/src/emqx_utils_maps.erl index 631c3914d..6bec32ae3 100644 --- a/apps/emqx/src/emqx_map_lib.erl +++ b/apps/emqx_utils/src/emqx_utils_maps.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_map_lib). +-module(emqx_utils_maps). -export([ deep_get/2, @@ -210,6 +210,7 @@ binary_string_kv(K, V, JsonableFun) -> {K1, V1} -> {binary_string(K1), V1} end. +%% [FIXME] this doesn't belong here binary_string([]) -> []; binary_string(Val) when is_list(Val) -> @@ -332,7 +333,7 @@ deep_filter(M, F) when is_map(M) -> if_only_to_toggle_enable(OldConf, Conf) -> #{added := Added, removed := Removed, changed := Updated} = - emqx_map_lib:diff_maps(OldConf, Conf), + emqx_utils_maps:diff_maps(OldConf, Conf), case {Added, Removed, Updated} of {Added, Removed, #{enable := _} = Updated} when map_size(Added) =:= 0, diff --git a/apps/emqx/test/emqx_misc_SUITE.erl b/apps/emqx_utils/test/emqx_utils_SUITE.erl similarity index 68% rename from apps/emqx/test/emqx_misc_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_SUITE.erl index c068a087e..99516b0eb 100644 --- a/apps/emqx/test/emqx_misc_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_misc_SUITE). +-module(emqx_utils_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -32,7 +32,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). t_merge_opts(_) -> - Opts = emqx_misc:merge_opts(?SOCKOPTS, [ + Opts = emqx_utils:merge_opts(?SOCKOPTS, [ raw, binary, {backlog, 1024}, @@ -57,58 +57,59 @@ t_merge_opts(_) -> ). t_maybe_apply(_) -> - ?assertEqual(undefined, emqx_misc:maybe_apply(fun(A) -> A end, undefined)), - ?assertEqual(a, emqx_misc:maybe_apply(fun(A) -> A end, a)). + ?assertEqual(undefined, emqx_utils:maybe_apply(fun(A) -> A end, undefined)), + ?assertEqual(a, emqx_utils:maybe_apply(fun(A) -> A end, a)). t_run_fold(_) -> - ?assertEqual(1, emqx_misc:run_fold([], 1, state)), + ?assertEqual(1, emqx_utils:run_fold([], 1, state)), Add = fun(I, St) -> I + St end, Mul = fun(I, St) -> I * St end, - ?assertEqual(6, emqx_misc:run_fold([Add, Mul], 1, 2)). + ?assertEqual(6, emqx_utils:run_fold([Add, Mul], 1, 2)). t_pipeline(_) -> - ?assertEqual({ok, input, state}, emqx_misc:pipeline([], input, state)), + ?assertEqual({ok, input, state}, emqx_utils:pipeline([], input, state)), Funs = [ fun(_I, _St) -> ok end, fun(_I, St) -> {ok, St + 1} end, fun(I, St) -> {ok, I + 1, St + 1} end, fun(I, St) -> {ok, I * 2, St * 2} end ], - ?assertEqual({ok, 4, 6}, emqx_misc:pipeline(Funs, 1, 1)), + ?assertEqual({ok, 4, 6}, emqx_utils:pipeline(Funs, 1, 1)), ?assertEqual( - {error, undefined, 1}, emqx_misc:pipeline([fun(_I) -> {error, undefined} end], 1, 1) + {error, undefined, 1}, emqx_utils:pipeline([fun(_I) -> {error, undefined} end], 1, 1) ), ?assertEqual( - {error, undefined, 2}, emqx_misc:pipeline([fun(_I, _St) -> {error, undefined, 2} end], 1, 1) + {error, undefined, 2}, + emqx_utils:pipeline([fun(_I, _St) -> {error, undefined, 2} end], 1, 1) ). t_start_timer(_) -> - TRef = emqx_misc:start_timer(1, tmsg), + TRef = emqx_utils:start_timer(1, tmsg), timer:sleep(2), ?assertEqual([{timeout, TRef, tmsg}], drain()), - ok = emqx_misc:cancel_timer(TRef). + ok = emqx_utils:cancel_timer(TRef). t_cancel_timer(_) -> - Timer = emqx_misc:start_timer(0, foo), - ok = emqx_misc:cancel_timer(Timer), + Timer = emqx_utils:start_timer(0, foo), + ok = emqx_utils:cancel_timer(Timer), ?assertEqual([], drain()), - ok = emqx_misc:cancel_timer(undefined). + ok = emqx_utils:cancel_timer(undefined). t_proc_name(_) -> - ?assertEqual(emqx_pool_1, emqx_misc:proc_name(emqx_pool, 1)). + ?assertEqual(emqx_pool_1, emqx_utils:proc_name(emqx_pool, 1)). t_proc_stats(_) -> Pid1 = spawn(fun() -> exit(normal) end), timer:sleep(10), - ?assertEqual([], emqx_misc:proc_stats(Pid1)), + ?assertEqual([], emqx_utils:proc_stats(Pid1)), Pid2 = spawn(fun() -> - ?assertMatch([{mailbox_len, 0} | _], emqx_misc:proc_stats()), + ?assertMatch([{mailbox_len, 0} | _], emqx_utils:proc_stats()), timer:sleep(200) end), timer:sleep(10), Pid2 ! msg, timer:sleep(10), - ?assertMatch([{mailbox_len, 1} | _], emqx_misc:proc_stats(Pid2)). + ?assertMatch([{mailbox_len, 1} | _], emqx_utils:proc_stats(Pid2)). t_drain_deliver(_) -> self() ! {deliver, t1, m1}, @@ -118,24 +119,24 @@ t_drain_deliver(_) -> {deliver, t1, m1}, {deliver, t2, m2} ], - emqx_misc:drain_deliver(2) + emqx_utils:drain_deliver(2) ). t_drain_down(_) -> {Pid1, _Ref1} = erlang:spawn_monitor(fun() -> ok end), {Pid2, _Ref2} = erlang:spawn_monitor(fun() -> ok end), timer:sleep(100), - ?assertEqual([Pid1, Pid2], lists:sort(emqx_misc:drain_down(2))), - ?assertEqual([], emqx_misc:drain_down(1)). + ?assertEqual([Pid1, Pid2], lists:sort(emqx_utils:drain_down(2))), + ?assertEqual([], emqx_utils:drain_down(1)). t_index_of(_) -> - try emqx_misc:index_of(a, []) of + try emqx_utils:index_of(a, []) of _ -> ct:fail(should_throw_error) catch error:Reason -> ?assertEqual(badarg, Reason) end, - ?assertEqual(3, emqx_misc:index_of(a, [b, c, a, e, f])). + ?assertEqual(3, emqx_utils:index_of(a, [b, c, a, e, f])). t_check(_) -> Policy = #{ @@ -144,11 +145,11 @@ t_check(_) -> enable => true }, [self() ! {msg, I} || I <- lists:seq(1, 5)], - ?assertEqual(ok, emqx_misc:check_oom(Policy)), + ?assertEqual(ok, emqx_utils:check_oom(Policy)), [self() ! {msg, I} || I <- lists:seq(1, 6)], ?assertEqual( {shutdown, #{reason => message_queue_too_long, value => 11, max => 10}}, - emqx_misc:check_oom(Policy) + emqx_utils:check_oom(Policy) ). drain() -> @@ -162,22 +163,22 @@ drain(Acc) -> end. t_rand_seed(_) -> - ?assert(is_tuple(emqx_misc:rand_seed())). + ?assert(is_tuple(emqx_utils:rand_seed())). t_now_to_secs(_) -> - ?assert(is_integer(emqx_misc:now_to_secs(os:timestamp()))). + ?assert(is_integer(emqx_utils:now_to_secs(os:timestamp()))). t_now_to_ms(_) -> - ?assert(is_integer(emqx_misc:now_to_ms(os:timestamp()))). + ?assert(is_integer(emqx_utils:now_to_ms(os:timestamp()))). t_gen_id(_) -> - ?assertEqual(10, length(emqx_misc:gen_id(10))), - ?assertEqual(20, length(emqx_misc:gen_id(20))). + ?assertEqual(10, length(emqx_utils:gen_id(10))), + ?assertEqual(20, length(emqx_utils:gen_id(20))). t_pmap_normal(_) -> ?assertEqual( [5, 7, 9], - emqx_misc:pmap( + emqx_utils:pmap( fun({A, B}) -> A + B end, [{2, 3}, {3, 4}, {4, 5}] ) @@ -186,7 +187,7 @@ t_pmap_normal(_) -> t_pmap_timeout(_) -> ?assertExit( timeout, - emqx_misc:pmap( + emqx_utils:pmap( fun (timeout) -> ct:sleep(1000); ({A, B}) -> A + B @@ -199,7 +200,7 @@ t_pmap_timeout(_) -> t_pmap_exception(_) -> ?assertError( foobar, - emqx_misc:pmap( + emqx_utils:pmap( fun (error) -> error(foobar); ({A, B}) -> A + B diff --git a/apps/emqx/test/emqx_api_lib_SUITE.erl b/apps/emqx_utils/test/emqx_utils_api_SUITE.erl similarity index 92% rename from apps/emqx/test/emqx_api_lib_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_api_SUITE.erl index 29f5c6095..3ed3cd250 100644 --- a/apps/emqx/test/emqx_api_lib_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_api_SUITE.erl @@ -14,12 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_api_lib_SUITE). +-module(emqx_utils_api_SUITE). -compile(export_all). -compile(nowarn_export_all). --include("emqx_api_lib.hrl"). +-include("emqx_utils_api.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(DUMMY, dummy_module). @@ -45,14 +45,14 @@ end_per_testcase(_Case, _Config) -> meck:unload(?DUMMY). t_with_node(_) -> - test_with(fun emqx_api_lib:with_node/2, [<<"all">>]). + test_with(fun emqx_utils_api:with_node/2, [<<"all">>]). t_with_node_or_cluster(_) -> - test_with(fun emqx_api_lib:with_node_or_cluster/2, []), + test_with(fun emqx_utils_api:with_node_or_cluster/2, []), meck:reset(?DUMMY), ?assertEqual( ?OK(success), - emqx_api_lib:with_node_or_cluster( + emqx_utils_api:with_node_or_cluster( <<"all">>, fun ?DUMMY:expect_success/1 ) diff --git a/apps/emqx_utils/test/emqx_utils_binary_tests.erl b/apps/emqx_utils/test/emqx_utils_binary_tests.erl new file mode 100644 index 000000000..79851dca5 --- /dev/null +++ b/apps/emqx_utils/test/emqx_utils_binary_tests.erl @@ -0,0 +1,213 @@ +%%-------------------------------------------------------------------- +%% Original file taken from https://github.com/arcusfelis/binary2 +%% Copyright (c) 2016 Michael Uvarov +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +-module(emqx_utils_binary_tests). + +-import(emqx_utils_binary, [ + trim/1, + ltrim/1, + rtrim/1, + trim/2, + ltrim/2, + rtrim/2, + reverse/1, + inverse/1, + join/2, + suffix/2, + prefix/2, + duplicate/2, + union/2, + intersection/2, + subtract/2, + optimize_patterns/1 +]). + +-include_lib("eunit/include/eunit.hrl"). + +trim1_test_() -> + [ + ?_assertEqual(trim(<<>>), <<>>), + ?_assertEqual(trim(<<0, 0, 0>>), <<>>), + ?_assertEqual(trim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(trim(<<0, 1, 2>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 0, 1, 2>>), <<1, 2>>), + ?_assertEqual(trim(<<1, 2, 0, 0>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 1, 2, 0>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<1, 2>>) + ]. + +ltrim1_test_() -> + [ + ?_assertEqual(ltrim(<<>>), <<>>), + ?_assertEqual(ltrim(<<0, 0, 0>>), <<>>), + ?_assertEqual(ltrim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(ltrim(<<0, 1, 2>>), <<1, 2>>), + ?_assertEqual(ltrim(<<0, 0, 1, 2>>), <<1, 2>>), + ?_assertEqual(ltrim(<<1, 2, 0, 0>>), <<1, 2, 0, 0>>), + ?_assertEqual(ltrim(<<0, 1, 2, 0>>), <<1, 2, 0>>), + ?_assertEqual(ltrim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<1, 2, 0, 0, 0>>) + ]. + +rtrim1_test_() -> + [ + ?_assertEqual(rtrim(<<>>), <<>>), + ?_assertEqual(rtrim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(rtrim(<<0, 0, 0>>), <<>>), + ?_assertEqual(rtrim(<<0, 1, 2>>), <<0, 1, 2>>), + ?_assertEqual(rtrim(<<0, 0, 1, 2>>), <<0, 0, 1, 2>>), + ?_assertEqual(rtrim(<<1, 2, 0, 0>>), <<1, 2>>), + ?_assertEqual(rtrim(<<0, 1, 2, 0>>), <<0, 1, 2>>), + ?_assertEqual(rtrim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<0, 0, 0, 1, 2>>) + ]. + +trim2_test_() -> + [ + ?_assertEqual(trim(<<5>>, 5), <<>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, 5), <<1, 2>>), + ?_assertEqual(trim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<1, 2, 0, 0, 0>>) + ]. + +ltrim2_test_() -> + [ + ?_assertEqual(ltrim(<<5>>, 5), <<>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, 5), <<1, 2, 5>>), + ?_assertEqual(ltrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<1, 2, 0, 0, 0>>) + ]. + +rtrim2_test_() -> + [ + ?_assertEqual(rtrim(<<5>>, 5), <<>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, 5), <<5, 1, 2>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<5, 5, 5, 1, 2, 0, 0, 0>>) + ]. + +mtrim2_test_() -> + [ + ?_assertEqual(trim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, [1, 5]), <<2>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, [1, 2, 5]), <<>>), + ?_assertEqual(trim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<2, 0, 0, 0>>) + ]. + +mltrim2_test_() -> + [ + ?_assertEqual(ltrim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, [1, 5]), <<2, 5>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, [2, 5]), <<1, 2, 5>>), + ?_assertEqual(ltrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<2, 0, 0, 0>>) + ]. + +mrtrim2_test_() -> + [ + ?_assertEqual(rtrim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, [1, 5]), <<5, 1, 2>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, [2, 5]), <<5, 1>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<5, 5, 5, 1, 2, 0, 0, 0>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [0, 5]), <<5, 5, 5, 1, 2>>) + ]. + +reverse_test_() -> + [?_assertEqual(reverse(<<0, 1, 2>>), <<2, 1, 0>>)]. + +join_test_() -> + [ + ?_assertEqual(join([<<1, 2>>, <<3, 4>>, <<5, 6>>], <<0>>), <<1, 2, 0, 3, 4, 0, 5, 6>>), + ?_assertEqual( + join([<<"abc">>, <<"def">>, <<"xyz">>], <<"|">>), + <<"abc|def|xyz">> + ), + ?_assertEqual( + join([<<>>, <<"|">>, <<"x|z">>], <<"|">>), + <<"|||x|z">> + ), + ?_assertEqual( + join([<<"abc">>, <<"def">>, <<"xyz">>], <<>>), + <<"abcdefxyz">> + ), + ?_assertEqual(join([], <<"|">>), <<>>) + ]. + +duplicate_test_() -> + [ + ?_assertEqual(duplicate(5, <<1, 2>>), <<1, 2, 1, 2, 1, 2, 1, 2, 1, 2>>), + ?_assertEqual(duplicate(50, <<0>>), <<0:400>>) + ]. + +suffix_test_() -> + [ + ?_assertEqual(suffix(<<1, 2, 3, 4, 5>>, 2), <<4, 5>>), + ?_assertError(badarg, prefix(<<1, 2, 3, 4, 5>>, 25)) + ]. + +prefix_test_() -> + [ + ?_assertEqual(prefix(<<1, 2, 3, 4, 5>>, 2), <<1, 2>>), + ?_assertError(badarg, prefix(<<1, 2, 3, 4, 5>>, 25)) + ]. + +union_test_() -> + [ + ?_assertEqual( + union( + <<2#0011011:7>>, + <<2#1011110:7>> + ), + <<2#1011111:7>> + ) + ]. + +inverse_test_() -> + [ + ?_assertEqual(inverse(inverse(<<0, 1, 2>>)), <<0, 1, 2>>), + ?_assertEqual(inverse(<<0>>), <<255>>), + ?_assertEqual(inverse(<<2#1:1>>), <<2#0:1>>), + ?_assertEqual(inverse(<<2#0:1>>), <<2#1:1>>), + ?_assertEqual( + inverse(<<2#01:2>>), + <<2#10:2>> + ), + ?_assertEqual( + inverse(<<2#0011011:7>>), + <<2#1100100:7>> + ) + ]. + +intersection_test_() -> + [ + ?_assertEqual( + intersection( + <<2#0011011>>, + <<2#1011110>> + ), + <<2#0011010>> + ) + ]. + +subtract_test_() -> + [ + ?_assertEqual( + subtract( + <<2#0011011>>, + <<2#1011110>> + ), + <<2#0000001>> + ) + ]. + +optimize_patterns_test_() -> + [ + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"t">>, <<"test">>]) + ), + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"t">>, <<"t">>, <<"test">>]) + ), + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"test">>, <<"t">>, <<"t">>]) + ) + ]. diff --git a/apps/emqx/test/emqx_tables_SUITE.erl b/apps/emqx_utils/test/emqx_utils_ets_SUITE.erl similarity index 73% rename from apps/emqx/test/emqx_tables_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_ets_SUITE.erl index ad53e7139..13bf427fd 100644 --- a/apps/emqx/test/emqx_tables_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_ets_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_tables_SUITE). +-module(emqx_utils_ets_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -26,19 +26,19 @@ all() -> emqx_common_test_helpers:all(?MODULE). t_new(_) -> - ok = emqx_tables:new(?TAB), - ok = emqx_tables:new(?TAB, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB), + ok = emqx_utils_ets:new(?TAB, [{read_concurrency, true}]), ?assertEqual(?TAB, ets:info(?TAB, name)). t_lookup_value(_) -> - ok = emqx_tables:new(?TAB, []), + ok = emqx_utils_ets:new(?TAB, []), true = ets:insert(?TAB, {key, val}), - ?assertEqual(val, emqx_tables:lookup_value(?TAB, key)), - ?assertEqual(undefined, emqx_tables:lookup_value(?TAB, badkey)). + ?assertEqual(val, emqx_utils_ets:lookup_value(?TAB, key)), + ?assertEqual(undefined, emqx_utils_ets:lookup_value(?TAB, badkey)). t_delete(_) -> - ok = emqx_tables:new(?TAB, []), + ok = emqx_utils_ets:new(?TAB, []), ?assertEqual(?TAB, ets:info(?TAB, name)), - ok = emqx_tables:delete(?TAB), - ok = emqx_tables:delete(?TAB), + ok = emqx_utils_ets:delete(?TAB), + ok = emqx_utils_ets:delete(?TAB), ?assertEqual(undefined, ets:info(?TAB, name)). diff --git a/apps/emqx/test/emqx_json_SUITE.erl b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl similarity index 83% rename from apps/emqx/test/emqx_json_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_json_SUITE.erl index a0bf48e4e..daf31b440 100644 --- a/apps/emqx/test/emqx_json_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_json_SUITE). +-module(emqx_utils_json_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -22,7 +22,7 @@ -include_lib("eunit/include/eunit.hrl"). -import( - emqx_json, + emqx_utils_json, [ encode/1, decode/1, @@ -51,7 +51,7 @@ %% #{<<"foo">> => <<"bar">>} -> {"foo": "bar"} -> #{<<"foo">> => <<"bar">>} %%-------------------------------------------------------------------- -%% but in emqx_json, we use the jsx style for it: +%% but in emqx_utils_json, we use the jsx style for it: %%-------------------------------------------------------------------- %% Erlang JSON Erlang %% ------------------------------------------------------------------- @@ -84,10 +84,10 @@ t_decode_encode(_) -> 1.25 = decode(encode(1.25)), [] = decode(encode([])), [true, 1] = decode(encode([true, 1])), - [{}] = decode(encode([{}])), - [{<<"foo">>, <<"bar">>}] = decode(encode([{foo, bar}])), - [{<<"foo">>, <<"bar">>}] = decode(encode([{<<"foo">>, <<"bar">>}])), - [[{<<"foo">>, <<"bar">>}]] = decode(encode([[{<<"foo">>, <<"bar">>}]])), + [{}] = decode(encode([{}]), []), + [{<<"foo">>, <<"bar">>}] = decode(encode([{foo, bar}]), []), + [{<<"foo">>, <<"bar">>}] = decode(encode([{<<"foo">>, <<"bar">>}]), []), + [[{<<"foo">>, <<"bar">>}]] = decode(encode([[{<<"foo">>, <<"bar">>}]]), []), [ [ {<<"foo">>, <<"bar">>}, @@ -101,7 +101,8 @@ t_decode_encode(_) -> {<<"a">>, <<"b">>} ], [{<<"x">>, <<"y">>}] - ]) + ]), + [] ), #{<<"foo">> := <<"bar">>} = decode(encode(#{<<"foo">> => <<"bar">>}), [return_maps]), JsonText = <<"{\"bool\":true,\"int\":10,\"foo\":\"bar\"}">>, @@ -110,8 +111,12 @@ t_decode_encode(_) -> <<"int">> => 10, <<"foo">> => <<"bar">> }, - ?assertEqual(JsonText, encode({decode(JsonText)})), - ?assertEqual(JsonMaps, decode(JsonText, [return_maps])). + ?assertEqual(JsonText, encode({decode(JsonText, [])})), + ?assertEqual(JsonMaps, decode(JsonText, [return_maps])), + ?assertEqual( + #{<<"foo">> => #{<<"bar">> => <<"baz">>}}, + decode(encode(#{<<"foo">> => [{<<"bar">>, <<"baz">>}]})) + ). t_safe_decode_encode(_) -> safe_encode_decode(null), @@ -123,16 +128,20 @@ t_safe_decode_encode(_) -> 1.25 = safe_encode_decode(1.25), [] = safe_encode_decode([]), [true, 1] = safe_encode_decode([true, 1]), - [{}] = decode(encode([{}])), + [{}] = decode(encode([{}]), []), [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{foo, bar}]), [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{<<"foo">>, <<"bar">>}]), [[{<<"foo">>, <<"bar">>}]] = safe_encode_decode([[{<<"foo">>, <<"bar">>}]]), - {ok, Json} = emqx_json:safe_encode(#{<<"foo">> => <<"bar">>}), - {ok, #{<<"foo">> := <<"bar">>}} = emqx_json:safe_decode(Json, [return_maps]). + {ok, Json} = emqx_utils_json:safe_encode(#{<<"foo">> => <<"bar">>}), + {ok, #{<<"foo">> := <<"bar">>}} = emqx_utils_json:safe_decode(Json, [return_maps]). safe_encode_decode(Term) -> - {ok, Json} = emqx_json:safe_encode(Term), - case emqx_json:safe_decode(Json) of + {ok, Json} = emqx_utils_json:safe_encode(Term), + case emqx_utils_json:safe_decode(Json, []) of {ok, {NTerm}} -> NTerm; {ok, NTerm} -> NTerm end. + +t_is_json(_) -> + ?assert(emqx_utils_json:is_json(<<"{}">>)), + ?assert(not emqx_utils_json:is_json(<<"foo">>)). diff --git a/apps/emqx/test/emqx_map_lib_tests.erl b/apps/emqx_utils/test/emqx_utils_maps_tests.erl similarity index 76% rename from apps/emqx/test/emqx_map_lib_tests.erl rename to apps/emqx_utils/test/emqx_utils_maps_tests.erl index 894811d7c..506851f0a 100644 --- a/apps/emqx/test/emqx_map_lib_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_maps_tests.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_map_lib_tests). +-module(emqx_utils_maps_tests). -include_lib("eunit/include/eunit.hrl"). best_effort_recursive_sum_test_() -> @@ -22,21 +22,21 @@ best_effort_recursive_sum_test_() -> [ ?_assertEqual( #{foo => 3}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1}, #{foo => 2}, DummyLogger) + emqx_utils_maps:best_effort_recursive_sum(#{foo => 1}, #{foo => 2}, DummyLogger) ), ?_assertEqual( #{foo => 3, bar => 6.0}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => 2.0}, #{foo => 2, bar => 4.0}, DummyLogger ) ), ?_assertEqual( #{foo => 1, bar => 2}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1}, #{bar => 2}, DummyLogger) + emqx_utils_maps:best_effort_recursive_sum(#{foo => 1}, #{bar => 2}, DummyLogger) ), ?_assertEqual( #{foo => #{bar => 42}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 2}}, #{foo => #{bar => 40}}, DummyLogger ) ), @@ -45,7 +45,9 @@ best_effort_recursive_sum_test_() -> Logger = fun(What) -> Self ! {log, What} end, ?assertEqual( #{foo => 1, bar => 2}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1, bar => 2}, #{bar => bar}, Logger) + emqx_utils_maps:best_effort_recursive_sum( + #{foo => 1, bar => 2}, #{bar => bar}, Logger + ) ), receive {log, Log} -> @@ -55,55 +57,55 @@ best_effort_recursive_sum_test_() -> end, ?_assertEqual( #{}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => foo}, #{foo => bar}, DummyLogger ) ), ?_assertEqual( #{foo => 1}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1}, #{foo => bar}, DummyLogger ) ), ?_assertEqual( #{foo => 1}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => bar}, #{foo => 1}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 1}}, #{foo => 1}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1}, #{foo => #{bar => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => ignored}, #{foo => #{bar => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 2}, bar => #{foo => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => #{foo => 1}}, #{foo => #{bar => 2}, bar => 2}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 2}, bar => #{foo => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 2}, bar => 2}, #{foo => 1, bar => #{foo => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => #{}}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => #{foo => []}}}, #{foo => 1}, DummyLogger ) ) diff --git a/apps/emqx/test/props/prop_emqx_json.erl b/apps/emqx_utils/test/props/prop_emqx_utils_json.erl similarity index 97% rename from apps/emqx/test/props/prop_emqx_json.erl rename to apps/emqx_utils/test/props/prop_emqx_utils_json.erl index 2bc079634..0be1508da 100644 --- a/apps/emqx/test/props/prop_emqx_json.erl +++ b/apps/emqx_utils/test/props/prop_emqx_utils_json.erl @@ -14,10 +14,10 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(prop_emqx_json). +-module(prop_emqx_utils_json). -import( - emqx_json, + emqx_utils_json, [ decode/1, decode/2, @@ -66,7 +66,7 @@ prop_object_proplist_to_proplist() -> begin {ok, J} = safe_encode(T), {ok, T} = safe_decode(J), - T = decode(encode(T)), + T = decode(encode(T), []), true end ). @@ -108,7 +108,7 @@ prop_object_map_to_proplist() -> T = to_list(T0), {ok, J} = safe_encode(T0), {ok, T} = safe_decode(J), - T = decode(encode(T0)), + T = decode(encode(T0), []), true end ). diff --git a/changes/ce/feat-10404.en.md b/changes/ce/feat-10404.en.md new file mode 100644 index 000000000..ad216336e --- /dev/null +++ b/changes/ce/feat-10404.en.md @@ -0,0 +1,2 @@ +Change the default queue mode for buffer workers to `memory_only`. +Before this change, the default queue mode was `volatile_offload`. When under high message rate pressure and when the resource is not keeping up with such rate, the buffer performance degraded a lot due to the constant disk operations. diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index afd90f622..b26df658a 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -3,6 +3,7 @@ , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + , {emqx_utils, {path, "../emqx_utils"}} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl index 78db8352a..26c6de04d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl @@ -63,7 +63,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl index 0b611c142..56671c586 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl @@ -61,7 +61,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl index e6a3d1a58..ba1fd0c70 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl @@ -56,7 +56,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index f3ed44247..7914c77e2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -57,7 +57,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl index 958bc3449..a5dcb19e6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl @@ -59,7 +59,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl index 78fd527d3..28b94a1a4 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl @@ -56,7 +56,7 @@ values(post) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }; values(put) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl index 49a5ed0ce..e216299c2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl @@ -60,7 +60,7 @@ values(post) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }; values(put) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl index 7a958d45f..54406541d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl @@ -58,7 +58,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index 208b68de6..3e442a926 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -261,7 +261,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(cassa_bridge_type, Config), Name = ?config(cassa_name, Config), BridgeConfig0 = ?config(cassa_config, Config), - BridgeConfig = emqx_map_lib:deep_merge(BridgeConfig0, Overrides), + BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), emqx_bridge:create(BridgeType, Name, BridgeConfig). delete_bridge(Config) -> @@ -273,7 +273,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 0c0dbbe04..5ebd9a89d 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -193,7 +193,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(dynamo_bridge_type, Config), Name = ?config(dynamo_name, Config), DynamoConfig0 = ?config(dynamo_config, Config), - DynamoConfig = emqx_map_lib:deep_merge(DynamoConfig0, Overrides), + DynamoConfig = emqx_utils_maps:deep_merge(DynamoConfig0, Overrides), emqx_bridge:create(BridgeType, Name, DynamoConfig). delete_all_bridges() -> @@ -208,7 +208,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -272,7 +272,7 @@ t_setup_via_config_and_publish(Config) -> {ok, _}, create_bridge(Config) ), - MsgId = emqx_misc:gen_id(), + MsgId = emqx_utils:gen_id(), SentData = #{id => MsgId, payload => ?PAYLOAD}, ?check_trace( begin @@ -309,7 +309,7 @@ t_setup_via_http_api_and_publish(Config) -> {ok, _}, create_bridge_http(PgsqlConfig) ), - MsgId = emqx_misc:gen_id(), + MsgId = emqx_utils:gen_id(), SentData = #{id => MsgId, payload => ?PAYLOAD}, ?check_trace( begin @@ -375,7 +375,7 @@ t_write_failure(Config) -> #{?snk_kind := resource_connected_enter}, 20_000 ), - SentData = #{id => emqx_misc:gen_id(), payload => ?PAYLOAD}, + SentData = #{id => emqx_utils:gen_id(), payload => ?PAYLOAD}, emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> ?assertMatch( {error, {resource_error, #{reason := timeout}}}, send_message(Config, SentData) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 75d2d2d8c..a785924d4 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -181,7 +181,7 @@ create_bridge(Config, GCPPubSubConfigOverrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(gcp_pubsub_name, Config), GCPPubSubConfig0 = ?config(gcp_pubsub_config, Config), - GCPPubSubConfig = emqx_map_lib:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), + GCPPubSubConfig = emqx_utils_maps:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), ct:pal("creating bridge: ~p", [GCPPubSubConfig]), Res = emqx_bridge:create(TypeBin, Name, GCPPubSubConfig), ct:pal("bridge creation result: ~p", [Res]), @@ -194,7 +194,7 @@ create_bridge_http(Config, GCPPubSubConfigOverrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(gcp_pubsub_name, Config), GCPPubSubConfig0 = ?config(gcp_pubsub_config, Config), - GCPPubSubConfig = emqx_map_lib:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), + GCPPubSubConfig = emqx_utils_maps:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), Params = GCPPubSubConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), @@ -204,7 +204,7 @@ create_bridge_http(Config, GCPPubSubConfigOverrides) -> ct:pal("probe result: ~p", [ProbeResult]), Res = case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res0} -> {ok, emqx_json:decode(Res0, [return_maps])}; + {ok, Res0} -> {ok, emqx_utils_json:decode(Res0, [return_maps])}; Error -> Error end, ct:pal("bridge creation result: ~p", [Res]), @@ -222,7 +222,7 @@ create_rule_and_action_http(Config) -> Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -234,7 +234,7 @@ success_http_handler() -> Rep = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{messageIds => [<<"6058891368195201">>]}), + emqx_utils_json:encode(#{messageIds => [<<"6058891368195201">>]}), Req ), {ok, Rep, State} @@ -274,7 +274,7 @@ gcp_pubsub_config(Config) -> PubSubTopic = proplists:get_value(pubsub_topic, Config, <<"mytopic">>), PipelineSize = proplists:get_value(pipeline_size, Config, 100), ServiceAccountJSON = proplists:get_value(pubsub_topic, Config, generate_service_account_json()), - ServiceAccountJSONStr = emqx_json:encode(ServiceAccountJSON), + ServiceAccountJSONStr = emqx_utils_json:encode(ServiceAccountJSON), GUID = emqx_guid:to_hexstr(emqx_guid:gen()), Name = <<(atom_to_binary(?MODULE))/binary, (GUID)/binary>>, ConfigString = @@ -463,7 +463,7 @@ assert_valid_request_headers(Headers, ServiceAccountJSON) -> end. assert_valid_request_body(Body) -> - BodyMap = emqx_json:decode(Body, [return_maps]), + BodyMap = emqx_utils_json:decode(Body, [return_maps]), ?assertMatch(#{<<"messages">> := [_ | _]}, BodyMap), #{<<"messages">> := Messages} = BodyMap, lists:map( @@ -471,7 +471,7 @@ assert_valid_request_body(Body) -> ?assertMatch(#{<<"data">> := <<_/binary>>}, Msg), #{<<"data">> := Content64} = Msg, Content = base64:decode(Content64), - Decoded = emqx_json:decode(Content, [return_maps]), + Decoded = emqx_utils_json:decode(Content, [return_maps]), ct:pal("decoded payload: ~p", [Decoded]), ?assert(is_map(Decoded)), Decoded @@ -809,7 +809,7 @@ test_publish_success_batch(Config) -> %% making 1-sized batches. also important to note that the pool %% size for the resource (replayq buffering) must be set to 1 to %% avoid further segmentation of batches. - emqx_misc:pmap(fun emqx:publish/1, Messages), + emqx_utils:pmap(fun emqx:publish/1, Messages), DecodedMessages0 = assert_http_request(ServiceAccountJSON), ?assertEqual(BatchSize, length(DecodedMessages0)), DecodedMessages1 = assert_http_request(ServiceAccountJSON), @@ -1014,7 +1014,7 @@ t_publish_timeout(Config) -> Rep = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{messageIds => [<<"6058891368195201">>]}), + emqx_utils_json:encode(#{messageIds => [<<"6058891368195201">>]}), Req ), {ok, Rep, State} @@ -1180,7 +1180,7 @@ t_failure_with_body(Config) -> Rep = cowboy_req:reply( 400, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{}), + emqx_utils_json:encode(#{}), Req ), {ok, Rep, State} diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl index 1b4b4aeb2..3def35920 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl @@ -354,7 +354,7 @@ create_bridge(Config, Overrides) -> Type = influxdb_type_bin(?config(influxdb_type, Config)), Name = ?config(influxdb_name, Config), InfluxDBConfig0 = ?config(influxdb_config, Config), - InfluxDBConfig = emqx_map_lib:deep_merge(InfluxDBConfig0, Overrides), + InfluxDBConfig = emqx_utils_maps:deep_merge(InfluxDBConfig0, Overrides), emqx_bridge:create(Type, Name, InfluxDBConfig). delete_bridge(Config) -> @@ -390,11 +390,11 @@ create_rule_and_action_http(Config, Overrides) -> sql => <<"SELECT * FROM \"t/topic\"">>, actions => [BridgeId] }, - Params = emqx_map_lib:deep_merge(Params0, Overrides), + Params = emqx_utils_maps:deep_merge(Params0, Overrides), Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -435,7 +435,7 @@ query_by_clientid(ClientId, Config) -> {"Content-Type", "application/json"} ], Body = - emqx_json:encode(#{ + emqx_utils_json:encode(#{ query => Query, dialect => #{ header => true, @@ -545,7 +545,7 @@ t_start_ok(Config) -> int_value => <<"-123">>, uint_value => <<"123">>, float_value => <<"24.5">>, - payload => emqx_json:encode(Payload) + payload => emqx_utils_json:encode(Payload) }, assert_persisted_data(ClientId, Expected, PersistedData), ok @@ -764,7 +764,7 @@ t_boolean_variants(Config) -> bool => atom_to_binary(Translation), int_value => <<"-123">>, uint_value => <<"123">>, - payload => emqx_json:encode(Payload) + payload => emqx_utils_json:encode(Payload) }, assert_persisted_data(ClientId, Expected, PersistedData), ok @@ -1024,9 +1024,9 @@ t_missing_field(Config) -> ClientId0 = emqx_guid:to_hexstr(emqx_guid:gen()), ClientId1 = emqx_guid:to_hexstr(emqx_guid:gen()), %% Message with the field that we "forgot" to select in the rule - Msg0 = emqx_message:make(ClientId0, <<"t/topic">>, emqx_json:encode(#{foo => 123})), + Msg0 = emqx_message:make(ClientId0, <<"t/topic">>, emqx_utils_json:encode(#{foo => 123})), %% Message without any fields - Msg1 = emqx_message:make(ClientId1, <<"t/topic">>, emqx_json:encode(#{})), + Msg1 = emqx_message:make(ClientId1, <<"t/topic">>, emqx_utils_json:encode(#{})), ?check_trace( begin emqx:publish(Msg0), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 116dcc729..0959e3c78 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -245,7 +245,7 @@ create_bridge(Config, Overrides) -> Type = mongo_type_bin(?config(mongo_type, Config)), Name = ?config(mongo_name, Config), MongoConfig0 = ?config(mongo_config, Config), - MongoConfig = emqx_map_lib:deep_merge(MongoConfig0, Overrides), + MongoConfig = emqx_utils_maps:deep_merge(MongoConfig0, Overrides), ct:pal("creating ~p bridge with config:\n ~p", [Type, MongoConfig]), emqx_bridge:create(Type, Name, MongoConfig). @@ -258,7 +258,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl index 38e31c7ae..be07a2bb7 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl @@ -226,7 +226,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -615,7 +615,7 @@ t_workload_fits_prepared_statement_limit(Config) -> create_bridge(Config) ), Results = lists:append( - emqx_misc:pmap( + emqx_utils:pmap( fun(_) -> [ begin diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl index 83cb8b1f3..d76149b16 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl @@ -229,7 +229,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(pgsql_bridge_type, Config), Name = ?config(pgsql_name, Config), PGConfig0 = ?config(pgsql_config, Config), - PGConfig = emqx_map_lib:deep_merge(PGConfig0, Overrides), + PGConfig = emqx_utils_maps:deep_merge(PGConfig0, Overrides), emqx_bridge:create(BridgeType, Name, PGConfig). delete_bridge(Config) -> @@ -241,7 +241,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl index 95ec47e7f..0cb14e5c3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl @@ -176,7 +176,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl index c1b33c3c9..68bf7a057 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -489,7 +489,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(sqlserver_bridge_type, Config), Name = ?config(sqlserver_name, Config), SSConfig0 = ?config(sqlserver_config, Config), - SSConfig = emqx_map_lib:deep_merge(SSConfig0, Overrides), + SSConfig = emqx_utils_maps:deep_merge(SSConfig0, Overrides), emqx_bridge:create(BridgeType, Name, SSConfig). delete_bridge(Config) -> @@ -501,7 +501,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl index c956a93c6..36ed10f38 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl @@ -207,7 +207,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(tdengine_bridge_type, Config), Name = ?config(tdengine_name, Config), TDConfig0 = ?config(tdengine_config, Config), - TDConfig = emqx_map_lib:deep_merge(TDConfig0, Overrides), + TDConfig = emqx_utils_maps:deep_merge(TDConfig0, Overrides), emqx_bridge:create(BridgeType, Name, TDConfig). delete_bridge(Config) -> @@ -219,7 +219,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index e754bd573..352c54629 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -6,7 +6,8 @@ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, - {emqx, {path, "../../apps/emqx"}} + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index ce964dd17..86b908038 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -102,7 +102,7 @@ on_start( ?SLOG(info, #{ msg => "starting_cassandra_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), Options = [ diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl index 3749ecc6e..2872e8cf0 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl @@ -139,7 +139,7 @@ on_start( ?SLOG(info, #{ msg => "starting_clickhouse_connector", connector => InstanceID, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), PoolName = emqx_plugin_libs_pool:pool_name(InstanceID), Options = [ @@ -181,7 +181,7 @@ log_start_error(Config, Reason, Stacktrace) -> #{ msg => "clickhouse_connector_start_failed", error_reason => Reason, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }, ?SLOG(info, maps:merge(LogMessage, StacktraceMap)), ?tp( @@ -318,7 +318,7 @@ do_get_status(PoolName, Timeout) -> Error end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of Results -> case [E || {error, _} = E <- Results] of [] -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 4703e0a21..85daefbb7 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -95,7 +95,7 @@ on_start( ?SLOG(info, #{ msg => "starting_dynamo_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), {Schema, Server} = get_host_schema(to_str(Url)), @@ -322,7 +322,7 @@ convert_to_item(Msg) when is_map(Msg), map_size(Msg) > 0 -> Msg ); convert_to_item(MsgBin) when is_binary(MsgBin) -> - Msg = emqx_json:decode(MsgBin), + Msg = emqx_utils_json:decode(MsgBin), convert_to_item(Msg); convert_to_item(Item) -> erlang:throw({invalid_item, Item}). @@ -334,7 +334,7 @@ convert2binary(Value) when is_binary(Value); is_number(Value) -> convert2binary(Value) when is_list(Value) -> unicode:characters_to_binary(Value); convert2binary(Value) when is_map(Value) -> - emqx_json:encode(Value). + emqx_utils_json:encode(Value). do_async_reply(Result, {ReplyFun, [Context]}) -> ReplyFun(Context, Result). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl index f07cbceab..2295f63ab 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl @@ -86,7 +86,7 @@ on_start( PoolType = random, Transport = tls, TransportOpts = emqx_tls_lib:to_client_opts(#{enable => true, verify => verify_none}), - NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + NTransportOpts = emqx_utils:ipv6_probe(TransportOpts), PoolOpts = [ {host, Host}, {port, Port}, @@ -334,14 +334,14 @@ ensure_jwt_worker(InstanceId, #{ encode_payload(_State = #{payload_template := PayloadTemplate}, Selected) -> Interpolated = case PayloadTemplate of - [] -> emqx_json:encode(Selected); + [] -> emqx_utils_json:encode(Selected); _ -> emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Selected) end, #{data => base64:encode(Interpolated)}. -spec to_pubsub_request([#{data := binary()}]) -> binary(). to_pubsub_request(Payloads) -> - emqx_json:encode(#{messages => Payloads}). + emqx_utils_json:encode(#{messages => Payloads}). -spec publish_path(state()) -> binary(). publish_path( @@ -587,7 +587,7 @@ do_get_status(InstanceId, PoolName, Timeout) -> false end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of [_ | _] = Status -> lists:all(fun(St) -> St =:= true end, Status); [] -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index afe17cae6..700eb2a81 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -216,8 +216,8 @@ start_client(InstId, Config) -> ?SLOG(info, #{ msg => "starting influxdb connector", connector => InstId, - config => emqx_misc:redact(Config), - client_config => emqx_misc:redact(ClientConfig) + config => emqx_utils:redact(Config), + client_config => emqx_utils:redact(ClientConfig) }), try do_start_client(InstId, ClientConfig, Config) @@ -353,7 +353,7 @@ password(_) -> []. redact_auth(Term) -> - emqx_misc:redact(Term, fun is_auth_key/1). + emqx_utils:redact(Term, fun is_auth_key/1). is_auth_key(Key) when is_binary(Key) -> string:equal("authorization", Key, true); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl index aa03863b0..4e7adcd6e 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl @@ -84,4 +84,4 @@ render_message(PayloadTemplate, Message) -> %% Note: mongo expects a map as a document, so the rendered result %% must be JSON-serializable Rendered = emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Message), - emqx_json:decode(Rendered, [return_maps]). + emqx_utils_json:decode(Rendered, [return_maps]). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index 84f2e2a89..205359bb8 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -248,14 +248,14 @@ get_topic_key([Query | _], RawTopic, TopicTks) -> apply_template({Key, Msg} = _Req, Templates) -> case maps:get(Key, Templates, undefined) of undefined -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); Template -> emqx_plugin_libs_rule:proc_tmpl(Template, Msg) end; apply_template([{Key, _} | _] = Reqs, Templates) -> case maps:get(Key, Templates, undefined) of undefined -> - [emqx_json:encode(Msg) || {_, Msg} <- Reqs]; + [emqx_utils_json:encode(Msg) || {_, Msg} <- Reqs]; Template -> [emqx_plugin_libs_rule:proc_tmpl(Template, Msg) || {_, Msg} <- Reqs] end. @@ -265,7 +265,7 @@ client_id(InstanceId) -> erlang:binary_to_atom(Name, utf8). redact(Msg) -> - emqx_misc:redact(Msg, fun is_sensitive_key/1). + emqx_utils:redact(Msg, fun is_sensitive_key/1). is_sensitive_key(security_token) -> true; diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index dc9759341..f11441a3b 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -188,7 +188,7 @@ on_start( ?SLOG(info, #{ msg => "starting_sqlserver_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), ODBCDir = code:priv_dir(odbc), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl index 746ab5485..9b6718882 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl @@ -93,7 +93,7 @@ on_start( ?SLOG(info, #{ msg => "starting_tdengine_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), {Host, Port} = emqx_schema:parse_server(Server, ?TD_HOST_OPTIONS), diff --git a/lib-ee/emqx_ee_schema_registry/rebar.config b/lib-ee/emqx_ee_schema_registry/rebar.config index b19fb05ae..223ebf533 100644 --- a/lib-ee/emqx_ee_schema_registry/rebar.config +++ b/lib-ee/emqx_ee_schema_registry/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../../apps/emqx_utils"}}, {erlavro, {git, "https://github.com/klarna/erlavro.git", {tag, "2.9.8"}}} ]}. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl index 3569b246e..4c00903d5 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -67,7 +67,7 @@ get_schema(SchemaName) -> -spec add_schema(schema_name(), schema()) -> ok | {error, term()}. add_schema(Name, Schema) -> - RawSchema = emqx_map_lib:binary_key_map(Schema), + RawSchema = emqx_utils_maps:binary_key_map(Schema), Res = emqx_conf:update( [?CONF_KEY_ROOT, schemas, Name], RawSchema, @@ -113,7 +113,7 @@ post_config_update( added := Added, changed := Changed0, removed := Removed - } = emqx_map_lib:diff_maps(NewSchemas, OldSchemas), + } = emqx_utils_maps:diff_maps(NewSchemas, OldSchemas), Changed = maps:map(fun(_N, {_Old, New}) -> New end, Changed0), RemovedNames = maps:keys(Removed), case RemovedNames of diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl index 897d29e07..c7cc6c99a 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl @@ -8,7 +8,7 @@ -include("emqx_ee_schema_registry.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_api_lib.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). -export([ namespace/0, diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl index 43145fb16..9835ec7c2 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl @@ -65,6 +65,6 @@ make_serde(avro, Name, Source0) -> inject_avro_name(Name, Source0) -> %% The schema checks that the source is a valid JSON when %% typechecking, so we shouldn't need to validate here. - Schema0 = emqx_json:decode(Source0, [return_maps]), + Schema0 = emqx_utils_json:decode(Source0, [return_maps]), Schema = Schema0#{<<"name">> => Name}, - emqx_json:encode(Schema). + emqx_utils_json:encode(Schema). diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 9b2f64c03..5bfba34b3 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -68,7 +68,7 @@ trace_rule(Data, Envs, _Args) -> make_trace_fn_action() -> persistent_term:put({?MODULE, test_pid}, self()), Fn = <<(atom_to_binary(?MODULE))/binary, ":trace_rule">>, - emqx_tables:new(recorded_actions, [named_table, public, ordered_set]), + emqx_utils_ets:new(recorded_actions, [named_table, public, ordered_set]), #{function => Fn, args => #{}}. create_rule_http(RuleParams) -> @@ -95,7 +95,7 @@ create_rule_http(RuleParams) -> Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -107,7 +107,7 @@ schema_params(avro) -> #{name => <<"s">>, type => <<"string">>} ] }, - SourceBin = emqx_json:encode(Source), + SourceBin = emqx_utils_json:encode(Source), #{type => avro, source => SourceBin}. create_serde(SerdeType, SerdeName) -> @@ -162,7 +162,7 @@ receive_published(Line) -> maps:update_with( payload, fun(Raw) -> - case emqx_json:safe_decode(Raw, [return_maps]) of + case emqx_utils_json:safe_decode(Raw, [return_maps]) of {ok, Decoded} -> Decoded; {error, _} -> Raw end @@ -218,7 +218,7 @@ start_cluster(Cluster) -> || {Name, Opts} <- Cluster ], on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -262,7 +262,7 @@ t_encode_decode(Config) -> {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode_decode1)}), on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, - PayloadBin = emqx_json:encode(Payload), + PayloadBin = emqx_utils_json:encode(Payload), emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), Res = receive_action_results(), ?assertMatch( @@ -311,7 +311,7 @@ t_encode(Config) -> {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode1)}), on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, - PayloadBin = emqx_json:encode(Payload), + PayloadBin = emqx_utils_json:encode(Payload), emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), Published = receive_published(?LINE), ?assertMatch( @@ -344,7 +344,7 @@ t_decode(Config) -> t_fail_rollback(Config) -> SerdeType = ?config(serde_type, Config), - OkSchema = emqx_map_lib:binary_key_map(schema_params(SerdeType)), + OkSchema = emqx_utils_maps:binary_key_map(schema_params(SerdeType)), BrokenSchema = OkSchema#{<<"source">> := <<"{}">>}, %% hopefully, for this small map, the key order is used. Serdes = #{ diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl index bbb6d5ef0..e7034d562 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl @@ -64,7 +64,7 @@ do_request(Method, PathParts, Body) -> {ok, Code, <<>>} -> {ok, Code, <<>>}; {ok, Code, Res1} -> - Res2 = emqx_json:decode(Res1, [return_maps]), + Res2 = emqx_utils_json:decode(Res1, [return_maps]), Res3 = try_decode_error_message(Res2), {ok, Code, Res3}; Error -> @@ -72,7 +72,7 @@ do_request(Method, PathParts, Body) -> end. try_decode_error_message(#{<<"message">> := Msg0} = Res0) -> - case emqx_json:safe_decode(Msg0, [return_maps]) of + case emqx_utils_json:safe_decode(Msg0, [return_maps]) of {ok, Msg} -> Res0#{<<"message">> := Msg}; {error, _} -> @@ -102,7 +102,7 @@ t_crud(_Config) -> #{name => <<"s">>, type => <<"string">>} ] }, - SourceBin = emqx_json:encode(Source), + SourceBin = emqx_utils_json:encode(Source), Params = #{ <<"type">> => <<"avro">>, <<"source">> => SourceBin, diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl index be62717d3..12798c6a2 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl @@ -59,7 +59,7 @@ schema_params(avro) -> #{name => <<"s">>, type => <<"string">>} ] }, - SourceBin = emqx_json:encode(Source), + SourceBin = emqx_utils_json:encode(Source), #{type => avro, source => SourceBin}. assert_roundtrip(SerdeName, Original) -> diff --git a/lib-ee/emqx_license/rebar.config b/lib-ee/emqx_license/rebar.config index 1cb5ace88..e7620387a 100644 --- a/lib-ee/emqx_license/rebar.config +++ b/lib-ee/emqx_license/rebar.config @@ -1,3 +1,6 @@ -{deps, [{emqx, {path, "../../apps/emqx"}}]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/lib-ee/emqx_license/src/emqx_license_http_api.erl index 0133cd637..ef0e1ee52 100644 --- a/lib-ee/emqx_license/src/emqx_license_http_api.erl +++ b/lib-ee/emqx_license/src/emqx_license_http_api.erl @@ -95,7 +95,7 @@ sample_license_info_response() -> }. error_msg(Code, Msg) -> - #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. %% read license info '/license'(get, _Params) -> diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index 11ac1ca89..3de5ae121 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -103,7 +103,7 @@ t_license_info(_Config) -> <<"start_at">> => <<"2022-01-11">>, <<"type">> => <<"trial">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), ok. @@ -128,7 +128,7 @@ t_license_upload_key_success(_Config) -> <<"start_at">> => <<"2022-01-11">>, <<"type">> => <<"trial">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), ?assertMatch( #{max_connections := 999}, @@ -150,7 +150,7 @@ t_license_upload_key_bad_key(_Config) -> <<"code">> => <<"BAD_REQUEST">>, <<"message">> => <<"Bad license key">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. @@ -168,7 +168,7 @@ t_license_upload_key_not_json(_Config) -> <<"code">> => <<"BAD_REQUEST">>, <<"message">> => <<"Invalid request params">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. diff --git a/mix.exs b/mix.exs index 55036d498..2b8de4c54 100644 --- a/mix.exs +++ b/mix.exs @@ -303,6 +303,7 @@ defmodule EMQXUmbrella.MixProject do tools: :load, covertool: :load, system_monitor: :load, + emqx_utils: :load, emqx_http_lib: :permanent, emqx_resource: :permanent, emqx_connector: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index 2ae6ffad4..7c00622c2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -391,6 +391,7 @@ relx_apps(ReleaseType, Edition) -> {covertool, load}, % started by emqx_machine {system_monitor, load}, + {emqx_utils, load}, emqx_http_lib, emqx_resource, emqx_connector, diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index c73f8b1aa..f4a9982bc 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -168,7 +168,7 @@ When disabled the messages are buffered in RAM only.""" } } - max_queue_bytes { + max_buffer_bytes { desc { en: """Maximum number of bytes to buffer for each buffer worker.""" zh: """每个缓存 worker 允许使用的最大字节数。""" @@ -179,5 +179,36 @@ When disabled the messages are buffered in RAM only.""" } } + buffer_seg_bytes { + desc { + en: "Applicable when buffer mode is set to volatile_offload.\n" + "This value is to specify the size of each on-disk buffer file." + zh: "当缓存模式是 volatile_offload 时适用。" + "该配置用于指定缓存到磁盘上的文件的大小。" + } + label { + en: "Segment File Bytes" + zh: "缓存文件大小" + } + } + + buffer_mode { + desc { + en: "Buffer operation mode.\n" + "memory_only: Buffer all messages in memory." + "volatile_offload: Buffer message in memory first, when up to certain limit" + " (see buffer_seg_bytes config for more information), then start offloading messages to disk" + zh: "队列操作模式。\n" + "memory_only: 所有的消息都缓存在内存里。" + "volatile_offload: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" + "（配置项 buffer_seg_bytes 指定该限制）后，" + " 消息会开始缓存到磁盘上。" + } + label { + en: "Buffer Mode" + zh: "缓存模式" + } + } + } diff --git a/scripts/test/influx/influx-bridge.conf b/scripts/test/influx/influx-bridge.conf index 0416e42b6..0574ac38a 100644 --- a/scripts/test/influx/influx-bridge.conf +++ b/scripts/test/influx/influx-bridge.conf @@ -11,7 +11,7 @@ bridges { batch_size = 100 batch_time = "10ms" health_check_interval = "15s" - max_queue_bytes = "1GB" + max_buffer_bytes = "1GB" query_mode = "sync" request_timeout = "15s" start_after_created = "true"