From f8e9e54393204114eaaa1afcccdf457d350e4147 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 14:02:37 +0200 Subject: [PATCH 01/32] refactor: move emqx_json to emqx_utils_json --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_alarm.erl | 2 +- apps/emqx/src/emqx_logger_jsonfmt.erl | 2 +- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx/src/emqx_sys.erl | 2 +- apps/emqx/test/emqx_common_test_helpers.erl | 6 +- apps/emqx/test/emqx_common_test_http.erl | 4 +- apps/emqx/test/emqx_config_SUITE.erl | 2 +- apps/emqx/test/emqx_crl_cache_SUITE.erl | 6 +- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 14 +-- apps/emqx/test/props/prop_emqx_json.erl | 4 +- apps/emqx_authn/rebar.config | 1 + .../src/simple_authn/emqx_authn_http.erl | 4 +- .../simple_authn/emqx_authn_jwks_client.erl | 2 +- .../src/simple_authn/emqx_authn_jwt.erl | 2 +- .../src/simple_authn/emqx_authn_mnesia.erl | 2 +- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 24 ++--- .../emqx_authn/test/emqx_authn_http_SUITE.erl | 20 ++-- .../test/emqx_authn_https_SUITE.erl | 2 +- apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl | 2 +- apps/emqx_authz/rebar.config | 1 + apps/emqx_authz/src/emqx_authz_utils.erl | 2 +- .../test/emqx_authz_api_cache_SUITE.erl | 4 +- .../test/emqx_authz_api_sources_SUITE.erl | 23 ++-- .../emqx_authz/test/emqx_authz_http_SUITE.erl | 6 +- apps/emqx_auto_subscribe/rebar.config | 5 +- .../test/emqx_auto_subscribe_SUITE.erl | 4 +- apps/emqx_bridge/rebar.config | 8 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- .../test/emqx_bridge_api_SUITE.erl | 2 +- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 18 ++-- .../emqx_bridge_kafka_impl_producer_SUITE.erl | 2 +- apps/emqx_connector/rebar.config | 1 + .../src/emqx_connector_http.erl | 2 +- .../src/mqtt/emqx_connector_mqtt_msg.erl | 2 +- apps/emqx_dashboard/rebar.config | 5 +- .../test/emqx_dashboard_SUITE.erl | 2 +- .../test/emqx_dashboard_error_code_SUITE.erl | 2 +- .../test/emqx_dashboard_monitor_SUITE.erl | 4 +- apps/emqx_exhook/rebar.config | 3 +- .../test/emqx_exhook_api_SUITE.erl | 4 +- apps/emqx_gateway/rebar.config | 3 +- apps/emqx_gateway/src/emqx_gateway_cli.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_http.erl | 2 +- .../test/emqx_gateway_auth_ct.erl | 2 +- .../test/emqx_gateway_authn_SUITE.erl | 2 +- .../test/emqx_gateway_authz_SUITE.erl | 6 +- .../test/emqx_gateway_test_utils.erl | 4 +- apps/emqx_gateway_coap/rebar.config | 8 +- .../test/emqx_coap_api_SUITE.erl | 2 +- apps/emqx_gateway_exproto/rebar.config | 8 +- .../test/emqx_exproto_echo_svr.erl | 20 ++-- .../src/emqx_lwm2m_message.erl | 2 +- .../src/emqx_lwm2m_session.erl | 4 +- .../test/emqx_lwm2m_SUITE.erl | 100 +++++++++--------- .../test/emqx_lwm2m_api_SUITE.erl | 8 +- apps/emqx_gateway_stomp/rebar.config | 8 +- apps/emqx_machine/rebar.config | 5 +- apps/emqx_machine/src/emqx_machine.app.src | 2 +- apps/emqx_machine/src/emqx_machine.erl | 2 +- apps/emqx_management/rebar.config | 5 +- .../src/emqx_mgmt_api_banned.erl | 2 +- .../test/emqx_mgmt_api_alarms_SUITE.erl | 2 +- .../test/emqx_mgmt_api_api_keys_SUITE.erl | 10 +- .../test/emqx_mgmt_api_banned_SUITE.erl | 4 +- .../test/emqx_mgmt_api_clients_SUITE.erl | 10 +- .../test/emqx_mgmt_api_configs_SUITE.erl | 10 +- .../test/emqx_mgmt_api_listeners_SUITE.erl | 2 +- .../test/emqx_mgmt_api_metrics_SUITE.erl | 4 +- .../test/emqx_mgmt_api_nodes_SUITE.erl | 10 +- .../test/emqx_mgmt_api_plugins_SUITE.erl | 10 +- .../test/emqx_mgmt_api_publish_SUITE.erl | 2 +- .../test/emqx_mgmt_api_stats_SUITE.erl | 4 +- .../test/emqx_mgmt_api_subscription_SUITE.erl | 4 +- .../test/emqx_mgmt_api_test_util.erl | 3 +- .../test/emqx_mgmt_api_topics_SUITE.erl | 6 +- .../test/emqx_mgmt_api_trace_SUITE.erl | 2 +- apps/emqx_modules/rebar.config | 1 + apps/emqx_modules/src/emqx_telemetry.erl | 2 +- apps/emqx_modules/src/emqx_telemetry_api.erl | 2 +- .../test/emqx_delayed_api_SUITE.erl | 2 +- .../test/emqx_rewrite_api_SUITE.erl | 2 +- .../test/emqx_telemetry_SUITE.erl | 2 +- apps/emqx_plugin_libs/rebar.config | 5 +- .../emqx_plugin_libs/src/emqx_placeholder.erl | 2 +- .../src/emqx_plugin_libs_rule.erl | 10 +- apps/emqx_prometheus/rebar.config | 1 + .../test/emqx_prometheus_api_SUITE.erl | 6 +- apps/emqx_retainer/rebar.config | 5 +- .../test/emqx_retainer_api_SUITE.erl | 10 +- apps/emqx_rule_engine/rebar.config | 3 +- .../src/emqx_rule_actions.erl | 2 +- .../src/emqx_rule_engine_api.erl | 2 +- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 4 +- apps/emqx_rule_engine/src/emqx_rule_maps.erl | 2 +- .../src/emqx_rule_runtime.erl | 2 +- .../test/emqx_rule_engine_SUITE.erl | 14 ++- .../test/emqx_rule_engine_api_SUITE.erl | 10 +- apps/emqx_slow_subs/rebar.config | 5 +- .../test/emqx_slow_subs_api_SUITE.erl | 6 +- apps/emqx_statsd/rebar.config | 1 + apps/emqx_statsd/test/emqx_statsd_SUITE.erl | 2 +- apps/emqx_utils/README.md | 43 ++++++++ apps/emqx_utils/rebar.config | 11 ++ apps/emqx_utils/src/emqx_utils.app.src | 25 +++++ .../src/emqx_utils_json.erl} | 12 +-- apps/emqx_utils/test/emqx_utils_SUITE.erl | 48 +++++++++ .../test/emqx_utils_json_SUITE.erl} | 14 +-- lib-ee/emqx_ee_bridge/rebar.config | 1 + .../test/emqx_ee_bridge_cassa_SUITE.erl | 2 +- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 2 +- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 16 +-- .../test/emqx_ee_bridge_influxdb_SUITE.erl | 12 +-- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 2 +- .../test/emqx_ee_bridge_mysql_SUITE.erl | 2 +- .../test/emqx_ee_bridge_pgsql_SUITE.erl | 2 +- .../test/emqx_ee_bridge_rocketmq_SUITE.erl | 2 +- .../test/emqx_ee_bridge_tdengine_SUITE.erl | 2 +- lib-ee/emqx_ee_connector/rebar.config | 3 +- .../src/emqx_ee_connector_dynamo.erl | 4 +- .../src/emqx_ee_connector_gcp_pubsub.erl | 4 +- .../src/emqx_ee_connector_mongodb.erl | 2 +- .../src/emqx_ee_connector_rocketmq.erl | 4 +- lib-ee/emqx_ee_schema_registry/rebar.config | 1 + .../src/emqx_ee_schema_registry_serde.erl | 4 +- .../test/emqx_ee_schema_registry_SUITE.erl | 10 +- ...emqx_ee_schema_registry_http_api_SUITE.erl | 6 +- .../emqx_ee_schema_registry_serde_SUITE.erl | 2 +- lib-ee/emqx_license/rebar.config | 5 +- .../test/emqx_license_http_api_SUITE.erl | 8 +- 130 files changed, 508 insertions(+), 331 deletions(-) create mode 100644 apps/emqx_utils/README.md create mode 100644 apps/emqx_utils/rebar.config create mode 100644 apps/emqx_utils/src/emqx_utils.app.src rename apps/{emqx/src/emqx_json.erl => emqx_utils/src/emqx_utils_json.erl} (90%) create mode 100644 apps/emqx_utils/test/emqx_utils_SUITE.erl rename apps/{emqx/test/emqx_json_SUITE.erl => emqx_utils/test/emqx_utils_json_SUITE.erl} (93%) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 9079322eb..a3eff6c16 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -22,9 +22,9 @@ %% This rebar.config is necessary because the app may be used as a %% `git_subdir` dependency in other projects. {deps, [ + {emqx_utils, {path, "../emqx_utils"}}, {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, - {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 84c40ef2a..6aa3cb95d 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -423,7 +423,7 @@ do_actions(deactivate, Alarm = #deactivated_alarm{name = Name}, [log | More]) -> do_actions(deactivate, Alarm, More); do_actions(Operation, Alarm, [publish | More]) -> Topic = topic(Operation), - {ok, Payload} = emqx_json:safe_encode(normalize(Alarm)), + {ok, Payload} = emqx_utils_json:safe_encode(normalize(Alarm)), Message = emqx_message:make( ?MODULE, 0, diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 22cf75153..f60f5a716 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -92,7 +92,7 @@ format(Msg, Meta, Config) -> } end, Data = maps:without([report_cb], Data0), - jiffy:encode(json_obj(Data, Config)). + emqx_utils_json:encode(json_obj(Data, Config)). maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) -> case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 851fa3d3b..8c91ae782 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2583,7 +2583,7 @@ to_url(Str) -> end. to_json_binary(Str) -> - case emqx_json:safe_decode(Str) of + case emqx_utils_json:safe_decode(Str) of {ok, _} -> {ok, iolist_to_binary(Str)}; Error -> diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index a5f14e32a..af2b052aa 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -348,7 +348,7 @@ publish(Event, Payload) when Event == unsubscribed -> Topic = event_topic(Event, Payload), - safe_publish(Topic, emqx_json:encode(Payload)). + safe_publish(Topic, emqx_utils_json:encode(Payload)). metric_topic(Name) -> translate_topic("metrics/", Name). diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 406183094..85cb46106 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -1036,7 +1036,7 @@ switch_proxy(Switch, Name, ProxyHost, ProxyPort) -> off -> #{<<"enabled">> => false}; on -> #{<<"enabled">> => true} end, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, @@ -1056,7 +1056,7 @@ timeout_proxy(on, Name, ProxyHost, ProxyPort) -> <<"toxicity">> => 1.0, <<"attributes">> => #{<<"timeout">> => 0} }, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, @@ -1091,7 +1091,7 @@ latency_up_proxy(on, Name, ProxyHost, ProxyPort) -> <<"jitter">> => 3_000 } }, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl index 575bed5c3..e9064715d 100644 --- a/apps/emqx/test/emqx_common_test_http.erl +++ b/apps/emqx/test/emqx_common_test_http.erl @@ -54,7 +54,7 @@ request_api(Method, Url, QueryParams, Auth, Body, HttpOpts) -> [] -> {NewUrl, [Auth]}; _ -> - {NewUrl, [Auth], "application/json", emqx_json:encode(Body)} + {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)} end, do_request_api(Method, Request, HttpOpts). @@ -70,7 +70,7 @@ do_request_api(Method, Request, HttpOpts) -> end. get_http_data(ResponseBody) -> - emqx_json:decode(ResponseBody, [return_maps]). + emqx_utils_json:decode(ResponseBody, [return_maps]). auth_header(User, Pass) -> Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index fe8a5fed8..7befd7a16 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -57,5 +57,5 @@ t_fill_default_values(_) -> WithDefaults ), %% ensure JSON compatible - _ = emqx_json:encode(WithDefaults), + _ = emqx_utils_json:encode(WithDefaults), ok. diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl index 01f9c7172..21bfc39df 100644 --- a/apps/emqx/test/emqx_crl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -402,7 +402,7 @@ request(Method, Url, QueryParams, Body) -> Opts = #{return_all => true}, case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of {ok, {Reason, Headers, BodyR}} -> - {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}}; + {ok, {Reason, Headers, emqx_utils_json:decode(BodyR, [return_maps])}}; Error -> Error end. @@ -1052,7 +1052,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} = - emqx_json:decode(ResRaw1, [return_maps]), + emqx_utils_json:decode(ResRaw1, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -1064,7 +1064,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw1, [return_maps]) + emqx_utils_json:decode(MsgRaw1, [return_maps]) ), ok. diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index 3c3fd0341..a9b80c276 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -430,7 +430,7 @@ request(Method, Url, QueryParams, Body) -> Opts = #{return_all => true}, case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of {ok, {Reason, Headers, BodyR}} -> - {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}}; + {ok, {Reason, Headers, emqx_utils_json:decode(BodyR, [return_maps])}}; Error -> Error end. @@ -827,7 +827,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} = - emqx_json:decode(ResRaw1, [return_maps]), + emqx_utils_json:decode(ResRaw1, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -839,7 +839,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw1, [return_maps]) + emqx_utils_json:decode(MsgRaw1, [return_maps]) ), ListenerData2 = @@ -857,7 +857,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw2}} = update_listener_via_api(ListenerId, ListenerData2), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw2} = - emqx_json:decode(ResRaw2, [return_maps]), + emqx_utils_json:decode(ResRaw2, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -869,7 +869,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw2, [return_maps]) + emqx_utils_json:decode(MsgRaw2, [return_maps]) ), ListenerData3a = @@ -889,7 +889,7 @@ do_t_validations(_Config) -> ListenerData3 = emqx_map_lib:deep_remove([<<"ssl_options">>, <<"certfile">>], ListenerData3a), {error, {_, _, ResRaw3}} = update_listener_via_api(ListenerId, ListenerData3), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw3} = - emqx_json:decode(ResRaw3, [return_maps]), + emqx_utils_json:decode(ResRaw3, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -901,7 +901,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw3, [return_maps]) + emqx_utils_json:decode(MsgRaw3, [return_maps]) ), ok. diff --git a/apps/emqx/test/props/prop_emqx_json.erl b/apps/emqx/test/props/prop_emqx_json.erl index 2bc079634..86f60b39c 100644 --- a/apps/emqx/test/props/prop_emqx_json.erl +++ b/apps/emqx/test/props/prop_emqx_json.erl @@ -14,10 +14,10 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(prop_emqx_json). +-module(prop_emqx_utils_json). -import( - emqx_json, + emqx_utils_json, [ decode/1, decode/2, diff --git a/apps/emqx_authn/rebar.config b/apps/emqx_authn/rebar.config index 8fd9cea0f..5f0043c39 100644 --- a/apps/emqx_authn/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_connector, {path, "../emqx_connector"}} ]}. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 9be1d0a33..3c34d878e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -357,7 +357,7 @@ qs([{K, V} | More], Acc) -> qs(More, [["&", uri_encode(K), "=", uri_encode(V)] | Acc]). serialize_body(<<"application/json">>, Body) -> - emqx_json:encode(Body); + emqx_utils_json:encode(Body); serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> qs(maps:to_list(Body)). @@ -395,7 +395,7 @@ safely_parse_body(ContentType, Body) -> end. parse_body(<<"application/json", _/binary>>, Body) -> - {ok, emqx_json:decode(Body, [return_maps])}; + {ok, emqx_utils_json:decode(Body, [return_maps])}; parse_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> Flags = [<<"result">>, <<"is_superuser">>], RawMap = maps:from_list(cow_qs:parse_qs(Body)), diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl index 5ee923859..23d939f7d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl @@ -99,7 +99,7 @@ handle_info( State1; {StatusLine, Headers, Body} -> try - JWKS = jose_jwk:from(emqx_json:decode(Body, [return_maps])), + JWKS = jose_jwk:from(emqx_utils_json:decode(Body, [return_maps])), {_, JWKs} = JWKS#jose_jwk.keys, State1#{jwks := JWKs} catch diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 73ba6a2c1..a891a55e2 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -407,7 +407,7 @@ do_verify(_JWT, [], _VerifyClaims) -> do_verify(JWT, [JWK | More], VerifyClaims) -> try jose_jws:verify(JWK, JWT) of {true, Payload, _JWT} -> - Claims0 = emqx_json:decode(Payload, [return_maps]), + Claims0 = emqx_utils_json:decode(Payload, [return_maps]), Claims = try_convert_to_num(Claims0, [<<"exp">>, <<"iat">>, <<"nbf">>]), case verify_claims(Claims, VerifyClaims) of ok -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index 25a3a5976..20b604e70 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -332,7 +332,7 @@ run_fuzzy_filter( %% Example: data/user-credentials.json import_users_from_json(Bin, #{user_group := UserGroup}) -> - case emqx_json:safe_decode(Bin, [return_maps]) of + case emqx_utils_json:safe_decode(Bin, [return_maps]) of {ok, List} -> trans(fun ?MODULE:import/2, [UserGroup, List]); {error, Reason} -> diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 11e2c6773..c7f718dfc 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -29,7 +29,7 @@ -define(assertAuthenticatorsMatch(Guard, Path), (fun() -> {ok, 200, Response} = request(get, uri(Path)), - ?assertMatch(Guard, jiffy:decode(Response, [return_maps])) + ?assertMatch(Guard, emqx_utils_json:decode(Response, [return_maps])) end)() ). @@ -234,7 +234,7 @@ test_authenticator(PathPrefix) -> get, uri(PathPrefix ++ [?CONF_NS, "password_based:http", "status"]) ), - {ok, RList} = emqx_json:safe_decode(Res), + {ok, RList} = emqx_utils_json:safe_decode(Res), Snd = fun({_, Val}) -> Val end, LookupVal = fun LookupV(List, RestJson) -> case List of @@ -353,7 +353,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 0, <<"nomatch">> := 1 } - } = jiffy:decode(PageData0, [return_maps]); + } = emqx_utils_json:decode(PageData0, [return_maps]); ["listeners", 'tcp:default'] -> #{ <<"metrics">> := #{ @@ -361,7 +361,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 0, <<"nomatch">> := 1 } - } = jiffy:decode(PageData0, [return_maps]) + } = emqx_utils_json:decode(PageData0, [return_maps]) end, InvalidUsers = [ @@ -384,7 +384,7 @@ test_authenticator_users(PathPrefix) -> lists:foreach( fun(User) -> {ok, 201, UserData} = request(post, UsersUri, User), - CreatedUser = jiffy:decode(UserData, [return_maps]), + CreatedUser = emqx_utils_json:decode(UserData, [return_maps]), ?assertMatch(#{<<"user_id">> := _}, CreatedUser) end, ValidUsers @@ -411,7 +411,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 1, <<"nomatch">> := 1 } - } = jiffy:decode(PageData01, [return_maps]); + } = emqx_utils_json:decode(PageData01, [return_maps]); ["listeners", 'tcp:default'] -> #{ <<"metrics">> := #{ @@ -419,7 +419,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 1, <<"nomatch">> := 1 } - } = jiffy:decode(PageData01, [return_maps]) + } = emqx_utils_json:decode(PageData01, [return_maps]) end, {ok, 200, Page1Data} = request(get, UsersUri ++ "?page=1&limit=2"), @@ -433,7 +433,7 @@ test_authenticator_users(PathPrefix) -> <<"count">> := 3 } } = - jiffy:decode(Page1Data, [return_maps]), + emqx_utils_json:decode(Page1Data, [return_maps]), {ok, 200, Page2Data} = request(get, UsersUri ++ "?page=2&limit=2"), @@ -445,7 +445,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 2, <<"count">> := 3 } - } = jiffy:decode(Page2Data, [return_maps]), + } = emqx_utils_json:decode(Page2Data, [return_maps]), ?assertEqual(2, length(Page1Users)), ?assertEqual(1, length(Page2Users)), @@ -465,7 +465,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 3, <<"count">> := 1 } - } = jiffy:decode(Super1Data, [return_maps]), + } = emqx_utils_json:decode(Super1Data, [return_maps]), ?assertEqual( [<<"u2">>], @@ -482,7 +482,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 3, <<"count">> := 2 } - } = jiffy:decode(Super2Data, [return_maps]), + } = emqx_utils_json:decode(Super2Data, [return_maps]), ?assertEqual( [<<"u1">>, <<"u3">>], @@ -509,7 +509,7 @@ test_authenticator_user(PathPrefix) -> {ok, 200, UserData} = request(get, UsersUri ++ "/u1"), - FetchedUser = jiffy:decode(UserData, [return_maps]), + FetchedUser = emqx_utils_json:decode(UserData, [return_maps]), ?assertMatch(#{<<"user_id">> := <<"u1">>}, FetchedUser), ?assertNotMatch(#{<<"password">> := _}, FetchedUser), diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 9a3c7c833..851e80f6d 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -41,7 +41,7 @@ -define(SERVER_RESPONSE_JSON(Result), ?SERVER_RESPONSE_JSON(Result, false)). -define(SERVER_RESPONSE_JSON(Result, IsSuperuser), - jiffy:encode(#{ + emqx_utils_json:encode(#{ result => Result, is_superuser => IsSuperuser }) @@ -172,11 +172,11 @@ t_no_value_for_placeholder(_Config) -> #{ <<"cert_subject">> := <<"">>, <<"cert_common_name">> := <<"">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -444,7 +444,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req0 ), {ok, Req, State} @@ -459,7 +459,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => true}), + emqx_utils_json:encode(#{result => allow, is_superuser => true}), Req0 ), {ok, Req, State} @@ -512,11 +512,11 @@ samples() -> #{ <<"username">> := <<"plain">>, <<"password">> := <<"plain">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -539,7 +539,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -565,11 +565,11 @@ samples() -> <<"peerhost">> := <<"127.0.0.1">>, <<"cert_subject">> := <<"cert_subject_data">>, <<"cert_common_name">> := <<"cert_common_name_data">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} diff --git a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl index 7d51ff425..c4315b69f 100644 --- a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl @@ -168,7 +168,7 @@ cowboy_handler(Req0, State) -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req0 ), {ok, Req, State}. diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 7a51d2bbb..94c07ca96 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -467,7 +467,7 @@ jwks_handler(Req0, State) -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(JWKS), + emqx_utils_json:encode(JWKS), Req0 ), {ok, Req, State}. diff --git a/apps/emqx_authz/rebar.config b/apps/emqx_authz/rebar.config index da2fa7807..9fd61b060 100644 --- a/apps/emqx_authz/rebar.config +++ b/apps/emqx_authz/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info, nowarn_unused_import]}. {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_connector, {path, "../emqx_connector"}} ]}. diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index df77673a2..560141d0a 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -144,7 +144,7 @@ parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> end; parse_http_resp_body(<<"application/json", _/binary>>, Body) -> try - result(emqx_json:decode(Body, [return_maps])) + result(emqx_utils_json:decode(Body, [return_maps])) catch _:_ -> error end. diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index 45e6d7287..cfd9f581f 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -67,12 +67,12 @@ t_clean_cahce(_) -> ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0), {ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(2, length(emqx_json:decode(Result3))), + ?assertEqual(2, length(emqx_utils_json:decode(Result3))), request(delete, uri(["authorization", "cache"])), {ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(0, length(emqx_json:decode(Result4))), + ?assertEqual(0, length(emqx_utils_json:decode(Result4))), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 411399d64..b717775a6 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -182,7 +182,7 @@ t_api(_) -> {ok, 404, ErrResult} = request(get, uri(["authorization", "sources", "http"]), []), ?assertMatch( #{<<"code">> := <<"NOT_FOUND">>, <<"message">> := <<"Not found: http">>}, - emqx_json:decode(ErrResult, [return_maps]) + emqx_utils_json:decode(ErrResult, [return_maps]) ), [ @@ -215,7 +215,8 @@ t_api(_) -> ), {ok, 200, Result3} = request(get, uri(["authorization", "sources", "http"]), []), ?assertMatch( - #{<<"type">> := <<"http">>, <<"enable">> := false}, emqx_json:decode(Result3, [return_maps]) + #{<<"type">> := <<"http">>, <<"enable">> := false}, + emqx_utils_json:decode(Result3, [return_maps]) ), Keyfile = emqx_common_test_helpers:app_path( @@ -253,7 +254,7 @@ t_api(_) -> <<"total">> := 0, <<"nomatch">> := 0 } - } = emqx_json:decode(Status4, [return_maps]), + } = emqx_utils_json:decode(Status4, [return_maps]), ?assertMatch( #{ <<"type">> := <<"mongodb">>, @@ -265,7 +266,7 @@ t_api(_) -> <<"verify">> := <<"verify_none">> } }, - emqx_json:decode(Result4, [return_maps]) + emqx_utils_json:decode(Result4, [return_maps]) ), {ok, Cacert} = file:read_file(Cacertfile), @@ -297,7 +298,7 @@ t_api(_) -> <<"verify">> := <<"verify_none">> } }, - emqx_json:decode(Result5, [return_maps]) + emqx_utils_json:decode(Result5, [return_maps]) ), {ok, 200, Status5_1} = request(get, uri(["authorization", "sources", "mongodb", "status"]), []), @@ -308,7 +309,7 @@ t_api(_) -> <<"total">> := 0, <<"nomatch">> := 0 } - } = emqx_json:decode(Status5_1, [return_maps]), + } = emqx_utils_json:decode(Status5_1, [return_maps]), #{ ssl := #{ @@ -355,7 +356,7 @@ t_api(_) -> <<"code">> := <<"BAD_REQUEST">>, <<"message">> := <<"Type mismatch", _/binary>> }, - emqx_json:decode(TypeMismatch, [return_maps]) + emqx_utils_json:decode(TypeMismatch, [return_maps]) ), lists:foreach( @@ -443,7 +444,7 @@ t_api(_) -> <<"total">> := 1, <<"nomatch">> := 0 } - } = emqx_json:decode(Status5, [return_maps]) + } = emqx_utils_json:decode(Status5, [return_maps]) end ), @@ -469,7 +470,7 @@ t_api(_) -> <<"total">> := 2, <<"nomatch">> := 0 } - } = emqx_json:decode(Status6, [return_maps]) + } = emqx_utils_json:decode(Status6, [return_maps]) end ), @@ -495,7 +496,7 @@ t_api(_) -> <<"total">> := 3, <<"nomatch">> := 0 } - } = emqx_json:decode(Status7, [return_maps]) + } = emqx_utils_json:decode(Status7, [return_maps]) end ), ok. @@ -621,7 +622,7 @@ t_aggregate_metrics(_) -> ). get_sources(Result) -> - maps:get(<<"sources">>, emqx_json:decode(Result, [return_maps])). + maps:get(<<"sources">>, emqx_utils_json:decode(Result, [return_maps])). data_dir() -> emqx:data_dir(). diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index 0757113f3..9ff84b805 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -311,7 +311,7 @@ t_json_body(_Config) -> <<"topic">> := <<"t">>, <<"action">> := <<"publish">> }, - jiffy:decode(RawBody, [return_maps]) + emqx_utils_json:decode(RawBody, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, @@ -366,7 +366,7 @@ t_placeholder_and_body(_Config) -> <<"CN">> := ?PH_CERT_CN_NAME, <<"CS">> := ?PH_CERT_SUBJECT }, - jiffy:decode(PostVars, [return_maps]) + emqx_utils_json:decode(PostVars, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, @@ -418,7 +418,7 @@ t_no_value_for_placeholder(_Config) -> #{ <<"mountpoint">> := <<"[]">> }, - jiffy:decode(RawBody, [return_maps]) + emqx_utils_json:decode(RawBody, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, diff --git a/apps/emqx_auto_subscribe/rebar.config b/apps/emqx_auto_subscribe/rebar.config index 33e077f50..a19783033 100644 --- a/apps/emqx_auto_subscribe/rebar.config +++ b/apps/emqx_auto_subscribe/rebar.config @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {shell, [ {apps, [emqx_auto_subscribe]} diff --git a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl index 5c5a3ee79..9d8d47bf2 100644 --- a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl +++ b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl @@ -141,7 +141,7 @@ t_update(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Body = [#{topic => ?TOPIC_S}], {ok, Response} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, Body), - ResponseMap = emqx_json:decode(Response, [return_maps]), + ResponseMap = emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(1, erlang:length(ResponseMap)), BadBody1 = #{topic => ?TOPIC_S}, @@ -177,7 +177,7 @@ t_update(_) -> emqtt:disconnect(Client), {ok, GETResponse} = emqx_mgmt_api_test_util:request_api(get, Path), - GETResponseMap = emqx_json:decode(GETResponse, [return_maps]), + GETResponseMap = emqx_utils_json:decode(GETResponse, [return_maps]), ?assertEqual(1, erlang:length(GETResponseMap)), ok. diff --git a/apps/emqx_bridge/rebar.config b/apps/emqx_bridge/rebar.config index 79f2caf50..864c45e9a 100644 --- a/apps/emqx_bridge/rebar.config +++ b/apps/emqx_bridge/rebar.config @@ -1,7 +1,9 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../emqx"}} - , {emqx_resource, {path, "../../apps/emqx_resource"}} - ]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}} + ]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index b29cefacd..675541311 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -1028,6 +1028,6 @@ deobfuscate(NewConf, OldConf) -> ). map_to_json(M) -> - emqx_json:encode( + emqx_utils_json:encode( emqx_map_lib:jsonable_map(M, fun(K, V) -> {K, emqx_map_lib:binary_string(V)} end) ). diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 4d5495ffe..3ec6061d8 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1304,4 +1304,4 @@ str(S) when is_list(S) -> S; str(S) when is_binary(S) -> binary_to_list(S). json(B) when is_binary(B) -> - emqx_json:decode(B, [return_maps]). + emqx_utils_json:decode(B, [return_maps]). diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index fb7cf524c..fc9a6dc29 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -705,7 +705,7 @@ create_bridge_api(Config, Overrides) -> Res = case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of {ok, {Status, Headers, Body0}} -> - {ok, {Status, Headers, emqx_json:decode(Body0, [return_maps])}}; + {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; Error -> Error end, @@ -728,7 +728,7 @@ update_bridge_api(Config, Overrides) -> ct:pal("updating bridge (via http): ~p", [Params]), Res = case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, Params, Opts) of - {ok, {_Status, _Headers, Body0}} -> {ok, emqx_json:decode(Body0, [return_maps])}; + {ok, {_Status, _Headers, Body0}} -> {ok, emqx_utils_json:decode(Body0, [return_maps])}; Error -> Error end, ct:pal("bridge update result: ~p", [Res]), @@ -776,7 +776,7 @@ do_wait_for_expected_published_messages(Messages, Acc, _Timeout) when map_size(M do_wait_for_expected_published_messages(Messages0, Acc0, Timeout) -> receive {publish, Msg0 = #{payload := Payload}} -> - case emqx_json:safe_decode(Payload, [return_maps]) of + case emqx_utils_json:safe_decode(Payload, [return_maps]) of {error, _} -> ct:pal("unexpected message: ~p; discarding", [Msg0]), do_wait_for_expected_published_messages(Messages0, Acc0, Timeout); @@ -928,7 +928,7 @@ create_rule_and_action_http(Config) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), ct:pal("rule action params: ~p", [Params]), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -1188,7 +1188,7 @@ t_start_and_consume_ok(Config) -> <<"offset">> := OffsetReply, <<"headers">> := #{<<"hkey">> := <<"hvalue">>} }, - emqx_json:decode(PayloadBin, [return_maps]), + emqx_utils_json:decode(PayloadBin, [return_maps]), #{ offset_reply => OffsetReply, kafka_topic => KafkaTopic, @@ -1300,7 +1300,7 @@ t_multiple_topic_mappings(Config) -> %% as configured. Payloads = lists:sort([ - case emqx_json:safe_decode(P, [return_maps]) of + case emqx_utils_json:safe_decode(P, [return_maps]) of {ok, Decoded} -> Decoded; {error, _} -> P end @@ -1441,7 +1441,7 @@ do_t_failed_creation_then_fixed(Config) -> <<"offset">> := _, <<"headers">> := #{<<"hkey">> := <<"hvalue">>} }, - emqx_json:decode(PayloadBin, [return_maps]), + emqx_utils_json:decode(PayloadBin, [return_maps]), #{ kafka_topic => KafkaTopic, payload => Payload @@ -1636,7 +1636,7 @@ t_bridge_rule_action_source(Config) -> <<"headers">> := #{<<"hkey">> := <<"hvalue">>}, <<"topic">> := KafkaTopic }, - emqx_json:decode(RawPayload, [return_maps]) + emqx_utils_json:decode(RawPayload, [return_maps]) ), ?retry( _Interval = 200, @@ -2004,7 +2004,7 @@ t_begin_offset_earliest(Config) -> %% the consumers Published = receive_published(#{n => NumMessages}), Payloads = lists:map( - fun(#{payload := P}) -> emqx_json:decode(P, [return_maps]) end, + fun(#{payload := P}) -> emqx_utils_json:decode(P, [return_maps]) end, Published ), ?assert( diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index 6e3ddf5bb..5e2a2a7a8 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -802,7 +802,7 @@ api_path(Parts) -> ?HOST ++ filename:join([?BASE_PATH | Parts]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. delete_all_bridges() -> diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 2ce6b00f8..03be87356 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -7,6 +7,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_resource, {path, "../emqx_resource"}}, {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}}, diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 401fc8812..dfa6dab81 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -516,7 +516,7 @@ process_request( }. process_request_body(undefined, Msg) -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); process_request_body(BodyTks, Msg) -> emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index defbbaea2..67fc40efa 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -123,7 +123,7 @@ process_payload(From, MapMsg) -> do_process_payload(maps:get(payload, From, undefined), MapMsg). do_process_payload(undefined, Msg) -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); do_process_payload(Tks, Msg) -> replace_vars_in_str(Tks, Msg). diff --git a/apps/emqx_dashboard/rebar.config b/apps/emqx_dashboard/rebar.config index 9657d0bbf..440fde465 100644 --- a/apps/emqx_dashboard/rebar.config +++ b/apps/emqx_dashboard/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index e951a9a2a..ee850ec17 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -246,5 +246,5 @@ api_path(Parts) -> ?HOST ++ filename:join([?BASE_PATH | Parts]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl index 19d3f471e..c0a772d2d 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl @@ -100,7 +100,7 @@ request(Url) -> {ok, {{"HTTP/1.1", Code, _}, _, Return}} when Code >= 200 andalso Code =< 299 -> - {ok, emqx_json:decode(Return, [return_maps])}; + {ok, emqx_utils_json:decode(Return, [return_maps])}; {ok, {Reason, _, _}} -> {error, Reason} end. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index f35652f8e..a24fc2337 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -137,10 +137,10 @@ do_request_api(Method, Request) -> Code >= 200 andalso Code =< 299 -> ct:pal("Resp ~p ~p~n", [Code, Return]), - {ok, emqx_json:decode(Return, [return_maps])}; + {ok, emqx_utils_json:decode(Return, [return_maps])}; {ok, {{"HTTP/1.1", Code, _}, _, Return}} -> ct:pal("Resp ~p ~p~n", [Code, Return]), - {error, {Code, emqx_json:decode(Return, [return_maps])}}; + {error, {Code, emqx_utils_json:decode(Return, [return_maps])}}; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index fad539ed1..7abc601b4 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -5,7 +5,8 @@ ]}. {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. {grpc, [ diff --git a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl index 8a4fb7a44..58f2f29b4 100644 --- a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl @@ -310,7 +310,7 @@ t_update(Cfg) -> ?assertMatch([], emqx_exhook_mgr:running()). decode_json(Data) -> - BinJosn = emqx_json:decode(Data, [return_maps]), + BinJosn = emqx_utils_json:decode(Data, [return_maps]), emqx_map_lib:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> @@ -332,7 +332,7 @@ request_api(Method, Url, QueryParams, Auth, Body) -> "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)}). do_request_api(Method, Request) -> case httpc:request(Method, Request, [], [{body_format, binary}]) of diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 7e5228a9e..2340a2dd8 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -1,5 +1,6 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index df808f295..fb4261065 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -74,7 +74,7 @@ gateway(["load", Name, Conf]) -> case emqx_gateway_conf:load_gateway( bin(Name), - emqx_json:decode(Conf, [return_maps]) + emqx_utils_json:decode(Conf, [return_maps]) ) of {ok, _} -> diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index a0155a126..5982334b4 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -404,7 +404,7 @@ return_http_error(Code, Msg) -> -spec reason2msg({atom(), map()} | any()) -> error | string(). reason2msg({badconf, #{key := Key, value := Value, reason := Reason}}) -> NValue = - case emqx_json:safe_encode(Value) of + case emqx_utils_json:safe_encode(Value) of {ok, Str} -> Str; {error, _} -> emqx_gateway_utils:stringfy(Value) end, diff --git a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl index d75bf80eb..0ed66a38d 100644 --- a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl +++ b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl @@ -153,7 +153,7 @@ on_start_auth(authn_http) -> Handler = fun(Req0, State) -> ct:pal("Authn Req:~p~nState:~p~n", [Req0, State]), Headers = #{<<"content-type">> => <<"application/json">>}, - Response = jiffy:encode(#{result => allow, is_superuser => false}), + Response = emqx_utils_json:encode(#{result => allow, is_superuser => false}), case cowboy_req:match_qs([username, password], Req0) of #{ username := <<"admin">>, diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 1a4bab5f3..1ab36f7b8 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -266,7 +266,7 @@ t_case_exproto(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, 5000), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(Expect), C(Recv)) end ) diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index 9bbcf2711..c62e840df 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -165,7 +165,7 @@ t_case_lwm2m(_) -> Test("lwm2m", fun(SubTopic, Msg) -> ?assertEqual(true, lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics())), Payload = emqx_message:payload(Msg), - Cmd = emqx_json:decode(Payload, [return_maps]), + Cmd = emqx_utils_json:decode(Payload, [return_maps]), ?assertMatch(#{<<"msgType">> := <<"register">>, <<"data">> := _}, Cmd) end), @@ -350,7 +350,7 @@ t_case_exproto_publish(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, 5000), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(SvrMod:frame_connack(0)), C(Recv)), Send = fun() -> @@ -387,7 +387,7 @@ t_case_exproto_subscribe(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, WaitTime), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(SvrMod:frame_connack(0)), C(Recv)), SubBin = SvrMod:frame_subscribe(Topic, 0), diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index 7ec0c9538..6d80b63d8 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -160,7 +160,7 @@ do_request(Mth, Req) -> #{}; _ -> emqx_map_lib:unsafe_atom_key_map( - emqx_json:decode(Resp, [return_maps]) + emqx_utils_json:decode(Resp, [return_maps]) ) end, {Code, NResp}; @@ -172,7 +172,7 @@ req(Path, Qs) -> {url(Path, Qs), auth([])}. req(Path, Qs, Body) -> - {url(Path, Qs), auth([]), "application/json", emqx_json:encode(Body)}. + {url(Path, Qs), auth([]), "application/json", emqx_utils_json:encode(Body)}. url(Path, []) -> lists:concat([?http_api_host, Path]); diff --git a/apps/emqx_gateway_coap/rebar.config b/apps/emqx_gateway_coap/rebar.config index c8675c3ba..3b070a72a 100644 --- a/apps/emqx_gateway_coap/rebar.config +++ b/apps/emqx_gateway_coap/rebar.config @@ -1,4 +1,6 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../../apps/emqx"}}, - {emqx_gateway, {path, "../../apps/emqx_gateway"}} - ]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../emqx_gateway"}} +]}. diff --git a/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl index a0d6dbaaa..cec09a016 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl @@ -92,7 +92,7 @@ t_send_request_api(_) -> Req ), #{<<"token">> := RToken, <<"payload">> := RPayload} = - emqx_json:decode(Response, [return_maps]), + emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(Token, RToken), ?assertEqual(Payload, RPayload) end, diff --git a/apps/emqx_gateway_exproto/rebar.config b/apps/emqx_gateway_exproto/rebar.config index 928949c69..473fa9b67 100644 --- a/apps/emqx_gateway_exproto/rebar.config +++ b/apps/emqx_gateway_exproto/rebar.config @@ -1,7 +1,9 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../../apps/emqx"}}, - {emqx_gateway, {path, "../../apps/emqx_gateway"}} - ]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../emqx_gateway"}} +]}. {plugins, [ {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} diff --git a/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl b/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl index b2e3ad4a7..e04990f5f 100644 --- a/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl +++ b/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl @@ -148,7 +148,7 @@ on_received_bytes(Stream, _Md) -> fun(Reqs) -> lists:foreach( fun(#{conn := Conn, bytes := Bytes}) -> - #{<<"type">> := Type} = Params = emqx_json:decode(Bytes, [return_maps]), + #{<<"type">> := Type} = Params = emqx_utils_json:decode(Bytes, [return_maps]), _ = handle_in(Conn, Type, Params) end, Reqs @@ -284,16 +284,16 @@ handle_out(Conn, ?TYPE_DISCONNECT) -> %% Frame frame_connect(ClientInfo, Password) -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ type => ?TYPE_CONNECT, clientinfo => ClientInfo, password => Password }). frame_connack(Code) -> - emqx_json:encode(#{type => ?TYPE_CONNACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_CONNACK, code => Code}). frame_publish(Topic, Qos, Payload) -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ type => ?TYPE_PUBLISH, topic => Topic, qos => Qos, @@ -301,19 +301,19 @@ frame_publish(Topic, Qos, Payload) -> }). frame_puback(Code) -> - emqx_json:encode(#{type => ?TYPE_PUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_PUBACK, code => Code}). frame_subscribe(Topic, Qos) -> - emqx_json:encode(#{type => ?TYPE_SUBSCRIBE, topic => Topic, qos => Qos}). + emqx_utils_json:encode(#{type => ?TYPE_SUBSCRIBE, topic => Topic, qos => Qos}). frame_suback(Code) -> - emqx_json:encode(#{type => ?TYPE_SUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_SUBACK, code => Code}). frame_unsubscribe(Topic) -> - emqx_json:encode(#{type => ?TYPE_UNSUBSCRIBE, topic => Topic}). + emqx_utils_json:encode(#{type => ?TYPE_UNSUBSCRIBE, topic => Topic}). frame_unsuback(Code) -> - emqx_json:encode(#{type => ?TYPE_UNSUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_UNSUBACK, code => Code}). frame_disconnect() -> - emqx_json:encode(#{type => ?TYPE_DISCONNECT}). + emqx_utils_json:encode(#{type => ?TYPE_DISCONNECT}). diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl index 90a0306b7..c3d23d4d9 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl @@ -351,7 +351,7 @@ opaque_to_json(BaseName, Binary) -> [#{path => BaseName, value => base64:encode(Binary)}]. translate_json(JSONBin) -> - JSONTerm = emqx_json:decode(JSONBin, [return_maps]), + JSONTerm = emqx_utils_json:decode(JSONBin, [return_maps]), BaseName = maps:get(<<"bn">>, JSONTerm, <<>>), ElementList = maps:get(<<"e">>, JSONTerm, []), translate_element(BaseName, ElementList, []). diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl index 2752804dc..c8bfa8974 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl @@ -737,7 +737,7 @@ proto_publish( Epn, Qos, MountedTopic, - emqx_json:encode(Payload), + emqx_utils_json:encode(Payload), #{}, Headers ), @@ -786,7 +786,7 @@ deliver_to_coap(AlternatePath, JsonData, MQTT, CacheMode, WithContext, Session) is_binary(JsonData) -> try - TermData = emqx_json:decode(JsonData, [return_maps]), + TermData = emqx_utils_json:decode(JsonData, [return_maps]), deliver_to_coap(AlternatePath, TermData, MQTT, CacheMode, WithContext, Session) catch ExClass:Error:ST -> diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl index 33ccec2c7..9f388b07c 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -402,7 +402,7 @@ case01_register_report(Config) -> timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -478,7 +478,7 @@ case02_update_deregister(Config) -> ?LOGT("Options got: ~p", [Opts]), Location = maps:get(location_path, Opts), - Register = emqx_json:encode( + Register = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -521,7 +521,7 @@ case02_update_deregister(Config) -> } = test_recv_coap_response(UdpSock), {ok, changed} = Method2, MsgId2 = RspId2, - Update = emqx_json:encode( + Update = emqx_utils_json:encode( #{ <<"msgType">> => <<"update">>, <<"data">> => #{ @@ -754,7 +754,7 @@ case08_reregister(Config) -> timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -871,7 +871,7 @@ case10_read(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -902,7 +902,7 @@ case10_read(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -957,7 +957,7 @@ case10_read_bad_request(Config) -> <<"path">> => <<"/3333/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -979,7 +979,7 @@ case10_read_bad_request(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode(#{ + ReadResult = emqx_utils_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"read">>, @@ -1015,7 +1015,7 @@ case10_read_separate_ack(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1032,7 +1032,7 @@ case10_read_separate_ack(Config) -> ?assertEqual(<<>>, Payload2), test_send_empty_ack(UdpSock, "127.0.0.1", ?PORT, Request2), - ReadResultACK = emqx_json:encode( + ReadResultACK = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1057,7 +1057,7 @@ case10_read_separate_ack(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1100,7 +1100,7 @@ case11_read_object_tlv(Config) -> <<"path">> => <<"/3/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1132,7 +1132,7 @@ case11_read_object_tlv(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1185,7 +1185,7 @@ case11_read_object_json(Config) -> <<"path">> => <<"/3/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1215,7 +1215,7 @@ case11_read_object_json(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1267,7 +1267,7 @@ case12_read_resource_opaque(Config) -> <<"path">> => <<"/3/0/8">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1293,7 +1293,7 @@ case12_read_resource_opaque(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1335,7 +1335,7 @@ case13_read_no_xml(Config) -> <<"msgType">> => <<"read">>, <<"data">> => #{<<"path">> => <<"/9723/0/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1360,7 +1360,7 @@ case13_read_no_xml(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1399,7 +1399,7 @@ case20_single_write(Config) -> <<"value">> => <<"12345">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1426,7 +1426,7 @@ case20_single_write(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1470,7 +1470,7 @@ case20_write(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1497,7 +1497,7 @@ case20_write(Config) -> ), timer:sleep(100), - WriteResult = emqx_json:encode( + WriteResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1547,7 +1547,7 @@ case21_write_object(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1574,7 +1574,7 @@ case21_write_object(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1618,7 +1618,7 @@ case22_write_error(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1639,7 +1639,7 @@ case22_write_error(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1677,7 +1677,7 @@ case_create_basic(Config) -> <<"basePath">> => <<"/5">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1703,7 +1703,7 @@ case_create_basic(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1738,7 +1738,7 @@ case_delete_basic(Config) -> <<"msgType">> => <<"delete">>, <<"data">> => #{<<"path">> => <<"/5/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1764,7 +1764,7 @@ case_delete_basic(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1804,7 +1804,7 @@ case30_execute(Config) -> <<"args">> => <<"2,7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1830,7 +1830,7 @@ case30_execute(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1868,7 +1868,7 @@ case31_execute_error(Config) -> <<"args">> => <<"2,7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1894,7 +1894,7 @@ case31_execute_error(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1931,7 +1931,7 @@ case40_discover(Config) -> <<"path">> => <<"/3/0/7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1961,7 +1961,7 @@ case40_discover(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2006,7 +2006,7 @@ case50_write_attribute(Config) -> <<"lt">> => <<"5">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(100), Request2 = test_recv_coap_request(UdpSock), @@ -2042,7 +2042,7 @@ case50_write_attribute(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2079,7 +2079,7 @@ case60_observe(Config) -> <<"msgType">> => <<"observe">>, <<"data">> => #{<<"path">> => <<"/3/0/10">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -2106,7 +2106,7 @@ case60_observe(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2141,7 +2141,7 @@ case60_observe(Config) -> timer:sleep(100), #coap_message{} = test_recv_coap_response(UdpSock), - ReadResult2 = emqx_json:encode( + ReadResult2 = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2173,7 +2173,7 @@ case60_observe(Config) -> <<"path">> => <<"/3/0/10">> } }, - CommandJson3 = emqx_json:encode(Command3), + CommandJson3 = emqx_utils_json:encode(Command3), test_mqtt_broker:publish(CommandTopic, CommandJson3, 0), timer:sleep(50), Request3 = test_recv_coap_request(UdpSock), @@ -2200,7 +2200,7 @@ case60_observe(Config) -> ), timer:sleep(100), - ReadResult3 = emqx_json:encode( + ReadResult3 = emqx_utils_json:encode( #{ <<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, @@ -2242,7 +2242,7 @@ case60_observe(Config) -> %% MsgId1), %% #coap_message{method = Method1} = test_recv_coap_response(UdpSock), %% ?assertEqual({ok,created}, Method1), -%% ReadResult = emqx_json:encode( +%% ReadResult = emqx_utils_json:encode( %% #{<<"msgType">> => <<"register">>, %% <<"data">> => #{ %% <<"alternatePath">> => <<"/">>, @@ -2268,7 +2268,7 @@ case60_observe(Config) -> %% <<"path">> => <<"/19/0/0">> %% } %% }, -%% CommandJson = emqx_json:encode(Command), +%% CommandJson = emqx_utils_json:encode(Command), %% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), %% timer:sleep(50), %% Request2 = test_recv_coap_request(UdpSock), @@ -2325,7 +2325,7 @@ case60_observe(Config) -> %% <<"value">> => base64:encode(<<12345:32>>) %% }}, %% -%% CommandJson = emqx_json:encode(Command), +%% CommandJson = emqx_utils_json:encode(Command), %% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), %% timer:sleep(50), %% Request2 = test_recv_coap_request(UdpSock), @@ -2342,7 +2342,7 @@ case60_observe(Config) -> %% {ok, changed}, #coap_content{}, Request2, true), %% timer:sleep(100), %% -%% ReadResult = emqx_json:encode( +%% ReadResult = emqx_utils_json:encode( %% #{<<"requestID">> => CmdId, %% <<"cacheID">> => CmdId, %% <<"data">> => #{ @@ -2502,7 +2502,7 @@ send_read_command_1(CmdId, _UdpSock) -> <<"msgType">> => <<"read">>, <<"data">> => #{<<"path">> => <<"/3/0/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50). @@ -2528,7 +2528,7 @@ verify_read_response_1(CmdId, UdpSock) -> true ), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl index b3c059b31..6fa46ebbc 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl @@ -131,7 +131,7 @@ t_lookup_read(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -178,7 +178,7 @@ t_lookup_discover(Config) -> <<"path">> => <<"/3/0/7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(200), @@ -350,10 +350,10 @@ no_received_request(ClientId, Path, Action) -> <<"codeMsg">> => <<"reply_not_received">>, <<"path">> => Path }, - ?assertEqual(NotReceived, emqx_json:decode(Response, [return_maps])). + ?assertEqual(NotReceived, emqx_utils_json:decode(Response, [return_maps])). normal_received_request(ClientId, Path, Action) -> Response = call_lookup_api(ClientId, Path, Action), - RCont = emqx_json:decode(Response, [return_maps]), + RCont = emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(list_to_binary(ClientId), maps:get(<<"clientid">>, RCont, undefined)), ?assertEqual(Path, maps:get(<<"path">>, RCont, undefined)), ?assertEqual(Action, maps:get(<<"action">>, RCont, undefined)), diff --git a/apps/emqx_gateway_stomp/rebar.config b/apps/emqx_gateway_stomp/rebar.config index c8675c3ba..cfeb0a195 100644 --- a/apps/emqx_gateway_stomp/rebar.config +++ b/apps/emqx_gateway_stomp/rebar.config @@ -1,4 +1,6 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../../apps/emqx"}}, - {emqx_gateway, {path, "../../apps/emqx_gateway"}} - ]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} +]}. diff --git a/apps/emqx_machine/rebar.config b/apps/emqx_machine/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_machine/rebar.config +++ b/apps/emqx_machine/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 0bee30e35..6bd36aab5 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index 243c4bb8c..6872b150c 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -88,7 +88,7 @@ start_sysmon() -> end. node_status() -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ backend => mria_rlog:backend(), role => mria_rlog:role() }). diff --git a/apps/emqx_management/rebar.config b/apps/emqx_management/rebar.config index 73cbf471f..b2f5a40af 100644 --- a/apps/emqx_management/rebar.config +++ b/apps/emqx_management/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 0a5cc3afe..508cf7d07 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -165,7 +165,7 @@ banned(post, #{body := Body}) -> {ok, Banned} -> {200, format(Banned)}; {error, {already_exist, Old}} -> - OldBannedFormat = emqx_json:encode(format(Old)), + OldBannedFormat = emqx_utils_json:encode(format(Old)), {400, 'ALREADY_EXISTS', OldBannedFormat} end end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl index 69ace16e8..dc7be7671 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl @@ -56,7 +56,7 @@ get_alarms(AssertCount, Activated) -> Qs = "activated=" ++ Activated, Headers = emqx_mgmt_api_test_util:auth_header_(), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, Qs, Headers), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, Data), Page = maps:get(<<"page">>, Meta), Limit = maps:get(<<"limit">>, Meta), diff --git a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl index 241a73dc4..1a396d795 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl @@ -228,7 +228,7 @@ list_app() -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key"]), case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. @@ -236,7 +236,7 @@ read_app(Name) -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key", Name]), case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -251,7 +251,7 @@ create_app(Name) -> enable => true }, case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -260,7 +260,7 @@ create_unexpired_app(Name, Params) -> Path = emqx_mgmt_api_test_util:api_path(["api_key"]), App = maps:merge(#{name => Name, desc => <<"Note"/utf8>>, enable => true}, Params), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -273,7 +273,7 @@ update_app(Name, Change) -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), UpdatePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]), case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of - {ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])}; + {ok, Update} -> {ok, emqx_utils_json:decode(Update, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl index c765f00bc..9f1b560f7 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl @@ -160,7 +160,7 @@ t_delete(_Config) -> list_banned() -> Path = emqx_mgmt_api_test_util:api_path(["banned"]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. @@ -168,7 +168,7 @@ create_banned(Banned) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["banned"]), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Banned) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 9f26f8542..6d7733b22 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -58,7 +58,7 @@ t_clients(_) -> %% get /clients ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), - ClientsResponse = emqx_json:decode(Clients, [return_maps]), + ClientsResponse = emqx_utils_json:decode(Clients, [return_maps]), ClientsMeta = maps:get(<<"meta">>, ClientsResponse), ClientsPage = maps:get(<<"page">>, ClientsMeta), ClientsLimit = maps:get(<<"limit">>, ClientsMeta), @@ -70,7 +70,7 @@ t_clients(_) -> %% get /clients/:clientid Client1Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1)]), {ok, Client1} = emqx_mgmt_api_test_util:request_api(get, Client1Path), - Client1Response = emqx_json:decode(Client1, [return_maps]), + Client1Response = emqx_utils_json:decode(Client1, [return_maps]), ?assertEqual(Username1, maps:get(<<"username">>, Client1Response)), ?assertEqual(ClientId1, maps:get(<<"clientid">>, Client1Response)), ?assertEqual(120, maps:get(<<"expiry_interval">>, Client1Response)), @@ -130,7 +130,7 @@ t_clients(_) -> "", AuthHeader ), - [SubscriptionsData] = emqx_json:decode(SubscriptionsRes, [return_maps]), + [SubscriptionsData] = emqx_utils_json:decode(SubscriptionsRes, [return_maps]), ?assertMatch( #{ <<"clientid">> := ClientId1, @@ -210,7 +210,7 @@ t_query_clients_with_time(_) -> GteParamRfc3339 ++ GteParamStamp ], DecodedResults = [ - emqx_json:decode(Response, [return_maps]) + emqx_utils_json:decode(Response, [return_maps]) || {ok, Response} <- RequestResults ], {LteResponseDecodeds, GteResponseDecodeds} = lists:split(4, DecodedResults), @@ -247,7 +247,7 @@ t_keepalive(_Config) -> {ok, C1} = emqtt:start_link(#{username => Username, clientid => ClientId}), {ok, _} = emqtt:connect(C1), {ok, NewClient} = emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body), - #{<<"keepalive">> := 11} = emqx_json:decode(NewClient, [return_maps]), + #{<<"keepalive">> := 11} = emqx_utils_json:decode(NewClient, [return_maps]), [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), #{conninfo := #{keepalive := Keepalive}} = emqx_connection:info(Pid), ?assertEqual(11, Keepalive), diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 2d24bce99..cc200a4d5 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -235,7 +235,7 @@ t_configs_node(_) -> ?assertEqual(error, ExpType), ?assertMatch({{_, 404, _}, _, _}, ExpRes), {_, _, Body} = ExpRes, - ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_json:decode(Body, [return_maps])), + ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_utils_json:decode(Body, [return_maps])), ?assertMatch({error, {_, 500, _}}, get_configs("bad_node")). @@ -245,7 +245,7 @@ get_config(Name) -> Path = emqx_mgmt_api_test_util:api_path(["configs", Name]), case emqx_mgmt_api_test_util:request_api(get, Path) of {ok, Res} -> - {ok, emqx_json:decode(Res, [return_maps])}; + {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -264,8 +264,8 @@ get_configs(Node, Opts) -> end, URI = emqx_mgmt_api_test_util:api_path(Path), case emqx_mgmt_api_test_util:request_api(get, URI, [], [], [], Opts) of - {ok, {_, _, Res}} -> {ok, emqx_json:decode(Res, [return_maps])}; - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, {_, _, Res}} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -273,7 +273,7 @@ update_config(Name, Change) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), UpdatePath = emqx_mgmt_api_test_util:api_path(["configs", Name]), case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of - {ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])}; + {ok, Update} -> {ok, emqx_utils_json:decode(Update, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index 3238588e2..90808d4be 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -385,7 +385,7 @@ action_listener(ID, Action, Running) -> request(Method, Url, QueryParams, Body) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body) of - {ok, Res} -> emqx_json:decode(Res, [return_maps]); + {ok, Res} -> emqx_utils_json:decode(Res, [return_maps]); Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl index 93cb69f0a..7ecfe9817 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl @@ -32,13 +32,13 @@ end_per_suite(_) -> t_metrics_api(_) -> {ok, MetricsResponse} = request_helper("metrics?aggregate=true"), - MetricsFromAPI = emqx_json:decode(MetricsResponse, [return_maps]), + MetricsFromAPI = emqx_utils_json:decode(MetricsResponse, [return_maps]), AggregateMetrics = emqx_mgmt:get_metrics(), match_helper(AggregateMetrics, MetricsFromAPI). t_single_node_metrics_api(_) -> {ok, MetricsResponse} = request_helper("metrics"), - [MetricsFromAPI] = emqx_json:decode(MetricsResponse, [return_maps]), + [MetricsFromAPI] = emqx_utils_json:decode(MetricsResponse, [return_maps]), LocalNodeMetrics = maps:from_list( emqx_mgmt:get_metrics(node()) ++ [{node, to_bin(node())}] ), diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 30313e555..7e4ccc541 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -53,7 +53,7 @@ end_per_testcase(_, Config) -> t_nodes_api(_) -> NodesPath = emqx_mgmt_api_test_util:api_path(["nodes"]), {ok, Nodes} = emqx_mgmt_api_test_util:request_api(get, NodesPath), - NodesResponse = emqx_json:decode(Nodes, [return_maps]), + NodesResponse = emqx_utils_json:decode(Nodes, [return_maps]), LocalNodeInfo = hd(NodesResponse), Node = binary_to_atom(maps:get(<<"node">>, LocalNodeInfo), utf8), ?assertEqual(Node, node()), @@ -63,7 +63,7 @@ t_nodes_api(_) -> NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]), {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), NodeNameResponse = - binary_to_atom(maps:get(<<"node">>, emqx_json:decode(NodeInfo, [return_maps])), utf8), + binary_to_atom(maps:get(<<"node">>, emqx_utils_json:decode(NodeInfo, [return_maps])), utf8), ?assertEqual(node(), NodeNameResponse), BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode"]), @@ -75,7 +75,7 @@ t_nodes_api(_) -> t_log_path(_) -> NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]), {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), - #{<<"log_path">> := Path} = emqx_json:decode(NodeInfo, [return_maps]), + #{<<"log_path">> := Path} = emqx_utils_json:decode(NodeInfo, [return_maps]), ?assertEqual( <<"log">>, filename:basename(Path) @@ -85,7 +85,7 @@ t_node_stats_api(_) -> StatsPath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "stats"]), SystemStats = emqx_mgmt:get_stats(), {ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath), - Stats = emqx_json:decode(StatsResponse, [return_maps]), + Stats = emqx_utils_json:decode(StatsResponse, [return_maps]), Fun = fun(Key) -> ?assertEqual(maps:get(Key, SystemStats), maps:get(atom_to_binary(Key, utf8), Stats)) @@ -103,7 +103,7 @@ t_node_metrics_api(_) -> emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "metrics"]), SystemMetrics = emqx_mgmt:get_metrics(), {ok, MetricsResponse} = emqx_mgmt_api_test_util:request_api(get, MetricsPath), - Metrics = emqx_json:decode(MetricsResponse, [return_maps]), + Metrics = emqx_utils_json:decode(MetricsResponse, [return_maps]), Fun = fun(Key) -> ?assertEqual(maps:get(Key, SystemMetrics), maps:get(atom_to_binary(Key, utf8), Metrics)) diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 24e55494d..62fed8211 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -136,14 +136,14 @@ t_bad_plugin(Config) -> list_plugins() -> Path = emqx_mgmt_api_test_util:api_path(["plugins"]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. describe_plugins(Name) -> Path = emqx_mgmt_api_test_util:api_path(["plugins", Name]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -172,7 +172,7 @@ update_boot_order(Name, MoveBody) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["plugins", Name, "move"]), case emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, MoveBody) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -206,7 +206,7 @@ create_renamed_package(PackagePath, NewNameVsn) -> NewPackagePath. update_release_json(["release.json"], FileContent, NewName) -> - ContentMap = emqx_json:decode(FileContent, [return_maps]), - emqx_json:encode(ContentMap#{<<"name">> => NewName}); + ContentMap = emqx_utils_json:decode(FileContent, [return_maps]), + emqx_utils_json:encode(ContentMap#{<<"name">> => NewName}); update_release_json(_FileName, FileContent, _NewName) -> FileContent. diff --git a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl index 44b8069f5..303a73b41 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl @@ -352,4 +352,4 @@ receive_assert(Topic, Qos, Payload) -> end. decode_json(In) -> - emqx_json:decode(In, [return_maps]). + emqx_utils_json:decode(In, [return_maps]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl index 7236ac9e4..4099426b8 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl @@ -33,7 +33,7 @@ end_per_suite(_) -> t_stats_api(_) -> S = emqx_mgmt_api_test_util:api_path(["stats?aggregate=false"]), {ok, S1} = emqx_mgmt_api_test_util:request_api(get, S), - [Stats1] = emqx_json:decode(S1, [return_maps]), + [Stats1] = emqx_utils_json:decode(S1, [return_maps]), SystemStats1 = emqx_mgmt:get_stats(), Fun1 = fun(Key) -> @@ -43,7 +43,7 @@ t_stats_api(_) -> StatsPath = emqx_mgmt_api_test_util:api_path(["stats?aggregate=true"]), SystemStats = emqx_mgmt:get_stats(), {ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath), - Stats = emqx_json:decode(StatsResponse, [return_maps]), + Stats = emqx_utils_json:decode(StatsResponse, [return_maps]), ?assertEqual(erlang:length(maps:keys(SystemStats)), erlang:length(maps:keys(Stats))), Fun = fun(Key) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index b9e9fffd8..a23d70f2f 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -55,7 +55,7 @@ t_subscription_api(Config) -> {ok, _, _} = emqtt:subscribe(Client, ?TOPIC2), Path = emqx_mgmt_api_test_util:api_path(["subscriptions"]), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, Data), ?assertEqual(1, maps:get(<<"page">>, Meta)), ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), @@ -158,7 +158,7 @@ t_list_with_internal_subscription(_Config) -> request_json(Method, Query, Headers) when is_list(Query) -> Qs = uri_string:compose_query(Query), {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers), - emqx_json:decode(MatchRes, [return_maps]). + emqx_utils_json:decode(MatchRes, [return_maps]). path() -> emqx_mgmt_api_test_util:api_path(["subscriptions"]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 782f493fe..5fe4ca937 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -108,7 +108,8 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when end, do_request_api( Method, - {NewUrl, build_http_header(AuthOrHeaders), "application/json", emqx_json:encode(Body)}, + {NewUrl, build_http_header(AuthOrHeaders), "application/json", + emqx_utils_json:encode(Body)}, Opts ). diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl index 0c2e684b4..659ae0d44 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl @@ -49,7 +49,7 @@ t_nodes_api(Config) -> %% list all Path = emqx_mgmt_api_test_util:api_path(["topics"]), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path), - RoutesData = emqx_json:decode(Response, [return_maps]), + RoutesData = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, RoutesData), ?assertEqual(1, maps:get(<<"page">>, Meta)), ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), @@ -68,7 +68,7 @@ t_nodes_api(Config) -> ]), Headers = emqx_mgmt_api_test_util:auth_header_(), {ok, MatchResponse} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers), - MatchData = emqx_json:decode(MatchResponse, [return_maps]), + MatchData = emqx_utils_json:decode(MatchResponse, [return_maps]), ?assertMatch( #{<<"count">> := 1, <<"page">> := 1, <<"limit">> := 100}, maps:get(<<"meta">>, MatchData) @@ -90,6 +90,6 @@ t_nodes_api(Config) -> [ #{<<"topic">> := Topic, <<"node">> := Node1}, #{<<"topic">> := Topic, <<"node">> := Node2} - ] = emqx_json:decode(RouteResponse, [return_maps]), + ] = emqx_utils_json:decode(RouteResponse, [return_maps]), ?assertEqual(lists:usort([Node, atom_to_binary(Slave)]), lists:usort([Node1, Node2])). diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 7922bbb40..0102eb56c 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -384,7 +384,7 @@ api_path(Path) -> emqx_mgmt_api_test_util:api_path([Path]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. load() -> diff --git a/apps/emqx_modules/rebar.config b/apps/emqx_modules/rebar.config index 9688d5043..ff542aed7 100644 --- a/apps/emqx_modules/rebar.config +++ b/apps/emqx_modules/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_conf, {path, "../emqx_conf"}} ]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl index 6d5c772f0..1d49fb3e7 100644 --- a/apps/emqx_modules/src/emqx_telemetry.erl +++ b/apps/emqx_modules/src/emqx_telemetry.erl @@ -356,7 +356,7 @@ get_telemetry(State0 = #state{node_uuid = NodeUUID, cluster_uuid = ClusterUUID}) report_telemetry(State0 = #state{url = URL}) -> {State, Data} = get_telemetry(State0), - case emqx_json:safe_encode(Data) of + case emqx_utils_json:safe_encode(Data) of {ok, Bin} -> httpc_request(post, URL, [], Bin), ?tp(debug, telemetry_data_reported, #{}); diff --git a/apps/emqx_modules/src/emqx_telemetry_api.erl b/apps/emqx_modules/src/emqx_telemetry_api.erl index 798c3ad17..b7209d146 100644 --- a/apps/emqx_modules/src/emqx_telemetry_api.erl +++ b/apps/emqx_modules/src/emqx_telemetry_api.erl @@ -243,7 +243,7 @@ status(put, #{body := Body}) -> data(get, _Request) -> case emqx_modules_conf:is_telemetry_enabled() of true -> - {200, emqx_json:encode(get_telemetry_data())}; + {200, emqx_utils_json:encode(get_telemetry_data())}; false -> {404, #{ code => ?NOT_FOUND, diff --git a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl index ed3cd9292..b2d854d22 100644 --- a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl @@ -229,7 +229,7 @@ t_large_payload(_) -> %%-------------------------------------------------------------------- decode_json(Data) -> - BinJson = emqx_json:decode(Data, [return_maps]), + BinJson = emqx_utils_json:decode(Data, [return_maps]), emqx_map_lib:unsafe_atom_key_map(BinJson). clear_all_record() -> diff --git a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl index ddb136f1e..68d12a2c1 100644 --- a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl @@ -75,7 +75,7 @@ t_mqtt_topic_rewrite(_) -> ?assertEqual( Rules, - emqx_json:decode(Result, [return_maps]) + emqx_utils_json:decode(Result, [return_maps]) ). t_mqtt_topic_rewrite_limit(_) -> diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl index a61781e13..bb5f39c1f 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -512,7 +512,7 @@ t_send_after_enable(_) -> ), receive {request, post, _URL, _Headers, Body} -> - {ok, Decoded} = emqx_json:safe_decode(Body, [return_maps]), + {ok, Decoded} = emqx_utils_json:safe_decode(Body, [return_maps]), ?assertMatch( #{ <<"uuid">> := _, diff --git a/apps/emqx_plugin_libs/rebar.config b/apps/emqx_plugin_libs/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_plugin_libs/rebar.config +++ b/apps/emqx_plugin_libs/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_plugin_libs/src/emqx_placeholder.erl b/apps/emqx_plugin_libs/src/emqx_placeholder.erl index 1f93c1d3e..18ef9e8fb 100644 --- a/apps/emqx_plugin_libs/src/emqx_placeholder.erl +++ b/apps/emqx_plugin_libs/src/emqx_placeholder.erl @@ -240,7 +240,7 @@ sql_data(Bin) when is_binary(Bin) -> Bin; sql_data(Num) when is_number(Num) -> Num; sql_data(Bool) when is_boolean(Bool) -> Bool; sql_data(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -sql_data(Map) when is_map(Map) -> emqx_json:encode(Map). +sql_data(Map) when is_map(Map) -> emqx_utils_json:encode(Map). -spec bin(term()) -> binary(). bin(Val) -> emqx_plugin_libs_rule:bin(Val). diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index d1a821895..40f88a0dc 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -236,11 +236,11 @@ tcp_connectivity(Host, Port, Timeout) -> str(Bin) when is_binary(Bin) -> binary_to_list(Bin); str(Num) when is_number(Num) -> number_to_list(Num); str(Atom) when is_atom(Atom) -> atom_to_list(Atom); -str(Map) when is_map(Map) -> binary_to_list(emqx_json:encode(Map)); +str(Map) when is_map(Map) -> binary_to_list(emqx_utils_json:encode(Map)); str(List) when is_list(List) -> case io_lib:printable_list(List) of true -> List; - false -> binary_to_list(emqx_json:encode(List)) + false -> binary_to_list(emqx_utils_json:encode(List)) end; str(Data) -> error({invalid_str, Data}). @@ -258,11 +258,11 @@ utf8_str(Str) -> bin(Bin) when is_binary(Bin) -> Bin; bin(Num) when is_number(Num) -> number_to_binary(Num); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -bin(Map) when is_map(Map) -> emqx_json:encode(Map); +bin(Map) when is_map(Map) -> emqx_utils_json:encode(Map); bin(List) when is_list(List) -> case io_lib:printable_list(List) of true -> list_to_binary(List); - false -> emqx_json:encode(List) + false -> emqx_utils_json:encode(List) end; bin(Data) -> error({invalid_bin, Data}). @@ -312,7 +312,7 @@ float2str(Float, Precision) when is_float(Float) and is_integer(Precision) -> float_to_binary(Float, [{decimals, Precision}, compact]). map(Bin) when is_binary(Bin) -> - case emqx_json:decode(Bin, [return_maps]) of + case emqx_utils_json:decode(Bin, [return_maps]) of Map = #{} -> Map; _ -> error({invalid_map, Bin}) end; diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 88b3d27a2..7b9a6cc48 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}} ]}. diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index c4867d9fd..e29d46720 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -66,7 +66,7 @@ t_prometheus_api(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), - Conf = emqx_json:decode(Response, [return_maps]), + Conf = emqx_utils_json:decode(Response, [return_maps]), ?assertMatch( #{ <<"push_gateway_server">> := _, @@ -84,7 +84,7 @@ t_prometheus_api(_) -> NewConf = Conf#{<<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"disabled">>}, {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf), - Conf2 = emqx_json:decode(Response2, [return_maps]), + Conf2 = emqx_utils_json:decode(Response2, [return_maps]), ?assertMatch(NewConf, Conf2), ?assertEqual({ok, []}, application:get_env(prometheus, vm_statistics_collector_metrics)), ?assertEqual({ok, all}, application:get_env(prometheus, vm_memory_collector_metrics)), @@ -106,7 +106,7 @@ t_stats_api(_) -> Headers = [{"accept", "application/json"}, Auth], {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Headers), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), ?assertMatch(#{<<"client">> := _, <<"delivery">> := _}, Data), {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index 65de71fdd..a178e10a1 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index ba96887a2..2b92b3f59 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -72,7 +72,7 @@ t_config(_Config) -> ), UpdateConf = fun(Enable) -> - RawConf = emqx_json:decode(ConfJson, [return_maps]), + RawConf = emqx_utils_json:decode(ConfJson, [return_maps]), UpdateJson = RawConf#{<<"enable">> := Enable}, {ok, UpdateResJson} = request_api( put, @@ -81,7 +81,7 @@ t_config(_Config) -> auth_header_(), UpdateJson ), - UpdateRawConf = emqx_json:decode(UpdateResJson, [return_maps]), + UpdateRawConf = emqx_utils_json:decode(UpdateResJson, [return_maps]), ?assertEqual(Enable, maps:get(<<"enable">>, UpdateRawConf)) end, @@ -224,7 +224,7 @@ t_lookup_and_delete(_) -> t_change_storage_type(_Config) -> Path = api_path(["mqtt", "retainer"]), {ok, ConfJson} = request_api(get, Path), - RawConf = emqx_json:decode(ConfJson, [return_maps]), + RawConf = emqx_utils_json:decode(ConfJson, [return_maps]), %% pre-conditions ?assertMatch( #{ @@ -271,7 +271,7 @@ t_change_storage_type(_Config) -> auth_header_(), ChangedConf ), - UpdatedRawConf = emqx_json:decode(UpdateResJson, [return_maps]), + UpdatedRawConf = emqx_utils_json:decode(UpdateResJson, [return_maps]), ?assertMatch( #{ <<"backend">> := #{ @@ -311,7 +311,7 @@ t_change_storage_type(_Config) -> %% HTTP Request %%-------------------------------------------------------------------- decode_json(Data) -> - BinJson = emqx_json:decode(Data, [return_maps]), + BinJson = emqx_utils_json:decode(Data, [return_maps]), emqx_map_lib:unsafe_atom_key_map(BinJson). %%-------------------------------------------------------------------- diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 110caa33d..07c53d3e3 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. {erl_opts, [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index c4a6e2e73..40158d342 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -213,7 +213,7 @@ replace_simple_var(Val, _Data, _Default) -> Val. format_msg([], Selected) -> - emqx_json:encode(Selected); + emqx_utils_json:encode(Selected); format_msg(Tokens, Selected) -> emqx_plugin_libs_rule:proc_tmpl(Tokens, Selected). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index f640f8303..c63d6275f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -468,7 +468,7 @@ err_msg(Msg) -> encode_nested_error(RuleError, Reason) when is_tuple(Reason) -> encode_nested_error(RuleError, element(1, Reason)); encode_nested_error(RuleError, Reason) -> - case emqx_json:safe_encode([{RuleError, Reason}]) of + case emqx_utils_json:safe_encode([{RuleError, Reason}]) of {ok, Json} -> Json; _ -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 79e0406c1..944aa02f8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -987,10 +987,10 @@ base64_decode(Data) when is_binary(Data) -> base64:decode(Data). json_encode(Data) -> - emqx_json:encode(Data). + emqx_utils_json:encode(Data). json_decode(Data) -> - emqx_json:decode(Data, [return_maps]). + emqx_utils_json:decode(Data, [return_maps]). term_encode(Term) -> erlang:term_to_binary(Term). diff --git a/apps/emqx_rule_engine/src/emqx_rule_maps.erl b/apps/emqx_rule_engine/src/emqx_rule_maps.erl index 13f99c88b..3dfffca46 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_maps.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_maps.erl @@ -86,7 +86,7 @@ general_map_put(Key, Val, Map, OrgData) -> ). general_find(KeyOrIndex, Data, OrgData, Handler) when is_binary(Data) -> - try emqx_json:decode(Data, [return_maps]) of + try emqx_utils_json:decode(Data, [return_maps]) of Json -> general_find(KeyOrIndex, Json, OrgData, Handler) catch _:_ -> Handler(not_found) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 51491df53..d7412d03c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -495,7 +495,7 @@ cache_payload(DecodedP) -> safe_decode_and_cache(MaybeJson) -> try - cache_payload(emqx_json:decode(MaybeJson, [return_maps])) + cache_payload(emqx_utils_json:decode(MaybeJson, [return_maps])) catch _:_ -> error({decode_json_failed, MaybeJson}) end. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 93d7c7352..eb253e516 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -614,7 +614,9 @@ t_event_client_disconnected_normal(_Config) -> receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), - ?assertMatch(#{<<"reason">> := <<"normal">>}, emqx_json:decode(Payload, [return_maps])) + ?assertMatch( + #{<<"reason">> := <<"normal">>}, emqx_utils_json:decode(Payload, [return_maps]) + ) after 1000 -> ct:fail(wait_for_repub_disconnected_normal) end, @@ -651,7 +653,9 @@ t_event_client_disconnected_kicked(_Config) -> receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), - ?assertMatch(#{<<"reason">> := <<"kicked">>}, emqx_json:decode(Payload, [return_maps])) + ?assertMatch( + #{<<"reason">> := <<"kicked">>}, emqx_utils_json:decode(Payload, [return_maps]) + ) after 1000 -> ct:fail(wait_for_repub_disconnected_kicked) end, @@ -692,7 +696,7 @@ t_event_client_disconnected_discarded(_Config) -> {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), ?assertMatch( - #{<<"reason">> := <<"discarded">>}, emqx_json:decode(Payload, [return_maps]) + #{<<"reason">> := <<"discarded">>}, emqx_utils_json:decode(Payload, [return_maps]) ) after 1000 -> ct:fail(wait_for_repub_disconnected_discarded) @@ -737,7 +741,7 @@ t_event_client_disconnected_takenover(_Config) -> {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), ?assertMatch( - #{<<"reason">> := <<"takenover">>}, emqx_json:decode(Payload, [return_maps]) + #{<<"reason">> := <<"takenover">>}, emqx_utils_json:decode(Payload, [return_maps]) ) after 1000 -> ct:fail(wait_for_repub_disconnected_discarded) @@ -2800,7 +2804,7 @@ verify_event(EventName) -> [ begin %% verify fields can be formatted to JSON string - _ = emqx_json:encode(Fields), + _ = emqx_utils_json:encode(Fields), %% verify metadata fields verify_metadata_fields(EventName, Fields), %% verify available fields for each event name diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index e94806a7b..8d7546fca 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -46,13 +46,13 @@ end_per_suite(_Config) -> ok. init_per_testcase(t_crud_rule_api, Config) -> - meck:new(emqx_json, [passthrough]), + meck:new(emqx_utils_json, [passthrough]), init_per_testcase(common, Config); init_per_testcase(_, Config) -> Config. end_per_testcase(t_crud_rule_api, Config) -> - meck:unload(emqx_json), + meck:unload(emqx_utils_json), end_per_testcase(common, Config); end_per_testcase(_, _Config) -> {200, #{data := Rules}} = @@ -136,7 +136,7 @@ t_crud_rule_api(_Config) -> ), ?assertMatch( #{<<"select_and_transform_error">> := <<"decode_json_failed">>}, - emqx_json:decode(SelectAndTransformJsonError, [return_maps]) + emqx_utils_json:decode(SelectAndTransformJsonError, [return_maps]) ), {400, #{ code := 'BAD_REQUEST', @@ -150,7 +150,7 @@ t_crud_rule_api(_Config) -> ), ?assertMatch( #{<<"select_and_transform_error">> := <<"badarg">>}, - emqx_json:decode(SelectAndTransformBadArgError, [return_maps]) + emqx_utils_json:decode(SelectAndTransformBadArgError, [return_maps]) ), {400, #{ code := 'BAD_REQUEST', @@ -162,7 +162,7 @@ t_crud_rule_api(_Config) -> ) ), ?assertMatch({match, _}, re:run(BadSqlMessage, "syntax error")), - meck:expect(emqx_json, safe_encode, 1, {error, foo}), + meck:expect(emqx_utils_json, safe_encode, 1, {error, foo}), ?assertMatch( {400, #{ code := 'BAD_REQUEST', diff --git a/apps/emqx_slow_subs/rebar.config b/apps/emqx_slow_subs/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_slow_subs/rebar.config +++ b/apps/emqx_slow_subs/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl index 6b0721e3d..ae74348e0 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl @@ -108,7 +108,7 @@ t_get_history(_) -> "page=1&limit=10", auth_header_() ), - #{<<"data">> := [First | _]} = emqx_json:decode(Data, [return_maps]), + #{<<"data">> := [First | _]} = emqx_utils_json:decode(Data, [return_maps]), ?assertMatch( #{ @@ -165,7 +165,7 @@ t_settting(_) -> ?assertEqual(Conf2#{stats_type := <<"internal">>}, GetReturn). decode_json(Data) -> - BinJosn = emqx_json:decode(Data, [return_maps]), + BinJosn = emqx_utils_json:decode(Data, [return_maps]), emqx_map_lib:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> @@ -187,7 +187,7 @@ request_api(Method, Url, QueryParams, Auth, Body) -> "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)}). do_request_api(Method, Request) -> ct:pal("Method: ~p, Request: ~p", [Method, Request]), diff --git a/apps/emqx_statsd/rebar.config b/apps/emqx_statsd/rebar.config index bb9a14272..a1383d920 100644 --- a/apps/emqx_statsd/rebar.config +++ b/apps/emqx_statsd/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {estatsd, {git, "https://github.com/emqx/estatsd", {tag, "0.1.0"}}} ]}. diff --git a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl index bcc710050..b5669e4b9 100644 --- a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl +++ b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl @@ -200,7 +200,7 @@ request(Method) -> request(Method, []). request(Method, Body) -> case request(Method, uri(["statsd"]), Body) of {ok, 200, Res} -> - {ok, emqx_json:decode(Res, [return_maps])}; + {ok, emqx_utils_json:decode(Res, [return_maps])}; {ok, _Status, _} -> error end. diff --git a/apps/emqx_utils/README.md b/apps/emqx_utils/README.md new file mode 100644 index 000000000..482b7a947 --- /dev/null +++ b/apps/emqx_utils/README.md @@ -0,0 +1,43 @@ +# [Application Name] - [Mandatory] +> 0. App overview introduction +> 1. let people know what your project can do specifically. Is it a base +> library dependency, or what kind of functionality is provided to the user? +> 2. Provide context and add a link to any reference visitors might be +> unfamiliar with. +> 3. Design details, implementation technology architecture, Roadmap, etc. + +# [Features] - [Optional] +> A List of features your application provided. If the feature is quite simple, just +> list in the previous section. + +# [Limitation] - [Optional] +> Explain the limitations of the implementation used in your application. + +# [Documention links] - [Mandatory] +> You can use the official docs link to provide a detailed explanation of +> the concept and functions of this application. + +# [Installation] - [Optional] +> How users can obtain this application. In most cases, this section is unnecessary. +> Most of the applications are released with the EMQX distribution package. +> Otherwise, it is necessary to provide step-by-step instructions on how +> to install this application. + +# [Usage] - [Optional] +> This section explains how users can use the features provided by the application. + +# [Basic Usage Guide] - [Optional] +> Simple and reproducible introduction on how to use/start it. + +# [Configurations] - [Mandatory] +> Most important configurations that this application depends on. +> It would be best to attach official documentation if it's available. + +# [HTTP APIs] - [Optional] +> Same as configuration. + +# [Other] - [Optional] +> Other topics that users may need to know can be placed here. + +# Contributing - [Mandatory] +Please see our [contributing.md](../../CONTRIBUTING.md). diff --git a/apps/emqx_utils/rebar.config b/apps/emqx_utils/rebar.config new file mode 100644 index 000000000..4c39cfe64 --- /dev/null +++ b/apps/emqx_utils/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang -*- + +{erl_opts, [ + debug_info +]}. + +{deps, [ + {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src new file mode 100644 index 000000000..e1f10c139 --- /dev/null +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -0,0 +1,25 @@ +%% -*- mode: erlang -*- +{application, emqx_utils, [ + {description, "An OTP application"}, + % strict semver, bump manually! + {vsn, "5.0.0"}, + {modules, [ + emqx_utils, + emqx_utils_binary, + emqx_utils_ets, + emqx_utils_json, + emqx_utils_maps + ]}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {licenses, ["Apache-2.0"]}, + {maintainers, ["EMQX Team "]}, + {links, [ + {"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx/emqx"} + ]} +]}. diff --git a/apps/emqx/src/emqx_json.erl b/apps/emqx_utils/src/emqx_utils_json.erl similarity index 90% rename from apps/emqx/src/emqx_json.erl rename to apps/emqx_utils/src/emqx_utils_json.erl index 7827b98c9..2f0c6f88b 100644 --- a/apps/emqx/src/emqx_json.erl +++ b/apps/emqx_utils/src/emqx_utils_json.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_json). +-module(emqx_utils_json). -compile(inline). @@ -46,11 +46,11 @@ ]} ). --type encode_options() :: jiffy:encode_options(). --type decode_options() :: jiffy:decode_options(). +-type encode_options() :: emqx_utils_json:encode_options(). +-type decode_options() :: emqx_utils_json:decode_options(). -type json_text() :: iolist() | binary(). --type json_term() :: jiffy:jiffy_decode_result(). +-type json_term() :: emqx_utils_json:jiffy_decode_result(). -export_type([json_text/0, json_term/0]). -export_type([decode_options/0, encode_options/0]). @@ -61,7 +61,7 @@ encode(Term) -> -spec encode(json_term(), encode_options()) -> json_text(). encode(Term, Opts) -> - to_binary(jiffy:encode(to_ejson(Term), Opts)). + to_binary(emqx_utils_json:encode(to_ejson(Term), Opts)). -spec safe_encode(json_term()) -> {ok, json_text()} | {error, Reason :: term()}. @@ -83,7 +83,7 @@ decode(Json) -> decode(Json, []). -spec decode(json_text(), decode_options()) -> json_term(). decode(Json, Opts) -> - from_ejson(jiffy:decode(Json, Opts)). + from_ejson(emqx_utils_json:decode(Json, Opts)). -spec safe_decode(json_text()) -> {ok, json_term()} | {error, Reason :: term()}. diff --git a/apps/emqx_utils/test/emqx_utils_SUITE.erl b/apps/emqx_utils/test/emqx_utils_SUITE.erl new file mode 100644 index 000000000..09a8d085c --- /dev/null +++ b/apps/emqx_utils/test/emqx_utils_SUITE.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_utils_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([emqx_conf, emqx_utils]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_conf, emqx_utils]), + ok. + +init_per_testcase(TestCase, Config) -> + emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config). + +end_per_testcase(TestCase, Config) -> + emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config). + +t_fail(init, Config) -> + Config; +t_fail('end', _Config) -> + ok. + +t_fail(_Config) -> + ?assert(false). diff --git a/apps/emqx/test/emqx_json_SUITE.erl b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl similarity index 93% rename from apps/emqx/test/emqx_json_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_json_SUITE.erl index a0bf48e4e..889cde595 100644 --- a/apps/emqx/test/emqx_json_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_json_SUITE). +-module(emqx_utils_json_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -22,7 +22,7 @@ -include_lib("eunit/include/eunit.hrl"). -import( - emqx_json, + emqx_utils_json, [ encode/1, decode/1, @@ -51,7 +51,7 @@ %% #{<<"foo">> => <<"bar">>} -> {"foo": "bar"} -> #{<<"foo">> => <<"bar">>} %%-------------------------------------------------------------------- -%% but in emqx_json, we use the jsx style for it: +%% but in emqx_utils_json, we use the jsx style for it: %%-------------------------------------------------------------------- %% Erlang JSON Erlang %% ------------------------------------------------------------------- @@ -127,12 +127,12 @@ t_safe_decode_encode(_) -> [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{foo, bar}]), [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{<<"foo">>, <<"bar">>}]), [[{<<"foo">>, <<"bar">>}]] = safe_encode_decode([[{<<"foo">>, <<"bar">>}]]), - {ok, Json} = emqx_json:safe_encode(#{<<"foo">> => <<"bar">>}), - {ok, #{<<"foo">> := <<"bar">>}} = emqx_json:safe_decode(Json, [return_maps]). + {ok, Json} = emqx_utils_json:safe_encode(#{<<"foo">> => <<"bar">>}), + {ok, #{<<"foo">> := <<"bar">>}} = emqx_utils_json:safe_decode(Json, [return_maps]). safe_encode_decode(Term) -> - {ok, Json} = emqx_json:safe_encode(Term), - case emqx_json:safe_decode(Json) of + {ok, Json} = emqx_utils_json:safe_encode(Term), + case emqx_utils_json:safe_decode(Json) of {ok, {NTerm}} -> NTerm; {ok, NTerm} -> NTerm end. diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index afd90f622..b26df658a 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -3,6 +3,7 @@ , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + , {emqx_utils, {path, "../emqx_utils"}} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index 208b68de6..f37d7fb6a 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -273,7 +273,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 0c0dbbe04..e899b0fcd 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -208,7 +208,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 75d2d2d8c..3cc4c75e4 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -204,7 +204,7 @@ create_bridge_http(Config, GCPPubSubConfigOverrides) -> ct:pal("probe result: ~p", [ProbeResult]), Res = case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res0} -> {ok, emqx_json:decode(Res0, [return_maps])}; + {ok, Res0} -> {ok, emqx_utils_json:decode(Res0, [return_maps])}; Error -> Error end, ct:pal("bridge creation result: ~p", [Res]), @@ -222,7 +222,7 @@ create_rule_and_action_http(Config) -> Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -234,7 +234,7 @@ success_http_handler() -> Rep = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{messageIds => [<<"6058891368195201">>]}), + emqx_utils_json:encode(#{messageIds => [<<"6058891368195201">>]}), Req ), {ok, Rep, State} @@ -274,7 +274,7 @@ gcp_pubsub_config(Config) -> PubSubTopic = proplists:get_value(pubsub_topic, Config, <<"mytopic">>), PipelineSize = proplists:get_value(pipeline_size, Config, 100), ServiceAccountJSON = proplists:get_value(pubsub_topic, Config, generate_service_account_json()), - ServiceAccountJSONStr = emqx_json:encode(ServiceAccountJSON), + ServiceAccountJSONStr = emqx_utils_json:encode(ServiceAccountJSON), GUID = emqx_guid:to_hexstr(emqx_guid:gen()), Name = <<(atom_to_binary(?MODULE))/binary, (GUID)/binary>>, ConfigString = @@ -463,7 +463,7 @@ assert_valid_request_headers(Headers, ServiceAccountJSON) -> end. assert_valid_request_body(Body) -> - BodyMap = emqx_json:decode(Body, [return_maps]), + BodyMap = emqx_utils_json:decode(Body, [return_maps]), ?assertMatch(#{<<"messages">> := [_ | _]}, BodyMap), #{<<"messages">> := Messages} = BodyMap, lists:map( @@ -471,7 +471,7 @@ assert_valid_request_body(Body) -> ?assertMatch(#{<<"data">> := <<_/binary>>}, Msg), #{<<"data">> := Content64} = Msg, Content = base64:decode(Content64), - Decoded = emqx_json:decode(Content, [return_maps]), + Decoded = emqx_utils_json:decode(Content, [return_maps]), ct:pal("decoded payload: ~p", [Decoded]), ?assert(is_map(Decoded)), Decoded @@ -1014,7 +1014,7 @@ t_publish_timeout(Config) -> Rep = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{messageIds => [<<"6058891368195201">>]}), + emqx_utils_json:encode(#{messageIds => [<<"6058891368195201">>]}), Req ), {ok, Rep, State} @@ -1180,7 +1180,7 @@ t_failure_with_body(Config) -> Rep = cowboy_req:reply( 400, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{}), + emqx_utils_json:encode(#{}), Req ), {ok, Rep, State} diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl index 1b4b4aeb2..4891d5d9b 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl @@ -394,7 +394,7 @@ create_rule_and_action_http(Config, Overrides) -> Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -435,7 +435,7 @@ query_by_clientid(ClientId, Config) -> {"Content-Type", "application/json"} ], Body = - emqx_json:encode(#{ + emqx_utils_json:encode(#{ query => Query, dialect => #{ header => true, @@ -545,7 +545,7 @@ t_start_ok(Config) -> int_value => <<"-123">>, uint_value => <<"123">>, float_value => <<"24.5">>, - payload => emqx_json:encode(Payload) + payload => emqx_utils_json:encode(Payload) }, assert_persisted_data(ClientId, Expected, PersistedData), ok @@ -764,7 +764,7 @@ t_boolean_variants(Config) -> bool => atom_to_binary(Translation), int_value => <<"-123">>, uint_value => <<"123">>, - payload => emqx_json:encode(Payload) + payload => emqx_utils_json:encode(Payload) }, assert_persisted_data(ClientId, Expected, PersistedData), ok @@ -1024,9 +1024,9 @@ t_missing_field(Config) -> ClientId0 = emqx_guid:to_hexstr(emqx_guid:gen()), ClientId1 = emqx_guid:to_hexstr(emqx_guid:gen()), %% Message with the field that we "forgot" to select in the rule - Msg0 = emqx_message:make(ClientId0, <<"t/topic">>, emqx_json:encode(#{foo => 123})), + Msg0 = emqx_message:make(ClientId0, <<"t/topic">>, emqx_utils_json:encode(#{foo => 123})), %% Message without any fields - Msg1 = emqx_message:make(ClientId1, <<"t/topic">>, emqx_json:encode(#{})), + Msg1 = emqx_message:make(ClientId1, <<"t/topic">>, emqx_utils_json:encode(#{})), ?check_trace( begin emqx:publish(Msg0), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 116dcc729..2d4ee9222 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -258,7 +258,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl index 38e31c7ae..69b1e26ce 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl @@ -226,7 +226,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl index 83cb8b1f3..8f0b9ad9c 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl @@ -241,7 +241,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl index 95ec47e7f..0cb14e5c3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl @@ -176,7 +176,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl index c956a93c6..8e058f2f1 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl @@ -219,7 +219,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index e754bd573..352c54629 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -6,7 +6,8 @@ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, - {emqx, {path, "../../apps/emqx"}} + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 4703e0a21..ee56e0c5f 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -322,7 +322,7 @@ convert_to_item(Msg) when is_map(Msg), map_size(Msg) > 0 -> Msg ); convert_to_item(MsgBin) when is_binary(MsgBin) -> - Msg = emqx_json:decode(MsgBin), + Msg = emqx_utils_json:decode(MsgBin), convert_to_item(Msg); convert_to_item(Item) -> erlang:throw({invalid_item, Item}). @@ -334,7 +334,7 @@ convert2binary(Value) when is_binary(Value); is_number(Value) -> convert2binary(Value) when is_list(Value) -> unicode:characters_to_binary(Value); convert2binary(Value) when is_map(Value) -> - emqx_json:encode(Value). + emqx_utils_json:encode(Value). do_async_reply(Result, {ReplyFun, [Context]}) -> ReplyFun(Context, Result). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl index f07cbceab..a2045b8c5 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl @@ -334,14 +334,14 @@ ensure_jwt_worker(InstanceId, #{ encode_payload(_State = #{payload_template := PayloadTemplate}, Selected) -> Interpolated = case PayloadTemplate of - [] -> emqx_json:encode(Selected); + [] -> emqx_utils_json:encode(Selected); _ -> emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Selected) end, #{data => base64:encode(Interpolated)}. -spec to_pubsub_request([#{data := binary()}]) -> binary(). to_pubsub_request(Payloads) -> - emqx_json:encode(#{messages => Payloads}). + emqx_utils_json:encode(#{messages => Payloads}). -spec publish_path(state()) -> binary(). publish_path( diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl index aa03863b0..4e7adcd6e 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl @@ -84,4 +84,4 @@ render_message(PayloadTemplate, Message) -> %% Note: mongo expects a map as a document, so the rendered result %% must be JSON-serializable Rendered = emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Message), - emqx_json:decode(Rendered, [return_maps]). + emqx_utils_json:decode(Rendered, [return_maps]). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index 84f2e2a89..d296b0b4f 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -248,14 +248,14 @@ get_topic_key([Query | _], RawTopic, TopicTks) -> apply_template({Key, Msg} = _Req, Templates) -> case maps:get(Key, Templates, undefined) of undefined -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); Template -> emqx_plugin_libs_rule:proc_tmpl(Template, Msg) end; apply_template([{Key, _} | _] = Reqs, Templates) -> case maps:get(Key, Templates, undefined) of undefined -> - [emqx_json:encode(Msg) || {_, Msg} <- Reqs]; + [emqx_utils_json:encode(Msg) || {_, Msg} <- Reqs]; Template -> [emqx_plugin_libs_rule:proc_tmpl(Template, Msg) || {_, Msg} <- Reqs] end. diff --git a/lib-ee/emqx_ee_schema_registry/rebar.config b/lib-ee/emqx_ee_schema_registry/rebar.config index b19fb05ae..223ebf533 100644 --- a/lib-ee/emqx_ee_schema_registry/rebar.config +++ b/lib-ee/emqx_ee_schema_registry/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../../apps/emqx_utils"}}, {erlavro, {git, "https://github.com/klarna/erlavro.git", {tag, "2.9.8"}}} ]}. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl index 43145fb16..9835ec7c2 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl @@ -65,6 +65,6 @@ make_serde(avro, Name, Source0) -> inject_avro_name(Name, Source0) -> %% The schema checks that the source is a valid JSON when %% typechecking, so we shouldn't need to validate here. - Schema0 = emqx_json:decode(Source0, [return_maps]), + Schema0 = emqx_utils_json:decode(Source0, [return_maps]), Schema = Schema0#{<<"name">> => Name}, - emqx_json:encode(Schema). + emqx_utils_json:encode(Schema). diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 9b2f64c03..3c26d1966 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -95,7 +95,7 @@ create_rule_http(RuleParams) -> Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -107,7 +107,7 @@ schema_params(avro) -> #{name => <<"s">>, type => <<"string">>} ] }, - SourceBin = emqx_json:encode(Source), + SourceBin = emqx_utils_json:encode(Source), #{type => avro, source => SourceBin}. create_serde(SerdeType, SerdeName) -> @@ -162,7 +162,7 @@ receive_published(Line) -> maps:update_with( payload, fun(Raw) -> - case emqx_json:safe_decode(Raw, [return_maps]) of + case emqx_utils_json:safe_decode(Raw, [return_maps]) of {ok, Decoded} -> Decoded; {error, _} -> Raw end @@ -262,7 +262,7 @@ t_encode_decode(Config) -> {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode_decode1)}), on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, - PayloadBin = emqx_json:encode(Payload), + PayloadBin = emqx_utils_json:encode(Payload), emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), Res = receive_action_results(), ?assertMatch( @@ -311,7 +311,7 @@ t_encode(Config) -> {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode1)}), on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, - PayloadBin = emqx_json:encode(Payload), + PayloadBin = emqx_utils_json:encode(Payload), emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), Published = receive_published(?LINE), ?assertMatch( diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl index bbb6d5ef0..e7034d562 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl @@ -64,7 +64,7 @@ do_request(Method, PathParts, Body) -> {ok, Code, <<>>} -> {ok, Code, <<>>}; {ok, Code, Res1} -> - Res2 = emqx_json:decode(Res1, [return_maps]), + Res2 = emqx_utils_json:decode(Res1, [return_maps]), Res3 = try_decode_error_message(Res2), {ok, Code, Res3}; Error -> @@ -72,7 +72,7 @@ do_request(Method, PathParts, Body) -> end. try_decode_error_message(#{<<"message">> := Msg0} = Res0) -> - case emqx_json:safe_decode(Msg0, [return_maps]) of + case emqx_utils_json:safe_decode(Msg0, [return_maps]) of {ok, Msg} -> Res0#{<<"message">> := Msg}; {error, _} -> @@ -102,7 +102,7 @@ t_crud(_Config) -> #{name => <<"s">>, type => <<"string">>} ] }, - SourceBin = emqx_json:encode(Source), + SourceBin = emqx_utils_json:encode(Source), Params = #{ <<"type">> => <<"avro">>, <<"source">> => SourceBin, diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl index be62717d3..12798c6a2 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl @@ -59,7 +59,7 @@ schema_params(avro) -> #{name => <<"s">>, type => <<"string">>} ] }, - SourceBin = emqx_json:encode(Source), + SourceBin = emqx_utils_json:encode(Source), #{type => avro, source => SourceBin}. assert_roundtrip(SerdeName, Original) -> diff --git a/lib-ee/emqx_license/rebar.config b/lib-ee/emqx_license/rebar.config index 1cb5ace88..e7620387a 100644 --- a/lib-ee/emqx_license/rebar.config +++ b/lib-ee/emqx_license/rebar.config @@ -1,3 +1,6 @@ -{deps, [{emqx, {path, "../../apps/emqx"}}]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index 11ac1ca89..3de5ae121 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -103,7 +103,7 @@ t_license_info(_Config) -> <<"start_at">> => <<"2022-01-11">>, <<"type">> => <<"trial">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), ok. @@ -128,7 +128,7 @@ t_license_upload_key_success(_Config) -> <<"start_at">> => <<"2022-01-11">>, <<"type">> => <<"trial">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), ?assertMatch( #{max_connections := 999}, @@ -150,7 +150,7 @@ t_license_upload_key_bad_key(_Config) -> <<"code">> => <<"BAD_REQUEST">>, <<"message">> => <<"Bad license key">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. @@ -168,7 +168,7 @@ t_license_upload_key_not_json(_Config) -> <<"code">> => <<"BAD_REQUEST">>, <<"message">> => <<"Invalid request params">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. From 1880da0a2e69df8a798de27d73bea17931da92c1 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 14:38:49 +0200 Subject: [PATCH 02/32] refactor: move binary_util to emqx_utils_binary --- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl | 2 +- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl | 4 ++-- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl | 14 +++++++------- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl | 6 +++--- .../src/emqx_utils_binary.erl} | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) rename apps/{emqx_gateway_lwm2m/src/binary_util.erl => emqx_utils/src/emqx_utils_binary.erl} (99%) diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl index 80afadb8e..ca32d03db 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl @@ -227,7 +227,7 @@ to_operations(Obj, ObjDefinition) -> }. path_list(Path) -> - case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(Path, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId, ResInstId] -> [ObjId, ObjInsId, ResId, ResInstId]; [ObjId, ObjInsId, ResId] -> [ObjId, ObjInsId, ResId]; [ObjId, ObjInsId] -> [ObjId, ObjInsId]; diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl index 53995c97c..8e4286343 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl @@ -335,7 +335,7 @@ remove_tmp_fields(Ref) -> -spec path_list(Path :: binary()) -> {[PathWord :: binary()], [Query :: binary()]}. path_list(Path) -> - case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(Path, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId, LastPart] -> {ResInstId, QueryList} = query_list(LastPart), {[ObjId, ObjInsId, ResId, ResInstId], QueryList}; @@ -389,7 +389,7 @@ observe_seq(Options) -> add_alternate_path_prefix(<<"/">>, PathList) -> PathList; add_alternate_path_prefix(AlternatePath, PathList) -> - [binary_util:trim(AlternatePath, $/) | PathList]. + [emqx_utils_binary:trim(AlternatePath, $/) | PathList]. extract_path(Ref = #{}) -> drop_query( diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl index c3d23d4d9..c65fff441 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl @@ -97,7 +97,7 @@ tlv_single_resource(BaseName, Id, Value, ObjDefinition) -> [#{path => BaseName, value => Val}]. basename(OldBaseName, _ObjectId, ObjectInstanceId, ResourceId, 3) -> - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary, $/, ResId/binary>>; [ObjId, ObjInsId] -> @@ -113,13 +113,13 @@ basename(OldBaseName, _ObjectId, ObjectInstanceId, ResourceId, 3) -> >> end; basename(OldBaseName, _ObjectId, ObjectInstanceId, _ResourceId, 2) -> - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, _ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; [ObjId, ObjInsId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; [ObjId] -> <<$/, ObjId/binary, $/, (integer_to_binary(ObjectInstanceId))/binary>> end. % basename(OldBaseName, _ObjectId, _ObjectInstanceId, _ResourceId, 1) -> -% case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of +% case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of % [ObjId, _ObjInsId, _ResId] -> <<$/, ObjId/binary>>; % [ObjId, _ObjInsId] -> <<$/, ObjId/binary>>; % [ObjId] -> <<$/, ObjId/binary>> @@ -129,7 +129,7 @@ make_path(RelativePath, Id) -> <>. object_id(BaseName) -> - case binary:split(binary_util:trim(BaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(BaseName, $/), [<<$/>>], [global]) of [ObjId] -> binary_to_integer(ObjId); [ObjId, _] -> binary_to_integer(ObjId); [ObjId, _, _] -> binary_to_integer(ObjId); @@ -137,7 +137,7 @@ object_id(BaseName) -> end. object_resource_id(BaseName) -> - case binary:split(binary_util:trim(BaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(BaseName, $/), [<<$/>>], [global]) of [_ObjIdBin1] -> error({invalid_basename, BaseName}); [_ObjIdBin2, _] -> @@ -371,8 +371,8 @@ translate_element(BaseName, [Element | ElementList], Acc) -> translate_element(BaseName, ElementList, NewAcc). full_path(BaseName, RelativePath) -> - Prefix = binary_util:rtrim(BaseName, $/), - Path = binary_util:ltrim(RelativePath, $/), + Prefix = emqx_utils_binary:rtrim(BaseName, $/), + Path = emqx_utils_binary:ltrim(RelativePath, $/), <>. get_element_value(#{<<"t">> := Value}) -> Value; diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl index c8bfa8974..e267692a6 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl @@ -386,11 +386,11 @@ is_alternate_path(LinkAttrs) -> LinkAttrs ). -trim(Str) -> binary_util:trim(Str, $\s). +trim(Str) -> emqx_utils_binary:trim(Str, $\s). delink(Str) -> - Ltrim = binary_util:ltrim(Str, $<), - binary_util:rtrim(Ltrim, $>). + Ltrim = emqx_utils_binary:ltrim(Str, $<), + emqx_utils_binary:rtrim(Ltrim, $>). get_lifetime(#{<<"lt">> := LT}) -> case LT of diff --git a/apps/emqx_gateway_lwm2m/src/binary_util.erl b/apps/emqx_utils/src/emqx_utils_binary.erl similarity index 99% rename from apps/emqx_gateway_lwm2m/src/binary_util.erl rename to apps/emqx_utils/src/emqx_utils_binary.erl index 68ac7a0d7..d9e504f46 100644 --- a/apps/emqx_gateway_lwm2m/src/binary_util.erl +++ b/apps/emqx_utils/src/emqx_utils_binary.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(binary_util). +-module(emqx_utils_binary). %% copied from https://github.com/arcusfelis/binary2 From 9c11bfce8041e8d110b870d0b8ebffbefc9ea5e0 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 14:45:18 +0200 Subject: [PATCH 03/32] refactor: rename emqx_misc to emqx_utils --- apps/emqx/src/emqx_api_lib.erl | 2 +- apps/emqx/src/emqx_authentication_config.erl | 4 +- apps/emqx/src/emqx_banned.erl | 6 +- apps/emqx/src/emqx_batch.erl | 2 +- apps/emqx/src/emqx_broker.erl | 2 +- apps/emqx/src/emqx_broker_helper.erl | 2 +- apps/emqx/src/emqx_channel.erl | 12 +- apps/emqx/src/emqx_cm.erl | 2 +- apps/emqx/src/emqx_connection.erl | 16 +- apps/emqx/src/emqx_crl_cache.erl | 8 +- apps/emqx/src/emqx_flapping.erl | 2 +- apps/emqx/src/emqx_guid.erl | 4 +- .../src/emqx_limiter/src/emqx_htb_limiter.erl | 2 +- apps/emqx/src/emqx_listeners.erl | 4 +- apps/emqx/src/emqx_ocsp_cache.erl | 4 +- apps/emqx/src/emqx_os_mon.erl | 6 +- apps/emqx/src/emqx_pool.erl | 2 +- apps/emqx/src/emqx_router.erl | 2 +- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx/src/emqx_session.erl | 2 +- apps/emqx/src/emqx_session_router.erl | 2 +- apps/emqx/src/emqx_stats.erl | 4 +- apps/emqx/src/emqx_sys.erl | 4 +- apps/emqx/src/emqx_sys_mon.erl | 4 +- apps/emqx/src/emqx_trace/emqx_trace.erl | 6 +- .../src/emqx_trace/emqx_trace_handler.erl | 2 +- apps/emqx/src/emqx_vm_mon.erl | 2 +- apps/emqx/src/emqx_ws_connection.erl | 14 +- apps/emqx/test/emqx_connection_SUITE.erl | 6 +- apps/emqx/test/emqx_misc_SUITE.erl | 209 ------------------ apps/emqx/test/emqx_router_SUITE.erl | 2 +- apps/emqx_authz/src/emqx_authz_file.erl | 2 +- .../test/emqx_authz_api_sources_SUITE.erl | 6 +- apps/emqx_bridge/src/emqx_bridge.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 8 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 10 +- .../test/emqx_bridge_api_SUITE.erl | 2 +- .../src/emqx_bridge_kafka_impl_consumer.erl | 4 +- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 8 +- apps/emqx_conf/src/emqx_cluster_rpc.erl | 10 +- .../src/emqx_cluster_rpc_cleaner.erl | 4 +- .../src/emqx_connector_http.erl | 6 +- .../src/emqx_connector_ldap.erl | 2 +- .../src/emqx_connector_mongo.erl | 2 +- .../src/emqx_connector_mqtt.erl | 2 +- .../src/emqx_connector_mysql.erl | 2 +- .../src/emqx_connector_pgsql.erl | 2 +- .../src/emqx_connector_redis.erl | 2 +- .../src/mqtt/emqx_connector_mqtt_msg.erl | 4 +- .../src/mqtt/emqx_connector_mqtt_worker.erl | 2 +- .../src/emqx_dashboard_swagger.erl | 2 +- apps/emqx_exhook/src/emqx_exhook_api.erl | 2 +- .../src/bhvrs/emqx_gateway_conn.erl | 18 +- .../src/emqx_gateway_api_clients.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_cm.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_http.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_utils.erl | 4 +- .../src/emqx_coap_channel.erl | 6 +- .../src/emqx_coap_session.erl | 2 +- apps/emqx_gateway_coap/src/emqx_coap_tm.erl | 6 +- .../src/emqx_coap_transport.erl | 2 +- .../src/emqx_exproto_channel.erl | 6 +- .../src/emqx_exproto_gcli.erl | 2 +- .../src/emqx_lwm2m_channel.erl | 8 +- .../src/emqx_mqttsn_channel.erl | 22 +- .../src/emqx_stomp_channel.erl | 12 +- apps/emqx_machine/src/emqx_global_gc.erl | 2 +- .../src/emqx_mgmt_api_clients.erl | 2 +- .../src/emqx_mgmt_api_subscriptions.erl | 2 +- .../src/emqx_mgmt_api_trace.erl | 2 +- apps/emqx_management/src/emqx_mgmt_auth.erl | 6 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 18 +- apps/emqx_modules/src/emqx_delayed.erl | 8 +- apps/emqx_modules/src/emqx_telemetry.erl | 4 +- .../src/emqx_plugin_libs_pool.erl | 2 +- .../src/emqx_plugin_libs_rule.erl | 2 +- apps/emqx_prometheus/src/emqx_prometheus.erl | 2 +- .../src/emqx_resource_buffer_worker.erl | 2 +- .../src/emqx_resource_manager.erl | 4 +- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- .../src/emqx_retainer_dispatcher.erl | 2 +- .../src/emqx_rule_actions.erl | 2 +- .../src/emqx_rule_engine_api.erl | 8 +- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 6 +- .../src/emqx_rule_sqltester.erl | 2 +- apps/emqx_statsd/src/emqx_statsd.app.src | 2 +- apps/emqx_statsd/src/emqx_statsd.erl | 2 +- .../src/emqx_utils.erl} | 10 +- apps/emqx_utils/test/emqx_utils_SUITE.erl | 199 +++++++++++++++-- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 6 +- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 2 +- .../test/emqx_ee_bridge_mysql_SUITE.erl | 2 +- .../src/emqx_ee_connector_cassa.erl | 2 +- .../src/emqx_ee_connector_clickhouse.erl | 6 +- .../src/emqx_ee_connector_dynamo.erl | 2 +- .../src/emqx_ee_connector_gcp_pubsub.erl | 4 +- .../src/emqx_ee_connector_influxdb.erl | 6 +- .../src/emqx_ee_connector_rocketmq.erl | 2 +- .../src/emqx_ee_connector_tdengine.erl | 2 +- .../test/emqx_ee_schema_registry_SUITE.erl | 2 +- .../src/emqx_license_http_api.erl | 2 +- 101 files changed, 401 insertions(+), 447 deletions(-) delete mode 100644 apps/emqx/test/emqx_misc_SUITE.erl rename apps/{emqx/src/emqx_misc.erl => emqx_utils/src/emqx_utils.erl} (99%) diff --git a/apps/emqx/src/emqx_api_lib.erl b/apps/emqx/src/emqx_api_lib.erl index f1d65f350..5eddf6188 100644 --- a/apps/emqx/src/emqx_api_lib.erl +++ b/apps/emqx/src/emqx_api_lib.erl @@ -51,7 +51,7 @@ with_node_or_cluster(Node, Fun) -> -spec lookup_node(atom() | binary()) -> {ok, atom()} | not_found. lookup_node(BinNode) when is_binary(BinNode) -> - case emqx_misc:safe_to_existing_atom(BinNode, utf8) of + case emqx_utils:safe_to_existing_atom(BinNode, utf8) of {ok, Node} -> is_running_node(Node); _Error -> diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 5471b66fc..be3b35f57 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -277,9 +277,9 @@ atom(Bin) -> binary_to_existing_atom(Bin, utf8). certs_dir(ChainName, ConfigOrID) -> DirName = dir(ChainName, ConfigOrID), SubDir = iolist_to_binary(filename:join(["authn", DirName])), - emqx_misc:safe_filename(SubDir). + emqx_utils:safe_filename(SubDir). dir(ChainName, ID) when is_binary(ID) -> - emqx_misc:safe_filename(iolist_to_binary([to_bin(ChainName), "-", ID])); + emqx_utils:safe_filename(iolist_to_binary([to_bin(ChainName), "-", ID])); dir(ChainName, Config) when is_map(Config) -> dir(ChainName, authenticator_id(Config)). diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index 758c570da..a0ccd93d7 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -243,7 +243,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{expiry_timer := TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -254,10 +254,10 @@ code_change(_OldVsn, State, _Extra) -> -ifdef(TEST). ensure_expiry_timer(State) -> - State#{expiry_timer := emqx_misc:start_timer(10, expire)}. + State#{expiry_timer := emqx_utils:start_timer(10, expire)}. -else. ensure_expiry_timer(State) -> - State#{expiry_timer := emqx_misc:start_timer(timer:minutes(1), expire)}. + State#{expiry_timer := emqx_utils:start_timer(timer:minutes(1), expire)}. -endif. expire_banned_items(Now) -> diff --git a/apps/emqx/src/emqx_batch.erl b/apps/emqx/src/emqx_batch.erl index 2fe09942c..22e812975 100644 --- a/apps/emqx/src/emqx_batch.erl +++ b/apps/emqx/src/emqx_batch.erl @@ -85,7 +85,7 @@ commit(Batch = #batch{batch_q = Q, commit_fun = Commit}) -> reset(Batch). reset(Batch = #batch{linger_timer = TRef}) -> - _ = emqx_misc:cancel_timer(TRef), + _ = emqx_utils:cancel_timer(TRef), Batch#batch{batch_q = [], linger_timer = undefined}. -spec size(batch()) -> non_neg_integer(). diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index d56620123..7e00c050e 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -92,7 +92,7 @@ start_link(Pool, Id) -> ok = create_tabs(), gen_server:start_link( - {local, emqx_misc:proc_name(?BROKER, Id)}, + {local, emqx_utils:proc_name(?BROKER, Id)}, ?MODULE, [Pool, Id], [] diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl index 91b4c4994..87b291361 100644 --- a/apps/emqx/src/emqx_broker_helper.erl +++ b/apps/emqx/src/emqx_broker_helper.erl @@ -131,7 +131,7 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #{pmon := PMon}) -> - SubPids = [SubPid | emqx_misc:drain_down(?BATCH_SIZE)], + SubPids = [SubPid | emqx_utils:drain_down(?BATCH_SIZE)], ok = emqx_pool:async_submit( fun lists:foreach/2, [fun clean_down/1, SubPids] ), diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 29a59e482..745429536 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -61,7 +61,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [ run_fold/3, pipeline/3, @@ -622,7 +622,7 @@ process_connect( NChannel = Channel#channel{session = Session}, handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, ensure_connected(NChannel)); {ok, #{session := Session, present := true, pendings := Pendings}} -> - Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), + Pendings1 = lists:usort(lists:append(Pendings, emqx_utils:drain_deliver())), NChannel = Channel#channel{ session = Session, resuming = true, @@ -1203,7 +1203,7 @@ handle_call( ) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) - Delivers = emqx_misc:drain_deliver(), + Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), disconnect_and_shutdown(takenover, AllPendings, Channel); handle_call(list_authz_cache, Channel) -> @@ -1402,7 +1402,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> @@ -2045,7 +2045,7 @@ clear_keepalive(Channel = #channel{timers = Timers}) -> undefined -> Channel; TRef -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), Channel#channel{timers = maps:without([alive_timer], Timers)} end. %%-------------------------------------------------------------------- @@ -2241,7 +2241,7 @@ get_mqtt_conf(Zone, Key, Default) -> %%-------------------------------------------------------------------- set_field(Name, Value, Channel) -> - Pos = emqx_misc:index_of(Name, record_info(fields, channel)), + Pos = emqx_utils:index_of(Name, record_info(fields, channel)), setelement(Pos + 1, Channel, Value). get_mqueue(#channel{session = Session}) -> diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index f8c510482..346e8dcb5 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -672,7 +672,7 @@ handle_cast(Msg, State) -> handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> ?tp(emqx_cm_process_down, #{stale_pid => Pid, reason => _Reason}), - ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], + ChanPids = [Pid | emqx_utils:drain_down(?BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), lists:foreach(fun mark_channel_disconnected/1, ChanPids), ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]), diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index e5002cab4..8d47f033c 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -77,7 +77,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [start_timer/2] ). @@ -260,7 +260,7 @@ stats(#state{ {error, _} -> [] end, ChanStats = emqx_channel:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ChanStats, ProcStats]). %% @doc Set TCP keepalive socket options to override system defaults. @@ -392,7 +392,7 @@ run_loop( emqx_channel:info(zone, Channel), [force_shutdown] ), - emqx_misc:tune_heap_size(ShutdownPolicy), + emqx_utils:tune_heap_size(ShutdownPolicy), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); @@ -472,7 +472,7 @@ ensure_stats_timer(_Timeout, State) -> -compile({inline, [cancel_stats_timer/1]}). cancel_stats_timer(State = #state{stats_timer = TRef}) when is_reference(TRef) -> ?tp(debug, cancel_stats_timer, #{}), - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), State#state{stats_timer = undefined}; cancel_stats_timer(State) -> State. @@ -558,7 +558,7 @@ handle_msg( {incoming, Packet = ?CONNECT_PACKET(ConnPkt)}, State = #state{idle_timer = IdleTimer} ) -> - ok = emqx_misc:cancel_timer(IdleTimer), + ok = emqx_utils:cancel_timer(IdleTimer), Serialize = emqx_frame:serialize_opts(ConnPkt), NState = State#state{ serialize = Serialize, @@ -593,7 +593,7 @@ handle_msg( #state{listener = {Type, Listener}} = State ) -> ActiveN = get_active_n(Type, Listener), - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent handle_msg({inet_reply, _Sock, ok}, State = #state{listener = {Type, Listener}}) -> @@ -1073,7 +1073,7 @@ check_oom(State = #state{channel = Channel}) -> emqx_channel:info(zone, Channel), [force_shutdown] ), ?tp(debug, check_oom, #{policy => ShutdownPolicy}), - case emqx_misc:check_oom(ShutdownPolicy) of + case emqx_utils:check_oom(ShutdownPolicy) of {shutdown, Reason} -> %% triggers terminate/2 callback immediately erlang:exit({shutdown, Reason}); @@ -1200,7 +1200,7 @@ inc_counter(Key, Inc) -> %%-------------------------------------------------------------------- set_field(Name, Value, State) -> - Pos = emqx_misc:index_of(Name, record_info(fields, state)), + Pos = emqx_utils:index_of(Name, record_info(fields, state)), setelement(Pos + 1, State, Value). get_state(Pid) -> diff --git a/apps/emqx/src/emqx_crl_cache.erl b/apps/emqx/src/emqx_crl_cache.erl index 79e47a6dc..084313420 100644 --- a/apps/emqx/src/emqx_crl_cache.erl +++ b/apps/emqx/src/emqx_crl_cache.erl @@ -117,7 +117,7 @@ handle_call(Call, _From, State) -> handle_cast({evict, URL}, State0 = #state{refresh_timers = RefreshTimers0}) -> emqx_ssl_crl_cache:delete(URL), MTimer = maps:get(URL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = maps:without([URL], RefreshTimers0), State = State0#state{refresh_timers = RefreshTimers}, ?tp( @@ -223,9 +223,9 @@ ensure_timer(URL, State = #state{refresh_interval = Timeout}) -> ensure_timer(URL, State = #state{refresh_timers = RefreshTimers0}, Timeout) -> ?tp(crl_cache_ensure_timer, #{url => URL, timeout => Timeout}), MTimer = maps:get(URL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = RefreshTimers0#{ - URL => emqx_misc:start_timer( + URL => emqx_utils:start_timer( Timeout, {refresh, URL} ) @@ -297,7 +297,7 @@ handle_cache_overflow(State0) -> {_Time, OldestURL, InsertionTimes} = gb_trees:take_smallest(InsertionTimes0), emqx_ssl_crl_cache:delete(OldestURL), MTimer = maps:get(OldestURL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = maps:remove(OldestURL, RefreshTimers0), CachedURLs = sets:del_element(OldestURL, CachedURLs0), ?tp(debug, crl_cache_overflow, #{oldest_url => OldestURL}), diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 64e4ed6c3..2cfed0a8e 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -184,7 +184,7 @@ code_change(_OldVsn, State, _Extra) -> start_timer(Zone) -> WindTime = maps:get(window_time, get_policy(Zone)), - emqx_misc:start_timer(WindTime, {garbage_collect, Zone}). + emqx_utils:start_timer(WindTime, {garbage_collect, Zone}). start_timers() -> lists:foreach( diff --git a/apps/emqx/src/emqx_guid.erl b/apps/emqx/src/emqx_guid.erl index fea4e70b0..d313723fb 100644 --- a/apps/emqx/src/emqx_guid.erl +++ b/apps/emqx/src/emqx_guid.erl @@ -145,10 +145,10 @@ npid() -> NPid. to_hexstr(I) when byte_size(I) =:= 16 -> - emqx_misc:bin_to_hexstr(I, upper). + emqx_utils:bin_to_hexstr(I, upper). from_hexstr(S) when byte_size(S) =:= 32 -> - emqx_misc:hexstr_to_bin(S). + emqx_utils:hexstr_to_bin(S). to_base62(<>) -> emqx_base62:encode(I). diff --git a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl index 83bc2ec72..bbebd9460 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl @@ -375,7 +375,7 @@ return_pause(infinity, PauseType, Fun, Diff, Limiter) -> {PauseType, ?MINIMUM_PAUSE, make_retry_context(Fun, Diff), Limiter}; return_pause(Rate, PauseType, Fun, Diff, Limiter) -> Val = erlang:round(Diff * emqx_limiter_schema:default_period() / Rate), - Pause = emqx_misc:clamp(Val, ?MINIMUM_PAUSE, ?MAXIMUM_PAUSE), + Pause = emqx_utils:clamp(Val, ?MINIMUM_PAUSE, ?MAXIMUM_PAUSE), {PauseType, Pause, make_retry_context(Fun, Diff), Limiter}. -spec make_retry_context(undefined | retry_fun(Limiter), non_neg_integer()) -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 4e5843166..96e28b1de 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -538,7 +538,7 @@ esockd_access_rules(StrRules) -> [A, CIDR] = string:tokens(S, " "), %% esockd rules only use words 'allow' and 'deny', both are existing %% comparison of strings may be better, but there is a loss of backward compatibility - case emqx_misc:safe_to_existing_atom(A) of + case emqx_utils:safe_to_existing_atom(A) of {ok, Action} -> [ { @@ -560,7 +560,7 @@ esockd_access_rules(StrRules) -> merge_default(Options) -> case lists:keytake(tcp_options, 1, Options) of {value, {tcp_options, TcpOpts}, Options1} -> - [{tcp_options, emqx_misc:merge_opts(?MQTT_SOCKOPTS, TcpOpts)} | Options1]; + [{tcp_options, emqx_utils:merge_opts(?MQTT_SOCKOPTS, TcpOpts)} | Options1]; false -> [{tcp_options, ?MQTT_SOCKOPTS} | Options] end. diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index 4e7ada044..7044d992e 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -476,9 +476,9 @@ ensure_timer(ListenerID, State, Timeout) -> ensure_timer(ListenerID, {refresh, ListenerID}, State, Timeout). ensure_timer(ListenerID, Message, State, Timeout) -> - emqx_misc:cancel_timer(maps:get(?REFRESH_TIMER(ListenerID), State, undefined)), + emqx_utils:cancel_timer(maps:get(?REFRESH_TIMER(ListenerID), State, undefined)), State#{ - ?REFRESH_TIMER(ListenerID) => emqx_misc:start_timer( + ?REFRESH_TIMER(ListenerID) => emqx_utils:start_timer( Timeout, Message ) diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index c5ce35bf9..4810798eb 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -180,8 +180,8 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- cancel_outdated_timer(#{mem_time_ref := MemRef, cpu_time_ref := CpuRef}) -> - emqx_misc:cancel_timer(MemRef), - emqx_misc:cancel_timer(CpuRef), + emqx_utils:cancel_timer(MemRef), + emqx_utils:cancel_timer(CpuRef), ok. start_cpu_check_timer() -> @@ -204,7 +204,7 @@ start_mem_check_timer() -> end. start_timer(Interval, Msg) -> - emqx_misc:start_timer(Interval, Msg). + emqx_utils:start_timer(Interval, Msg). update_mem_alarm_status(HWM) when HWM > 1.0 orelse HWM < 0.0 -> ?SLOG(warning, #{msg => "discarded_out_of_range_mem_alarm_threshold", value => HWM}), diff --git a/apps/emqx/src/emqx_pool.erl b/apps/emqx/src/emqx_pool.erl index 1691a533a..1cb5f429c 100644 --- a/apps/emqx/src/emqx_pool.erl +++ b/apps/emqx/src/emqx_pool.erl @@ -57,7 +57,7 @@ -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 7c9cc61b0..42430af5d 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -98,7 +98,7 @@ mnesia(boot) -> -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8c91ae782..96d9aea34 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2327,7 +2327,7 @@ mqtt_ssl_listener_ssl_options_validator(Conf) -> fun ocsp_outer_validator/1, fun crl_outer_validator/1 ], - case emqx_misc:pipeline(Checks, Conf, not_used) of + case emqx_utils:pipeline(Checks, Conf, not_used) of {ok, _, _} -> ok; {error, Reason, _NotUsed} -> diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index a13dfe491..8b15340e9 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -941,7 +941,7 @@ age(Now, Ts) -> Now - Ts. %%-------------------------------------------------------------------- set_field(Name, Value, Session) -> - Pos = emqx_misc:index_of(Name, record_info(fields, session)), + Pos = emqx_utils:index_of(Name, record_info(fields, session)), setelement(Pos + 1, Session, Value). get_mqueue(#session{mqueue = Q}) -> diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl index 0d4972e8c..8a21d4d03 100644 --- a/apps/emqx/src/emqx_session_router.erl +++ b/apps/emqx/src/emqx_session_router.erl @@ -104,7 +104,7 @@ create_init_tab() -> -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index ed901d9a9..017b83116 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -213,7 +213,7 @@ init(#{tick_ms := TickMs}) -> {ok, start_timer(#state{updates = [], tick_ms = TickMs}), hibernate}. start_timer(#state{tick_ms = Ms} = State) -> - State#state{timer = emqx_misc:start_timer(Ms, tick)}. + State#state{timer = emqx_utils:start_timer(Ms, tick)}. handle_call(stop, _From, State) -> {stop, normal, ok, State}; @@ -301,7 +301,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #state{timer = TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index af2b052aa..509429796 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -62,7 +62,7 @@ -endif. -import(emqx_topic, [systop/1]). --import(emqx_misc, [start_timer/2]). +-import(emqx_utils, [start_timer/2]). -record(state, { heartbeat :: maybe(reference()), @@ -222,7 +222,7 @@ handle_info(Info, State) -> terminate(_Reason, #state{heartbeat = TRef1, ticker = TRef2}) -> _ = emqx_config_handler:remove_handler(?CONF_KEY_PATH), unload_event_hooks(sys_event_messages()), - lists:foreach(fun emqx_misc:cancel_timer/1, [TRef1, TRef2]). + lists:foreach(fun emqx_utils:cancel_timer/1, [TRef1, TRef2]). unload_event_hooks([]) -> ok; diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index 6ff68820e..f1190f586 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -77,7 +77,7 @@ init([]) -> {ok, start_timer(#{timer => undefined, events => []})}. start_timer(State) -> - State#{timer := emqx_misc:start_timer(timer:seconds(2), reset)}. + State#{timer := emqx_utils:start_timer(timer:seconds(2), reset)}. sysm_opts(VM) -> sysm_opts(maps:to_list(VM), []). @@ -204,7 +204,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), ok. code_change(_OldVsn, State, _Extra) -> diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index f14dc0c15..91194772f 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -272,7 +272,7 @@ handle_info({timeout, TRef, update_trace}, #{timer := TRef} = State) -> ?tp(update_trace_done, #{}), {noreply, State#{timer => NextTRef}}; handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), handle_info({timeout, TRef, update_trace}, State); handle_info(Info, State) -> ?SLOG(error, #{unexpected_info => Info}), @@ -280,7 +280,7 @@ handle_info(Info, State) -> terminate(_Reason, #{timer := TRef}) -> _ = mnesia:unsubscribe({table, ?TRACE, simple}), - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), stop_all_trace_handler(), update_trace_handler(), _ = file:del_dir_r(zip_dir()), @@ -302,7 +302,7 @@ update_trace(Traces) -> ok = stop_trace(NeedStop, Started), clean_stale_trace_files(), NextTime = find_closest_time(Traces, Now), - emqx_misc:start_timer(NextTime, update_trace). + emqx_utils:start_timer(NextTime, update_trace). stop_all_trace_handler() -> lists:foreach( diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index 231fd5e7b..528bc4d42 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -196,7 +196,7 @@ handler_id(Name, Type) -> do_handler_id(Name, Type) catch _:_ -> - Hash = emqx_misc:bin_to_hexstr(crypto:hash(md5, Name), lower), + Hash = emqx_utils:bin_to_hexstr(crypto:hash(md5, Name), lower), do_handler_id(Hash, Type) end. diff --git a/apps/emqx/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl index 1327a1bb0..d90d4139b 100644 --- a/apps/emqx/src/emqx_vm_mon.erl +++ b/apps/emqx/src/emqx_vm_mon.erl @@ -107,7 +107,7 @@ code_change(_OldVsn, State, _Extra) -> start_check_timer() -> Interval = emqx:get_config([sysmon, vm, process_check_interval]), - emqx_misc:start_timer(Interval, check). + emqx_utils:start_timer(Interval, check). usage(Percent) -> integer_to_list(floor(Percent * 100)) ++ "%". diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index ead609ed8..20962809f 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -52,7 +52,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [ maybe_apply/2, start_timer/2 @@ -172,7 +172,7 @@ stats(WsPid) when is_pid(WsPid) -> stats(#state{channel = Channel}) -> SockStats = emqx_pd:get_counters(?SOCK_STATS), ChanStats = emqx_channel:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ChanStats, ProcStats]). %% kick|discard|takeover @@ -340,7 +340,7 @@ tune_heap_size(Channel) -> ) of #{enable := false} -> ok; - ShutdownPolicy -> emqx_misc:tune_heap_size(ShutdownPolicy) + ShutdownPolicy -> emqx_utils:tune_heap_size(ShutdownPolicy) end. get_stats_enable(Zone) -> @@ -454,7 +454,7 @@ websocket_info( State = #state{listener = {Type, Listener}} ) -> ActiveN = get_active_n(Type, Listener), - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); websocket_info( {timeout, _, limit_timeout}, @@ -678,7 +678,7 @@ check_oom(State = #state{channel = Channel}) -> #{enable := false} -> State; #{enable := true} -> - case emqx_misc:check_oom(ShutdownPolicy) of + case emqx_utils:check_oom(ShutdownPolicy) of Shutdown = {shutdown, _Reason} -> postpone(Shutdown, State); _Other -> @@ -913,7 +913,7 @@ inc_qos_stats_key(_, _) -> undefined. %% Cancel idle timer cancel_idle_timer(State = #state{idle_timer = IdleTimer}) -> - ok = emqx_misc:cancel_timer(IdleTimer), + ok = emqx_utils:cancel_timer(IdleTimer), State#state{idle_timer = undefined}. %%-------------------------------------------------------------------- @@ -1046,7 +1046,7 @@ check_max_connection(Type, Listener) -> %%-------------------------------------------------------------------- set_field(Name, Value, State) -> - Pos = emqx_misc:index_of(Name, record_info(fields, state)), + Pos = emqx_utils:index_of(Name, record_info(fields, state)), setelement(Pos + 1, State, Value). %% ensure lowercase letters in headers diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index cc9e03168..21ed45119 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -496,16 +496,16 @@ t_get_conn_info(_) -> t_oom_shutdown(init, Config) -> ok = snabbkaffe:start_trace(), - ok = meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]), + ok = meck:new(emqx_utils, [non_strict, passthrough, no_history, no_link]), meck:expect( - emqx_misc, + emqx_utils, check_oom, fun(_) -> {shutdown, "fake_oom"} end ), Config; t_oom_shutdown('end', _Config) -> snabbkaffe:stop(), - meck:unload(emqx_misc), + meck:unload(emqx_utils), ok. t_oom_shutdown(_) -> diff --git a/apps/emqx/test/emqx_misc_SUITE.erl b/apps/emqx/test/emqx_misc_SUITE.erl deleted file mode 100644 index c068a087e..000000000 --- a/apps/emqx/test/emqx_misc_SUITE.erl +++ /dev/null @@ -1,209 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_misc_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --define(SOCKOPTS, [ - binary, - {packet, raw}, - {reuseaddr, true}, - {backlog, 512}, - {nodelay, true} -]). - -all() -> emqx_common_test_helpers:all(?MODULE). - -t_merge_opts(_) -> - Opts = emqx_misc:merge_opts(?SOCKOPTS, [ - raw, - binary, - {backlog, 1024}, - {nodelay, false}, - {max_clients, 1024}, - {acceptors, 16} - ]), - ?assertEqual(1024, proplists:get_value(backlog, Opts)), - ?assertEqual(1024, proplists:get_value(max_clients, Opts)), - ?assertEqual( - [ - binary, - raw, - {acceptors, 16}, - {backlog, 1024}, - {max_clients, 1024}, - {nodelay, false}, - {packet, raw}, - {reuseaddr, true} - ], - lists:sort(Opts) - ). - -t_maybe_apply(_) -> - ?assertEqual(undefined, emqx_misc:maybe_apply(fun(A) -> A end, undefined)), - ?assertEqual(a, emqx_misc:maybe_apply(fun(A) -> A end, a)). - -t_run_fold(_) -> - ?assertEqual(1, emqx_misc:run_fold([], 1, state)), - Add = fun(I, St) -> I + St end, - Mul = fun(I, St) -> I * St end, - ?assertEqual(6, emqx_misc:run_fold([Add, Mul], 1, 2)). - -t_pipeline(_) -> - ?assertEqual({ok, input, state}, emqx_misc:pipeline([], input, state)), - Funs = [ - fun(_I, _St) -> ok end, - fun(_I, St) -> {ok, St + 1} end, - fun(I, St) -> {ok, I + 1, St + 1} end, - fun(I, St) -> {ok, I * 2, St * 2} end - ], - ?assertEqual({ok, 4, 6}, emqx_misc:pipeline(Funs, 1, 1)), - ?assertEqual( - {error, undefined, 1}, emqx_misc:pipeline([fun(_I) -> {error, undefined} end], 1, 1) - ), - ?assertEqual( - {error, undefined, 2}, emqx_misc:pipeline([fun(_I, _St) -> {error, undefined, 2} end], 1, 1) - ). - -t_start_timer(_) -> - TRef = emqx_misc:start_timer(1, tmsg), - timer:sleep(2), - ?assertEqual([{timeout, TRef, tmsg}], drain()), - ok = emqx_misc:cancel_timer(TRef). - -t_cancel_timer(_) -> - Timer = emqx_misc:start_timer(0, foo), - ok = emqx_misc:cancel_timer(Timer), - ?assertEqual([], drain()), - ok = emqx_misc:cancel_timer(undefined). - -t_proc_name(_) -> - ?assertEqual(emqx_pool_1, emqx_misc:proc_name(emqx_pool, 1)). - -t_proc_stats(_) -> - Pid1 = spawn(fun() -> exit(normal) end), - timer:sleep(10), - ?assertEqual([], emqx_misc:proc_stats(Pid1)), - Pid2 = spawn(fun() -> - ?assertMatch([{mailbox_len, 0} | _], emqx_misc:proc_stats()), - timer:sleep(200) - end), - timer:sleep(10), - Pid2 ! msg, - timer:sleep(10), - ?assertMatch([{mailbox_len, 1} | _], emqx_misc:proc_stats(Pid2)). - -t_drain_deliver(_) -> - self() ! {deliver, t1, m1}, - self() ! {deliver, t2, m2}, - ?assertEqual( - [ - {deliver, t1, m1}, - {deliver, t2, m2} - ], - emqx_misc:drain_deliver(2) - ). - -t_drain_down(_) -> - {Pid1, _Ref1} = erlang:spawn_monitor(fun() -> ok end), - {Pid2, _Ref2} = erlang:spawn_monitor(fun() -> ok end), - timer:sleep(100), - ?assertEqual([Pid1, Pid2], lists:sort(emqx_misc:drain_down(2))), - ?assertEqual([], emqx_misc:drain_down(1)). - -t_index_of(_) -> - try emqx_misc:index_of(a, []) of - _ -> ct:fail(should_throw_error) - catch - error:Reason -> - ?assertEqual(badarg, Reason) - end, - ?assertEqual(3, emqx_misc:index_of(a, [b, c, a, e, f])). - -t_check(_) -> - Policy = #{ - max_message_queue_len => 10, - max_heap_size => 1024 * 1024 * 8, - enable => true - }, - [self() ! {msg, I} || I <- lists:seq(1, 5)], - ?assertEqual(ok, emqx_misc:check_oom(Policy)), - [self() ! {msg, I} || I <- lists:seq(1, 6)], - ?assertEqual( - {shutdown, #{reason => message_queue_too_long, value => 11, max => 10}}, - emqx_misc:check_oom(Policy) - ). - -drain() -> - drain([]). - -drain(Acc) -> - receive - Msg -> drain([Msg | Acc]) - after 0 -> - lists:reverse(Acc) - end. - -t_rand_seed(_) -> - ?assert(is_tuple(emqx_misc:rand_seed())). - -t_now_to_secs(_) -> - ?assert(is_integer(emqx_misc:now_to_secs(os:timestamp()))). - -t_now_to_ms(_) -> - ?assert(is_integer(emqx_misc:now_to_ms(os:timestamp()))). - -t_gen_id(_) -> - ?assertEqual(10, length(emqx_misc:gen_id(10))), - ?assertEqual(20, length(emqx_misc:gen_id(20))). - -t_pmap_normal(_) -> - ?assertEqual( - [5, 7, 9], - emqx_misc:pmap( - fun({A, B}) -> A + B end, - [{2, 3}, {3, 4}, {4, 5}] - ) - ). - -t_pmap_timeout(_) -> - ?assertExit( - timeout, - emqx_misc:pmap( - fun - (timeout) -> ct:sleep(1000); - ({A, B}) -> A + B - end, - [{2, 3}, {3, 4}, timeout], - 100 - ) - ). - -t_pmap_exception(_) -> - ?assertError( - foobar, - emqx_misc:pmap( - fun - (error) -> error(foobar); - ({A, B}) -> A + B - end, - [{2, 3}, {3, 4}, error] - ) - ). diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index 298a33fe8..2db0acf82 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -119,7 +119,7 @@ t_has_routes(_) -> ?R:delete_route(<<"devices/+/messages">>). t_unexpected(_) -> - Router = emqx_misc:proc_name(?R, 1), + Router = emqx_utils:proc_name(?R, 1), ?assertEqual(ignored, gen_server:call(Router, bad_request)), ?assertEqual(ok, gen_server:cast(Router, bad_message)), Router ! bad_info. diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index 9aa2d506f..ede4a9582 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -47,7 +47,7 @@ create(#{path := Path} = Source) -> ?SLOG(alert, #{ msg => failed_to_read_acl_file, path => Path, - explain => emqx_misc:explain_posix(Reason) + explain => emqx_utils:explain_posix(Reason) }), throw(failed_to_read_acl_file); {error, Reason} -> diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index b717775a6..7a7dbb7e9 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -148,8 +148,8 @@ set_special_configs(_App) -> ok. init_per_testcase(t_api, Config) -> - meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_misc, gen_id, fun() -> "fake" end), + meck:new(emqx_utils, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_utils, gen_id, fun() -> "fake" end), meck:new(emqx, [non_strict, passthrough, no_history, no_link]), meck:expect( @@ -165,7 +165,7 @@ init_per_testcase(_, Config) -> Config. end_per_testcase(t_api, _Config) -> - meck:unload(emqx_misc), + meck:unload(emqx_utils), meck:unload(emqx), ok; end_per_testcase(_, _Config) -> diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 2a20c5994..087bc6a3f 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -296,7 +296,7 @@ create(BridgeType, BridgeName, RawConf) -> brige_action => create, bridge_type => BridgeType, bridge_name => BridgeName, - bridge_raw_config => emqx_misc:redact(RawConf) + bridge_raw_config => emqx_utils:redact(RawConf) }), emqx_conf:update( emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 675541311..814fcdeb7 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -668,7 +668,7 @@ get_metrics_from_local_node(BridgeType, BridgeName) -> false -> ?BRIDGE_NOT_ENABLED; true -> - case emqx_misc:safe_to_existing_atom(Node, utf8) of + case emqx_utils:safe_to_existing_atom(Node, utf8) of {ok, TargetNode} -> call_operation(TargetNode, OperFunc, [ TargetNode, BridgeType, BridgeName @@ -835,7 +835,7 @@ format_resource_data(ResData) -> format_resource_data(error, undefined, Result) -> Result; format_resource_data(error, Error, Result) -> - Result#{status_reason => emqx_misc:readable_error_msg(Error)}; + Result#{status_reason => emqx_utils:readable_error_msg(Error)}; format_resource_data(K, V, Result) -> Result#{K => V}. @@ -1004,7 +1004,7 @@ supported_versions(get_metrics_from_all_nodes) -> [4]; supported_versions(_Call) -> [1, 2, 3, 4]. redact(Term) -> - emqx_misc:redact(Term). + emqx_utils:redact(Term). deobfuscate(NewConf, OldConf) -> maps:fold( @@ -1015,7 +1015,7 @@ deobfuscate(NewConf, OldConf) -> {ok, OldV} when is_map(V), is_map(OldV) -> Acc#{K => deobfuscate(V, OldV)}; {ok, OldV} -> - case emqx_misc:is_redacted(K, V) of + case emqx_utils:is_redacted(K, V) of true -> Acc#{K => OldV}; _ -> diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index b43cbe0ec..903e86443 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -157,7 +157,7 @@ create(Type, Name, Conf, Opts0) -> msg => "create bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), Opts = override_start_after_created(Conf, Opts0), {ok, _Data} = emqx_resource:create_local( @@ -192,7 +192,7 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> msg => "update bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), case recreate(Type, Name, Conf, Opts) of {ok, _} -> @@ -202,7 +202,7 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> msg => "updating_a_non_existing_bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), create(Type, Name, Conf, Opts); {error, Reason} -> @@ -236,8 +236,8 @@ recreate(Type, Name, Conf, Opts) -> ). create_dry_run(Type, Conf0) -> - TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_misc:gen_id(8)]), - TmpPath = emqx_misc:safe_filename(TmpPath0), + TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), + TmpPath = emqx_utils:safe_filename(TmpPath0), Conf = emqx_map_lib:safe_atom_key_map(Conf0), case emqx_connector_ssl:convert_certs(TmpPath, Conf) of {error, Reason} -> diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 3ec6061d8..ef997f7e3 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -975,7 +975,7 @@ t_with_redact_update(Config) -> ), %% update with redacted config - BridgeConf = emqx_misc:redact(Template), + BridgeConf = emqx_utils:redact(Template), BridgeID = emqx_bridge_resource:bridge_id(Type, Name), {ok, 200, _} = request(put, uri(["bridges", BridgeID]), BridgeConf, Config), ?assertEqual( diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index 2dc43a130..fdfa3300c 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -156,7 +156,7 @@ on_start(InstanceId, Config) -> msg => "failed_to_start_kafka_consumer_client", instance_id => InstanceId, kafka_hosts => BootstrapHosts, - reason => emqx_misc:redact(Reason) + reason => emqx_utils:redact(Reason) }), throw(?CLIENT_DOWN_MESSAGE) end, @@ -344,7 +344,7 @@ start_consumer(Config, InstanceId, ClientID) -> msg => "failed_to_start_kafka_consumer", instance_id => InstanceId, kafka_hosts => emqx_bridge_kafka_impl:hosts(BootstrapHosts0), - reason => emqx_misc:redact(Reason2) + reason => emqx_utils:redact(Reason2) }), stop_client(ClientID), throw(failed_to_start_kafka_consumer) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index fc9a6dc29..254284b75 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -299,7 +299,7 @@ init_per_testcase(TestCase, Config) when common_init_per_testcase(TestCase, Config) end; init_per_testcase(t_cluster_group = TestCase, Config0) -> - Config = emqx_misc:merge_opts(Config0, [{num_partitions, 6}]), + Config = emqx_utils:merge_opts(Config0, [{num_partitions, 6}]), common_init_per_testcase(TestCase, Config); init_per_testcase(t_multiple_topic_mappings = TestCase, Config0) -> KafkaTopicBase = @@ -1543,7 +1543,7 @@ do_t_receive_after_recovery(Config) -> %% 2) publish messages while the consumer is down. %% we use `pmap' to avoid wolff sending the whole %% batch to a single partition. - emqx_misc:pmap(fun(Msg) -> publish(Config, [Msg]) end, Messages1), + emqx_utils:pmap(fun(Msg) -> publish(Config, [Msg]) end, Messages1), ok end), %% 3) restore and consume messages @@ -1667,7 +1667,7 @@ t_cluster_group(Config) -> || {Name, Opts} <- Cluster ], on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1889,7 +1889,7 @@ t_cluster_node_down(Config) -> Cluster ), on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 0382045d4..c82191bc3 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -501,15 +501,17 @@ log_and_alarm(IsSuccess, Res, #{kind := ?APPLY_KIND_INITIATE} = Meta) -> %% because nothing is committed case IsSuccess of true -> - ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_result", result => emqx_misc:redact(Res)}); + ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_result", result => emqx_utils:redact(Res)}); false -> - ?SLOG(warning, Meta#{msg => "cluster_rpc_apply_result", result => emqx_misc:redact(Res)}) + ?SLOG(warning, Meta#{ + msg => "cluster_rpc_apply_result", result => emqx_utils:redact(Res) + }) end; log_and_alarm(true, Res, Meta) -> - ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_ok", result => emqx_misc:redact(Res)}), + ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_ok", result => emqx_utils:redact(Res)}), do_alarm(deactivate, Res, Meta); log_and_alarm(false, Res, Meta) -> - ?SLOG(error, Meta#{msg => "cluster_rpc_apply_failed", result => emqx_misc:redact(Res)}), + ?SLOG(error, Meta#{msg => "cluster_rpc_apply_failed", result => emqx_utils:redact(Res)}), do_alarm(activate, Res, Meta). do_alarm(Fun, Res, #{tnx_id := Id} = Meta) -> diff --git a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl index bce866c2d..fe72cd65b 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl @@ -73,7 +73,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -82,7 +82,7 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- ensure_timer(State = #{cleanup_ms := Ms}) -> - State#{timer := emqx_misc:start_timer(Ms, del_stale_mfa)}. + State#{timer := emqx_utils:start_timer(Ms, del_stale_mfa)}. %% @doc Keep the latest completed 100 records for querying and troubleshooting. del_stale_mfa(MaxHistory) -> diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index dfa6dab81..ef2e11eb7 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -219,7 +219,7 @@ on_start( SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, Config)), {tls, SSLOpts} end, - NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + NTransportOpts = emqx_utils:ipv6_probe(TransportOpts), PoolOpts = [ {host, Host}, {port, Port}, @@ -425,7 +425,7 @@ do_get_status(PoolName, Timeout) -> Error end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of % we crash in case of non-empty lists since we don't know what to do in that case [_ | _] = Results -> case [E || {error, _} = E <- Results] of @@ -603,7 +603,7 @@ is_sensitive_key(_) -> %% Function that will do a deep traversal of Data and remove sensitive %% information (i.e., passwords) redact(Data) -> - emqx_misc:redact(Data, fun is_sensitive_key/1). + emqx_utils:redact(Data, fun is_sensitive_key/1). %% because the body may contain some sensitive data %% and at the same time the redact function will not scan the binary data diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 82d622e09..ac2af301e 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -65,7 +65,7 @@ on_start( ?SLOG(info, #{ msg => "starting_ldap_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), Servers = emqx_schema:parse_servers(Servers0, ?LDAP_HOST_OPTIONS), SslOpts = diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index ead3e2e49..a5873bcf6 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -162,7 +162,7 @@ on_start( rs -> "starting_mongodb_replica_set_connector"; sharded -> "starting_mongodb_sharded_connector" end, - ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_misc:redact(Config)}), + ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_utils:redact(Config)}), NConfig = #{hosts := Hosts} = maybe_resolve_srv_and_txt_records(Config), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 8f2d06517..5b488825b 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -149,7 +149,7 @@ on_start(InstanceId, Conf) -> ?SLOG(info, #{ msg => "starting_mqtt_connector", connector => InstanceId, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), BasicConf = basic_config(Conf), BridgeConf = BasicConf#{ diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index c69cf5ca0..6600a5f77 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -102,7 +102,7 @@ on_start( ?SLOG(info, #{ msg => "starting_mysql_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 14cbbc80f..8796e00a5 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -95,7 +95,7 @@ on_start( ?SLOG(info, #{ msg => "starting_postgresql_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index c70e766af..4ef778e6b 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -123,7 +123,7 @@ on_start( ?SLOG(info, #{ msg => "starting_redis_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), ConfKey = case Type of diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index 67fc40efa..df1114483 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -85,7 +85,7 @@ to_remote_msg(MapMsg, #{ qos = QoS, retain = Retain, topic = topic(Mountpoint, Topic), - props = emqx_misc:pub_props_to_packet(PubProps), + props = emqx_utils:pub_props_to_packet(PubProps), payload = Payload }; to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> @@ -112,7 +112,7 @@ to_broker_msg( Retain = replace_simple_var(RetainToken, MapMsg), PubProps = maps:get(pub_props, MapMsg, #{}), set_headers( - Props#{properties => emqx_misc:pub_props_to_packet(PubProps)}, + Props#{properties => emqx_utils:pub_props_to_packet(PubProps)}, emqx_message:set_flags( #{dup => Dup, retain => Retain}, emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload) diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 8e41a9f0f..5197a35df 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -124,7 +124,7 @@ start_link(Name, BridgeOpts) -> {error, Reason} = Error -> ?SLOG(error, #{ msg => "client_start_failed", - config => emqx_misc:redact(BridgeOpts), + config => emqx_utils:redact(BridgeOpts), reason => Reason }), Error diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index eb7f6c741..22cf484ff 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -915,4 +915,4 @@ schema_converter(Options) -> maps:get(schema_converter, Options, fun hocon_schema_to_spec/2). hocon_error_msg(Reason) -> - emqx_misc:readable_error_msg(Reason). + emqx_utils:readable_error_msg(Reason). diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl index aa5d1897f..9bfae9579 100644 --- a/apps/emqx_exhook/src/emqx_exhook_api.erl +++ b/apps/emqx_exhook/src/emqx_exhook_api.erl @@ -478,7 +478,7 @@ call_cluster(Fun) -> %%-------------------------------------------------------------------- %% Internal Funcs %%-------------------------------------------------------------------- -err_msg(Msg) -> emqx_misc:readable_error_msg(Msg). +err_msg(Msg) -> emqx_utils:readable_error_msg(Msg). get_raw_config() -> RawConfig = emqx:get_raw_config([exhook, servers], []), diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 7f37061ac..4145a92a7 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -173,7 +173,7 @@ stats(#state{ end, ConnStats = emqx_pd:get_counters(?CONN_STATS), ChanStats = ChannMod:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ConnStats, ChanStats, ProcStats]). call(Pid, Req) -> @@ -297,7 +297,7 @@ init_state(WrappedSock, Peername, Options, FrameMod, ChannMod) -> StatsTimer = emqx_gateway_utils:stats_timer(Options), IdleTimeout = emqx_gateway_utils:idle_timeout(Options), OomPolicy = emqx_gateway_utils:oom_policy(Options), - IdleTimer = emqx_misc:start_timer(IdleTimeout, idle_timeout), + IdleTimer = emqx_utils:start_timer(IdleTimeout, idle_timeout), #state{ socket = WrappedSock, peername = Peername, @@ -327,7 +327,7 @@ run_loop( } ) -> emqx_logger:set_metadata_peername(esockd:format(Peername)), - _ = emqx_misc:tune_heap_size(OomPolicy), + _ = emqx_utils:tune_heap_size(OomPolicy), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); @@ -383,14 +383,14 @@ wakeup_from_hib(Parent, State) -> %% Ensure/cancel stats timer ensure_stats_timer(Timeout, State = #state{stats_timer = undefined}) -> - State#state{stats_timer = emqx_misc:start_timer(Timeout, emit_stats)}; + State#state{stats_timer = emqx_utils:start_timer(Timeout, emit_stats)}; ensure_stats_timer(_Timeout, State) -> State. cancel_stats_timer(State = #state{stats_timer = TRef}) when is_reference(TRef) -> - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), State#state{stats_timer = undefined}; cancel_stats_timer(State) -> State. @@ -471,7 +471,7 @@ handle_msg( State = #state{idle_timer = IdleTimer} ) -> IdleTimer /= undefined andalso - emqx_misc:cancel_timer(IdleTimer), + emqx_utils:cancel_timer(IdleTimer), NState = State#state{idle_timer = undefined}, handle_incoming(Packet, NState); handle_msg({outgoing, Data}, State) -> @@ -501,7 +501,7 @@ handle_msg( Deliver = {deliver, _Topic, _Msg}, State = #state{active_n = ActiveN} ) -> - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent %% TODO: Who will deliver this message? @@ -904,7 +904,7 @@ handle_info(Info, State) -> %% msg => "reach_rate_limit", %% pause => Time %% }), -%% TRef = emqx_misc:start_timer(Time, limit_timeout), +%% TRef = emqx_utils:start_timer(Time, limit_timeout), %% State#state{ %% sockstate = blocked, %% limiter = Limiter1, @@ -928,7 +928,7 @@ run_gc(Stats, State = #state{gc_state = GcSt}) -> end. check_oom(State = #state{oom_policy = OomPolicy}) -> - case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of + case ?ENABLED(OomPolicy) andalso emqx_utils:check_oom(OomPolicy) of {shutdown, Reason} -> %% triggers terminate/2 callback immediately erlang:exit({shutdown, Reason}); diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index e64e918b4..cd387e3bb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -115,7 +115,7 @@ clients(get, #{ fun ?MODULE:format_channel_info/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> QStringWithoutNode = maps:without([<<"node">>], QString), emqx_mgmt_api:node_query( diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 71ec4bf59..5de6da63f 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -802,7 +802,7 @@ handle_info( {'DOWN', _MRef, process, Pid, _Reason}, State = #state{gwname = GwName, chan_pmon = PMon} ) -> - ChanPids = [Pid | emqx_misc:drain_down(?DEFAULT_BATCH_SIZE)], + ChanPids = [Pid | emqx_utils:drain_down(?DEFAULT_BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), CmTabs = cmtabs(GwName), diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 5982334b4..9bb5821f5 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -495,7 +495,7 @@ reason2msg( reason2msg( {#{roots := [{gateway, _}]}, [_ | _]} = Error ) -> - Bin = emqx_misc:readable_error_msg(Error), + Bin = emqx_utils:readable_error_msg(Error), <<"Invalid configurations: ", Bin/binary>>; reason2msg(_) -> error. diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 9d80de00e..07185ef42 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -223,7 +223,7 @@ merge_default(Udp, Options) -> case lists:keytake(Key, 1, Options) of {value, {Key, TcpOpts}, Options1} -> [ - {Key, emqx_misc:merge_opts(Default, TcpOpts)} + {Key, emqx_utils:merge_opts(Default, TcpOpts)} | Options1 ]; false -> @@ -482,7 +482,7 @@ frame_options(Options) -> -spec init_gc_state(map()) -> emqx_gc:gc_state() | undefined. init_gc_state(Options) -> - emqx_misc:maybe_apply(fun emqx_gc:init/1, force_gc_policy(Options)). + emqx_utils:maybe_apply(fun emqx_gc:init/1, force_gc_policy(Options)). -spec force_gc_policy(map()) -> emqx_gc:opts() | undefined. force_gc_policy(Options) -> diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index 4cf362d9d..b90fd630d 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -111,7 +111,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_coap_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_coap_session:info/1, Session); info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> ClientId; info(ctx, #channel{ctx = Ctx}) -> @@ -366,7 +366,7 @@ ensure_timer(Name, Time, Msg, #channel{timers = Timers} = Channel) -> end. make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. ensure_keepalive_timer(Channel) -> @@ -710,7 +710,7 @@ process_connection( ) -> Queries = emqx_coap_message:get_option(uri_query, Req), case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun run_conn_hooks/2, diff --git a/apps/emqx_gateway_coap/src/emqx_coap_session.erl b/apps/emqx_gateway_coap/src/emqx_coap_session.erl index 688defcbb..5ae169675 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_session.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_session.erl @@ -81,7 +81,7 @@ %%%------------------------------------------------------------------- -spec new() -> session(). new() -> - _ = emqx_misc:rand_seed(), + _ = emqx_utils:rand_seed(), #session{ transport_manager = emqx_coap_tm:new(), observe_manager = emqx_coap_observe_res:new_manager(), diff --git a/apps/emqx_gateway_coap/src/emqx_coap_tm.erl b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl index 82f616b25..68a7ae237 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_tm.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl @@ -272,12 +272,12 @@ cancel_state_timer(#state_machine{timers = Timers} = Machine) -> undefined -> Machine; Ref -> - _ = emqx_misc:cancel_timer(Ref), + _ = emqx_utils:cancel_timer(Ref), Machine#state_machine{timers = maps:remove(state_timer, Timers)} end. process_timer(SeqId, {Type, Interval, Msg}, Timers) -> - Ref = emqx_misc:start_timer(Interval, {state_machine, {SeqId, Type, Msg}}), + Ref = emqx_utils:start_timer(Interval, {state_machine, {SeqId, Type, Msg}}), Timers#{Type => Ref}. -spec delete_machine(manager_key(), manager()) -> manager(). @@ -293,7 +293,7 @@ delete_machine(Id, Manager) -> } -> lists:foreach( fun({_, Ref}) -> - emqx_misc:cancel_timer(Ref) + emqx_utils:cancel_timer(Ref) end, maps:to_list(Timers) ), diff --git a/apps/emqx_gateway_coap/src/emqx_coap_transport.erl b/apps/emqx_gateway_coap/src/emqx_coap_transport.erl index 1948c969d..daea13ba8 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_transport.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_transport.erl @@ -119,7 +119,7 @@ idle(out, #coap_message{type = non} = Msg, _) -> timeouts => [{stop_timeout, ?NON_LIFETIME}] }); idle(out, Msg, Transport) -> - _ = emqx_misc:rand_seed(), + _ = emqx_utils:rand_seed(), Timeout = ?ACK_TIMEOUT + rand:uniform(?ACK_RANDOM_FACTOR), out(Msg, #{ next => wait_ack, diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl index 7234e7a2f..3b2c8d73b 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl @@ -681,14 +681,14 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> ensure_timer(Name, remove_timer_ref(Name, Channel)). cancel_timer(Name, Channel = #channel{timers = Timers}) -> - emqx_misc:cancel_timer(maps:get(Name, Timers, undefined)), + emqx_utils:cancel_timer(maps:get(Name, Timers, undefined)), remove_timer_ref(Name, Channel). remove_timer_ref(Name, Channel = #channel{timers = Timers}) -> @@ -792,4 +792,4 @@ proto_name_to_protocol(ProtoName) when is_binary(ProtoName) -> binary_to_atom(ProtoName). anonymous_clientid() -> - iolist_to_binary(["exproto-", emqx_misc:gen_id()]). + iolist_to_binary(["exproto-", emqx_utils:gen_id()]). diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl index af15ef9d3..34883cdce 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl @@ -50,7 +50,7 @@ start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [] diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl index a424375dd..77652744a 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl @@ -105,7 +105,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_lwm2m_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_lwm2m_session:info/1, Session); info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> ClientId; info(ctx, #channel{ctx = Ctx}) -> @@ -286,7 +286,7 @@ handle_call(discard, _From, Channel) -> % pendings = Pendings}) -> % ok = emqx_session:takeover(Session), % %% TODO: Should not drain deliver here (side effect) -% Delivers = emqx_misc:drain_deliver(), +% Delivers = emqx_utils:drain_deliver(), % AllPendings = lists:append(Delivers, Pendings), % shutdown_and_reply(takenover, AllPendings, Channel); @@ -390,7 +390,7 @@ set_peercert_infos(Peercert, ClientInfo) -> ClientInfo#{dn => DN, cn => CN}. make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. update_life_timer(#channel{session = Session, timers = Timers} = Channel) -> @@ -413,7 +413,7 @@ do_takeover(_DesireId, Msg, Channel) -> do_connect(Req, Result, Channel, Iter) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun check_lwm2m_version/2, fun enrich_conninfo/2, diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index c27c0ba3f..1ccc8b95a 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -218,7 +218,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_session:info/1, Session); info(will_msg, #channel{will_msg = WillMsg}) -> WillMsg; info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> @@ -282,7 +282,7 @@ enrich_clientinfo( feedvar(Override, Packet, ConnInfo, ClientInfo0), ClientInfo0 ), - {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + {ok, NPacket, NClientInfo} = emqx_utils:pipeline( [ fun maybe_assign_clientid/2, %% FIXME: CALL After authentication successfully @@ -414,7 +414,7 @@ process_connect( Channel#channel{session = Session} ); {ok, #{session := Session, present := true, pendings := Pendings}} -> - Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), + Pendings1 = lists:usort(lists:append(Pendings, emqx_utils:drain_deliver())), NChannel = Channel#channel{ session = Session, resuming = true, @@ -595,7 +595,7 @@ handle_in( Channel = #channel{conn_state = idle} ) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun run_conn_hooks/2, @@ -718,7 +718,7 @@ handle_in(PubPkt = ?SN_PUBLISH_MSG(_Flags, TopicId0, MsgId, _Data), Channel) -> Id end, case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun check_qos3_enable/2, fun preproc_pub_pkt/2, @@ -877,7 +877,7 @@ handle_in( end; handle_in(SubPkt = ?SN_SUBSCRIBE_MSG(_, MsgId, _), Channel) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun preproc_subs_type/2, fun check_subscribe_authz/2, @@ -911,7 +911,7 @@ handle_in( Channel ) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun preproc_unsub_type/2, fun run_client_unsub_hook/2, @@ -1823,7 +1823,7 @@ handle_call( ) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) - Delivers = emqx_misc:drain_deliver(), + Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), shutdown_and_reply(takenover, AllPendings, Channel); %handle_call(list_authz_cache, _From, Channel) -> @@ -2247,7 +2247,7 @@ ensure_register_timer(Channel) -> ensure_register_timer(RetryTimes, Channel = #channel{timers = Timers}) -> Msg = maps:get(register_timer, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(?REGISTER_TIMEOUT, {Msg, RetryTimes}), + TRef = emqx_utils:start_timer(?REGISTER_TIMEOUT, {Msg, RetryTimes}), Channel#channel{timers = Timers#{register_timer => TRef}}. cancel_timer(Name, Channel = #channel{timers = Timers}) -> @@ -2255,7 +2255,7 @@ cancel_timer(Name, Channel = #channel{timers = Timers}) -> undefined -> Channel; TRef -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), Channel#channel{timers = maps:without([Name], Timers)} end. @@ -2270,7 +2270,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index 13b70348a..316432dea 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -252,7 +252,7 @@ enrich_clientinfo( feedvar(Override, Packet, ConnInfo, ClientInfo0), ClientInfo0 ), - {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + {ok, NPacket, NClientInfo} = emqx_utils:pipeline( [ fun maybe_assign_clientid/2, fun parse_heartbeat/2, @@ -416,7 +416,7 @@ handle_in( {error, unexpected_connect, Channel}; handle_in(Packet = ?PACKET(?CMD_CONNECT), Channel) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun negotiate_version/2, @@ -474,7 +474,7 @@ handle_in( Topic = header(<<"destination">>, Headers), Ack = header(<<"ack">>, Headers, <<"auto">>), case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun parse_topic_filter/2, fun check_subscribed_status/2, @@ -796,7 +796,7 @@ handle_call( reply({error, no_subid}, Channel); SubId -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun parse_topic_filter/2, fun check_subscribed_status/2 @@ -869,7 +869,7 @@ handle_call(discard, _From, Channel) -> % pendings = Pendings}) -> % ok = emqx_session:takeover(Session), % %% TODO: Should not drain deliver here (side effect) -% Delivers = emqx_misc:drain_deliver(), +% Delivers = emqx_utils:drain_deliver(), % AllPendings = lists:append(Delivers, Pendings), % shutdown_and_reply(takenover, AllPendings, Channel); @@ -1289,7 +1289,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> diff --git a/apps/emqx_machine/src/emqx_global_gc.erl b/apps/emqx_machine/src/emqx_global_gc.erl index f17ab2d16..121855e68 100644 --- a/apps/emqx_machine/src/emqx_global_gc.erl +++ b/apps/emqx_machine/src/emqx_global_gc.erl @@ -86,7 +86,7 @@ ensure_timer(State) -> disabled -> State; Interval when is_integer(Interval) -> - TRef = emqx_misc:start_timer(Interval, run), + TRef = emqx_utils:start_timer(Interval, run), State#{timer := TRef} end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index cac3edaed..cd4829342 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -644,7 +644,7 @@ list_clients(QString) -> fun ?MODULE:format_channel_info/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> QStringWithoutNode = maps:without([<<"node">>], QString), emqx_mgmt_api:node_query( diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 409af4d95..1b69835f9 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -149,7 +149,7 @@ subscriptions(get, #{query_string := QString}) -> fun ?MODULE:format/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> emqx_mgmt_api:node_query( Node1, diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index a69837eb3..5df641add 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -498,7 +498,7 @@ download_trace_log(get, #{bindings := #{name := Name}, query_string := Query}) - %% We generate a session ID so that we name files %% with unique names. Then we won't cause %% overwrites for concurrent requests. - SessionId = emqx_misc:gen_id(), + SessionId = emqx_utils:gen_id(), ZipDir = filename:join([emqx_trace:zip_dir(), SessionId]), ok = file:make_dir(ZipDir), %% Write files to ZipDir and create an in-memory zip file diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 6f2a27414..12a7a6641 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -174,7 +174,7 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) -> desc = Desc, created_at = erlang:system_time(second), api_secret_hash = emqx_dashboard_admin:hash(ApiSecret), - api_key = list_to_binary(emqx_misc:gen_id(16)) + api_key = list_to_binary(emqx_utils:gen_id(16)) }, case create_app(App) of {ok, Res} -> @@ -213,7 +213,7 @@ do_force_create_app(App, ApiKey, NamePrefix) -> end. generate_unique_name(NamePrefix) -> - New = list_to_binary(NamePrefix ++ emqx_misc:gen_id(16)), + New = list_to_binary(NamePrefix ++ emqx_utils:gen_id(16)), case mnesia:read(?APP, New) of [] -> New; _ -> generate_unique_name(NamePrefix) @@ -246,7 +246,7 @@ init_bootstrap_file(File) -> {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), init_bootstrap_file(File, Dev, MP); {error, Reason0} -> - Reason = emqx_misc:explain_posix(Reason0), + Reason = emqx_utils:explain_posix(Reason0), ?SLOG( error, #{ diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 442d5c7de..253d527ac 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -356,7 +356,7 @@ mnesia(_) -> %% @doc Logger Command log(["set-level", Level]) -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> case emqx_logger:set_log_level(Level1) of ok -> emqx_ctl:print("~ts~n", [Level]); @@ -369,7 +369,7 @@ log(["primary-level"]) -> Level = emqx_logger:get_primary_log_level(), emqx_ctl:print("~ts~n", [Level]); log(["primary-level", Level]) -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> _ = emqx_logger:set_primary_log_level(Level1), ok; @@ -392,7 +392,7 @@ log(["handlers", "list"]) -> ], ok; log(["handlers", "start", HandlerId]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> case emqx_logger:start_log_handler(HandlerId1) of ok -> @@ -406,7 +406,7 @@ log(["handlers", "start", HandlerId]) -> emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(["handlers", "stop", HandlerId]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> case emqx_logger:stop_log_handler(HandlerId1) of ok -> @@ -420,9 +420,9 @@ log(["handlers", "stop", HandlerId]) -> emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(["handlers", "set-level", HandlerId, Level]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> case emqx_logger:set_log_handler_level(HandlerId1, Level1) of ok -> @@ -628,7 +628,7 @@ listeners([]) -> emqx_listeners:list() ); listeners(["stop", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:stop_listener(ListenerId1) of ok -> @@ -640,7 +640,7 @@ listeners(["stop", ListenerId]) -> emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(["start", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:start_listener(ListenerId1) of ok -> @@ -652,7 +652,7 @@ listeners(["start", ListenerId]) -> emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(["restart", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:restart_listener(ListenerId1) of ok -> diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 91cdbedd3..85313c181 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -328,7 +328,7 @@ handle_info(Info, State) -> terminate(_Reason, #{stats_timer := StatsTimer} = State) -> emqx_conf:remove_handler([delayed]), - emqx_misc:cancel_timer(StatsTimer), + emqx_utils:cancel_timer(StatsTimer), do_load_or_unload(false, State). code_change(_Vsn, State, _Extra) -> @@ -370,14 +370,14 @@ ensure_publish_timer({Ts, _Id}, State = #{publish_timer := undefined}) -> ensure_publish_timer({Ts, _Id}, State = #{publish_timer := TRef, publish_at := PubAt}) when Ts < PubAt -> - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), ensure_publish_timer(Ts, ?NOW, State); ensure_publish_timer(_Key, State) -> State. ensure_publish_timer(Ts, Now, State) -> Interval = max(1, Ts - Now), - TRef = emqx_misc:start_timer(Interval, do_publish), + TRef = emqx_utils:start_timer(Interval, do_publish), State#{publish_timer := TRef, publish_at := Now + Interval}. do_publish(Key, Now) -> @@ -418,7 +418,7 @@ do_load_or_unload(true, State) -> State; do_load_or_unload(false, #{publish_timer := PubTimer} = State) -> emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), - emqx_misc:cancel_timer(PubTimer), + emqx_utils:cancel_timer(PubTimer), ets:delete_all_objects(?TAB), State#{publish_timer := undefined, publish_at := 0}; do_load_or_unload(_, State) -> diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl index 1d49fb3e7..3b27302df 100644 --- a/apps/emqx_modules/src/emqx_telemetry.erl +++ b/apps/emqx_modules/src/emqx_telemetry.erl @@ -161,7 +161,7 @@ handle_call(enable, _From, State) -> FirstReportTimeoutMS = timer:seconds(10), {reply, ok, ensure_report_timer(FirstReportTimeoutMS, State)}; handle_call(disable, _From, State = #state{timer = Timer}) -> - emqx_misc:cancel_timer(Timer), + emqx_utils:cancel_timer(Timer), {reply, ok, State#state{timer = undefined}}; handle_call(get_node_uuid, _From, State = #state{node_uuid = UUID}) -> {reply, {ok, UUID}, State}; @@ -205,7 +205,7 @@ ensure_report_timer(State = #state{report_interval = ReportInterval}) -> ensure_report_timer(ReportInterval, State). ensure_report_timer(ReportInterval, State) -> - State#state{timer = emqx_misc:start_timer(ReportInterval, time_to_report_telemetry_data)}. + State#state{timer = emqx_utils:start_timer(ReportInterval, time_to_report_telemetry_data)}. os_info() -> case erlang:system_info(os_type) of diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl index 9b286f360..a3048e5dd 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl @@ -79,7 +79,7 @@ health_check_ecpool_workers(PoolName, CheckFunc, Timeout) -> false end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of [_ | _] = Status -> lists:all(fun(St) -> St =:= true end, Status); [] -> diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index 40f88a0dc..8844fe586 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -225,7 +225,7 @@ tcp_connectivity(Host, Port) -> ) -> ok | {error, Reason :: term()}. tcp_connectivity(Host, Port, Timeout) -> - case gen_tcp:connect(Host, Port, emqx_misc:ipv6_probe([]), Timeout) of + case gen_tcp:connect(Host, Port, emqx_utils:ipv6_probe([]), Timeout) of {ok, Sock} -> gen_tcp:close(Sock), ok; diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 60d52f58b..05d9508b6 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -144,7 +144,7 @@ terminate(_Reason, _State) -> ok. ensure_timer(Interval) -> - emqx_misc:start_timer(Interval, ?TIMER_MSG). + emqx_utils:start_timer(Interval, ?TIMER_MSG). %%-------------------------------------------------------------------- %% prometheus callbacks diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 0fa4c0bd8..331480064 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1535,7 +1535,7 @@ queue_count(Q) -> disk_queue_dir(Id, Index) -> QDir0 = binary_to_list(Id) ++ ":" ++ integer_to_list(Index), QDir = filename:join([emqx:data_dir(), "bufs", node(), QDir0]), - emqx_misc:safe_filename(QDir). + emqx_utils:safe_filename(QDir). clear_disk_queue_dir(Id, Index) -> ReplayQDir = disk_queue_dir(Id, Index), diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 9fe0fbdf2..7ecf56c18 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -539,7 +539,7 @@ stop_resource(#data{state = ResState, id = ResId} = Data) -> Data#data{status = stopped}. make_test_id() -> - RandId = iolist_to_binary(emqx_misc:gen_id(16)), + RandId = iolist_to_binary(emqx_utils:gen_id(16)), <>. handle_manually_health_check(From, Data) -> @@ -613,7 +613,7 @@ maybe_alarm(_Status, ResId, Error) -> HrError = case Error of undefined -> <<"Unknown reason">>; - _Else -> emqx_misc:readable_error_msg(Error) + _Else -> emqx_utils:readable_error_msg(Error) end, emqx_alarm:activate( ResId, diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 11013cdd3..7bfc8ee4e 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl index 454a65eb3..32cbba084 100644 --- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl +++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl @@ -91,7 +91,7 @@ worker() -> | ignore. start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 40158d342..a9f24ddcd 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -175,7 +175,7 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) -> flags = Flags, headers = #{ republish_by => RuleId, - properties => emqx_misc:pub_props_to_packet(PubProps) + properties => emqx_utils:pub_props_to_packet(PubProps) }, topic = Topic, payload = Payload, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index c63d6275f..d66f2c1c9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -343,7 +343,7 @@ param_path_id() -> {200, Result} end; '/rules'(post, #{body := Params0}) -> - case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of + case maps:get(<<"id">>, Params0, list_to_binary(emqx_utils:gen_id(8))) of <<>> -> {400, #{code => 'BAD_REQUEST', message => <<"empty rule id is not allowed">>}}; Id -> @@ -459,11 +459,11 @@ param_path_id() -> %%------------------------------------------------------------------------------ err_msg({RuleError, {_E, Reason, _S}}) -> - emqx_misc:readable_error_msg(encode_nested_error(RuleError, Reason)); + emqx_utils:readable_error_msg(encode_nested_error(RuleError, Reason)); err_msg({Reason, _Details}) -> - emqx_misc:readable_error_msg(Reason); + emqx_utils:readable_error_msg(Reason); err_msg(Msg) -> - emqx_misc:readable_error_msg(Msg). + emqx_utils:readable_error_msg(Msg). encode_nested_error(RuleError, Reason) when is_tuple(Reason) -> encode_nested_error(RuleError, element(1, Reason)); diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 944aa02f8..81435d9ac 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -643,10 +643,10 @@ map(Data) -> emqx_plugin_libs_rule:map(Data). bin2hexstr(Bin) when is_binary(Bin) -> - emqx_misc:bin_to_hexstr(Bin, upper). + emqx_utils:bin_to_hexstr(Bin, upper). hexstr2bin(Str) when is_binary(Str) -> - emqx_misc:hexstr_to_bin(Str). + emqx_utils:hexstr_to_bin(Str). %%------------------------------------------------------------------------------ %% NULL Funcs @@ -944,7 +944,7 @@ sha256(S) when is_binary(S) -> hash(sha256, S). hash(Type, Data) -> - emqx_misc:bin_to_hexstr(crypto:hash(Type, Data), lower). + emqx_utils:bin_to_hexstr(crypto:hash(Type, Data), lower). %%------------------------------------------------------------------------------ %% gzip Funcs diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index e97c45b35..455efe389 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -48,7 +48,7 @@ test(#{sql := Sql, context := Context}) -> end. test_rule(Sql, Select, Context, EventTopics) -> - RuleId = iolist_to_binary(["sql_tester:", emqx_misc:gen_id(16)]), + RuleId = iolist_to_binary(["sql_tester:", emqx_utils:gen_id(16)]), ok = emqx_rule_engine:maybe_add_metrics_for_rule(RuleId), Rule = #{ id => RuleId, diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 9d40c6857..412e0b685 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_statsd, [ {description, "EMQX Statsd"}, - {vsn, "5.0.7"}, + {vsn, "5.0.8"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, [ diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl index c2e1819ac..c5a7fc1c8 100644 --- a/apps/emqx_statsd/src/emqx_statsd.erl +++ b/apps/emqx_statsd/src/emqx_statsd.erl @@ -144,7 +144,7 @@ flush_interval(_FlushInterval, SampleInterval) -> SampleInterval. ensure_timer(State = #{sample_time_interval := SampleTimeInterval}) -> - State#{timer => emqx_misc:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}. + State#{timer => emqx_utils:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}. check_multicall_result({Results, []}) -> case diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx_utils/src/emqx_utils.erl similarity index 99% rename from apps/emqx/src/emqx_misc.erl rename to apps/emqx_utils/src/emqx_utils.erl index d0c7397c8..21dbd339d 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -14,14 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_misc). +-module(emqx_utils). -compile(inline). +%% [TODO] Cleanup so the instruction below is not necessary. -elvis([{elvis_style, god_modules, disable}]). --include("types.hrl"). --include("logger.hrl"). - -export([ merge_opts/2, maybe_apply/2, @@ -71,6 +69,8 @@ -export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]). +-type maybe(T) :: undefined | T. + -dialyzer({nowarn_function, [nolink_apply/2]}). -define(SHORT, 8). @@ -221,6 +221,7 @@ drain_down(Cnt, Acc) -> %% `ok': There is nothing out of the ordinary. %% `shutdown': Some numbers (message queue length hit the limit), %% hence shutdown for greater good (system stability). +%% [FIXME] cross-dependency on `emqx_types`. -spec check_oom(emqx_types:oom_policy()) -> ok | {shutdown, term()}. check_oom(Policy) -> check_oom(self(), Policy). @@ -279,6 +280,7 @@ proc_name(Mod, Id) -> list_to_atom(lists:concat([Mod, "_", Id])). %% Get Proc's Stats. +%% [FIXME] cross-dependency on `emqx_types`. -spec proc_stats() -> emqx_types:stats(). proc_stats() -> proc_stats(self()). diff --git a/apps/emqx_utils/test/emqx_utils_SUITE.erl b/apps/emqx_utils/test/emqx_utils_SUITE.erl index 09a8d085c..598f05a21 100644 --- a/apps/emqx_utils/test/emqx_utils_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,29 +20,188 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). -all() -> - emqx_common_test_helpers:all(?MODULE). +-define(SOCKOPTS, [ + binary, + {packet, raw}, + {reuseaddr, true}, + {backlog, 512}, + {nodelay, true} +]). -init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([emqx_conf, emqx_utils]), - Config. +all() -> emqx_common_test_helpers:all(?MODULE). -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_conf, emqx_utils]), - ok. +t_merge_opts(_) -> + Opts = emqx_utils:merge_opts(?SOCKOPTS, [ + raw, + binary, + {backlog, 1024}, + {nodelay, false}, + {max_clients, 1024}, + {acceptors, 16} + ]), + ?assertEqual(1024, proplists:get_value(backlog, Opts)), + ?assertEqual(1024, proplists:get_value(max_clients, Opts)), + ?assertEqual( + [ + binary, + raw, + {acceptors, 16}, + {backlog, 1024}, + {max_clients, 1024}, + {nodelay, false}, + {packet, raw}, + {reuseaddr, true} + ], + lists:sort(Opts) + ). -init_per_testcase(TestCase, Config) -> - emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config). +t_maybe_apply(_) -> + ?assertEqual(undefined, emqx_utils:maybe_apply(fun(A) -> A end, undefined)), + ?assertEqual(a, emqx_utils:maybe_apply(fun(A) -> A end, a)). -end_per_testcase(TestCase, Config) -> - emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config). +t_run_fold(_) -> + ?assertEqual(1, emqx_utils:run_fold([], 1, state)), + Add = fun(I, St) -> I + St end, + Mul = fun(I, St) -> I * St end, + ?assertEqual(6, emqx_utils:run_fold([Add, Mul], 1, 2)). -t_fail(init, Config) -> - Config; -t_fail('end', _Config) -> - ok. +t_pipeline(_) -> + ?assertEqual({ok, input, state}, emqx_utils:pipeline([], input, state)), + Funs = [ + fun(_I, _St) -> ok end, + fun(_I, St) -> {ok, St + 1} end, + fun(I, St) -> {ok, I + 1, St + 1} end, + fun(I, St) -> {ok, I * 2, St * 2} end + ], + ?assertEqual({ok, 4, 6}, emqx_utils:pipeline(Funs, 1, 1)), + ?assertEqual( + {error, undefined, 1}, emqx_utils:pipeline([fun(_I) -> {error, undefined} end], 1, 1) + ), + ?assertEqual( + {error, undefined, 2}, + emqx_utils:pipeline([fun(_I, _St) -> {error, undefined, 2} end], 1, 1) + ). -t_fail(_Config) -> - ?assert(false). +t_start_timer(_) -> + TRef = emqx_utils:start_timer(1, tmsg), + timer:sleep(2), + ?assertEqual([{timeout, TRef, tmsg}], drain()), + ok = emqx_utils:cancel_timer(TRef). + +t_cancel_timer(_) -> + Timer = emqx_utils:start_timer(0, foo), + ok = emqx_utils:cancel_timer(Timer), + ?assertEqual([], drain()), + ok = emqx_utils:cancel_timer(undefined). + +t_proc_name(_) -> + ?assertEqual(emqx_pool_1, emqx_utils:proc_name(emqx_pool, 1)). + +t_proc_stats(_) -> + Pid1 = spawn(fun() -> exit(normal) end), + timer:sleep(10), + ?assertEqual([], emqx_utils:proc_stats(Pid1)), + Pid2 = spawn(fun() -> + ?assertMatch([{mailbox_len, 0} | _], emqx_utils:proc_stats()), + timer:sleep(200) + end), + timer:sleep(10), + Pid2 ! msg, + timer:sleep(10), + ?assertMatch([{mailbox_len, 1} | _], emqx_utils:proc_stats(Pid2)). + +t_drain_deliver(_) -> + self() ! {deliver, t1, m1}, + self() ! {deliver, t2, m2}, + ?assertEqual( + [ + {deliver, t1, m1}, + {deliver, t2, m2} + ], + emqx_utils:drain_deliver(2) + ). + +t_drain_down(_) -> + {Pid1, _Ref1} = erlang:spawn_monitor(fun() -> ok end), + {Pid2, _Ref2} = erlang:spawn_monitor(fun() -> ok end), + timer:sleep(100), + ?assertEqual([Pid1, Pid2], lists:sort(emqx_utils:drain_down(2))), + ?assertEqual([], emqx_utils:drain_down(1)). + +t_index_of(_) -> + try emqx_utils:index_of(a, []) of + _ -> ct:fail(should_throw_error) + catch + error:Reason -> + ?assertEqual(badarg, Reason) + end, + ?assertEqual(3, emqx_utils:index_of(a, [b, c, a, e, f])). + +t_check(_) -> + Policy = #{ + max_message_queue_len => 10, + max_heap_size => 1024 * 1024 * 8, + enable => true + }, + [self() ! {msg, I} || I <- lists:seq(1, 5)], + ?assertEqual(ok, emqx_utils:check_oom(Policy)), + [self() ! {msg, I} || I <- lists:seq(1, 6)], + ?assertEqual({shutdown, message_queue_too_long}, emqx_utils:check_oom(Policy)). + +drain() -> + drain([]). + +drain(Acc) -> + receive + Msg -> drain([Msg | Acc]) + after 0 -> + lists:reverse(Acc) + end. + +t_rand_seed(_) -> + ?assert(is_tuple(emqx_utils:rand_seed())). + +t_now_to_secs(_) -> + ?assert(is_integer(emqx_utils:now_to_secs(os:timestamp()))). + +t_now_to_ms(_) -> + ?assert(is_integer(emqx_utils:now_to_ms(os:timestamp()))). + +t_gen_id(_) -> + ?assertEqual(10, length(emqx_utils:gen_id(10))), + ?assertEqual(20, length(emqx_utils:gen_id(20))). + +t_pmap_normal(_) -> + ?assertEqual( + [5, 7, 9], + emqx_utils:pmap( + fun({A, B}) -> A + B end, + [{2, 3}, {3, 4}, {4, 5}] + ) + ). + +t_pmap_timeout(_) -> + ?assertExit( + timeout, + emqx_utils:pmap( + fun + (timeout) -> ct:sleep(1000); + ({A, B}) -> A + B + end, + [{2, 3}, {3, 4}, timeout], + 100 + ) + ). + +t_pmap_exception(_) -> + ?assertError( + foobar, + emqx_utils:pmap( + fun + (error) -> error(foobar); + ({A, B}) -> A + B + end, + [{2, 3}, {3, 4}, error] + ) + ). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index e899b0fcd..133ef0430 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -272,7 +272,7 @@ t_setup_via_config_and_publish(Config) -> {ok, _}, create_bridge(Config) ), - MsgId = emqx_misc:gen_id(), + MsgId = emqx_utils:gen_id(), SentData = #{id => MsgId, payload => ?PAYLOAD}, ?check_trace( begin @@ -309,7 +309,7 @@ t_setup_via_http_api_and_publish(Config) -> {ok, _}, create_bridge_http(PgsqlConfig) ), - MsgId = emqx_misc:gen_id(), + MsgId = emqx_utils:gen_id(), SentData = #{id => MsgId, payload => ?PAYLOAD}, ?check_trace( begin @@ -375,7 +375,7 @@ t_write_failure(Config) -> #{?snk_kind := resource_connected_enter}, 20_000 ), - SentData = #{id => emqx_misc:gen_id(), payload => ?PAYLOAD}, + SentData = #{id => emqx_utils:gen_id(), payload => ?PAYLOAD}, emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> ?assertMatch( {error, {resource_error, #{reason := timeout}}}, send_message(Config, SentData) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 3cc4c75e4..49bc444c6 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -809,7 +809,7 @@ test_publish_success_batch(Config) -> %% making 1-sized batches. also important to note that the pool %% size for the resource (replayq buffering) must be set to 1 to %% avoid further segmentation of batches. - emqx_misc:pmap(fun emqx:publish/1, Messages), + emqx_utils:pmap(fun emqx:publish/1, Messages), DecodedMessages0 = assert_http_request(ServiceAccountJSON), ?assertEqual(BatchSize, length(DecodedMessages0)), DecodedMessages1 = assert_http_request(ServiceAccountJSON), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl index 69b1e26ce..be07a2bb7 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl @@ -615,7 +615,7 @@ t_workload_fits_prepared_statement_limit(Config) -> create_bridge(Config) ), Results = lists:append( - emqx_misc:pmap( + emqx_utils:pmap( fun(_) -> [ begin diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index ce964dd17..86b908038 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -102,7 +102,7 @@ on_start( ?SLOG(info, #{ msg => "starting_cassandra_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), Options = [ diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl index 3749ecc6e..2872e8cf0 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl @@ -139,7 +139,7 @@ on_start( ?SLOG(info, #{ msg => "starting_clickhouse_connector", connector => InstanceID, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), PoolName = emqx_plugin_libs_pool:pool_name(InstanceID), Options = [ @@ -181,7 +181,7 @@ log_start_error(Config, Reason, Stacktrace) -> #{ msg => "clickhouse_connector_start_failed", error_reason => Reason, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }, ?SLOG(info, maps:merge(LogMessage, StacktraceMap)), ?tp( @@ -318,7 +318,7 @@ do_get_status(PoolName, Timeout) -> Error end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of Results -> case [E || {error, _} = E <- Results] of [] -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index ee56e0c5f..85daefbb7 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -95,7 +95,7 @@ on_start( ?SLOG(info, #{ msg => "starting_dynamo_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), {Schema, Server} = get_host_schema(to_str(Url)), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl index a2045b8c5..2295f63ab 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl @@ -86,7 +86,7 @@ on_start( PoolType = random, Transport = tls, TransportOpts = emqx_tls_lib:to_client_opts(#{enable => true, verify => verify_none}), - NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + NTransportOpts = emqx_utils:ipv6_probe(TransportOpts), PoolOpts = [ {host, Host}, {port, Port}, @@ -587,7 +587,7 @@ do_get_status(InstanceId, PoolName, Timeout) -> false end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of [_ | _] = Status -> lists:all(fun(St) -> St =:= true end, Status); [] -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index afe17cae6..700eb2a81 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -216,8 +216,8 @@ start_client(InstId, Config) -> ?SLOG(info, #{ msg => "starting influxdb connector", connector => InstId, - config => emqx_misc:redact(Config), - client_config => emqx_misc:redact(ClientConfig) + config => emqx_utils:redact(Config), + client_config => emqx_utils:redact(ClientConfig) }), try do_start_client(InstId, ClientConfig, Config) @@ -353,7 +353,7 @@ password(_) -> []. redact_auth(Term) -> - emqx_misc:redact(Term, fun is_auth_key/1). + emqx_utils:redact(Term, fun is_auth_key/1). is_auth_key(Key) when is_binary(Key) -> string:equal("authorization", Key, true); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index d296b0b4f..205359bb8 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -265,7 +265,7 @@ client_id(InstanceId) -> erlang:binary_to_atom(Name, utf8). redact(Msg) -> - emqx_misc:redact(Msg, fun is_sensitive_key/1). + emqx_utils:redact(Msg, fun is_sensitive_key/1). is_sensitive_key(security_token) -> true; diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl index 746ab5485..9b6718882 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl @@ -93,7 +93,7 @@ on_start( ?SLOG(info, #{ msg => "starting_tdengine_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), {Host, Port} = emqx_schema:parse_server(Server, ?TD_HOST_OPTIONS), diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 3c26d1966..43ccd9fc1 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -218,7 +218,7 @@ start_cluster(Cluster) -> || {Name, Opts} <- Cluster ], on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/lib-ee/emqx_license/src/emqx_license_http_api.erl index 0133cd637..ef0e1ee52 100644 --- a/lib-ee/emqx_license/src/emqx_license_http_api.erl +++ b/lib-ee/emqx_license/src/emqx_license_http_api.erl @@ -95,7 +95,7 @@ sample_license_info_response() -> }. error_msg(Code, Msg) -> - #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. %% read license info '/license'(get, _Params) -> From d0df086c80bce1c9aa9c3d2ad4de0e98f947cd09 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 14:50:30 +0200 Subject: [PATCH 04/32] refactor: rename emqx_api_lib to emqx_utils_api --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl | 6 +++--- apps/emqx_management/src/emqx_mgmt_api_nodes.erl | 6 +++--- .../include/emqx_utils_api.hrl} | 0 apps/emqx_utils/src/emqx_utils.app.src | 1 + .../src/emqx_utils_api.erl} | 4 ++-- .../test/emqx_utils_api_SUITE.erl} | 10 +++++----- .../src/emqx_ee_schema_registry_http_api.erl | 2 +- 8 files changed, 16 insertions(+), 15 deletions(-) rename apps/{emqx/include/emqx_api_lib.hrl => emqx_utils/include/emqx_utils_api.hrl} (100%) rename apps/{emqx/src/emqx_api_lib.erl => emqx_utils/src/emqx_utils_api.erl} (97%) rename apps/{emqx/test/emqx_api_lib_SUITE.erl => emqx_utils/test/emqx_utils_api_SUITE.erl} (92%) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 814fcdeb7..a1841b705 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -20,7 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_api_lib.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). -import(hoconsc, [mk/2, array/1, enum/1]). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index c2dd4a9dd..c0e162b62 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -122,7 +122,7 @@ fields(sampler_current) -> monitor(get, #{query_string := QS, bindings := Bindings}) -> Latest = maps:get(<<"latest">>, QS, infinity), RawNode = maps:get(node, Bindings, <<"all">>), - emqx_api_lib:with_node_or_cluster(RawNode, dashboard_samplers_fun(Latest)). + emqx_utils_api:with_node_or_cluster(RawNode, dashboard_samplers_fun(Latest)). dashboard_samplers_fun(Latest) -> fun(NodeOrCluster) -> @@ -133,10 +133,10 @@ dashboard_samplers_fun(Latest) -> end. monitor_current(get, #{bindings := []}) -> - emqx_api_lib:with_node_or_cluster(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); + emqx_utils_api:with_node_or_cluster(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); monitor_current(get, #{bindings := Bindings}) -> RawNode = maps:get(node, Bindings, <<"all">>), - emqx_api_lib:with_node_or_cluster(RawNode, fun current_rate/1). + emqx_utils_api:with_node_or_cluster(RawNode, fun current_rate/1). current_rate(Node) -> case emqx_dashboard_monitor:current_rate(Node) of diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index a4173f5b0..ecf465f43 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -247,13 +247,13 @@ nodes(get, _Params) -> list_nodes(#{}). node(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun get_node/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun get_node/1)). node_metrics(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_metrics/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_metrics/1)). node_stats(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_stats/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_stats/1)). %%-------------------------------------------------------------------- %% api apply diff --git a/apps/emqx/include/emqx_api_lib.hrl b/apps/emqx_utils/include/emqx_utils_api.hrl similarity index 100% rename from apps/emqx/include/emqx_api_lib.hrl rename to apps/emqx_utils/include/emqx_utils_api.hrl diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index e1f10c139..08a802d1b 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -5,6 +5,7 @@ {vsn, "5.0.0"}, {modules, [ emqx_utils, + emqx_utils_api, emqx_utils_binary, emqx_utils_ets, emqx_utils_json, diff --git a/apps/emqx/src/emqx_api_lib.erl b/apps/emqx_utils/src/emqx_utils_api.erl similarity index 97% rename from apps/emqx/src/emqx_api_lib.erl rename to apps/emqx_utils/src/emqx_utils_api.erl index 5eddf6188..e6bd07272 100644 --- a/apps/emqx/src/emqx_api_lib.erl +++ b/apps/emqx_utils/src/emqx_utils_api.erl @@ -14,14 +14,14 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_api_lib). +-module(emqx_utils_api). -export([ with_node/2, with_node_or_cluster/2 ]). --include("emqx_api_lib.hrl"). +-include("emqx_utils_api.hrl"). -define(NODE_NOT_FOUND(NODE), ?NOT_FOUND(<<"Node not found: ", NODE/binary>>)). diff --git a/apps/emqx/test/emqx_api_lib_SUITE.erl b/apps/emqx_utils/test/emqx_utils_api_SUITE.erl similarity index 92% rename from apps/emqx/test/emqx_api_lib_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_api_SUITE.erl index 29f5c6095..3ed3cd250 100644 --- a/apps/emqx/test/emqx_api_lib_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_api_SUITE.erl @@ -14,12 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_api_lib_SUITE). +-module(emqx_utils_api_SUITE). -compile(export_all). -compile(nowarn_export_all). --include("emqx_api_lib.hrl"). +-include("emqx_utils_api.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(DUMMY, dummy_module). @@ -45,14 +45,14 @@ end_per_testcase(_Case, _Config) -> meck:unload(?DUMMY). t_with_node(_) -> - test_with(fun emqx_api_lib:with_node/2, [<<"all">>]). + test_with(fun emqx_utils_api:with_node/2, [<<"all">>]). t_with_node_or_cluster(_) -> - test_with(fun emqx_api_lib:with_node_or_cluster/2, []), + test_with(fun emqx_utils_api:with_node_or_cluster/2, []), meck:reset(?DUMMY), ?assertEqual( ?OK(success), - emqx_api_lib:with_node_or_cluster( + emqx_utils_api:with_node_or_cluster( <<"all">>, fun ?DUMMY:expect_success/1 ) diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl index 897d29e07..c7cc6c99a 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl @@ -8,7 +8,7 @@ -include("emqx_ee_schema_registry.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_api_lib.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). -export([ namespace/0, From 6e8665365baa952dff6161bd66290dd53c1bbaa8 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 14:57:45 +0200 Subject: [PATCH 05/32] refactor: rename emqx_tables to emqx_utils_ets --- apps/emqx/src/emqx_broker.erl | 8 ++++---- apps/emqx/src/emqx_broker_helper.erl | 10 +++++----- apps/emqx/src/emqx_cm.erl | 8 ++++---- apps/emqx/src/emqx_flapping.erl | 2 +- apps/emqx/src/emqx_hooks.erl | 2 +- apps/emqx/src/emqx_metrics.erl | 2 +- apps/emqx/src/emqx_ocsp_cache.erl | 2 +- apps/emqx/src/emqx_sequence.erl | 2 +- apps/emqx/src/emqx_session_router.erl | 10 +++++----- apps/emqx/src/emqx_shared_sub.erl | 8 +++++--- apps/emqx/src/emqx_stats.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_cm.erl | 6 +++--- apps/emqx_gateway/src/emqx_gateway_metrics.erl | 2 +- apps/emqx_modules/src/emqx_topic_metrics.erl | 2 +- .../src/emqx_utils_ets.erl} | 2 +- .../test/emqx_utils_ets_SUITE.erl} | 18 +++++++++--------- .../test/emqx_ee_schema_registry_SUITE.erl | 2 +- 17 files changed, 45 insertions(+), 43 deletions(-) rename apps/{emqx/src/emqx_tables.erl => emqx_utils/src/emqx_utils_ets.erl} (98%) rename apps/{emqx/test/emqx_tables_SUITE.erl => emqx_utils/test/emqx_utils_ets_SUITE.erl} (73%) diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index 7e00c050e..5f7c4aaf5 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -71,7 +71,7 @@ code_change/3 ]). --import(emqx_tables, [lookup_value/2, lookup_value/3]). +-import(emqx_utils_ets, [lookup_value/2, lookup_value/3]). -ifdef(TEST). -compile(export_all). @@ -107,15 +107,15 @@ create_tabs() -> TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}], %% SubOption: {Topic, SubPid} -> SubOption - ok = emqx_tables:new(?SUBOPTION, [ordered_set | TabOpts]), + ok = emqx_utils_ets:new(?SUBOPTION, [ordered_set | TabOpts]), %% Subscription: SubPid -> Topic1, Topic2, Topic3, ... %% duplicate_bag: o(1) insert - ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), + ok = emqx_utils_ets:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... %% bag: o(n) insert:( - ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). + ok = emqx_utils_ets:new(?SUBSCRIBER, [bag | TabOpts]). %%------------------------------------------------------------------------------ %% Subscribe API diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl index 87b291361..06f249678 100644 --- a/apps/emqx/src/emqx_broker_helper.erl +++ b/apps/emqx/src/emqx_broker_helper.erl @@ -73,11 +73,11 @@ register_sub(SubPid, SubId) when is_pid(SubPid) -> -spec lookup_subid(pid()) -> maybe(emqx_types:subid()). lookup_subid(SubPid) when is_pid(SubPid) -> - emqx_tables:lookup_value(?SUBMON, SubPid). + emqx_utils_ets:lookup_value(?SUBMON, SubPid). -spec lookup_subpid(emqx_types:subid()) -> maybe(pid()). lookup_subpid(SubId) -> - emqx_tables:lookup_value(?SUBID, SubId). + emqx_utils_ets:lookup_value(?SUBID, SubId). -spec get_sub_shard(pid(), emqx_types:topic()) -> non_neg_integer(). get_sub_shard(SubPid, Topic) -> @@ -105,15 +105,15 @@ reclaim_seq(Topic) -> init([]) -> %% Helper table - ok = emqx_tables:new(?HELPER, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?HELPER, [{read_concurrency, true}]), %% Shards: CPU * 32 true = ets:insert(?HELPER, {shards, emqx_vm:schedulers() * 32}), %% SubSeq: Topic -> SeqId ok = emqx_sequence:create(?SUBSEQ), %% SubId: SubId -> SubPid - ok = emqx_tables:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), %% SubMon: SubPid -> SubId - ok = emqx_tables:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), %% Stats timer ok = emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), {ok, #{pmon => emqx_pmon:new()}}. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 346e8dcb5..0290b57d3 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -651,10 +651,10 @@ cast(Msg) -> gen_server:cast(?CM, Msg). init([]) -> TabOpts = [public, {write_concurrency, true}], - ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]), - ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]), - ok = emqx_tables:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]), - ok = emqx_tables:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_CONN_TAB, [bag | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]), ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0), State = #{chan_pmon => emqx_pmon:new()}, {ok, State}. diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 2cfed0a8e..46193fb16 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -99,7 +99,7 @@ now_diff(TS) -> erlang:system_time(millisecond) - TS. %%-------------------------------------------------------------------- init([]) -> - ok = emqx_tables:new(?FLAPPING_TAB, [ + ok = emqx_utils_ets:new(?FLAPPING_TAB, [ public, set, {keypos, #flapping.clientid}, diff --git a/apps/emqx/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl index 1784d8ea3..0b8dc0941 100644 --- a/apps/emqx/src/emqx_hooks.erl +++ b/apps/emqx/src/emqx_hooks.erl @@ -229,7 +229,7 @@ lookup(HookPoint) -> %%-------------------------------------------------------------------- init([]) -> - ok = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), {ok, #{}}. handle_call({add, HookPoint, Callback = #callback{action = {M, F, _}}}, _From, State) -> diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index c2e297623..21a114c0f 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -541,7 +541,7 @@ init([]) -> CRef = counters:new(?MAX_SIZE, [write_concurrency]), ok = persistent_term:put(?MODULE, CRef), % Create index mapping table - ok = emqx_tables:new(?TAB, [{keypos, 2}, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{keypos, 2}, {read_concurrency, true}]), Metrics = lists:append([ ?BYTES_METRICS, ?PACKET_METRICS, diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index 7044d992e..ccf3e0c40 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -120,7 +120,7 @@ inject_sni_fun(ListenerID, Conf0) -> init(_Args) -> logger:set_process_metadata(#{domain => [emqx, ocsp, cache]}), - emqx_tables:new(?CACHE_TAB, [ + emqx_utils_ets:new(?CACHE_TAB, [ named_table, public, {heir, whereis(emqx_kernel_sup), none}, diff --git a/apps/emqx/src/emqx_sequence.erl b/apps/emqx/src/emqx_sequence.erl index 60596324a..7acc87256 100644 --- a/apps/emqx/src/emqx_sequence.erl +++ b/apps/emqx/src/emqx_sequence.erl @@ -39,7 +39,7 @@ %% @doc Create a sequence. -spec create(name()) -> ok. create(Name) -> - emqx_tables:new(Name, [public, set, {write_concurrency, true}]). + emqx_utils_ets:new(Name, [public, set, {write_concurrency, true}]). %% @doc Next value of the sequence. -spec nextval(name(), key()) -> seqid(). diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl index 8a21d4d03..0435ddca3 100644 --- a/apps/emqx/src/emqx_session_router.erl +++ b/apps/emqx/src/emqx_session_router.erl @@ -95,7 +95,7 @@ create_table(Tab, Storage) -> %%-------------------------------------------------------------------- create_init_tab() -> - emqx_tables:new(?SESSION_INIT_TAB, [ + emqx_utils_ets:new(?SESSION_INIT_TAB, [ public, {read_concurrency, true}, {write_concurrency, true} @@ -182,7 +182,7 @@ pending(SessionID, MarkerIDs) -> call(pick(SessionID), {pending, SessionID, MarkerIDs}). buffer(SessionID, STopic, Msg) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> ok; Worker -> emqx_session_router_worker:buffer(Worker, STopic, Msg) end. @@ -194,7 +194,7 @@ resume_begin(From, SessionID) when is_pid(From), is_binary(SessionID) -> -spec resume_end(pid(), binary()) -> {'ok', [emqx_types:message()]} | {'error', term()}. resume_end(From, SessionID) when is_pid(From), is_binary(SessionID) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> ?tp(ps_session_not_found, #{sid => SessionID}), {error, not_found}; @@ -249,7 +249,7 @@ handle_cast({delete_routes, SessionID, Subscriptions}, State) -> ok = lists:foreach(Fun, maps:to_list(Subscriptions)), {noreply, State}; handle_cast({resume_end, SessionID, Pid}, State) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> skip; P when P =:= Pid -> ets:delete(?SESSION_INIT_TAB, SessionID); P when is_pid(P) -> skip @@ -283,7 +283,7 @@ init_resume_worker(RemotePid, SessionID, #{pmon := Pmon} = State) -> error; {ok, Pid} -> Pmon1 = emqx_pmon:monitor(Pid, Pmon), - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> {ok, Pid, State#{pmon => Pmon1}}; {_, OldPid} -> diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 061f2a42f..d7dc8c5a6 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -399,9 +399,11 @@ init([]) -> ok = mria:wait_for_tables([?TAB]), {ok, _} = mnesia:subscribe({table, ?TAB, simple}), {atomic, PMon} = mria:transaction(?SHARED_SUB_SHARD, fun ?MODULE:init_monitors/0), - ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), - ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), - ok = emqx_tables:new(?SHARED_SUBS_ROUND_ROBIN_COUNTER, [public, set, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SHARED_SUBS, [protected, bag]), + ok = emqx_utils_ets:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?SHARED_SUBS_ROUND_ROBIN_COUNTER, [ + public, set, {write_concurrency, true} + ]), {ok, update_stats(#state{pmon = PMon})}. init_monitors() -> diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index 017b83116..ef9109e33 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -201,7 +201,7 @@ cast(Msg) -> gen_server:cast(?SERVER, Msg). %%-------------------------------------------------------------------- init(#{tick_ms := TickMs}) -> - ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [public, set, {write_concurrency, true}]), Stats = lists:append([ ?CONNECTION_STATS, ?CHANNEL_STATS, diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 5de6da63f..837600811 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -766,9 +766,9 @@ init(Options) -> TabOpts = [public, {write_concurrency, true}], {ChanTab, ConnTab, InfoTab} = cmtabs(GwName), - ok = emqx_tables:new(ChanTab, [bag, {read_concurrency, true} | TabOpts]), - ok = emqx_tables:new(ConnTab, [bag | TabOpts]), - ok = emqx_tables:new(InfoTab, [ordered_set, compressed | TabOpts]), + ok = emqx_utils_ets:new(ChanTab, [bag, {read_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(ConnTab, [bag | TabOpts]), + ok = emqx_utils_ets:new(InfoTab, [ordered_set, compressed | TabOpts]), %% Start link cm-registry process %% XXX: Should I hang it under a higher level supervisor? diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl index e94510387..0aa2ff210 100644 --- a/apps/emqx_gateway/src/emqx_gateway_metrics.erl +++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl @@ -89,7 +89,7 @@ tabname(GwName) -> init([GwName]) -> TabOpts = [public, {write_concurrency, true}], - ok = emqx_tables:new(tabname(GwName), [set | TabOpts]), + ok = emqx_utils_ets:new(tabname(GwName), [set | TabOpts]), {ok, #state{}}. handle_call(_Request, _From, State) -> diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl index dfc6b07ab..de09e568f 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics.erl @@ -201,7 +201,7 @@ reset() -> init([Opts]) -> erlang:process_flag(trap_exit, true), - ok = emqx_tables:new(?TAB, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{read_concurrency, true}]), erlang:send_after(timer:seconds(?TICKING_INTERVAL), self(), ticking), Fun = fun(#{topic := Topic}, CurrentSpeeds) -> diff --git a/apps/emqx/src/emqx_tables.erl b/apps/emqx_utils/src/emqx_utils_ets.erl similarity index 98% rename from apps/emqx/src/emqx_tables.erl rename to apps/emqx_utils/src/emqx_utils_ets.erl index ffdf7d891..099152675 100644 --- a/apps/emqx/src/emqx_tables.erl +++ b/apps/emqx_utils/src/emqx_utils_ets.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_tables). +-module(emqx_utils_ets). -export([ new/1, diff --git a/apps/emqx/test/emqx_tables_SUITE.erl b/apps/emqx_utils/test/emqx_utils_ets_SUITE.erl similarity index 73% rename from apps/emqx/test/emqx_tables_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_ets_SUITE.erl index ad53e7139..13bf427fd 100644 --- a/apps/emqx/test/emqx_tables_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_ets_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_tables_SUITE). +-module(emqx_utils_ets_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -26,19 +26,19 @@ all() -> emqx_common_test_helpers:all(?MODULE). t_new(_) -> - ok = emqx_tables:new(?TAB), - ok = emqx_tables:new(?TAB, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB), + ok = emqx_utils_ets:new(?TAB, [{read_concurrency, true}]), ?assertEqual(?TAB, ets:info(?TAB, name)). t_lookup_value(_) -> - ok = emqx_tables:new(?TAB, []), + ok = emqx_utils_ets:new(?TAB, []), true = ets:insert(?TAB, {key, val}), - ?assertEqual(val, emqx_tables:lookup_value(?TAB, key)), - ?assertEqual(undefined, emqx_tables:lookup_value(?TAB, badkey)). + ?assertEqual(val, emqx_utils_ets:lookup_value(?TAB, key)), + ?assertEqual(undefined, emqx_utils_ets:lookup_value(?TAB, badkey)). t_delete(_) -> - ok = emqx_tables:new(?TAB, []), + ok = emqx_utils_ets:new(?TAB, []), ?assertEqual(?TAB, ets:info(?TAB, name)), - ok = emqx_tables:delete(?TAB), - ok = emqx_tables:delete(?TAB), + ok = emqx_utils_ets:delete(?TAB), + ok = emqx_utils_ets:delete(?TAB), ?assertEqual(undefined, ets:info(?TAB, name)). diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 43ccd9fc1..325d3d499 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -68,7 +68,7 @@ trace_rule(Data, Envs, _Args) -> make_trace_fn_action() -> persistent_term:put({?MODULE, test_pid}, self()), Fn = <<(atom_to_binary(?MODULE))/binary, ":trace_rule">>, - emqx_tables:new(recorded_actions, [named_table, public, ordered_set]), + emqx_utils_ets:new(recorded_actions, [named_table, public, ordered_set]), #{function => Fn, args => #{}}. create_rule_http(RuleParams) -> From 062ce5f81991cbd0103a081815bd67c64ceed452 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 15:02:41 +0200 Subject: [PATCH 06/32] refactor: rename emqx_map_lib to emqx_utils_maps --- apps/emqx/src/emqx.erl | 18 +++--- apps/emqx/src/emqx_config.erl | 64 +++++++++---------- apps/emqx/src/emqx_config_handler.erl | 18 +++--- .../emqx_limiter/src/emqx_limiter_server.erl | 2 +- apps/emqx/src/emqx_listeners.erl | 6 +- apps/emqx/src/emqx_ocsp_cache.erl | 4 +- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx/src/emqx_tls_lib.erl | 20 +++--- apps/emqx/test/emqx_crl_cache_SUITE.erl | 4 +- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 22 ++++--- apps/emqx/test/emqx_tls_lib_tests.erl | 6 +- apps/emqx_authn/src/emqx_authn_api.erl | 4 +- .../emqx_authz/src/emqx_authz_api_sources.erl | 2 +- apps/emqx_authz/src/emqx_authz_schema.erl | 2 +- apps/emqx_bridge/src/emqx_bridge.erl | 4 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 4 +- .../src/schema/emqx_bridge_schema.erl | 2 +- .../test/emqx_bridge_api_SUITE.erl | 2 +- .../test/emqx_bridge_mqtt_SUITE.erl | 2 +- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 6 +- .../test/emqx_bridge_kafka_tests.erl | 4 +- apps/emqx_conf/src/emqx_conf.erl | 28 ++++---- .../src/proto/emqx_conf_proto_v1.erl | 8 +-- .../src/proto/emqx_conf_proto_v2.erl | 8 +-- .../emqx_connector/src/emqx_connector_ssl.erl | 2 +- .../src/mqtt/emqx_connector_mqtt_worker.erl | 2 +- .../src/emqx_dashboard_listener.erl | 11 ++-- .../src/emqx_dashboard_swagger.erl | 2 +- apps/emqx_exhook/src/emqx_exhook_mgr.erl | 8 +-- .../test/emqx_exhook_api_SUITE.erl | 2 +- .../src/emqx_gateway_api_listeners.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_conf.erl | 60 ++++++++--------- apps/emqx_gateway/src/emqx_gateway_http.erl | 4 +- .../test/emqx_gateway_api_SUITE.erl | 12 ++-- .../test/emqx_gateway_authn_SUITE.erl | 2 +- .../test/emqx_gateway_conf_SUITE.erl | 6 +- .../test/emqx_gateway_test_utils.erl | 2 +- .../src/emqx_gateway_exproto.erl | 4 +- .../src/emqx_mgmt_api_clients.erl | 6 +- .../src/emqx_mgmt_api_configs.erl | 4 +- .../src/emqx_mgmt_api_listeners.erl | 2 +- .../test/emqx_mgmt_api_configs_SUITE.erl | 38 ++++++----- .../test/emqx_mgmt_api_listeners_SUITE.erl | 6 +- .../test/emqx_mgmt_api_nodes_SUITE.erl | 6 +- .../test/emqx_delayed_api_SUITE.erl | 2 +- .../src/emqx_retainer_dispatcher.erl | 2 +- .../test/emqx_retainer_api_SUITE.erl | 4 +- .../emqx_rule_engine/src/emqx_rule_engine.erl | 2 +- .../test/emqx_slow_subs_api_SUITE.erl | 2 +- .../src/emqx_utils_maps.erl} | 4 +- .../test/emqx_utils_maps_tests.erl} | 32 +++++----- .../test/emqx_ee_bridge_cassa_SUITE.erl | 2 +- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 2 +- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 4 +- .../test/emqx_ee_bridge_influxdb_SUITE.erl | 4 +- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 2 +- .../test/emqx_ee_bridge_pgsql_SUITE.erl | 2 +- .../test/emqx_ee_bridge_tdengine_SUITE.erl | 2 +- .../src/emqx_ee_schema_registry.erl | 4 +- .../test/emqx_ee_schema_registry_SUITE.erl | 2 +- 61 files changed, 256 insertions(+), 241 deletions(-) rename apps/{emqx/src/emqx_map_lib.erl => emqx_utils/src/emqx_utils_maps.erl} (99%) rename apps/{emqx/test/emqx_map_lib_tests.erl => emqx_utils/test/emqx_utils_maps_tests.erl} (76%) diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 6e4aa9922..ef870685a 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -164,29 +164,29 @@ run_hook(HookPoint, Args) -> run_fold_hook(HookPoint, Args, Acc) -> emqx_hooks:run_fold(HookPoint, Args, Acc). --spec get_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_config(emqx_utils_maps:config_key_path()) -> term(). get_config(KeyPath) -> emqx_config:get(KeyPath). --spec get_config(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_config(KeyPath, Default) -> emqx_config:get(KeyPath, Default). --spec get_raw_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw_config(emqx_utils_maps:config_key_path()) -> term(). get_raw_config(KeyPath) -> emqx_config:get_raw(KeyPath). --spec get_raw_config(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw_config(KeyPath, Default) -> emqx_config:get_raw(KeyPath, Default). --spec update_config(emqx_map_lib:config_key_path(), emqx_config:update_request()) -> +-spec update_config(emqx_utils_maps:config_key_path(), emqx_config:update_request()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_config(KeyPath, UpdateReq) -> update_config(KeyPath, UpdateReq, #{}). -spec update_config( - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -198,12 +198,12 @@ update_config([RootName | _] = KeyPath, UpdateReq, Opts) -> {{update, UpdateReq}, Opts} ). --spec remove_config(emqx_map_lib:config_key_path()) -> +-spec remove_config(emqx_utils_maps:config_key_path()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove_config(KeyPath) -> remove_config(KeyPath, #{}). --spec remove_config(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove_config([RootName | _] = KeyPath, Opts) -> emqx_config_handler:update_config( @@ -212,7 +212,7 @@ remove_config([RootName | _] = KeyPath, Opts) -> {remove, Opts} ). --spec reset_config(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset_config([RootName | _] = KeyPath, Opts) -> case emqx_config:get_default_value(KeyPath) of diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index bf3134568..b7effa240 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -142,7 +142,7 @@ -type app_envs() :: [proplists:property()]. %% @doc For the given path, get root value enclosed in a single-key map. --spec get_root(emqx_map_lib:config_key_path()) -> map(). +-spec get_root(emqx_utils_maps:config_key_path()) -> map(). get_root([RootName | _]) -> #{RootName => do_get(?CONF, [RootName], #{})}. @@ -153,14 +153,14 @@ get_root_raw([RootName | _]) -> %% @doc Get a config value for the given path. %% The path should at least include root config name. --spec get(emqx_map_lib:config_key_path()) -> term(). +-spec get(emqx_utils_maps:config_key_path()) -> term(). get(KeyPath) -> do_get(?CONF, KeyPath). --spec get(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get(emqx_utils_maps:config_key_path(), term()) -> term(). get(KeyPath, Default) -> do_get(?CONF, KeyPath, Default). --spec find(emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find(emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find([]) -> Ref = make_ref(), case do_get(?CONF, [], Ref) of @@ -170,12 +170,12 @@ find([]) -> find(KeyPath) -> atom_conf_path( KeyPath, - fun(AtomKeyPath) -> emqx_map_lib:deep_find(AtomKeyPath, get_root(KeyPath)) end, + fun(AtomKeyPath) -> emqx_utils_maps:deep_find(AtomKeyPath, get_root(KeyPath)) end, {return, {not_found, KeyPath}} ). --spec find_raw(emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find_raw(emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find_raw([]) -> Ref = make_ref(), case do_get_raw([], Ref) of @@ -183,9 +183,9 @@ find_raw([]) -> Res -> {ok, Res} end; find_raw(KeyPath) -> - emqx_map_lib:deep_find([bin(Key) || Key <- KeyPath], get_root_raw(KeyPath)). + emqx_utils_maps:deep_find([bin(Key) || Key <- KeyPath], get_root_raw(KeyPath)). --spec get_zone_conf(atom(), emqx_map_lib:config_key_path()) -> term(). +-spec get_zone_conf(atom(), emqx_utils_maps:config_key_path()) -> term(). get_zone_conf(Zone, KeyPath) -> case find(?ZONE_CONF_PATH(Zone, KeyPath)) of %% not found in zones, try to find the global config @@ -195,7 +195,7 @@ get_zone_conf(Zone, KeyPath) -> Value end. --spec get_zone_conf(atom(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_zone_conf(atom(), emqx_utils_maps:config_key_path(), term()) -> term(). get_zone_conf(Zone, KeyPath, Default) -> case find(?ZONE_CONF_PATH(Zone, KeyPath)) of %% not found in zones, try to find the global config @@ -205,24 +205,24 @@ get_zone_conf(Zone, KeyPath, Default) -> Value end. --spec put_zone_conf(atom(), emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_zone_conf(atom(), emqx_utils_maps:config_key_path(), term()) -> ok. put_zone_conf(Zone, KeyPath, Conf) -> ?MODULE:put(?ZONE_CONF_PATH(Zone, KeyPath), Conf). --spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> term(). +-spec get_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path()) -> term(). get_listener_conf(Type, Listener, KeyPath) -> ?MODULE:get(?LISTENER_CONF_PATH(Type, Listener, KeyPath)). --spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path(), term()) -> term(). get_listener_conf(Type, Listener, KeyPath, Default) -> ?MODULE:get(?LISTENER_CONF_PATH(Type, Listener, KeyPath), Default). --spec put_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path(), term()) -> ok. put_listener_conf(Type, Listener, KeyPath, Conf) -> ?MODULE:put(?LISTENER_CONF_PATH(Type, Listener, KeyPath), Conf). --spec find_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find_listener_conf(Type, Listener, KeyPath) -> find(?LISTENER_CONF_PATH(Type, Listener, KeyPath)). @@ -241,20 +241,20 @@ erase(RootName) -> persistent_term:erase(?PERSIS_KEY(?RAW_CONF, bin(RootName))), ok. --spec put(emqx_map_lib:config_key_path(), term()) -> ok. +-spec put(emqx_utils_maps:config_key_path(), term()) -> ok. put(KeyPath, Config) -> Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_put(Path, Map, Value) + emqx_utils_maps:deep_put(Path, Map, Value) end, do_put(?CONF, Putter, KeyPath, Config). %% Puts value into configuration even if path doesn't exist %% For paths of non-existing atoms use force_put(KeyPath, Config, unsafe) --spec force_put(emqx_map_lib:config_key_path(), term()) -> ok. +-spec force_put(emqx_utils_maps:config_key_path(), term()) -> ok. force_put(KeyPath, Config) -> force_put(KeyPath, Config, safe). --spec force_put(emqx_map_lib:config_key_path(), term(), safe | unsafe) -> ok. +-spec force_put(emqx_utils_maps:config_key_path(), term(), safe | unsafe) -> ok. force_put(KeyPath0, Config, Safety) -> KeyPath = case Safety of @@ -262,19 +262,19 @@ force_put(KeyPath0, Config, Safety) -> unsafe -> [unsafe_atom(Key) || Key <- KeyPath0] end, Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_force_put(Path, Map, Value) + emqx_utils_maps:deep_force_put(Path, Map, Value) end, do_put(?CONF, Putter, KeyPath, Config). --spec get_default_value(emqx_map_lib:config_key_path()) -> {ok, term()} | {error, term()}. +-spec get_default_value(emqx_utils_maps:config_key_path()) -> {ok, term()} | {error, term()}. get_default_value([RootName | _] = KeyPath) -> BinKeyPath = [bin(Key) || Key <- KeyPath], case find_raw([RootName]) of {ok, RawConf} -> - RawConf1 = emqx_map_lib:deep_remove(BinKeyPath, #{bin(RootName) => RawConf}), + RawConf1 = emqx_utils_maps:deep_remove(BinKeyPath, #{bin(RootName) => RawConf}), try fill_defaults(get_schema_mod(RootName), RawConf1, #{}) of FullConf -> - case emqx_map_lib:deep_find(BinKeyPath, FullConf) of + case emqx_utils_maps:deep_find(BinKeyPath, FullConf) of {not_found, _, _} -> {error, no_default_value}; {ok, Val} -> {ok, Val} end @@ -285,10 +285,10 @@ get_default_value([RootName | _] = KeyPath) -> {error, {rootname_not_found, RootName}} end. --spec get_raw(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path()) -> term(). get_raw(KeyPath) -> do_get_raw(KeyPath). --spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw(KeyPath, Default) -> do_get_raw(KeyPath, Default). -spec put_raw(map()) -> ok. @@ -301,10 +301,10 @@ put_raw(Config) -> hocon_maps:ensure_plain(Config) ). --spec put_raw(emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_raw(emqx_utils_maps:config_key_path(), term()) -> ok. put_raw(KeyPath, Config) -> Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_force_put(Path, Map, Value) + emqx_utils_maps:deep_force_put(Path, Map, Value) end, do_put(?RAW_CONF, Putter, KeyPath, Config). @@ -430,7 +430,7 @@ do_check_config(SchemaMod, RawConf, Opts0) -> Opts = maps:merge(Opts0, Opts1), {AppEnvs, CheckedConf} = hocon_tconf:map_translate(SchemaMod, RawConf, Opts), - {AppEnvs, emqx_map_lib:unsafe_atom_key_map(CheckedConf)}. + {AppEnvs, emqx_utils_maps:unsafe_atom_key_map(CheckedConf)}. fill_defaults(RawConf) -> fill_defaults(RawConf, #{}). @@ -645,11 +645,11 @@ do_put(Type, Putter, [RootName | KeyPath], DeepValue) -> do_deep_get(?CONF, KeyPath, Map, Default) -> atom_conf_path( KeyPath, - fun(AtomKeyPath) -> emqx_map_lib:deep_get(AtomKeyPath, Map, Default) end, + fun(AtomKeyPath) -> emqx_utils_maps:deep_get(AtomKeyPath, Map, Default) end, {return, Default} ); do_deep_get(?RAW_CONF, KeyPath, Map, Default) -> - emqx_map_lib:deep_get([bin(Key) || Key <- KeyPath], Map, Default). + emqx_utils_maps:deep_get([bin(Key) || Key <- KeyPath], Map, Default). do_deep_put(?CONF, Putter, KeyPath, Map, Value) -> atom_conf_path( diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index a0a99b62e..b440d2af7 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -232,7 +232,7 @@ process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) -> BinKeyPath = bin_path(ConfKeyPath), case check_permissions(remove, BinKeyPath, OldRawConf, Opts) of allow -> - NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), + NewRawConf = emqx_utils_maps:deep_remove(BinKeyPath, OldRawConf), OverrideConf = remove_from_override_config(BinKeyPath, Opts), {ok, NewRawConf, OverrideConf, Opts}; {deny, Reason} -> @@ -445,7 +445,7 @@ remove_from_override_config(_BinKeyPath, #{persistent := false}) -> undefined; remove_from_override_config(BinKeyPath, Opts) -> OldConf = emqx_config:read_override_conf(Opts), - emqx_map_lib:deep_remove(BinKeyPath, OldConf). + emqx_utils_maps:deep_remove(BinKeyPath, OldConf). %% apply new config on top of override config merge_to_override_config(_RawConf, #{persistent := false}) -> @@ -467,7 +467,7 @@ return_change_result(_ConfKeyPath, {remove, _Opts}) -> return_rawconf(ConfKeyPath, #{rawconf_with_defaults := true}) -> FullRawConf = emqx_config:fill_defaults(emqx_config:get_raw([])), - emqx_map_lib:deep_get(bin_path(ConfKeyPath), FullRawConf); + emqx_utils_maps:deep_get(bin_path(ConfKeyPath), FullRawConf); return_rawconf(ConfKeyPath, _) -> emqx_config:get_raw(ConfKeyPath). @@ -485,16 +485,16 @@ atom(Atom) when is_atom(Atom) -> -dialyzer({nowarn_function, do_remove_handler/2}). do_remove_handler(ConfKeyPath, Handlers) -> - NewHandlers = emqx_map_lib:deep_remove(ConfKeyPath ++ [?MOD], Handlers), + NewHandlers = emqx_utils_maps:deep_remove(ConfKeyPath ++ [?MOD], Handlers), remove_empty_leaf(ConfKeyPath, NewHandlers). remove_empty_leaf([], Handlers) -> Handlers; remove_empty_leaf(KeyPath, Handlers) -> - case emqx_map_lib:deep_find(KeyPath, Handlers) =:= {ok, #{}} of + case emqx_utils_maps:deep_find(KeyPath, Handlers) =:= {ok, #{}} of %% empty leaf true -> - Handlers1 = emqx_map_lib:deep_remove(KeyPath, Handlers), + Handlers1 = emqx_utils_maps:deep_remove(KeyPath, Handlers), SubKeyPath = lists:sublist(KeyPath, length(KeyPath) - 1), remove_empty_leaf(SubKeyPath, Handlers1); false -> @@ -511,7 +511,7 @@ assert_callback_function(Mod) -> end, ok. --spec schema(module(), emqx_map_lib:config_key_path()) -> hocon_schema:schema(). +-spec schema(module(), emqx_utils_maps:config_key_path()) -> hocon_schema:schema(). schema(SchemaModule, [RootKey | _]) -> Roots = hocon_schema:roots(SchemaModule), {Field, Translations} = @@ -550,10 +550,10 @@ save_handlers(Handlers) -> check_permissions(_Action, _ConfKeyPath, _NewRawConf, #{override_to := local}) -> allow; check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) -> - case emqx_map_lib:deep_find(ConfKeyPath, NewRawConf) of + case emqx_utils_maps:deep_find(ConfKeyPath, NewRawConf) of {ok, NewRaw} -> LocalOverride = emqx_config:read_override_conf(#{override_to => local}), - case emqx_map_lib:deep_find(ConfKeyPath, LocalOverride) of + case emqx_utils_maps:deep_find(ConfKeyPath, LocalOverride) of {ok, LocalRaw} -> case is_mutable(Action, NewRaw, LocalRaw) of ok -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 44663ceeb..f1daeaaeb 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -572,7 +572,7 @@ find_limiter_cfg(Type, #{rate := _} = Cfg) -> find_limiter_cfg(Type, Cfg) -> { maps:get(Type, Cfg, undefined), - find_client_cfg(Type, emqx_map_lib:deep_get([client, Type], Cfg, undefined)) + find_client_cfg(Type, emqx_utils_maps:deep_get([client, Type], Cfg, undefined)) }. find_client_cfg(Type, BucketCfg) -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 96e28b1de..f82aebe7c 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -427,12 +427,12 @@ pre_config_update([listeners, _Type, _Name], {create, _NewConf}, _RawConf) -> pre_config_update([listeners, _Type, _Name], {update, _Request}, undefined) -> {error, not_found}; pre_config_update([listeners, Type, Name], {update, Request}, RawConf) -> - NewConfT = emqx_map_lib:deep_merge(RawConf, Request), + NewConfT = emqx_utils_maps:deep_merge(RawConf, Request), NewConf = ensure_override_limiter_conf(NewConfT, Request), CertsDir = certs_dir(Type, Name), {ok, convert_certs(CertsDir, NewConf)}; pre_config_update([listeners, _Type, _Name], {action, _Action, Updated}, RawConf) -> - NewConf = emqx_map_lib:deep_merge(RawConf, Updated), + NewConf = emqx_utils_maps:deep_merge(RawConf, Updated), {ok, NewConf}; pre_config_update(_Path, _Request, RawConf) -> {ok, RawConf}. @@ -500,7 +500,7 @@ esockd_opts(ListenerId, Type, Opts0) -> ws_opts(Type, ListenerName, Opts) -> WsPaths = [ - {emqx_map_lib:deep_get([websocket, mqtt_path], Opts, "/mqtt"), emqx_ws_connection, #{ + {emqx_utils_maps:deep_get([websocket, mqtt_path], Opts, "/mqtt"), emqx_ws_connection, #{ zone => zone(Opts), listener => {Type, ListenerName}, limiter => limiter(Opts), diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index ccf3e0c40..3bb10ee5c 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -110,7 +110,7 @@ register_listener(ListenerID, Opts) -> -spec inject_sni_fun(emqx_listeners:listener_id(), map()) -> map(). inject_sni_fun(ListenerID, Conf0) -> SNIFun = emqx_const_v1:make_sni_fun(ListenerID), - Conf = emqx_map_lib:deep_merge(Conf0, #{ssl_options => #{sni_fun => SNIFun}}), + Conf = emqx_utils_maps:deep_merge(Conf0, #{ssl_options => #{sni_fun => SNIFun}}), ok = ?MODULE:register_listener(ListenerID, Conf), Conf. @@ -149,7 +149,7 @@ handle_call({register_listener, ListenerID, Conf}, _From, State0) -> msg => "registering_ocsp_cache", listener_id => ListenerID }), - RefreshInterval0 = emqx_map_lib:deep_get([ssl_options, ocsp, refresh_interval], Conf), + RefreshInterval0 = emqx_utils_maps:deep_get([ssl_options, ocsp, refresh_interval], Conf), RefreshInterval = max(RefreshInterval0, ?MIN_REFRESH_INTERVAL), State = State0#{{refresh_interval, ListenerID} => RefreshInterval}, %% we need to pass the config along because this might be called diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 96d9aea34..9934e09c9 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2348,7 +2348,7 @@ ocsp_outer_validator(_Conf) -> ok. ocsp_inner_validator(#{enable_ocsp_stapling := _} = Conf) -> - ocsp_inner_validator(emqx_map_lib:binary_key_map(Conf)); + ocsp_inner_validator(emqx_utils_maps:binary_key_map(Conf)); ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := false} = _Conf) -> ok; ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := true} = Conf) -> diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 47797b326..d1c57bf0d 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -317,7 +317,9 @@ ensure_ssl_files(Dir, SSL, Opts) -> ensure_ssl_files(_Dir, SSL, [], _Opts) -> {ok, SSL}; ensure_ssl_files(Dir, SSL, [KeyPath | KeyPaths], Opts) -> - case ensure_ssl_file(Dir, KeyPath, SSL, emqx_map_lib:deep_get(KeyPath, SSL, undefined), Opts) of + case + ensure_ssl_file(Dir, KeyPath, SSL, emqx_utils_maps:deep_get(KeyPath, SSL, undefined), Opts) + of {ok, NewSSL} -> ensure_ssl_files(Dir, NewSSL, KeyPaths, Opts); {error, Reason} -> @@ -332,7 +334,7 @@ delete_ssl_files(Dir, NewOpts0, OldOpts0) -> {ok, OldOpts} = ensure_ssl_files(Dir, OldOpts0, #{dry_run => DryRun}), Get = fun (_KP, undefined) -> undefined; - (KP, Opts) -> emqx_map_lib:deep_get(KP, Opts, undefined) + (KP, Opts) -> emqx_utils_maps:deep_get(KP, Opts, undefined) end, lists:foreach( fun(KeyPath) -> delete_old_file(Get(KeyPath, NewOpts), Get(KeyPath, OldOpts)) end, @@ -372,7 +374,7 @@ do_ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, DryRun) -> true -> case save_pem_file(Dir, KeyPath, MaybePem, DryRun) of {ok, Path} -> - NewSSL = emqx_map_lib:deep_put(KeyPath, SSL, Path), + NewSSL = emqx_utils_maps:deep_put(KeyPath, SSL, Path), {ok, NewSSL}; {error, Reason} -> {error, Reason} @@ -482,9 +484,9 @@ is_valid_pem_file(Path) -> %% so they are forced to upload a cert file, or use an existing file path. -spec drop_invalid_certs(map()) -> map(). drop_invalid_certs(#{enable := False} = SSL) when ?IS_FALSE(False) -> - lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS_A); + lists:foldl(fun emqx_utils_maps:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS_A); drop_invalid_certs(#{<<"enable">> := False} = SSL) when ?IS_FALSE(False) -> - lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS); + lists:foldl(fun emqx_utils_maps:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS); drop_invalid_certs(#{enable := True} = SSL) when ?IS_TRUE(True) -> do_drop_invalid_certs(?SSL_FILE_OPT_PATHS_A, SSL); drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> @@ -493,7 +495,7 @@ drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> do_drop_invalid_certs([], SSL) -> SSL; do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> - case emqx_map_lib:deep_get(KeyPath, SSL, undefined) of + case emqx_utils_maps:deep_get(KeyPath, SSL, undefined) of undefined -> do_drop_invalid_certs(KeyPaths, SSL); PemOrPath -> @@ -501,7 +503,7 @@ do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> true -> do_drop_invalid_certs(KeyPaths, SSL); {error, _} -> - do_drop_invalid_certs(KeyPaths, emqx_map_lib:deep_remove(KeyPath, SSL)) + do_drop_invalid_certs(KeyPaths, emqx_utils_maps:deep_remove(KeyPath, SSL)) end end. @@ -586,7 +588,9 @@ ensure_ssl_file_key(_SSL, []) -> ok; ensure_ssl_file_key(SSL, RequiredKeyPaths) -> NotFoundRef = make_ref(), - Filter = fun(KeyPath) -> NotFoundRef =:= emqx_map_lib:deep_get(KeyPath, SSL, NotFoundRef) end, + Filter = fun(KeyPath) -> + NotFoundRef =:= emqx_utils_maps:deep_get(KeyPath, SSL, NotFoundRef) + end, case lists:filter(Filter, RequiredKeyPaths) of [] -> ok; Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl index 21bfc39df..dd3eb29e7 100644 --- a/apps/emqx/test/emqx_crl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -997,7 +997,7 @@ do_t_update_listener(Config) -> <<"enable_crl_check">> => true } }, - ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, CRLConfig), + ListenerData1 = emqx_utils_maps:deep_merge(ListenerData0, CRLConfig), {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1), ?assertMatch( #{ @@ -1040,7 +1040,7 @@ do_t_validations(_Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ListenerData1 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index a9b80c276..dff8ce5a7 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -143,7 +143,7 @@ init_per_testcase(t_ocsp_responder_error_responses, Config) -> } }, Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_map_lib:binary_key_map(Conf), + ConfBin = emqx_utils_maps:binary_key_map(Conf), hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), snabbkaffe:start_trace(), @@ -184,7 +184,7 @@ init_per_testcase(_TestCase, Config) -> } }, Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_map_lib:binary_key_map(Conf), + ConfBin = emqx_utils_maps:binary_key_map(Conf), hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), snabbkaffe:start_trace(), @@ -679,7 +679,7 @@ do_t_update_listener(Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ?assertEqual( undefined, - emqx_map_lib:deep_get([<<"ssl_options">>, <<"ocsp">>], ListenerData0, undefined) + emqx_utils_maps:deep_get([<<"ssl_options">>, <<"ocsp">>], ListenerData0, undefined) ), assert_no_http_get(), @@ -702,7 +702,7 @@ do_t_update_listener(Config) -> } } }, - ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, OCSPConfig), + ListenerData1 = emqx_utils_maps:deep_merge(ListenerData0, OCSPConfig), {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1), ?assertMatch( #{ @@ -722,14 +722,14 @@ do_t_update_listener(Config) -> %% location ?assertNotEqual( IssuerPemPath, - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [<<"ssl_options">>, <<"ocsp">>, <<"issuer_pem">>], ListenerData2 ) ), ?assertNotEqual( IssuerPem, - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [<<"ssl_options">>, <<"ocsp">>, <<"issuer_pem">>], ListenerData2 ) @@ -818,7 +818,7 @@ do_t_validations(_Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ListenerData1 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -843,7 +843,7 @@ do_t_validations(_Config) -> ), ListenerData2 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -873,7 +873,7 @@ do_t_validations(_Config) -> ), ListenerData3a = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -886,7 +886,9 @@ do_t_validations(_Config) -> } } ), - ListenerData3 = emqx_map_lib:deep_remove([<<"ssl_options">>, <<"certfile">>], ListenerData3a), + ListenerData3 = emqx_utils_maps:deep_remove( + [<<"ssl_options">>, <<"certfile">>], ListenerData3a + ), {error, {_, _, ResRaw3}} = update_listener_via_api(ListenerId, ListenerData3), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw3} = emqx_utils_json:decode(ResRaw3, [return_maps]), diff --git a/apps/emqx/test/emqx_tls_lib_tests.erl b/apps/emqx/test/emqx_tls_lib_tests.erl index 5510e4027..ad9598107 100644 --- a/apps/emqx/test/emqx_tls_lib_tests.erl +++ b/apps/emqx/test/emqx_tls_lib_tests.erl @@ -191,7 +191,7 @@ ssl_files_save_delete_test() -> FileKey = maps:get(<<"keyfile">>, SSL), ?assertMatch(<<"/tmp/ssl-test-dir/key-", _:16/binary>>, FileKey), ?assertEqual({ok, bin(test_key())}, file:read_file(FileKey)), - FileIssuerPem = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL), + FileIssuerPem = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL), ?assertMatch(<<"/tmp/ssl-test-dir/ocsp_issuer_pem-", _:16/binary>>, FileIssuerPem), ?assertEqual({ok, bin(test_key())}, file:read_file(FileIssuerPem)), %% no old file to delete @@ -251,8 +251,8 @@ ssl_file_replace_test() -> {ok, SSL3} = emqx_tls_lib:ensure_ssl_files(Dir, SSL1), File1 = maps:get(<<"keyfile">>, SSL2), File2 = maps:get(<<"keyfile">>, SSL3), - IssuerPem1 = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2), - IssuerPem2 = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL3), + IssuerPem1 = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2), + IssuerPem2 = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL3), ?assert(filelib:is_regular(File1)), ?assert(filelib:is_regular(File2)), ?assert(filelib:is_regular(IssuerPem1)), diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index fc026a671..de856f163 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -929,7 +929,7 @@ aggregate_metrics([]) -> aggregate_metrics([HeadMetrics | AllMetrics]) -> ErrorLogger = fun(Reason) -> ?SLOG(info, #{msg => "bad_metrics_value", error => Reason}) end, Fun = fun(ElemMap, AccMap) -> - emqx_map_lib:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) + emqx_utils_maps:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) end, lists:foldl(Fun, HeadMetrics, AllMetrics). @@ -1069,7 +1069,7 @@ update_user(ChainName, AuthenticatorID, UserID, UserInfo0) -> true -> serialize_error({missing_parameter, password}); false -> - UserInfo = emqx_map_lib:safe_atom_key_map(UserInfo0), + UserInfo = emqx_utils_maps:safe_atom_key_map(UserInfo0), case emqx_authentication:update_user(ChainName, AuthenticatorID, UserID, UserInfo) of {ok, User} -> {200, User}; diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 0c2dee340..2220e8f6e 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -403,7 +403,7 @@ aggregate_metrics([]) -> aggregate_metrics([HeadMetrics | AllMetrics]) -> ErrorLogger = fun(Reason) -> ?SLOG(info, #{msg => "bad_metrics_value", error => Reason}) end, Fun = fun(ElemMap, AccMap) -> - emqx_map_lib:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) + emqx_utils_maps:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) end, lists:foldl(Fun, HeadMetrics, AllMetrics). diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 6630ed526..f03ae52a8 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -337,7 +337,7 @@ check_ssl_opts(Conf) -> (#{<<"url">> := Url} = Source) -> case emqx_authz_http:parse_url(Url) of {<<"https", _/binary>>, _, _} -> - case emqx_map_lib:deep_find([<<"ssl">>, <<"enable">>], Source) of + case emqx_utils_maps:deep_find([<<"ssl">>, <<"enable">>], Source) of {ok, true} -> true; {ok, false} -> throw({ssl_not_enable, Url}); _ -> throw({ssl_enable_not_found, Url}) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 087bc6a3f..08b8222f2 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -207,7 +207,7 @@ send_message(BridgeId, Message) -> end. query_opts(Config) -> - case emqx_map_lib:deep_get([resource_opts, request_timeout], Config, false) of + case emqx_utils_maps:deep_get([resource_opts, request_timeout], Config, false) of Timeout when is_integer(Timeout) -> %% request_timeout is configured #{timeout => Timeout}; @@ -367,7 +367,7 @@ perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) -> perform_bridge_changes(Tasks, Result). diff_confs(NewConfs, OldConfs) -> - emqx_map_lib:diff_maps( + emqx_utils_maps:diff_maps( flatten_confs(NewConfs), flatten_confs(OldConfs) ). diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index a1841b705..205fcfcf9 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -1029,5 +1029,5 @@ deobfuscate(NewConf, OldConf) -> map_to_json(M) -> emqx_utils_json:encode( - emqx_map_lib:jsonable_map(M, fun(K, V) -> {K, emqx_map_lib:binary_string(V)} end) + emqx_utils_maps:jsonable_map(M, fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end) ). diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 903e86443..347f9d973 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -186,7 +186,7 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> %% without restarting the bridge. %% Opts = override_start_after_created(Conf, Opts0), - case emqx_map_lib:if_only_to_toggle_enable(OldConf, Conf) of + case emqx_utils_maps:if_only_to_toggle_enable(OldConf, Conf) of false -> ?SLOG(info, #{ msg => "update bridge", @@ -238,7 +238,7 @@ recreate(Type, Name, Conf, Opts) -> create_dry_run(Type, Conf0) -> TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), TmpPath = emqx_utils:safe_filename(TmpPath0), - Conf = emqx_map_lib:safe_atom_key_map(Conf0), + Conf = emqx_utils_maps:safe_atom_key_map(Conf0), case emqx_connector_ssl:convert_certs(TmpPath, Conf) of {error, Reason} -> {error, Reason}; diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 52000ba7f..e5def2d64 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -251,7 +251,7 @@ do_convert_webhook_config( case {MReqTRoot, MReqTResource} of {{ok, ReqTRoot}, {ok, ReqTResource}} -> {_Parsed, ReqTRaw} = max({ReqTRoot, ReqTRootRaw}, {ReqTResource, ReqTResourceRaw}), - Conf1 = emqx_map_lib:deep_merge( + Conf1 = emqx_utils_maps:deep_merge( Conf0, #{ <<"request_timeout">> => ReqTRaw, diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index ef997f7e3..3afe17080 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1221,7 +1221,7 @@ t_inconsistent_webhook_request_timeouts(Config) -> URL1 = ?URL(Port, "path1"), Name = ?BRIDGE_NAME, BadBridgeParams = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ?HTTP_BRIDGE(URL1, Name), #{ <<"request_timeout">> => <<"1s">>, diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index bd6af9323..ab58ce383 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -270,7 +270,7 @@ t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> ?SERVER_CONF(<<"user1">>)#{ <<"type">> => ?TYPE_MQTT, <<"name">> => BridgeName, - <<"ingress">> => emqx_map_lib:deep_merge( + <<"ingress">> => emqx_utils_maps:deep_merge( ?INGRESS_CONF, #{<<"remote">> => #{<<"qos">> => 2}} ) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 254284b75..08fbf5e15 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -673,7 +673,7 @@ create_bridge(Config, Overrides) -> Type = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), emqx_bridge:create(Type, Name, KafkaConfig). delete_bridge(Config) -> @@ -696,7 +696,7 @@ create_bridge_api(Config, Overrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), @@ -719,7 +719,7 @@ update_bridge_api(Config, Overrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), BridgeId = emqx_bridge_resource:bridge_id(TypeBin, Name), Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId]), diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl index fa352ce89..b16df854f 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl @@ -92,7 +92,7 @@ kafka_consumer_test() -> ), %% Bad: can't repeat kafka topics. - BadConf1 = emqx_map_lib:deep_put( + BadConf1 = emqx_utils_maps:deep_put( [<<"bridges">>, <<"kafka_consumer">>, <<"my_consumer">>, <<"topic_mapping">>], Conf1, [ @@ -121,7 +121,7 @@ kafka_consumer_test() -> ), %% Bad: there must be at least 1 mapping. - BadConf2 = emqx_map_lib:deep_put( + BadConf2 = emqx_utils_maps:deep_put( [<<"bridges">>, <<"kafka_consumer">>, <<"my_consumer">>, <<"topic_mapping">>], Conf1, [] diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index d03cf9c27..246875ab6 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -43,50 +43,50 @@ add_handler(ConfKeyPath, HandlerName) -> remove_handler(ConfKeyPath) -> emqx_config_handler:remove_handler(ConfKeyPath). --spec get(emqx_map_lib:config_key_path()) -> term(). +-spec get(emqx_utils_maps:config_key_path()) -> term(). get(KeyPath) -> emqx:get_config(KeyPath). --spec get(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get(emqx_utils_maps:config_key_path(), term()) -> term(). get(KeyPath, Default) -> emqx:get_config(KeyPath, Default). --spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw(KeyPath, Default) -> emqx_config:get_raw(KeyPath, Default). --spec get_raw(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path()) -> term(). get_raw(KeyPath) -> emqx_config:get_raw(KeyPath). %% @doc Returns all values in the cluster. --spec get_all(emqx_map_lib:config_key_path()) -> #{node() => term()}. +-spec get_all(emqx_utils_maps:config_key_path()) -> #{node() => term()}. get_all(KeyPath) -> {ResL, []} = emqx_conf_proto_v2:get_all(KeyPath), maps:from_list(ResL). %% @doc Returns the specified node's KeyPath, or exception if not found --spec get_by_node(node(), emqx_map_lib:config_key_path()) -> term(). +-spec get_by_node(node(), emqx_utils_maps:config_key_path()) -> term(). get_by_node(Node, KeyPath) when Node =:= node() -> emqx:get_config(KeyPath); get_by_node(Node, KeyPath) -> emqx_conf_proto_v2:get_config(Node, KeyPath). %% @doc Returns the specified node's KeyPath, or the default value if not found --spec get_by_node(node(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_by_node(node(), emqx_utils_maps:config_key_path(), term()) -> term(). get_by_node(Node, KeyPath, Default) when Node =:= node() -> emqx:get_config(KeyPath, Default); get_by_node(Node, KeyPath, Default) -> emqx_conf_proto_v2:get_config(Node, KeyPath, Default). %% @doc Returns the specified node's KeyPath, or config_not_found if key path not found --spec get_node_and_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_node_and_config(emqx_utils_maps:config_key_path()) -> term(). get_node_and_config(KeyPath) -> {node(), emqx:get_config(KeyPath, config_not_found)}. %% @doc Update all value of key path in cluster-override.conf or local-override.conf. -spec update( - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -97,7 +97,7 @@ update(KeyPath, UpdateReq, Opts) -> %% @doc Update the specified node's key path in local-override.conf. -spec update( node(), - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -108,13 +108,13 @@ update(Node, KeyPath, UpdateReq, Opts) -> emqx_conf_proto_v2:update(Node, KeyPath, UpdateReq, Opts). %% @doc remove all value of key path in cluster-override.conf or local-override.conf. --spec remove(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove(KeyPath, Opts) -> emqx_conf_proto_v2:remove_config(KeyPath, Opts). %% @doc remove the specified node's key path in local-override.conf. --spec remove(node(), emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove(node(), emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove(Node, KeyPath, Opts) when Node =:= node() -> emqx:remove_config(KeyPath, Opts#{override_to => local}); @@ -122,13 +122,13 @@ remove(Node, KeyPath, Opts) -> emqx_conf_proto_v2:remove_config(Node, KeyPath, Opts). %% @doc reset all value of key path in cluster-override.conf or local-override.conf. --spec reset(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset(KeyPath, Opts) -> emqx_conf_proto_v2:reset(KeyPath, Opts). %% @doc reset the specified node's key path in local-override.conf. --spec reset(node(), emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset(node(), emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset(Node, KeyPath, Opts) when Node =:= node() -> emqx:reset_config(KeyPath, Opts#{override_to => local}); diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl index 84687e314..b66307a1b 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl @@ -38,22 +38,22 @@ -include_lib("emqx/include/bpapi.hrl"). --type update_config_key_path() :: [emqx_map_lib:config_key(), ...]. +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. introduced_in() -> "5.0.0". --spec get_config(node(), emqx_map_lib:config_key_path()) -> +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath) -> rpc:call(Node, emqx, get_config, [KeyPath]). --spec get_config(node(), emqx_map_lib:config_key_path(), _Default) -> +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath, Default) -> rpc:call(Node, emqx, get_config, [KeyPath, Default]). --spec get_all(emqx_map_lib:config_key_path()) -> emqx_rpc:multicall_result(). +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). get_all(KeyPath) -> rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl index dd8d2fedd..97446ee9f 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -44,19 +44,19 @@ introduced_in() -> -spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). sync_data_from_node(Node) -> rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). --type update_config_key_path() :: [emqx_map_lib:config_key(), ...]. +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. --spec get_config(node(), emqx_map_lib:config_key_path()) -> +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath) -> rpc:call(Node, emqx, get_config, [KeyPath]). --spec get_config(node(), emqx_map_lib:config_key_path(), _Default) -> +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath, Default) -> rpc:call(Node, emqx, get_config, [KeyPath, Default]). --spec get_all(emqx_map_lib:config_key_path()) -> emqx_rpc:multicall_result(). +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). get_all(KeyPath) -> rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl index 54dc0e022..e07d95d51 100644 --- a/apps/emqx_connector/src/emqx_connector_ssl.erl +++ b/apps/emqx_connector/src/emqx_connector_ssl.erl @@ -74,7 +74,7 @@ new_ssl_config(Config, _NewSSL) -> normalize_key_to_bin(undefined) -> undefined; normalize_key_to_bin(Map) when is_map(Map) -> - emqx_map_lib:binary_key_map(Map). + emqx_utils_maps:binary_key_map(Map). try_map_get(Key, Map, Default) when is_map(Map) -> maps:get(Key, Map, Default); diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 5197a35df..880a99313 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -410,7 +410,7 @@ handle_disconnect(_Reason) -> ok. maybe_publish_local(Msg, Vars, Props) -> - case emqx_map_lib:deep_get([local, topic], Vars, undefined) of + case emqx_utils_maps:deep_get([local, topic], Vars, undefined) of %% local topic is not set, discard it undefined -> ok; diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl index eac4f845f..01d96bdf0 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl @@ -119,7 +119,7 @@ remove_handler() -> pre_config_update(_Path, UpdateConf0, RawConf) -> UpdateConf = remove_sensitive_data(UpdateConf0), - NewConf = emqx_map_lib:deep_merge(RawConf, UpdateConf), + NewConf = emqx_utils_maps:deep_merge(RawConf, UpdateConf), ensure_ssl_cert(NewConf). -define(SENSITIVE_PASSWORD, <<"******">>). @@ -134,7 +134,7 @@ remove_sensitive_data(Conf0) -> end, case Conf1 of #{<<"listeners">> := #{<<"https">> := #{<<"password">> := ?SENSITIVE_PASSWORD}}} -> - emqx_map_lib:deep_remove([<<"listeners">>, <<"https">>, <<"password">>], Conf1); + emqx_utils_maps:deep_remove([<<"listeners">>, <<"https">>, <<"password">>], Conf1); _ -> Conf1 end. @@ -152,7 +152,7 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) -> ok. get_listener(Type, Conf) -> - emqx_map_lib:deep_get([listeners, Type], Conf, undefined). + emqx_utils_maps:deep_get([listeners, Type], Conf, undefined). diff_listeners(_, Listener, Listener) -> {#{}, #{}}; diff_listeners(Type, undefined, Start) -> {#{}, #{Type => Start}}; @@ -162,13 +162,14 @@ diff_listeners(Type, Stop, Start) -> {#{Type => Stop}, #{Type => Start}}. -define(DIR, <<"dashboard">>). ensure_ssl_cert(#{<<"listeners">> := #{<<"https">> := #{<<"enable">> := true}}} = Conf) -> - Https = emqx_map_lib:deep_get([<<"listeners">>, <<"https">>], Conf, undefined), + Https = emqx_utils_maps:deep_get([<<"listeners">>, <<"https">>], Conf, undefined), Opts = #{required_keys => [[<<"keyfile">>], [<<"certfile">>], [<<"cacertfile">>]]}, case emqx_tls_lib:ensure_ssl_files(?DIR, Https, Opts) of {ok, undefined} -> {error, <<"ssl_cert_not_found">>}; {ok, NewHttps} -> - {ok, emqx_map_lib:deep_merge(Conf, #{<<"listeners">> => #{<<"https">> => NewHttps}})}; + {ok, + emqx_utils_maps:deep_merge(Conf, #{<<"listeners">> => #{<<"https">> => NewHttps}})}; {error, Reason} -> ?SLOG(error, Reason#{msg => "bad_ssl_config"}), {error, Reason} diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 22cf484ff..b2ad69997 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -474,7 +474,7 @@ maybe_add_summary_from_label(Spec, Hocon) -> get_i18n(Key, Struct, Default) -> {ok, #{cache := Cache, lang := Lang}} = emqx_dashboard:get_i18n(), Desc = hocon_schema:resolve_schema(Struct, Cache), - emqx_map_lib:deep_get([Key, Lang], Desc, Default). + emqx_utils_maps:deep_get([Key, Lang], Desc, Default). trans_label(Spec, Hocon, Default) -> Label = diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl index 77937a835..0647c80ea 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mgr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl @@ -507,11 +507,11 @@ sort_name_by_order(Names, Orders) -> lists:sort( fun (A, B) when is_binary(A) -> - emqx_map_lib:deep_get([A, order], Orders) < - emqx_map_lib:deep_get([B, order], Orders); + emqx_utils_maps:deep_get([A, order], Orders) < + emqx_utils_maps:deep_get([B, order], Orders); (#{name := A}, #{name := B}) -> - emqx_map_lib:deep_get([A, order], Orders) < - emqx_map_lib:deep_get([B, order], Orders) + emqx_utils_maps:deep_get([A, order], Orders) < + emqx_utils_maps:deep_get([B, order], Orders) end, Names ). diff --git a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl index 58f2f29b4..c03b3f231 100644 --- a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl @@ -311,7 +311,7 @@ t_update(Cfg) -> decode_json(Data) -> BinJosn = emqx_utils_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJosn). + emqx_utils_maps:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 14b80a500..d90bf3689 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -96,7 +96,7 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> LName = binary_to_atom(maps:get(<<"name">>, LConf)), Path = [listeners, Type, LName], - case emqx_map_lib:deep_get(Path, RunningConf, undefined) of + case emqx_utils_maps:deep_get(Path, RunningConf, undefined) of undefined -> ListenerId = emqx_gateway_utils:listener_id( GwName, Type, LName diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 07a4c1e2c..56a3e2068 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -106,7 +106,7 @@ unconvert_listeners(Ls) when is_list(Ls) -> {[Type, Name], Lis1} = maps_key_take([<<"type">>, <<"name">>], Lis), _ = vaildate_listener_name(Name), NLis1 = maps:without([<<"id">>, <<"running">>], Lis1), - emqx_map_lib:deep_merge(Acc, #{Type => #{Name => NLis1}}) + emqx_utils_maps:deep_merge(Acc, #{Type => #{Name => NLis1}}) end, #{}, Ls @@ -160,8 +160,8 @@ gateway(GwName0) -> RawConf = emqx_config:fill_defaults( emqx_config:get_root_raw(Path) ), - Confs = emqx_map_lib:jsonable_map( - emqx_map_lib:deep_get(Path, RawConf) + Confs = emqx_utils_maps:jsonable_map( + emqx_utils_maps:deep_get(Path, RawConf) ), LsConf = maps:get(<<"listeners">>, Confs, #{}), Confs#{<<"listeners">> => convert_listeners(GwName, LsConf)}. @@ -198,8 +198,8 @@ listeners(GwName0) -> RawConf = emqx_config:fill_defaults( emqx_config:get_root_raw([<<"gateway">>]) ), - Listeners = emqx_map_lib:jsonable_map( - emqx_map_lib:deep_get( + Listeners = emqx_utils_maps:jsonable_map( + emqx_utils_maps:deep_get( [<<"gateway">>, GwName, <<"listeners">>], RawConf ) ), @@ -213,12 +213,12 @@ listener(ListenerId) -> ), try Path = [<<"gateway">>, GwName, <<"listeners">>, Type, LName], - LConf = emqx_map_lib:deep_get(Path, RootConf), + LConf = emqx_utils_maps:deep_get(Path, RootConf), Running = emqx_gateway_utils:is_running( binary_to_existing_atom(ListenerId), LConf ), {ok, - emqx_map_lib:jsonable_map( + emqx_utils_maps:jsonable_map( LConf#{ id => ListenerId, type => Type, @@ -305,8 +305,8 @@ ret_ok_err({ok, _}) -> ok; ret_ok_err(Err) -> Err. ret_gw(GwName, {ok, #{raw_config := GwConf}}) -> - GwConf1 = emqx_map_lib:deep_get([bin(GwName)], GwConf), - LsConf = emqx_map_lib:deep_get( + GwConf1 = emqx_utils_maps:deep_get([bin(GwName)], GwConf), + LsConf = emqx_utils_maps:deep_get( [bin(GwName), <<"listeners">>], GwConf, #{} @@ -331,7 +331,7 @@ ret_gw(_GwName, Err) -> Err. ret_authn(GwName, {ok, #{raw_config := GwConf}}) -> - Authn = emqx_map_lib:deep_get( + Authn = emqx_utils_maps:deep_get( [bin(GwName), <<"authentication">>], GwConf ), @@ -340,7 +340,7 @@ ret_authn(_GwName, Err) -> Err. ret_authn(GwName, {LType, LName}, {ok, #{raw_config := GwConf}}) -> - Authn = emqx_map_lib:deep_get( + Authn = emqx_utils_maps:deep_get( [ bin(GwName), <<"listeners">>, @@ -355,7 +355,7 @@ ret_authn(_, _, Err) -> Err. ret_listener_or_err(GwName, {LType, LName}, {ok, #{raw_config := GwConf}}) -> - LConf = emqx_map_lib:deep_get( + LConf = emqx_utils_maps:deep_get( [bin(GwName), <<"listeners">>, bin(LType), bin(LName)], GwConf ), @@ -377,7 +377,7 @@ pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) -> case maps:get(GwName, RawConf, undefined) of undefined -> NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf), - {ok, emqx_map_lib:deep_put([GwName], RawConf, NConf)}; + {ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf)}; _ -> badres_gateway(already_exist, GwName) end; @@ -389,7 +389,7 @@ pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) -> Conf1 = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf), NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf1), NConf1 = maps:merge(GwRawConf, NConf), - {ok, emqx_map_lib:deep_put([GwName], RawConf, NConf1)} + {ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf1)} end; pre_config_update(_, {unload_gateway, GwName}, RawConf) -> _ = tune_gw_certs( @@ -400,7 +400,7 @@ pre_config_update(_, {unload_gateway, GwName}, RawConf) -> {ok, maps:remove(GwName, RawConf)}; pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined ) of @@ -408,7 +408,7 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> NConf = convert_certs(certs_dir(GwName), Conf), NListener = #{LType => #{LName => NConf}}, {ok, - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( RawConf, #{GwName => #{<<"listeners">> => NListener}} )}; @@ -417,7 +417,7 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined ) of @@ -425,7 +425,7 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) - badres_listener(not_found, GwName, LType, LName); OldConf -> NConf = convert_certs(certs_dir(GwName), Conf, OldConf), - NRawConf = emqx_map_lib:deep_put( + NRawConf = emqx_utils_maps:deep_put( [GwName, <<"listeners">>, LType, LName], RawConf, NConf @@ -434,22 +434,22 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) - end; pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) -> Path = [GwName, <<"listeners">>, LType, LName], - case emqx_map_lib:deep_get(Path, RawConf, undefined) of + case emqx_utils_maps:deep_get(Path, RawConf, undefined) of undefined -> {ok, RawConf}; OldConf -> clear_certs(certs_dir(GwName), OldConf), - {ok, emqx_map_lib:deep_remove(Path, RawConf)} + {ok, emqx_utils_maps:deep_remove(Path, RawConf)} end; pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, ?AUTHN_BIN], RawConf, undefined ) of undefined -> {ok, - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( RawConf, #{GwName => #{?AUTHN_BIN => Conf}} )}; @@ -458,7 +458,7 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> end; pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined @@ -477,25 +477,25 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> #{LType => #{LName => NListener}} } }, - {ok, emqx_map_lib:deep_merge(RawConf, NGateway)}; + {ok, emqx_utils_maps:deep_merge(RawConf, NGateway)}; _ -> badres_listener_authn(already_exist, GwName, LType, LName) end end; pre_config_update(_, {update_authn, GwName, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, ?AUTHN_BIN], RawConf, undefined ) of undefined -> badres_authn(not_found, GwName); _Authn -> - {ok, emqx_map_lib:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf)} + {ok, emqx_utils_maps:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf)} end; pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined @@ -514,7 +514,7 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> Listener ), {ok, - emqx_map_lib:deep_put( + emqx_utils_maps:deep_put( [GwName, <<"listeners">>, LType, LName], RawConf, NListener @@ -523,12 +523,12 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {remove_authn, GwName}, RawConf) -> {ok, - emqx_map_lib:deep_remove( + emqx_utils_maps:deep_remove( [GwName, ?AUTHN_BIN], RawConf )}; pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) -> Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN], - {ok, emqx_map_lib:deep_remove(Path, RawConf)}; + {ok, emqx_utils_maps:deep_remove(Path, RawConf)}; pre_config_update(_, UnknownReq, _RawConf) -> logger:error("Unknown configuration update request: ~0p", [UnknownReq]), {error, badreq}. diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 9bb5821f5..7aaaee9cb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -240,7 +240,7 @@ authn(GwName) -> ChainName = emqx_gateway_utils:global_chain(GwName), wrap_chain_name( ChainName, - emqx_map_lib:jsonable_map(emqx:get_raw_config(Path)) + emqx_utils_maps:jsonable_map(emqx:get_raw_config(Path)) ). -spec authn(gateway_name(), binary()) -> map(). @@ -250,7 +250,7 @@ authn(GwName, ListenerId) -> ChainName = emqx_gateway_utils:listener_chain(GwName, Type, Name), wrap_chain_name( ChainName, - emqx_map_lib:jsonable_map(emqx:get_raw_config(Path)) + emqx_utils_maps:jsonable_map(emqx:get_raw_config(Path)) ). wrap_chain_name(ChainName, Conf) -> diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 10adcc428..fb648062a 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -164,7 +164,7 @@ t_gateway_stomp(_) -> {204, _} = request(put, "/gateways/stomp", GwConf), {200, ConfResp} = request(get, "/gateways/stomp"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{frame => #{max_headers => 10}}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{frame => #{max_headers => 10}}), {204, _} = request(put, "/gateways/stomp", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/stomp"), assert_confs(GwConf2, ConfResp2), @@ -186,7 +186,7 @@ t_gateway_mqttsn(_) -> {204, _} = request(put, "/gateways/mqttsn", GwConf), {200, ConfResp} = request(get, "/gateways/mqttsn"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{predefined => []}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{predefined => []}), {204, _} = request(put, "/gateways/mqttsn", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/mqttsn"), assert_confs(GwConf2, ConfResp2), @@ -206,7 +206,7 @@ t_gateway_coap(_) -> {204, _} = request(put, "/gateways/coap", GwConf), {200, ConfResp} = request(get, "/gateways/coap"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{heartbeat => <<"10s">>}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{heartbeat => <<"10s">>}), {204, _} = request(put, "/gateways/coap", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/coap"), assert_confs(GwConf2, ConfResp2), @@ -244,7 +244,7 @@ t_gateway_lwm2m(_) -> {204, _} = request(put, "/gateways/lwm2m", GwConf), {200, ConfResp} = request(get, "/gateways/lwm2m"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}), {204, _} = request(put, "/gateways/lwm2m", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/lwm2m"), assert_confs(GwConf2, ConfResp2), @@ -264,7 +264,7 @@ t_gateway_exproto(_) -> {204, _} = request(put, "/gateways/exproto", GwConf), {200, ConfResp} = request(get, "/gateways/exproto"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}), {204, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/exproto"), assert_confs(GwConf2, ConfResp2), @@ -293,7 +293,7 @@ t_gateway_exproto_with_ssl(_) -> {204, _} = request(put, "/gateways/exproto", GwConf), {200, ConfResp} = request(get, "/gateways/exproto"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{ + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{ server => #{ bind => <<"9200">>, ssl_options => SslCliOpts diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 1ab36f7b8..149e6acd6 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -282,7 +282,7 @@ t_case_exproto(_) -> disable_authn(GwName, Type, Name) -> RawCfg = emqx_conf:get_raw([gateway, GwName], #{}), - ListenerCfg = emqx_map_lib:deep_get( + ListenerCfg = emqx_utils_maps:deep_get( [<<"listeners">>, atom_to_binary(Type), atom_to_binary(Name)], RawCfg ), {ok, _} = emqx_gateway_conf:update_listener(GwName, {Type, Name}, ListenerCfg#{ diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index 33c307770..1e947e793 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -413,7 +413,7 @@ t_load_gateway_with_certs_content(_) -> ), {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), - SslConf = emqx_map_lib:deep_get( + SslConf = emqx_utils_maps:deep_get( [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], emqx:get_raw_config([gateway, stomp]) ), @@ -436,7 +436,7 @@ t_load_gateway_with_certs_content(_) -> % ), % {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), % assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), -% SslConf = emqx_map_lib:deep_get( +% SslConf = emqx_utils_maps:deep_get( % [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], % emqx:get_raw_config([gateway, stomp]) % ), @@ -471,7 +471,7 @@ t_add_listener_with_certs_content(_) -> emqx:get_raw_config([gateway, stomp]) ), - SslConf = emqx_map_lib:deep_get( + SslConf = emqx_utils_maps:deep_get( [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], emqx:get_raw_config([gateway, stomp]) ), diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index 6d80b63d8..bb378ef10 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -159,7 +159,7 @@ do_request(Mth, Req) -> <<>> -> #{}; _ -> - emqx_map_lib:unsafe_atom_key_map( + emqx_utils_maps:unsafe_atom_key_map( emqx_utils_json:decode(Resp, [return_maps]) ) end, diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl index 7860f1220..ff105b931 100644 --- a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl @@ -149,7 +149,7 @@ start_grpc_server(GwName, Options = #{bind := ListenOn}) -> } }, SvrOptions = - case emqx_map_lib:deep_get([ssl, enable], Options, false) of + case emqx_utils_maps:deep_get([ssl, enable], Options, false) of false -> []; true -> @@ -201,7 +201,7 @@ start_grpc_client_channel(GwName, Options = #{address := Address}) -> }} ) end, - case emqx_map_lib:deep_get([ssl, enable], Options, false) of + case emqx_utils_maps:deep_get([ssl, enable], Options, false) of false -> SvrAddr = compose_http_uri(http, Host, Port), grpc_client_sup:create_channel_pool(GwName, SvrAddr, #{}); diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index cd4829342..681c851bf 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -860,8 +860,8 @@ format_channel_info(ChannInfo = {_, _ClientInfo, _ClientStats}) -> format_channel_info(WhichNode, {_, ClientInfo0, ClientStats}) -> Node = maps:get(node, ClientInfo0, WhichNode), - ClientInfo1 = emqx_map_lib:deep_remove([conninfo, clientid], ClientInfo0), - ClientInfo2 = emqx_map_lib:deep_remove([conninfo, username], ClientInfo1), + ClientInfo1 = emqx_utils_maps:deep_remove([conninfo, clientid], ClientInfo0), + ClientInfo2 = emqx_utils_maps:deep_remove([conninfo, username], ClientInfo1), StatsMap = maps:without( [memory, next_pkt_id, total_heap_size], maps:from_list(ClientStats) @@ -958,4 +958,4 @@ format_authz_cache({{PubSub, Topic}, {AuthzResult, Timestamp}}) -> to_topic_info(Data) -> M = maps:with([<<"topic">>, <<"qos">>, <<"nl">>, <<"rap">>, <<"rh">>], Data), - emqx_map_lib:safe_atom_key_map(M). + emqx_utils_maps:safe_atom_key_map(M). diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 55cc50597..b37a47be6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -215,7 +215,7 @@ fields(Field) -> %% HTTP API Callbacks config(get, _Params, Req) -> Path = conf_path(Req), - {ok, Conf} = emqx_map_lib:deep_find(Path, get_full_config()), + {ok, Conf} = emqx_utils_maps:deep_find(Path, get_full_config()), {200, Conf}; config(put, #{body := Body}, Req) -> Path = conf_path(Req), @@ -349,7 +349,7 @@ gen_schema(_Conf) -> #{type => string}. with_default_value(Type, Value) -> - Type#{example => emqx_map_lib:binary_string(Value)}. + Type#{example => emqx_utils_maps:binary_string(Value)}. global_zone_roots() -> lists:map(fun({K, _}) -> K end, global_zone_schema()). diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index c126cfe19..d7f3ff321 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -390,7 +390,7 @@ crud_listeners_by_id(put, #{bindings := #{id := Id}, body := Body0}) -> undefined -> {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}}; PrevConf -> - MergeConfT = emqx_map_lib:deep_merge(PrevConf, Conf), + MergeConfT = emqx_utils_maps:deep_merge(PrevConf, Conf), MergeConf = emqx_listeners:ensure_override_limiter_conf(MergeConfT, Conf), case update(Path, MergeConf) of {ok, #{raw_config := _RawConf}} -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index cc200a4d5..e5704b817 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -55,7 +55,7 @@ t_update(_Config) -> %% update ok {ok, SysMon} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort}} = SysMon, - NewSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, not BusyPort), + NewSysMon = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, not BusyPort), {ok, #{}} = update_config(<<"sysmon">>, NewSysMon), {ok, SysMon1} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort1}} = SysMon1, @@ -63,7 +63,7 @@ t_update(_Config) -> assert_busy_port(BusyPort1), %% update failed - ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), + ErrorSysMon = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), ?assertMatch( {error, {"HTTP/1.1", 400, _}}, update_config(<<"sysmon">>, ErrorSysMon) @@ -78,7 +78,7 @@ t_update(_Config) -> assert_busy_port(true), %% reset no_default_value config - NewSysMon1 = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false), + NewSysMon1 = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false), {ok, #{}} = update_config(<<"sysmon">>, NewSysMon1), ?assertMatch({error, {"HTTP/1.1", 400, _}}, reset_config(<<"sysmon">>, "")), {ok, SysMon4} = get_config(<<"sysmon">>), @@ -94,27 +94,33 @@ t_log(_Config) -> {ok, Log} = get_config("log"), File = "log/emqx-test.log", %% update handler - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), {ok, #{}} = update_config(<<"log">>, Log2), {ok, Log3} = logger:get_handler_config(default), ?assertMatch(#{config := #{file := File}}, Log3), - ErrLog1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, 1), + ErrLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, 1), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog1)), - ErrLog2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enabfe">>], Log, true), + ErrLog2 = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"default">>, <<"enabfe">>], Log, true + ), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog2)), %% add new handler File1 = "log/emqx-test1.log", - Handler = emqx_map_lib:deep_get([<<"file_handlers">>, <<"default">>], Log2), - NewLog1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>], Log2, Handler), - NewLog2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>, <<"file">>], NewLog1, File1), + Handler = emqx_utils_maps:deep_get([<<"file_handlers">>, <<"default">>], Log2), + NewLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"new">>], Log2, Handler), + NewLog2 = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"new">>, <<"file">>], NewLog1, File1 + ), {ok, #{}} = update_config(<<"log">>, NewLog2), {ok, Log4} = logger:get_handler_config(new), ?assertMatch(#{config := #{file := File1}}, Log4), %% disable new handler - Disable = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>, <<"enable">>], NewLog2, false), + Disable = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"new">>, <<"enable">>], NewLog2, false + ), {ok, #{}} = update_config(<<"log">>, Disable), ?assertEqual({error, {not_found, new}}, logger:get_handler_config(new)), ok. @@ -125,25 +131,25 @@ t_global_zone(_Config) -> ?assertEqual(lists:usort(ZonesKeys), lists:usort(maps:keys(Zones))), ?assertEqual( emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed]), - emqx_map_lib:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones) + emqx_utils_maps:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones) ), - NewZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), + NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), {ok, #{}} = update_global_zone(NewZones), ?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])), - BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), + BadZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)), %% Remove max_qos_allowed from raw config, but we still get default value(2). Mqtt0 = emqx_conf:get_raw([<<"mqtt">>]), - ?assertEqual(1, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt0)), + ?assertEqual(1, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt0)), Mqtt1 = maps:remove(<<"max_qos_allowed">>, Mqtt0), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt1), Mqtt2 = emqx_conf:get_raw([<<"mqtt">>]), ?assertNot(maps:is_key(<<"max_qos_allowed">>, Mqtt2), Mqtt2), {ok, #{<<"mqtt">> := Mqtt3}} = get_global_zone(), %% the default value is 2 - ?assertEqual(2, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt3)), + ?assertEqual(2, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt3)), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0), ok. diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index 90808d4be..62f689a84 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -193,10 +193,10 @@ t_clear_certs(Config) when is_list(Config) -> }, %% create, make sure the cert files are created - NewConf = emqx_map_lib:deep_put( + NewConf = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"certfile">>], ConfTemp, cert_file("certfile") ), - NewConf2 = emqx_map_lib:deep_put( + NewConf2 = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"keyfile">>], NewConf, cert_file("keyfile") ), @@ -205,7 +205,7 @@ t_clear_certs(Config) when is_list(Config) -> ?assertMatch({ok, [_, _]}, ListResult1), %% update - UpdateConf = emqx_map_lib:deep_put( + UpdateConf = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"keyfile">>], NewConf2, cert_file("keyfile2") ), _ = request(put, NewPath, [], UpdateConf), diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 7e4ccc541..1f14d075e 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -34,8 +34,8 @@ init_per_testcase(t_log_path, Config) -> emqx_config_logger:add_handler(), Log = emqx_conf:get_raw([log], #{}), File = "log/emqx-test.log", - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), {ok, #{}} = emqx_conf:update([log], Log2, #{rawconf_with_defaults => true}), Config; init_per_testcase(_, Config) -> @@ -43,7 +43,7 @@ init_per_testcase(_, Config) -> end_per_testcase(t_log_path, Config) -> Log = emqx_conf:get_raw([log], #{}), - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, false), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, false), {ok, #{}} = emqx_conf:update([log], Log1, #{rawconf_with_defaults => true}), emqx_config_logger:remove_handler(), Config; diff --git a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl index b2d854d22..4ae3dec88 100644 --- a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl @@ -230,7 +230,7 @@ t_large_payload(_) -> decode_json(Data) -> BinJson = emqx_utils_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJson). + emqx_utils_maps:unsafe_atom_key_map(BinJson). clear_all_record() -> ets:delete_all_objects(emqx_delayed). diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl index 32cbba084..7b0a0dc2a 100644 --- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl +++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl @@ -156,7 +156,7 @@ handle_cast({dispatch, Context, Pid, Topic}, #{limiter := Limiter} = State) -> {ok, Limiter2} = dispatch(Context, Pid, Topic, undefined, Limiter), {noreply, State#{limiter := Limiter2}}; handle_cast({refresh_limiter, Conf}, State) -> - BucketCfg = emqx_map_lib:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), + BucketCfg = emqx_utils_maps:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), {ok, Limiter} = emqx_limiter_server:connect(?APP, internal, BucketCfg), {noreply, State#{limiter := Limiter}}; handle_cast(Msg, State) -> diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index 2b92b3f59..61eee0510 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -257,7 +257,7 @@ t_change_storage_type(_Config) -> #{data := Msgs0, meta := _} = decode_json(MsgsJson0), ?assertEqual(1, length(Msgs0)), - ChangedConf = emqx_map_lib:deep_merge( + ChangedConf = emqx_utils_maps:deep_merge( RawConf, #{ <<"backend">> => @@ -312,7 +312,7 @@ t_change_storage_type(_Config) -> %%-------------------------------------------------------------------- decode_json(Data) -> BinJson = emqx_utils_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJson). + emqx_utils_maps:unsafe_atom_key_map(BinJson). %%-------------------------------------------------------------------- %% Internal funcs diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 44b49a75b..ada52c5aa 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -115,7 +115,7 @@ start_link() -> %%------------------------------------------------------------------------------ post_config_update(_, _Req, NewRules, OldRules, _AppEnvs) -> #{added := Added, removed := Removed, changed := Updated} = - emqx_map_lib:diff_maps(NewRules, OldRules), + emqx_utils_maps:diff_maps(NewRules, OldRules), maps_foreach( fun({Id, {_Old, New}}) -> {ok, _} = update_rule(New#{id => bin(Id)}) diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl index ae74348e0..5196868c7 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl @@ -166,7 +166,7 @@ t_settting(_) -> decode_json(Data) -> BinJosn = emqx_utils_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJosn). + emqx_utils_maps:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx_utils/src/emqx_utils_maps.erl similarity index 99% rename from apps/emqx/src/emqx_map_lib.erl rename to apps/emqx_utils/src/emqx_utils_maps.erl index 631c3914d..d64075511 100644 --- a/apps/emqx/src/emqx_map_lib.erl +++ b/apps/emqx_utils/src/emqx_utils_maps.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_map_lib). +-module(emqx_utils_maps). -export([ deep_get/2, @@ -332,7 +332,7 @@ deep_filter(M, F) when is_map(M) -> if_only_to_toggle_enable(OldConf, Conf) -> #{added := Added, removed := Removed, changed := Updated} = - emqx_map_lib:diff_maps(OldConf, Conf), + emqx_utils_maps:diff_maps(OldConf, Conf), case {Added, Removed, Updated} of {Added, Removed, #{enable := _} = Updated} when map_size(Added) =:= 0, diff --git a/apps/emqx/test/emqx_map_lib_tests.erl b/apps/emqx_utils/test/emqx_utils_maps_tests.erl similarity index 76% rename from apps/emqx/test/emqx_map_lib_tests.erl rename to apps/emqx_utils/test/emqx_utils_maps_tests.erl index 894811d7c..506851f0a 100644 --- a/apps/emqx/test/emqx_map_lib_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_maps_tests.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_map_lib_tests). +-module(emqx_utils_maps_tests). -include_lib("eunit/include/eunit.hrl"). best_effort_recursive_sum_test_() -> @@ -22,21 +22,21 @@ best_effort_recursive_sum_test_() -> [ ?_assertEqual( #{foo => 3}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1}, #{foo => 2}, DummyLogger) + emqx_utils_maps:best_effort_recursive_sum(#{foo => 1}, #{foo => 2}, DummyLogger) ), ?_assertEqual( #{foo => 3, bar => 6.0}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => 2.0}, #{foo => 2, bar => 4.0}, DummyLogger ) ), ?_assertEqual( #{foo => 1, bar => 2}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1}, #{bar => 2}, DummyLogger) + emqx_utils_maps:best_effort_recursive_sum(#{foo => 1}, #{bar => 2}, DummyLogger) ), ?_assertEqual( #{foo => #{bar => 42}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 2}}, #{foo => #{bar => 40}}, DummyLogger ) ), @@ -45,7 +45,9 @@ best_effort_recursive_sum_test_() -> Logger = fun(What) -> Self ! {log, What} end, ?assertEqual( #{foo => 1, bar => 2}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1, bar => 2}, #{bar => bar}, Logger) + emqx_utils_maps:best_effort_recursive_sum( + #{foo => 1, bar => 2}, #{bar => bar}, Logger + ) ), receive {log, Log} -> @@ -55,55 +57,55 @@ best_effort_recursive_sum_test_() -> end, ?_assertEqual( #{}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => foo}, #{foo => bar}, DummyLogger ) ), ?_assertEqual( #{foo => 1}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1}, #{foo => bar}, DummyLogger ) ), ?_assertEqual( #{foo => 1}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => bar}, #{foo => 1}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 1}}, #{foo => 1}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1}, #{foo => #{bar => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => ignored}, #{foo => #{bar => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 2}, bar => #{foo => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => #{foo => 1}}, #{foo => #{bar => 2}, bar => 2}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 2}, bar => #{foo => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 2}, bar => 2}, #{foo => 1, bar => #{foo => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => #{}}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => #{foo => []}}}, #{foo => 1}, DummyLogger ) ) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index f37d7fb6a..3e442a926 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -261,7 +261,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(cassa_bridge_type, Config), Name = ?config(cassa_name, Config), BridgeConfig0 = ?config(cassa_config, Config), - BridgeConfig = emqx_map_lib:deep_merge(BridgeConfig0, Overrides), + BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), emqx_bridge:create(BridgeType, Name, BridgeConfig). delete_bridge(Config) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 133ef0430..5ebd9a89d 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -193,7 +193,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(dynamo_bridge_type, Config), Name = ?config(dynamo_name, Config), DynamoConfig0 = ?config(dynamo_config, Config), - DynamoConfig = emqx_map_lib:deep_merge(DynamoConfig0, Overrides), + DynamoConfig = emqx_utils_maps:deep_merge(DynamoConfig0, Overrides), emqx_bridge:create(BridgeType, Name, DynamoConfig). delete_all_bridges() -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 49bc444c6..a785924d4 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -181,7 +181,7 @@ create_bridge(Config, GCPPubSubConfigOverrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(gcp_pubsub_name, Config), GCPPubSubConfig0 = ?config(gcp_pubsub_config, Config), - GCPPubSubConfig = emqx_map_lib:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), + GCPPubSubConfig = emqx_utils_maps:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), ct:pal("creating bridge: ~p", [GCPPubSubConfig]), Res = emqx_bridge:create(TypeBin, Name, GCPPubSubConfig), ct:pal("bridge creation result: ~p", [Res]), @@ -194,7 +194,7 @@ create_bridge_http(Config, GCPPubSubConfigOverrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(gcp_pubsub_name, Config), GCPPubSubConfig0 = ?config(gcp_pubsub_config, Config), - GCPPubSubConfig = emqx_map_lib:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), + GCPPubSubConfig = emqx_utils_maps:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), Params = GCPPubSubConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl index 4891d5d9b..3def35920 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl @@ -354,7 +354,7 @@ create_bridge(Config, Overrides) -> Type = influxdb_type_bin(?config(influxdb_type, Config)), Name = ?config(influxdb_name, Config), InfluxDBConfig0 = ?config(influxdb_config, Config), - InfluxDBConfig = emqx_map_lib:deep_merge(InfluxDBConfig0, Overrides), + InfluxDBConfig = emqx_utils_maps:deep_merge(InfluxDBConfig0, Overrides), emqx_bridge:create(Type, Name, InfluxDBConfig). delete_bridge(Config) -> @@ -390,7 +390,7 @@ create_rule_and_action_http(Config, Overrides) -> sql => <<"SELECT * FROM \"t/topic\"">>, actions => [BridgeId] }, - Params = emqx_map_lib:deep_merge(Params0, Overrides), + Params = emqx_utils_maps:deep_merge(Params0, Overrides), Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 2d4ee9222..0959e3c78 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -245,7 +245,7 @@ create_bridge(Config, Overrides) -> Type = mongo_type_bin(?config(mongo_type, Config)), Name = ?config(mongo_name, Config), MongoConfig0 = ?config(mongo_config, Config), - MongoConfig = emqx_map_lib:deep_merge(MongoConfig0, Overrides), + MongoConfig = emqx_utils_maps:deep_merge(MongoConfig0, Overrides), ct:pal("creating ~p bridge with config:\n ~p", [Type, MongoConfig]), emqx_bridge:create(Type, Name, MongoConfig). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl index 8f0b9ad9c..d76149b16 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl @@ -229,7 +229,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(pgsql_bridge_type, Config), Name = ?config(pgsql_name, Config), PGConfig0 = ?config(pgsql_config, Config), - PGConfig = emqx_map_lib:deep_merge(PGConfig0, Overrides), + PGConfig = emqx_utils_maps:deep_merge(PGConfig0, Overrides), emqx_bridge:create(BridgeType, Name, PGConfig). delete_bridge(Config) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl index 8e058f2f1..36ed10f38 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl @@ -207,7 +207,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(tdengine_bridge_type, Config), Name = ?config(tdengine_name, Config), TDConfig0 = ?config(tdengine_config, Config), - TDConfig = emqx_map_lib:deep_merge(TDConfig0, Overrides), + TDConfig = emqx_utils_maps:deep_merge(TDConfig0, Overrides), emqx_bridge:create(BridgeType, Name, TDConfig). delete_bridge(Config) -> diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl index 3569b246e..4c00903d5 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -67,7 +67,7 @@ get_schema(SchemaName) -> -spec add_schema(schema_name(), schema()) -> ok | {error, term()}. add_schema(Name, Schema) -> - RawSchema = emqx_map_lib:binary_key_map(Schema), + RawSchema = emqx_utils_maps:binary_key_map(Schema), Res = emqx_conf:update( [?CONF_KEY_ROOT, schemas, Name], RawSchema, @@ -113,7 +113,7 @@ post_config_update( added := Added, changed := Changed0, removed := Removed - } = emqx_map_lib:diff_maps(NewSchemas, OldSchemas), + } = emqx_utils_maps:diff_maps(NewSchemas, OldSchemas), Changed = maps:map(fun(_N, {_Old, New}) -> New end, Changed0), RemovedNames = maps:keys(Removed), case RemovedNames of diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 325d3d499..5bfba34b3 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -344,7 +344,7 @@ t_decode(Config) -> t_fail_rollback(Config) -> SerdeType = ?config(serde_type, Config), - OkSchema = emqx_map_lib:binary_key_map(schema_params(SerdeType)), + OkSchema = emqx_utils_maps:binary_key_map(schema_params(SerdeType)), BrokenSchema = OkSchema#{<<"source">> := <<"{}">>}, %% hopefully, for this small map, the key order is used. Serdes = #{ From 19981757aec4a4cdb215786571ece255c22f6673 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 15:49:12 +0200 Subject: [PATCH 07/32] fix: add is_json --- apps/emqx_utils/src/emqx_utils_json.erl | 16 ++++++++++++---- apps/emqx_utils/test/emqx_utils_json_SUITE.erl | 4 ++++ .../test/props/prop_emqx_utils_json.erl} | 0 3 files changed, 16 insertions(+), 4 deletions(-) rename apps/{emqx/test/props/prop_emqx_json.erl => emqx_utils/test/props/prop_emqx_utils_json.erl} (100%) diff --git a/apps/emqx_utils/src/emqx_utils_json.erl b/apps/emqx_utils/src/emqx_utils_json.erl index 2f0c6f88b..191063d38 100644 --- a/apps/emqx_utils/src/emqx_utils_json.erl +++ b/apps/emqx_utils/src/emqx_utils_json.erl @@ -46,11 +46,15 @@ ]} ). --type encode_options() :: emqx_utils_json:encode_options(). --type decode_options() :: emqx_utils_json:decode_options(). +-export([is_json/1]). + +-compile({inline, [is_json/1]}). + +-type encode_options() :: jiffy:encode_options(). +-type decode_options() :: jiffy:decode_options(). -type json_text() :: iolist() | binary(). --type json_term() :: emqx_utils_json:jiffy_decode_result(). +-type json_term() :: jiffy:jiffy_decode_result(). -export_type([json_text/0, json_term/0]). -export_type([decode_options/0, encode_options/0]). @@ -79,7 +83,7 @@ safe_encode(Term, Opts) -> end. -spec decode(json_text()) -> json_term(). -decode(Json) -> decode(Json, []). +decode(Json) -> decode(Json, [return_maps]). -spec decode(json_text(), decode_options()) -> json_term(). decode(Json, Opts) -> @@ -100,6 +104,10 @@ safe_decode(Json, Opts) -> {error, Reason} end. +-spec is_json(json_text()) -> boolean(). +is_json(Json) -> + element(1, safe_decode(Json)) =:= ok. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- diff --git a/apps/emqx_utils/test/emqx_utils_json_SUITE.erl b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl index 889cde595..ba7b5226a 100644 --- a/apps/emqx_utils/test/emqx_utils_json_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl @@ -136,3 +136,7 @@ safe_encode_decode(Term) -> {ok, {NTerm}} -> NTerm; {ok, NTerm} -> NTerm end. + +t_is_json(_) -> + ?assert(emqx_utils_json:is_json(<<"{}">>)), + ?assert(not emqx_utils_json:is_json(<<"foo">>)). diff --git a/apps/emqx/test/props/prop_emqx_json.erl b/apps/emqx_utils/test/props/prop_emqx_utils_json.erl similarity index 100% rename from apps/emqx/test/props/prop_emqx_json.erl rename to apps/emqx_utils/test/props/prop_emqx_utils_json.erl From 4f80690162e59f69532ba435e3c1355b3ecc65df Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 15:50:34 +0200 Subject: [PATCH 08/32] fix: byebye jsx --- apps/emqx/src/emqx.app.src | 1 - apps/emqx/src/emqx_logger_jsonfmt.erl | 4 ++-- apps/emqx/test/emqx_common_test_helpers.erl | 2 +- apps/emqx_authz/src/emqx_authz_http.erl | 2 +- .../test/emqx_authz_api_cache_SUITE.erl | 4 ++-- .../test/emqx_authz_api_mnesia_SUITE.erl | 22 ++++++++--------- .../test/emqx_authz_api_settings_SUITE.erl | 4 ++-- .../test/emqx_bridge_mqtt_SUITE.erl | 24 +++++++++---------- apps/emqx_conf/src/emqx_conf.erl | 6 ++--- .../test/emqx_dashboard_SUITE.erl | 2 +- .../test/emqx_dashboard_api_test_helpers.erl | 2 +- .../src/emqx_lwm2m_message.erl | 2 +- .../test/emqx_mgmt_api_test_util.erl | 2 +- .../test/emqx_telemetry_api_SUITE.erl | 6 ++--- .../test/emqx_topic_metrics_api_SUITE.erl | 10 ++++---- apps/emqx_utils/src/emqx_utils.app.src | 3 ++- apps/emqx_utils/src/emqx_utils_json.erl | 6 +++-- .../emqx_utils/test/emqx_utils_json_SUITE.erl | 23 +++++++++++------- 18 files changed, 66 insertions(+), 59 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 7014c8381..c4894c933 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -16,7 +16,6 @@ cowboy, sasl, os_mon, - jiffy, lc, hocon ]}, diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index f60f5a716..e231b76dd 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -62,11 +62,11 @@ %% The JSON object is pretty-printed. %% NOTE: do not use this function for logging. best_effort_json(Input) -> - best_effort_json(Input, [space, {indent, 4}]). + best_effort_json(Input, [pretty]). best_effort_json(Input, Opts) -> Config = #{depth => unlimited, single_line => true}, JsonReady = best_effort_json_obj(Input, Config), - jsx:encode(JsonReady, Opts). + emqx_utils_json:encode(JsonReady, Opts). -spec format(logger:log_event(), config()) -> iodata(). format(#{level := Level, msg := Msg, meta := Meta} = Event, Config0) when is_map(Config0) -> diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 85cb46106..6628401cc 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -477,7 +477,7 @@ copy_certs(_, _) -> load_config(SchemaModule, Config, Opts) -> ConfigBin = case is_map(Config) of - true -> jsx:encode(Config); + true -> emqx_utils_json:encode(Config); false -> Config end, ok = emqx_config:delete_override_conf_files(), diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 852a667c8..53378d9c2 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -227,7 +227,7 @@ encode_path(Path) -> lists:flatten(["/" ++ Part || Part <- lists:map(fun uri_encode/1, Parts)]). serialize_body(<<"application/json">>, Body) -> - jsx:encode(Body); + emqx_utils_json:encode(Body); serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> query_string(Body). diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index cfd9f581f..672ddc73c 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -67,12 +67,12 @@ t_clean_cahce(_) -> ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0), {ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(2, length(emqx_utils_json:decode(Result3))), + ?assertEqual(2, maps:size(emqx_utils_json:decode(Result3))), request(delete, uri(["authorization", "cache"])), {ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(0, length(emqx_utils_json:decode(Result4))), + ?assertEqual(0, maps:size(emqx_utils_json:decode(Result4))), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index 2aa2d9545..3775b9a1c 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -95,7 +95,7 @@ t_api(_) -> <<"page">> := 1, <<"hasnext">> := false } - } = jsx:decode(Request1), + } = emqx_utils_json:decode(Request1), ?assertEqual(3, length(Rules1)), {ok, 200, Request1_1} = @@ -119,7 +119,7 @@ t_api(_) -> <<"hasnext">> => false } }, - jsx:decode(Request1_1) + emqx_utils_json:decode(Request1_1) ), {ok, 200, Request2} = @@ -128,7 +128,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users", "user1"]), [] ), - #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = jsx:decode(Request2), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = emqx_utils_json:decode(Request2), {ok, 204, _} = request( @@ -142,7 +142,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users", "user1"]), [] ), - #{<<"username">> := <<"user1">>, <<"rules">> := Rules2} = jsx:decode(Request3), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules2} = emqx_utils_json:decode(Request3), ?assertEqual(0, length(Rules2)), {ok, 204, _} = @@ -202,8 +202,8 @@ t_api(_) -> <<"data">> := [#{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3}], <<"meta">> := #{<<"count">> := 1, <<"limit">> := 100, <<"page">> := 1} } = - jsx:decode(Request4), - #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = jsx:decode(Request5), + emqx_utils_json:decode(Request4), + #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = emqx_utils_json:decode(Request5), ?assertEqual(3, length(Rules3)), {ok, 204, _} = @@ -218,7 +218,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "clients", "client1"]), [] ), - #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules4} = jsx:decode(Request6), + #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules4} = emqx_utils_json:decode(Request6), ?assertEqual(0, length(Rules4)), {ok, 204, _} = @@ -252,7 +252,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "all"]), [] ), - #{<<"rules">> := Rules5} = jsx:decode(Request7), + #{<<"rules">> := Rules5} = emqx_utils_json:decode(Request7), ?assertEqual(3, length(Rules5)), {ok, 204, _} = @@ -267,7 +267,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "all"]), [] ), - #{<<"rules">> := Rules6} = jsx:decode(Request8), + #{<<"rules">> := Rules6} = emqx_utils_json:decode(Request8), ?assertEqual(0, length(Rules6)), {ok, 204, _} = @@ -285,7 +285,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users?page=2&limit=5"]), [] ), - #{<<"data">> := Data1} = jsx:decode(Request9), + #{<<"data">> := Data1} = emqx_utils_json:decode(Request9), ?assertEqual(5, length(Data1)), {ok, 204, _} = @@ -303,7 +303,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "clients?limit=5"]), [] ), - #{<<"data">> := Data2} = jsx:decode(Request10), + #{<<"data">> := Data2} = emqx_utils_json:decode(Request10), ?assertEqual(5, length(Data2)), {ok, 400, Msg1} = diff --git a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl index 41eba109e..e3412e169 100644 --- a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl @@ -76,7 +76,7 @@ t_api(_) -> {ok, 200, Result1} = request(put, uri(["authorization", "settings"]), Settings1), {ok, 200, Result1} = request(get, uri(["authorization", "settings"]), []), - ?assertEqual(Settings1, jsx:decode(Result1)), + ?assertEqual(Settings1, emqx_utils_json:decode(Result1)), Settings2 = #{ <<"no_match">> => <<"allow">>, @@ -90,7 +90,7 @@ t_api(_) -> {ok, 200, Result2} = request(put, uri(["authorization", "settings"]), Settings2), {ok, 200, Result2} = request(get, uri(["authorization", "settings"]), []), - ?assertEqual(Settings2, jsx:decode(Result2)), + ?assertEqual(Settings2, emqx_utils_json:decode(Result2)), ok. diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index ab58ce383..bd5cda3f0 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -201,7 +201,7 @@ t_mqtt_conn_bridge_ingress(_) -> #{ <<"type">> := ?TYPE_MQTT, <<"name">> := ?BRIDGE_NAME_INGRESS - } = jsx:decode(Bridge), + } = emqx_utils_json:decode(Bridge), BridgeIDIngress = emqx_bridge_resource:bridge_id(?TYPE_MQTT, ?BRIDGE_NAME_INGRESS), @@ -313,7 +313,7 @@ t_mqtt_conn_bridge_ingress_no_payload_template(_) -> emqx:publish(emqx_message:make(RemoteTopic, Payload)), %% we should receive a message on the local broker, with specified topic Msg = assert_mqtt_msg_received(LocalTopic), - ?assertMatch(#{<<"payload">> := Payload}, jsx:decode(Msg#message.payload)), + ?assertMatch(#{<<"payload">> := Payload}, emqx_utils_json:decode(Msg#message.payload)), %% verify the metrics of the bridge ?assertMetrics( @@ -402,7 +402,7 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> Msg = assert_mqtt_msg_received(RemoteTopic), %% the MapMsg is all fields outputed by Rule-Engine. it's a binary coded json here. ?assertMatch(<>, Msg#message.from), - ?assertMatch(#{<<"payload">> := Payload}, jsx:decode(Msg#message.payload)), + ?assertMatch(#{<<"payload">> := Payload}, emqx_utils_json:decode(Msg#message.payload)), %% verify the metrics of the bridge ?retry( @@ -545,7 +545,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> <<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">> } ), - #{<<"id">> := RuleId} = jsx:decode(Rule), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule), %% we now test if the bridge works as expected @@ -562,7 +562,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> %% and also the rule should be matched, with matched + 1: {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), {ok, 200, Metrics} = request(get, uri(["rules", RuleId, "metrics"]), []), - ?assertMatch(#{<<"id">> := RuleId}, jsx:decode(Rule1)), + ?assertMatch(#{<<"id">> := RuleId}, emqx_utils_json:decode(Rule1)), ?assertMatch( #{ <<"metrics">> := #{ @@ -581,7 +581,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> <<"actions.failed.unknown">> := 0 } }, - jsx:decode(Metrics) + emqx_utils_json:decode(Metrics) ), %% we also check if the actions of the rule is triggered @@ -630,7 +630,7 @@ t_egress_mqtt_bridge_with_rules(_) -> <<"sql">> => <<"SELECT * from \"t/1\"">> } ), - #{<<"id">> := RuleId} = jsx:decode(Rule), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule), %% we now test if the bridge works as expected LocalTopic = <>, @@ -653,7 +653,7 @@ t_egress_mqtt_bridge_with_rules(_) -> timer:sleep(100), emqx:publish(emqx_message:make(RuleTopic, Payload2)), {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), - ?assertMatch(#{<<"id">> := RuleId, <<"name">> := _}, jsx:decode(Rule1)), + ?assertMatch(#{<<"id">> := RuleId, <<"name">> := _}, emqx_utils_json:decode(Rule1)), {ok, 200, Metrics} = request(get, uri(["rules", RuleId, "metrics"]), []), ?assertMatch( #{ @@ -673,7 +673,7 @@ t_egress_mqtt_bridge_with_rules(_) -> <<"actions.failed.unknown">> := 0 } }, - jsx:decode(Metrics) + emqx_utils_json:decode(Metrics) ), %% we should receive a message on the "remote" broker, with specified topic @@ -911,17 +911,17 @@ create_bridge(Config = #{<<"type">> := Type, <<"name">> := Name}) -> <<"type">> := Type, <<"name">> := Name }, - jsx:decode(Bridge) + emqx_utils_json:decode(Bridge) ), emqx_bridge_resource:bridge_id(Type, Name). request_bridge(BridgeID) -> {ok, 200, Bridge} = request(get, uri(["bridges", BridgeID]), []), - jsx:decode(Bridge). + emqx_utils_json:decode(Bridge). request_bridge_metrics(BridgeID) -> {ok, 200, BridgeMetrics} = request(get, uri(["bridges", BridgeID, "metrics"]), []), - jsx:decode(BridgeMetrics). + emqx_utils_json:decode(BridgeMetrics). request(Method, Url, Body) -> request(<<"connector_admin">>, Method, Url, Body). diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 246875ab6..1ecda913d 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -166,7 +166,7 @@ gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> io:format(user, "===< Including fields from importance level: ~p~n", [IncludeImportance]), Opts = #{desc_file => I18nFile, lang => Lang, include_importance_up_from => IncludeImportance}, JsonMap = hocon_schema_json:gen(SchemaModule, Opts), - IoData = jsx:encode(JsonMap, [space, {indent, 4}]), + IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]), ok = file:write_file(SchemaJsonFile, IoData). gen_api_schema_json(Dir, I18nFile, Lang) -> @@ -268,13 +268,13 @@ do_gen_api_schema_json(File, SchemaMod, SchemaInfo) -> ApiSpec0 ), Components = lists:foldl(fun(M, Acc) -> maps:merge(M, Acc) end, #{}, Components0), - IoData = jsx:encode( + IoData = emqx_utils_json:encode( #{ info => SchemaInfo, paths => ApiSpec, components => #{schemas => Components} }, - [space, {indent, 4}] + [pretty, force_utf8] ), file:write_file(File, IoData). diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index ee850ec17..1f14b02c0 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -142,7 +142,7 @@ t_swagger_json(_Config) -> %% with auth Auth = auth_header_(<<"admin">>, <<"public_www1">>), {ok, 200, Body1} = request_api(get, Url, Auth), - ?assert(jsx:is_json(Body1)), + ?assert(emqx_utils_json:is_json(Body1)), %% without auth {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, Body2}} = httpc:request(get, {Url, []}, [], [{body_format, binary}]), diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index 8df130897..91c7729d3 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -82,7 +82,7 @@ request(Username, Method, Url, Body) -> -> {Url, [auth_header(Username)]}; _ -> - {Url, [auth_header(Username)], "application/json", jsx:encode(Body)} + {Url, [auth_header(Username)], "application/json", emqx_utils_json:encode(Body)} end, ct:pal("Method: ~p, Request: ~p", [Method, Request]), case httpc:request(Method, Request, [], [{body_format, binary}]) of diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl index c65fff441..8b9ba2491 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl @@ -152,7 +152,7 @@ object_resource_id(BaseName) -> value(Value, ResourceId, ObjDefinition) -> case emqx_lwm2m_xml_object:get_resource_type(ResourceId, ObjDefinition) of "String" -> - % keep binary type since it is same as a string for jsx + % keep binary type since it is same as a string for emqx_utils_json Value; "Integer" -> Size = byte_size(Value) * 8, diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 5fe4ca937..985b95d5b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -157,7 +157,7 @@ api_path_without_base_path(Parts) -> %% %% Usage with RequestData: %% Payload = [{upload_type, <<"user_picture">>}], -%% PayloadContent = jsx:encode(Payload), +%% PayloadContent = emqx_utils_json:encode(Payload), %% RequestData = [ %% {<<"payload">>, PayloadContent} %% ] diff --git a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl index 16f942bc0..ac6d12039 100644 --- a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl @@ -113,7 +113,7 @@ t_status(_) -> ?assertEqual( #{<<"enable">> => false}, - jsx:decode(Result0) + emqx_utils_json:decode(Result0) ), ?assertMatch( @@ -139,7 +139,7 @@ t_status(_) -> ?assertEqual( #{<<"enable">> => true}, - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -180,7 +180,7 @@ t_data(_) -> <<"uuid">> := _, <<"vm_specs">> := _ }, - jsx:decode(Result) + emqx_utils_json:decode(Result) ), {ok, 200, _} = diff --git a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl index ea85d1fe9..5d64f123d 100644 --- a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl @@ -74,7 +74,7 @@ t_mqtt_topic_metrics_collection(_) -> ?assertEqual( [], - jsx:decode(Result0) + emqx_utils_json:decode(Result0) ), {ok, 200, _} = request( @@ -95,7 +95,7 @@ t_mqtt_topic_metrics_collection(_) -> <<"metrics">> := #{} } ], - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -150,7 +150,7 @@ t_mqtt_topic_metrics(_) -> uri(["mqtt", "topic_metrics"]) ), - ?assertMatch([_], jsx:decode(Result0)), + ?assertMatch([_], emqx_utils_json:decode(Result0)), {ok, 200, Result1} = request( get, @@ -162,7 +162,7 @@ t_mqtt_topic_metrics(_) -> <<"topic">> := <<"topic/1/2">>, <<"metrics">> := #{} }, - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -288,7 +288,7 @@ t_node_aggregation(_) -> <<"topic">> := <<"topic/1/2">>, <<"metrics">> := #{<<"messages.dropped.count">> := 3} }, - jsx:decode(Result) + emqx_utils_json:decode(Result) ), meck:unload(emqx_topic_metrics_proto_v1). diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index 08a802d1b..b0dedbd72 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -14,7 +14,8 @@ {registered, []}, {applications, [ kernel, - stdlib + stdlib, + jiffy ]}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_utils/src/emqx_utils_json.erl b/apps/emqx_utils/src/emqx_utils_json.erl index 191063d38..df7388c94 100644 --- a/apps/emqx_utils/src/emqx_utils_json.erl +++ b/apps/emqx_utils/src/emqx_utils_json.erl @@ -65,7 +65,7 @@ encode(Term) -> -spec encode(json_term(), encode_options()) -> json_text(). encode(Term, Opts) -> - to_binary(emqx_utils_json:encode(to_ejson(Term), Opts)). + to_binary(jiffy:encode(to_ejson(Term), Opts)). -spec safe_encode(json_term()) -> {ok, json_text()} | {error, Reason :: term()}. @@ -87,7 +87,7 @@ decode(Json) -> decode(Json, [return_maps]). -spec decode(json_text(), decode_options()) -> json_term(). decode(Json, Opts) -> - from_ejson(emqx_utils_json:decode(Json, Opts)). + from_ejson(jiffy:decode(Json, Opts)). -spec safe_decode(json_text()) -> {ok, json_term()} | {error, Reason :: term()}. @@ -125,6 +125,8 @@ to_ejson([{_, _} | _] = L) -> {[{K, to_ejson(V)} || {K, V} <- L]}; to_ejson(L) when is_list(L) -> [to_ejson(E) || E <- L]; +to_ejson(M) when is_map(M) -> + maps:map(fun(_K, V) -> to_ejson(V) end, M); to_ejson(T) -> T. diff --git a/apps/emqx_utils/test/emqx_utils_json_SUITE.erl b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl index ba7b5226a..daf31b440 100644 --- a/apps/emqx_utils/test/emqx_utils_json_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl @@ -84,10 +84,10 @@ t_decode_encode(_) -> 1.25 = decode(encode(1.25)), [] = decode(encode([])), [true, 1] = decode(encode([true, 1])), - [{}] = decode(encode([{}])), - [{<<"foo">>, <<"bar">>}] = decode(encode([{foo, bar}])), - [{<<"foo">>, <<"bar">>}] = decode(encode([{<<"foo">>, <<"bar">>}])), - [[{<<"foo">>, <<"bar">>}]] = decode(encode([[{<<"foo">>, <<"bar">>}]])), + [{}] = decode(encode([{}]), []), + [{<<"foo">>, <<"bar">>}] = decode(encode([{foo, bar}]), []), + [{<<"foo">>, <<"bar">>}] = decode(encode([{<<"foo">>, <<"bar">>}]), []), + [[{<<"foo">>, <<"bar">>}]] = decode(encode([[{<<"foo">>, <<"bar">>}]]), []), [ [ {<<"foo">>, <<"bar">>}, @@ -101,7 +101,8 @@ t_decode_encode(_) -> {<<"a">>, <<"b">>} ], [{<<"x">>, <<"y">>}] - ]) + ]), + [] ), #{<<"foo">> := <<"bar">>} = decode(encode(#{<<"foo">> => <<"bar">>}), [return_maps]), JsonText = <<"{\"bool\":true,\"int\":10,\"foo\":\"bar\"}">>, @@ -110,8 +111,12 @@ t_decode_encode(_) -> <<"int">> => 10, <<"foo">> => <<"bar">> }, - ?assertEqual(JsonText, encode({decode(JsonText)})), - ?assertEqual(JsonMaps, decode(JsonText, [return_maps])). + ?assertEqual(JsonText, encode({decode(JsonText, [])})), + ?assertEqual(JsonMaps, decode(JsonText, [return_maps])), + ?assertEqual( + #{<<"foo">> => #{<<"bar">> => <<"baz">>}}, + decode(encode(#{<<"foo">> => [{<<"bar">>, <<"baz">>}]})) + ). t_safe_decode_encode(_) -> safe_encode_decode(null), @@ -123,7 +128,7 @@ t_safe_decode_encode(_) -> 1.25 = safe_encode_decode(1.25), [] = safe_encode_decode([]), [true, 1] = safe_encode_decode([true, 1]), - [{}] = decode(encode([{}])), + [{}] = decode(encode([{}]), []), [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{foo, bar}]), [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{<<"foo">>, <<"bar">>}]), [[{<<"foo">>, <<"bar">>}]] = safe_encode_decode([[{<<"foo">>, <<"bar">>}]]), @@ -132,7 +137,7 @@ t_safe_decode_encode(_) -> safe_encode_decode(Term) -> {ok, Json} = emqx_utils_json:safe_encode(Term), - case emqx_utils_json:safe_decode(Json) of + case emqx_utils_json:safe_decode(Json, []) of {ok, {NTerm}} -> NTerm; {ok, NTerm} -> NTerm end. From 90520a5382a3b871e97de066ff469135b2af9322 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 11:43:53 +0200 Subject: [PATCH 09/32] docs: add an actual README --- apps/emqx_utils/README.md | 55 +++++++++++++-------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/apps/emqx_utils/README.md b/apps/emqx_utils/README.md index 482b7a947..046b66321 100644 --- a/apps/emqx_utils/README.md +++ b/apps/emqx_utils/README.md @@ -1,43 +1,24 @@ -# [Application Name] - [Mandatory] -> 0. App overview introduction -> 1. let people know what your project can do specifically. Is it a base -> library dependency, or what kind of functionality is provided to the user? -> 2. Provide context and add a link to any reference visitors might be -> unfamiliar with. -> 3. Design details, implementation technology architecture, Roadmap, etc. +# emqx_utils - Erlang utility library for EMQ X -# [Features] - [Optional] -> A List of features your application provided. If the feature is quite simple, just -> list in the previous section. +## Overview -# [Limitation] - [Optional] -> Explain the limitations of the implementation used in your application. +`emqx_utils` is a collection of utility functions for EMQ X, organized into +several modules. It provides various functionalities to make it easier to work +with EMQ X, such as binary manipulations, maps, JSON en- and decoding, ets table +handling, data conversions, and more. -# [Documention links] - [Mandatory] -> You can use the official docs link to provide a detailed explanation of -> the concept and functions of this application. +## Features -# [Installation] - [Optional] -> How users can obtain this application. In most cases, this section is unnecessary. -> Most of the applications are released with the EMQX distribution package. -> Otherwise, it is necessary to provide step-by-step instructions on how -> to install this application. +- `emqx_utils`: unsorted helper functions, formerly known as `emqx_misc` - NEEDS WORK +- `emqx_utils_api`: collection of helper functions for API responses +- `emqx_utils_binary`: binary reverse, join, trim etc +- `emqx_utils_ets`: convenience functions for creating and looking up data in ets tables. +- `emqx_utils_json`: JSON encoding and decoding +- `emqx_utils_maps`: convenience functions for map lookup and manipulation like + deep_get etc. -# [Usage] - [Optional] -> This section explains how users can use the features provided by the application. +## Contributing -# [Basic Usage Guide] - [Optional] -> Simple and reproducible introduction on how to use/start it. - -# [Configurations] - [Mandatory] -> Most important configurations that this application depends on. -> It would be best to attach official documentation if it's available. - -# [HTTP APIs] - [Optional] -> Same as configuration. - -# [Other] - [Optional] -> Other topics that users may need to know can be placed here. - -# Contributing - [Mandatory] -Please see our [contributing.md](../../CONTRIBUTING.md). +Please see our [contributing guidelines](../../CONTRIBUTING.md) for information +on how to contribute to `emqx_utils`. We welcome bug reports, feature requests, +and pull requests. From d98f7222ff2ccdba5f6830a720961f1ea649015c Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 11:52:21 +0200 Subject: [PATCH 10/32] style: add comment to binary_string/1 --- apps/emqx_utils/src/emqx_utils_maps.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_utils/src/emqx_utils_maps.erl b/apps/emqx_utils/src/emqx_utils_maps.erl index d64075511..6bec32ae3 100644 --- a/apps/emqx_utils/src/emqx_utils_maps.erl +++ b/apps/emqx_utils/src/emqx_utils_maps.erl @@ -210,6 +210,7 @@ binary_string_kv(K, V, JsonableFun) -> {K1, V1} -> {binary_string(K1), V1} end. +%% [FIXME] this doesn't belong here binary_string([]) -> []; binary_string(Val) when is_list(Val) -> From badf96280019f1b89482cdb9de96cbca418fcd01 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 11:53:05 +0200 Subject: [PATCH 11/32] fix: stale call to emqx_misc --- apps/emqx_utils/include/emqx_utils_api.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_utils/include/emqx_utils_api.hrl b/apps/emqx_utils/include/emqx_utils_api.hrl index 549b0f94c..bfc8e0a53 100644 --- a/apps/emqx_utils/include/emqx_utils_api.hrl +++ b/apps/emqx_utils/include/emqx_utils_api.hrl @@ -17,7 +17,7 @@ -ifndef(EMQX_API_LIB_HRL). -define(EMQX_API_LIB_HRL, true). --define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_misc:readable_error_msg(REASON)}). +-define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_utils:readable_error_msg(REASON)}). -define(OK(CONTENT), {200, CONTENT}). From 0f162fb50a4cc38a8a599785ca9dde95c8f7aa0b Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 11:54:02 +0200 Subject: [PATCH 12/32] test: add tests for emqx_utils_binary from original site --- .../test/emqx_utils_binary_tests.erl | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 apps/emqx_utils/test/emqx_utils_binary_tests.erl diff --git a/apps/emqx_utils/test/emqx_utils_binary_tests.erl b/apps/emqx_utils/test/emqx_utils_binary_tests.erl new file mode 100644 index 000000000..514e0874a --- /dev/null +++ b/apps/emqx_utils/test/emqx_utils_binary_tests.erl @@ -0,0 +1,212 @@ +-module(emqx_utils_binary_tests). + +-import(emqx_utils_binary, [ + trim/1, + ltrim/1, + rtrim/1, + trim/2, + ltrim/2, + rtrim/2, + reverse/1, + inverse/1, + join/2, + suffix/2, + prefix/2, + duplicate/2, + union/2, + intersection/2, + subtract/2, + optimize_patterns/1 +]). + +-include_lib("eunit/include/eunit.hrl"). + +-ifdef(TEST). + +trim1_test_() -> + [ + ?_assertEqual(trim(<<>>), <<>>), + ?_assertEqual(trim(<<0, 0, 0>>), <<>>), + ?_assertEqual(trim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(trim(<<0, 1, 2>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 0, 1, 2>>), <<1, 2>>), + ?_assertEqual(trim(<<1, 2, 0, 0>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 1, 2, 0>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<1, 2>>) + ]. + +ltrim1_test_() -> + [ + ?_assertEqual(ltrim(<<>>), <<>>), + ?_assertEqual(ltrim(<<0, 0, 0>>), <<>>), + ?_assertEqual(ltrim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(ltrim(<<0, 1, 2>>), <<1, 2>>), + ?_assertEqual(ltrim(<<0, 0, 1, 2>>), <<1, 2>>), + ?_assertEqual(ltrim(<<1, 2, 0, 0>>), <<1, 2, 0, 0>>), + ?_assertEqual(ltrim(<<0, 1, 2, 0>>), <<1, 2, 0>>), + ?_assertEqual(ltrim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<1, 2, 0, 0, 0>>) + ]. + +rtrim1_test_() -> + [ + ?_assertEqual(rtrim(<<>>), <<>>), + ?_assertEqual(rtrim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(rtrim(<<0, 0, 0>>), <<>>), + ?_assertEqual(rtrim(<<0, 1, 2>>), <<0, 1, 2>>), + ?_assertEqual(rtrim(<<0, 0, 1, 2>>), <<0, 0, 1, 2>>), + ?_assertEqual(rtrim(<<1, 2, 0, 0>>), <<1, 2>>), + ?_assertEqual(rtrim(<<0, 1, 2, 0>>), <<0, 1, 2>>), + ?_assertEqual(rtrim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<0, 0, 0, 1, 2>>) + ]. + +trim2_test_() -> + [ + ?_assertEqual(trim(<<5>>, 5), <<>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, 5), <<1, 2>>), + ?_assertEqual(trim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<1, 2, 0, 0, 0>>) + ]. + +ltrim2_test_() -> + [ + ?_assertEqual(ltrim(<<5>>, 5), <<>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, 5), <<1, 2, 5>>), + ?_assertEqual(ltrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<1, 2, 0, 0, 0>>) + ]. + +rtrim2_test_() -> + [ + ?_assertEqual(rtrim(<<5>>, 5), <<>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, 5), <<5, 1, 2>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<5, 5, 5, 1, 2, 0, 0, 0>>) + ]. + +mtrim2_test_() -> + [ + ?_assertEqual(trim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, [1, 5]), <<2>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, [1, 2, 5]), <<>>), + ?_assertEqual(trim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<2, 0, 0, 0>>) + ]. + +mltrim2_test_() -> + [ + ?_assertEqual(ltrim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, [1, 5]), <<2, 5>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, [2, 5]), <<1, 2, 5>>), + ?_assertEqual(ltrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<2, 0, 0, 0>>) + ]. + +mrtrim2_test_() -> + [ + ?_assertEqual(rtrim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, [1, 5]), <<5, 1, 2>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, [2, 5]), <<5, 1>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<5, 5, 5, 1, 2, 0, 0, 0>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [0, 5]), <<5, 5, 5, 1, 2>>) + ]. + +reverse_test_() -> + [?_assertEqual(reverse(<<0, 1, 2>>), <<2, 1, 0>>)]. + +join_test_() -> + [ + ?_assertEqual(join([<<1, 2>>, <<3, 4>>, <<5, 6>>], <<0>>), <<1, 2, 0, 3, 4, 0, 5, 6>>), + ?_assertEqual( + join([<<"abc">>, <<"def">>, <<"xyz">>], <<"|">>), + <<"abc|def|xyz">> + ), + ?_assertEqual( + join([<<>>, <<"|">>, <<"x|z">>], <<"|">>), + <<"|||x|z">> + ), + ?_assertEqual( + join([<<"abc">>, <<"def">>, <<"xyz">>], <<>>), + <<"abcdefxyz">> + ), + ?_assertEqual(join([], <<"|">>), <<>>) + ]. + +duplicate_test_() -> + [ + ?_assertEqual(duplicate(5, <<1, 2>>), <<1, 2, 1, 2, 1, 2, 1, 2, 1, 2>>), + ?_assertEqual(duplicate(50, <<0>>), <<0:400>>) + ]. + +suffix_test_() -> + [ + ?_assertEqual(suffix(<<1, 2, 3, 4, 5>>, 2), <<4, 5>>), + ?_assertError(badarg, prefix(<<1, 2, 3, 4, 5>>, 25)) + ]. + +prefix_test_() -> + [ + ?_assertEqual(prefix(<<1, 2, 3, 4, 5>>, 2), <<1, 2>>), + ?_assertError(badarg, prefix(<<1, 2, 3, 4, 5>>, 25)) + ]. + +union_test_() -> + [ + ?_assertEqual( + union( + <<2#0011011:7>>, + <<2#1011110:7>> + ), + <<2#1011111:7>> + ) + ]. + +inverse_test_() -> + [ + ?_assertEqual(inverse(inverse(<<0, 1, 2>>)), <<0, 1, 2>>), + ?_assertEqual(inverse(<<0>>), <<255>>), + ?_assertEqual(inverse(<<2#1:1>>), <<2#0:1>>), + ?_assertEqual(inverse(<<2#0:1>>), <<2#1:1>>), + ?_assertEqual( + inverse(<<2#01:2>>), + <<2#10:2>> + ), + ?_assertEqual( + inverse(<<2#0011011:7>>), + <<2#1100100:7>> + ) + ]. + +intersection_test_() -> + [ + ?_assertEqual( + intersection( + <<2#0011011>>, + <<2#1011110>> + ), + <<2#0011010>> + ) + ]. + +subtract_test_() -> + [ + ?_assertEqual( + subtract( + <<2#0011011>>, + <<2#1011110>> + ), + <<2#0000001>> + ) + ]. + +optimize_patterns_test_() -> + [ + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"t">>, <<"test">>]) + ), + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"t">>, <<"t">>, <<"test">>]) + ), + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"test">>, <<"t">>, <<"t">>]) + ) + ]. + +-endif. From 92ca2f66f5a648799a29838a2db7fcab2a94f252 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 12:06:52 +0200 Subject: [PATCH 13/32] style: add original copyright header --- apps/emqx_utils/src/emqx_utils_binary.erl | 4 ++-- apps/emqx_utils/test/emqx_utils_binary_tests.erl | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/emqx_utils/src/emqx_utils_binary.erl b/apps/emqx_utils/src/emqx_utils_binary.erl index d9e504f46..26976496d 100644 --- a/apps/emqx_utils/src/emqx_utils_binary.erl +++ b/apps/emqx_utils/src/emqx_utils_binary.erl @@ -1,4 +1,6 @@ %%-------------------------------------------------------------------- +%% Original file taken from https://github.com/arcusfelis/binary2 +%% Copyright (c) 2016 Michael Uvarov %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,8 +18,6 @@ -module(emqx_utils_binary). -%% copied from https://github.com/arcusfelis/binary2 - %% Bytes -export([ reverse/1, diff --git a/apps/emqx_utils/test/emqx_utils_binary_tests.erl b/apps/emqx_utils/test/emqx_utils_binary_tests.erl index 514e0874a..3709fb014 100644 --- a/apps/emqx_utils/test/emqx_utils_binary_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_binary_tests.erl @@ -1,3 +1,8 @@ +%%-------------------------------------------------------------------- +%% Original file taken from https://github.com/arcusfelis/binary2 +%% Copyright (c) 2016 Michael Uvarov +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% -module(emqx_utils_binary_tests). -import(emqx_utils_binary, [ From 1bad6ca67db338c939193b539aced8ac6f41cce6 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 12:13:29 +0200 Subject: [PATCH 14/32] fix: jiffy pretty print is a bit different from jsx --- apps/emqx/src/emqx_logger_jsonfmt.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index e231b76dd..0e72fd2b5 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -62,7 +62,7 @@ %% The JSON object is pretty-printed. %% NOTE: do not use this function for logging. best_effort_json(Input) -> - best_effort_json(Input, [pretty]). + best_effort_json(Input, [pretty, force_utf8]). best_effort_json(Input, Opts) -> Config = #{depth => unlimited, single_line => true}, JsonReady = best_effort_json_obj(Input, Config), @@ -378,15 +378,15 @@ p_config() -> best_effort_json_test() -> ?assertEqual( - <<"{}">>, + <<"{\n \n}">>, emqx_logger_jsonfmt:best_effort_json([]) ), ?assertEqual( - <<"{\n \"key\": []\n}">>, + <<"{\n \"key\" : [\n \n ]\n}">>, emqx_logger_jsonfmt:best_effort_json(#{key => []}) ), ?assertEqual( - <<"[\n {\n \"key\": []\n }\n]">>, + <<"[\n {\n \"key\" : [\n \n ]\n }\n]">>, emqx_logger_jsonfmt:best_effort_json([#{key => []}]) ), ok. From b5a496aacdc4b81619f74e16f73efbc65b3b1cd4 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 14:02:56 +0200 Subject: [PATCH 15/32] fix: include emqx_utils in distribution --- rebar.config.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/rebar.config.erl b/rebar.config.erl index 2ae6ffad4..b6b6a7c18 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -391,6 +391,7 @@ relx_apps(ReleaseType, Edition) -> {covertool, load}, % started by emqx_machine {system_monitor, load}, + emqx_utils, emqx_http_lib, emqx_resource, emqx_connector, From a295d0f1341e17fe73fbddb44ce29894f18f308e Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 14:29:25 +0200 Subject: [PATCH 16/32] fix: add rebar3_path_deps plugin --- apps/emqx/rebar.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index a3eff6c16..bf509f059 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -18,7 +18,7 @@ ]}. %% Deps here may duplicate with emqx.git root level rebar.config -%% but there not be any descrpancy. +%% but there may not be any discrepancy. %% This rebar.config is necessary because the app may be used as a %% `git_subdir` dependency in other projects. {deps, [ @@ -36,7 +36,7 @@ {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} ]}. -{plugins, [{rebar3_proper, "0.12.1"}]}. +{plugins, [{rebar3_proper, "0.12.1"}, rebar3_path_deps]}. {extra_src_dirs, [{"etc", [recursive]}]}. {profiles, [ {test, [ From 4de13d280079725d3822883b55f5eaff83ea6a1f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 09:31:33 -0300 Subject: [PATCH 17/32] feat(buffer_worker): change default max queue bytes to 256 MB --- apps/emqx_resource/include/emqx_resource.hrl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 7f88e1440..904eeffa5 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -88,8 +88,8 @@ -define(DEFAULT_QUEUE_SEG_SIZE, 10 * 1024 * 1024). -define(DEFAULT_QUEUE_SEG_SIZE_RAW, <<"10MB">>). --define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024). --define(DEFAULT_QUEUE_SIZE_RAW, <<"100MB">>). +-define(DEFAULT_QUEUE_SIZE, 256 * 1024 * 1024). +-define(DEFAULT_QUEUE_SIZE_RAW, <<"256MB">>). -define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(15)). From 55661faac66a2787e29db0156fd2ad8cabf571dc Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 14:44:54 +0200 Subject: [PATCH 18/32] fix: make elixir happy: --- mix.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/mix.exs b/mix.exs index 582860039..299f23ce6 100644 --- a/mix.exs +++ b/mix.exs @@ -303,6 +303,7 @@ defmodule EMQXUmbrella.MixProject do tools: :load, covertool: :load, system_monitor: :load, + emqx_utils: :permanent, emqx_http_lib: :permanent, emqx_resource: :permanent, emqx_connector: :permanent, From 4f2f0ebb6a25817e468681bcec03357948d6f477 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 14:58:19 +0200 Subject: [PATCH 19/32] fix: bad ref to emqx_misc after rebase --- lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index dc9759341..f11441a3b 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -188,7 +188,7 @@ on_start( ?SLOG(info, #{ msg => "starting_sqlserver_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), ODBCDir = code:priv_dir(odbc), From a9976287b53e3d759136445eddddff4c6a7e5486 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 16:26:32 +0200 Subject: [PATCH 20/32] fix: force proper calling decode to return proplist --- apps/emqx_utils/test/props/prop_emqx_utils_json.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_utils/test/props/prop_emqx_utils_json.erl b/apps/emqx_utils/test/props/prop_emqx_utils_json.erl index 86f60b39c..0be1508da 100644 --- a/apps/emqx_utils/test/props/prop_emqx_utils_json.erl +++ b/apps/emqx_utils/test/props/prop_emqx_utils_json.erl @@ -66,7 +66,7 @@ prop_object_proplist_to_proplist() -> begin {ok, J} = safe_encode(T), {ok, T} = safe_decode(J), - T = decode(encode(T)), + T = decode(encode(T), []), true end ). @@ -108,7 +108,7 @@ prop_object_map_to_proplist() -> T = to_list(T0), {ok, J} = safe_encode(T0), {ok, T} = safe_decode(J), - T = decode(encode(T0)), + T = decode(encode(T0), []), true end ). From 53871e3a2ce8b2e947a89f611ebe502c9a35a5c9 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 16:30:27 +0200 Subject: [PATCH 21/32] fix: stale ref to emqx_json after rebase --- .../test/emqx_bridge_kafka_impl_producer_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index 5e2a2a7a8..a2111b1a8 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -320,7 +320,7 @@ kafka_bridge_rest_api_helper(Config) -> <<"sql">> => <<"SELECT * from \"kafka_bridge_topic/#\"">> } ), - #{<<"id">> := RuleId} = emqx_json:decode(Rule, [return_maps]), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule, [return_maps]), %% counters should be empty before ?assertEqual(0, emqx_resource_metrics:matched_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:success_get(ResourceId)), From edd1bc579f6eaba4119126dd46a3f71f18f5e898 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 16:32:42 +0200 Subject: [PATCH 22/32] fix: stale ref to emqx_json after rebase --- lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl index c1b33c3c9..6e6d7bf03 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -501,7 +501,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. From 14ed4a7adae564497719e1fc3bc66ec504beeb82 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 10:05:20 -0300 Subject: [PATCH 23/32] feat(buffer_worker): set default queue mode to `memory_only` Fixes https://emqx.atlassian.net/browse/EMQX-9367 For better user experience and performance for the average bridge, we should change the default queue mode to `memory_only`, as was the behavior of most bridges in e4.x. This leads to better performance when message rate is high enough and the remote resource is not keeping up with EMQX. Also, we set the default segment size to equal max queue bytes. --- apps/emqx_resource/include/emqx_resource.hrl | 3 - .../src/emqx_resource_buffer_worker.erl | 43 ++++--- .../src/schema/emqx_resource_schema.erl | 17 ++- .../test/emqx_resource_SUITE.erl | 112 ++++++++++++++++++ changes/ce/feat-10404.en.md | 2 + rel/i18n/emqx_resource_schema.hocon | 38 ++++++ 6 files changed, 196 insertions(+), 19 deletions(-) create mode 100644 changes/ce/feat-10404.en.md diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 904eeffa5..f8d671b40 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -85,9 +85,6 @@ -define(WORKER_POOL_SIZE, 16). --define(DEFAULT_QUEUE_SEG_SIZE, 10 * 1024 * 1024). --define(DEFAULT_QUEUE_SEG_SIZE_RAW, <<"10MB">>). - -define(DEFAULT_QUEUE_SIZE, 256 * 1024 * 1024). -define(DEFAULT_QUEUE_SIZE_RAW, <<"256MB">>). diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 0fa4c0bd8..c53627ca4 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -178,20 +178,7 @@ init({Id, Index, Opts}) -> process_flag(trap_exit, true), true = gproc_pool:connect_worker(Id, {Id, Index}), BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), - SegBytes0 = maps:get(queue_seg_bytes, Opts, ?DEFAULT_QUEUE_SEG_SIZE), - TotalBytes = maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE), - SegBytes = min(SegBytes0, TotalBytes), - QueueOpts = - #{ - dir => disk_queue_dir(Id, Index), - marshaller => fun ?MODULE:queue_item_marshaller/1, - max_total_bytes => TotalBytes, - %% we don't want to retain the queue after - %% resource restarts. - offload => {true, volatile}, - seg_bytes => SegBytes, - sizer => fun ?MODULE:estimate_size/1 - }, + QueueOpts = replayq_opts(Id, Index, Opts), Queue = replayq:open(QueueOpts), emqx_resource_metrics:queuing_set(Id, Index, queue_count(Queue)), emqx_resource_metrics:inflight_set(Id, Index, 0), @@ -214,7 +201,7 @@ init({Id, Index, Opts}) -> resume_interval => ResumeInterval, tref => undefined }, - ?tp(buffer_worker_init, #{id => Id, index => Index}), + ?tp(buffer_worker_init, #{id => Id, index => Index, queue_opts => QueueOpts}), {ok, running, Data}. running(enter, _, #{tref := _Tref} = Data) -> @@ -1679,6 +1666,32 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) -> end, BatchTime. +replayq_opts(Id, Index, Opts) -> + QueueMode = maps:get(queue_mode, Opts, memory_only), + TotalBytes = maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE), + case QueueMode of + memory_only -> + #{ + mem_only => true, + marshaller => fun ?MODULE:queue_item_marshaller/1, + max_total_bytes => TotalBytes, + sizer => fun ?MODULE:estimate_size/1 + }; + volatile_offload -> + SegBytes0 = maps:get(queue_seg_bytes, Opts, TotalBytes), + SegBytes = min(SegBytes0, TotalBytes), + #{ + dir => disk_queue_dir(Id, Index), + marshaller => fun ?MODULE:queue_item_marshaller/1, + max_total_bytes => TotalBytes, + %% we don't want to retain the queue after + %% resource restarts. + offload => {true, volatile}, + seg_bytes => SegBytes, + sizer => fun ?MODULE:estimate_size/1 + } + end. + %% The request timeout should be greater than the resume interval, as %% it defines how often the buffer worker tries to unblock. If request %% timeout is <= resume interval and the buffer worker is ever diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 647a40fed..9116927fa 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -40,6 +40,7 @@ fields("resource_opts") -> ]; fields("creation_opts") -> [ + {queue_mode, fun queue_mode/1}, {worker_pool_size, fun worker_pool_size/1}, {health_check_interval, fun health_check_interval/1}, {resume_interval, fun resume_interval/1}, @@ -53,7 +54,8 @@ fields("creation_opts") -> {batch_size, fun batch_size/1}, {batch_time, fun batch_time/1}, {enable_queue, fun enable_queue/1}, - {max_queue_bytes, fun max_queue_bytes/1} + {max_queue_bytes, fun max_queue_bytes/1}, + {queue_seg_bytes, fun queue_seg_bytes/1} ]. resource_opts_meta() -> @@ -149,4 +151,17 @@ max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW; max_queue_bytes(required) -> false; max_queue_bytes(_) -> undefined. +queue_mode(type) -> enum([memory_only, volatile_offload]); +queue_mode(desc) -> ?DESC("queue_mode"); +queue_mode(default) -> memory_only; +queue_mode(required) -> false; +queue_mode(importance) -> ?IMPORTANCE_HIDDEN; +queue_mode(_) -> undefined. + +queue_seg_bytes(type) -> emqx_schema:bytesize(); +queue_seg_bytes(desc) -> ?DESC("queue_seg_bytes"); +queue_seg_bytes(required) -> false; +queue_seg_bytes(importance) -> ?IMPORTANCE_HIDDEN; +queue_seg_bytes(_) -> undefined. + desc("creation_opts") -> ?DESC("creation_opts"). diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 5d8e85697..087a39eee 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1314,6 +1314,7 @@ t_delete_and_re_create_with_same_name(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => NumBufferWorkers, + queue_mode => volatile_offload, queue_seg_bytes => 100, resume_interval => 1_000 } @@ -2639,6 +2640,117 @@ t_call_mode_uncoupled_from_query_mode(_Config) -> end ). +%% The default mode is currently `memory_only'. +t_volatile_offload_mode(_Config) -> + MaxQueueBytes = 1_000, + DefaultOpts = #{ + max_queue_bytes => MaxQueueBytes, + worker_pool_size => 1 + }, + ?check_trace( + begin + emqx_connector_demo:set_callback_mode(async_if_possible), + %% Create without any specified segment bytes; should + %% default to equal max bytes. + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{queue_mode => volatile_offload} + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + %% Create with segment bytes < max bytes + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + queue_mode => volatile_offload, + queue_seg_bytes => MaxQueueBytes div 2 + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + %% Create with segment bytes = max bytes + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + queue_mode => volatile_offload, + queue_seg_bytes => MaxQueueBytes + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + %% Create with segment bytes > max bytes; should normalize + %% to max bytes. + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + queue_mode => volatile_offload, + queue_seg_bytes => 2 * MaxQueueBytes + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + ok + end, + fun(Trace) -> + HalfMaxQueueBytes = MaxQueueBytes div 2, + ?assertMatch( + [ + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + %% uses the specified value since it's smaller + %% than max bytes. + seg_bytes := HalfMaxQueueBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + } + ], + ?projection(queue_opts, ?of_kind(buffer_worker_init, Trace)) + ), + ok + end + ). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/changes/ce/feat-10404.en.md b/changes/ce/feat-10404.en.md new file mode 100644 index 000000000..ad216336e --- /dev/null +++ b/changes/ce/feat-10404.en.md @@ -0,0 +1,2 @@ +Change the default queue mode for buffer workers to `memory_only`. +Before this change, the default queue mode was `volatile_offload`. When under high message rate pressure and when the resource is not keeping up with such rate, the buffer performance degraded a lot due to the constant disk operations. diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index c73f8b1aa..7b693c256 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -179,5 +179,43 @@ When disabled the messages are buffered in RAM only.""" } } + queue_seg_bytes { + desc { + en: "Applicable when buffer mode is set to volatile_offload.\n" + "This value is to specify the size of each on-disk buffer file." + zh: "当缓存模式是 volatile_offload 时适用。" + "该配置用于指定缓存到磁盘上的文件的大小。" + } + label { + en: "Segment File Bytes" + zh: "缓存文件大小" + } + } + + queue_mode { + desc { + en: "Queue operation mode.\n" + "\n" + "memory_only: Buffer all messages in memory." + " The messages will be lost in case of EMQX node restart.\n" + "volatile_offload: Buffer message in memory first, when up to certain limit" + " (see queue_seg_bytes config for more information), then start offloading" + " messages to disk, Like memory_only mode, the messages will be lost in case of" + " EMQX node restart." + zh: "队列操作模式。\n" + "\n" + "memory_only: 所有的消息都缓存在内存里。" + " 如果 EMQX 服务重启，缓存的消息会丢失。\n" + "hybrid: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" + "（配置项 queue_seg_bytes 描述了该限制）后，" + " 后续的消息会缓存到磁盘上。与 memory_only 模式一样，" + " 如果 EMQX 服务重启，缓存的消息会丢失。" + } + label { + en: "Queue Mode" + zh: "排队模式" + } + } + } From dd381227972f0965d680bd4d8872f6f15765f70a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 11:22:04 -0300 Subject: [PATCH 24/32] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- rel/i18n/emqx_resource_schema.hocon | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index 7b693c256..885a6c4dd 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -195,25 +195,18 @@ When disabled the messages are buffered in RAM only.""" queue_mode { desc { en: "Queue operation mode.\n" - "\n" "memory_only: Buffer all messages in memory." - " The messages will be lost in case of EMQX node restart.\n" "volatile_offload: Buffer message in memory first, when up to certain limit" - " (see queue_seg_bytes config for more information), then start offloading" - " messages to disk, Like memory_only mode, the messages will be lost in case of" - " EMQX node restart." + " (see buffer_seg_bytes config for more information), then start offloading messages to disk" zh: "队列操作模式。\n" - "\n" "memory_only: 所有的消息都缓存在内存里。" - " 如果 EMQX 服务重启，缓存的消息会丢失。\n" - "hybrid: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" - "（配置项 queue_seg_bytes 描述了该限制）后，" - " 后续的消息会缓存到磁盘上。与 memory_only 模式一样，" - " 如果 EMQX 服务重启，缓存的消息会丢失。" + "volatile_offload: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" + "（配置项 buffer_seg_bytes 该限制）后，" + " 消息会缓存到磁盘上" } label { - en: "Queue Mode" - zh: "排队模式" + en: "Buffer Mode" + zh: "缓存模式" } } From e073bc90bc76d3bea1387a0f4196a3fe6b23ac9a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 11:29:52 -0300 Subject: [PATCH 25/32] refactor(buffer_worker): rename `s/queue/buffer/g` --- apps/emqx_bridge/src/emqx_bridge_api.erl | 4 +- .../schema/emqx_bridge_compatible_config.erl | 2 +- .../test/emqx_bridge_webhook_SUITE.erl | 2 +- apps/emqx_resource/include/emqx_resource.hrl | 6 +-- .../src/emqx_resource_buffer_worker.erl | 8 ++-- .../src/schema/emqx_resource_schema.erl | 39 ++++++++++--------- .../test/emqx_resource_SUITE.erl | 30 +++++++------- .../src/emqx_ee_bridge_cassa.erl | 2 +- .../src/emqx_ee_bridge_clickhouse.erl | 2 +- .../src/emqx_ee_bridge_dynamo.erl | 2 +- .../src/emqx_ee_bridge_mysql.erl | 2 +- .../src/emqx_ee_bridge_pgsql.erl | 2 +- .../src/emqx_ee_bridge_rocketmq.erl | 2 +- .../src/emqx_ee_bridge_sqlserver.erl | 2 +- .../src/emqx_ee_bridge_tdengine.erl | 2 +- rel/i18n/emqx_resource_schema.hocon | 8 ++-- scripts/test/influx/influx-bridge.conf | 2 +- 17 files changed, 59 insertions(+), 58 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index b29cefacd..137cd1241 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -220,7 +220,7 @@ info_example_basic(webhook) -> auto_restart_interval => 15000, query_mode => async, inflight_window => 100, - max_queue_bytes => 100 * 1024 * 1024 + max_buffer_bytes => 100 * 1024 * 1024 } }; info_example_basic(mqtt) -> @@ -245,7 +245,7 @@ mqtt_main_example() -> health_check_interval => <<"15s">>, auto_restart_interval => <<"60s">>, query_mode => sync, - max_queue_bytes => 100 * 1024 * 1024 + max_buffer_bytes => 100 * 1024 * 1024 }, ssl => #{ enable => false diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl index fe173fa89..595b75ecf 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl @@ -89,7 +89,7 @@ default_resource_opts() -> <<"inflight_window">> => 100, <<"auto_restart_interval">> => <<"60s">>, <<"health_check_interval">> => <<"15s">>, - <<"max_queue_bytes">> => <<"1GB">>, + <<"max_buffer_bytes">> => <<"1GB">>, <<"query_mode">> => <<"sync">>, %% there is only one underlying MQTT connection %% doesn't make a lot of sense to have a large pool diff --git a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl index e222190d2..f08c87b6e 100644 --- a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl @@ -175,7 +175,7 @@ bridge_async_config(#{port := Port} = Config) -> " inflight_window = 100\n" " auto_restart_interval = \"60s\"\n" " health_check_interval = \"15s\"\n" - " max_queue_bytes = \"1GB\"\n" + " max_buffer_bytes = \"1GB\"\n" " query_mode = \"~s\"\n" " request_timeout = \"~s\"\n" " start_after_created = \"true\"\n" diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index f8d671b40..91572eac3 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -70,7 +70,7 @@ auto_restart_interval => pos_integer(), batch_size => pos_integer(), batch_time => pos_integer(), - max_queue_bytes => pos_integer(), + max_buffer_bytes => pos_integer(), query_mode => query_mode(), resume_interval => pos_integer(), inflight_window => pos_integer() @@ -85,8 +85,8 @@ -define(WORKER_POOL_SIZE, 16). --define(DEFAULT_QUEUE_SIZE, 256 * 1024 * 1024). --define(DEFAULT_QUEUE_SIZE_RAW, <<"256MB">>). +-define(DEFAULT_BUFFER_BYTES, 256 * 1024 * 1024). +-define(DEFAULT_BUFFER_BYTES_RAW, <<"256MB">>). -define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(15)). diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index c53627ca4..a77335140 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1667,9 +1667,9 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) -> BatchTime. replayq_opts(Id, Index, Opts) -> - QueueMode = maps:get(queue_mode, Opts, memory_only), - TotalBytes = maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE), - case QueueMode of + BufferMode = maps:get(buffer_mode, Opts, memory_only), + TotalBytes = maps:get(max_buffer_bytes, Opts, ?DEFAULT_BUFFER_BYTES), + case BufferMode of memory_only -> #{ mem_only => true, @@ -1678,7 +1678,7 @@ replayq_opts(Id, Index, Opts) -> sizer => fun ?MODULE:estimate_size/1 }; volatile_offload -> - SegBytes0 = maps:get(queue_seg_bytes, Opts, TotalBytes), + SegBytes0 = maps:get(buffer_seg_bytes, Opts, TotalBytes), SegBytes = min(SegBytes0, TotalBytes), #{ dir => disk_queue_dir(Id, Index), diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 9116927fa..3b4fb66e5 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -40,7 +40,7 @@ fields("resource_opts") -> ]; fields("creation_opts") -> [ - {queue_mode, fun queue_mode/1}, + {buffer_mode, fun buffer_mode/1}, {worker_pool_size, fun worker_pool_size/1}, {health_check_interval, fun health_check_interval/1}, {resume_interval, fun resume_interval/1}, @@ -54,8 +54,8 @@ fields("creation_opts") -> {batch_size, fun batch_size/1}, {batch_time, fun batch_time/1}, {enable_queue, fun enable_queue/1}, - {max_queue_bytes, fun max_queue_bytes/1}, - {queue_seg_bytes, fun queue_seg_bytes/1} + {max_buffer_bytes, fun max_buffer_bytes/1}, + {buffer_seg_bytes, fun buffer_seg_bytes/1} ]. resource_opts_meta() -> @@ -145,23 +145,24 @@ batch_time(default) -> ?DEFAULT_BATCH_TIME_RAW; batch_time(required) -> false; batch_time(_) -> undefined. -max_queue_bytes(type) -> emqx_schema:bytesize(); -max_queue_bytes(desc) -> ?DESC("max_queue_bytes"); -max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW; -max_queue_bytes(required) -> false; -max_queue_bytes(_) -> undefined. +max_buffer_bytes(type) -> emqx_schema:bytesize(); +max_buffer_bytes(aliases) -> [max_queue_bytes]; +max_buffer_bytes(desc) -> ?DESC("max_buffer_bytes"); +max_buffer_bytes(default) -> ?DEFAULT_BUFFER_BYTES_RAW; +max_buffer_bytes(required) -> false; +max_buffer_bytes(_) -> undefined. -queue_mode(type) -> enum([memory_only, volatile_offload]); -queue_mode(desc) -> ?DESC("queue_mode"); -queue_mode(default) -> memory_only; -queue_mode(required) -> false; -queue_mode(importance) -> ?IMPORTANCE_HIDDEN; -queue_mode(_) -> undefined. +buffer_mode(type) -> enum([memory_only, volatile_offload]); +buffer_mode(desc) -> ?DESC("buffer_mode"); +buffer_mode(default) -> memory_only; +buffer_mode(required) -> false; +buffer_mode(importance) -> ?IMPORTANCE_HIDDEN; +buffer_mode(_) -> undefined. -queue_seg_bytes(type) -> emqx_schema:bytesize(); -queue_seg_bytes(desc) -> ?DESC("queue_seg_bytes"); -queue_seg_bytes(required) -> false; -queue_seg_bytes(importance) -> ?IMPORTANCE_HIDDEN; -queue_seg_bytes(_) -> undefined. +buffer_seg_bytes(type) -> emqx_schema:bytesize(); +buffer_seg_bytes(desc) -> ?DESC("buffer_seg_bytes"); +buffer_seg_bytes(required) -> false; +buffer_seg_bytes(importance) -> ?IMPORTANCE_HIDDEN; +buffer_seg_bytes(_) -> undefined. desc("creation_opts") -> ?DESC("creation_opts"). diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 087a39eee..385b4cb91 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1314,8 +1314,8 @@ t_delete_and_re_create_with_same_name(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => NumBufferWorkers, - queue_mode => volatile_offload, - queue_seg_bytes => 100, + buffer_mode => volatile_offload, + buffer_seg_bytes => 100, resume_interval => 1_000 } ), @@ -1374,7 +1374,7 @@ t_delete_and_re_create_with_same_name(_Config) -> query_mode => async, batch_size => 1, worker_pool_size => 2, - queue_seg_bytes => 100, + buffer_seg_bytes => 100, resume_interval => 1_000 } ), @@ -1406,7 +1406,7 @@ t_always_overflow(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => 1, - max_queue_bytes => 1, + max_buffer_bytes => 1, resume_interval => 1_000 } ), @@ -2642,9 +2642,9 @@ t_call_mode_uncoupled_from_query_mode(_Config) -> %% The default mode is currently `memory_only'. t_volatile_offload_mode(_Config) -> - MaxQueueBytes = 1_000, + MaxBufferBytes = 1_000, DefaultOpts = #{ - max_queue_bytes => MaxQueueBytes, + max_buffer_bytes => MaxBufferBytes, worker_pool_size => 1 }, ?check_trace( @@ -2659,7 +2659,7 @@ t_volatile_offload_mode(_Config) -> ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, #{name => test_resource}, - DefaultOpts#{queue_mode => volatile_offload} + DefaultOpts#{buffer_mode => volatile_offload} ) ), ?assertEqual(ok, emqx_resource:remove_local(?ID)), @@ -2673,8 +2673,8 @@ t_volatile_offload_mode(_Config) -> ?TEST_RESOURCE, #{name => test_resource}, DefaultOpts#{ - queue_mode => volatile_offload, - queue_seg_bytes => MaxQueueBytes div 2 + buffer_mode => volatile_offload, + buffer_seg_bytes => MaxBufferBytes div 2 } ) ), @@ -2688,8 +2688,8 @@ t_volatile_offload_mode(_Config) -> ?TEST_RESOURCE, #{name => test_resource}, DefaultOpts#{ - queue_mode => volatile_offload, - queue_seg_bytes => MaxQueueBytes + buffer_mode => volatile_offload, + buffer_seg_bytes => MaxBufferBytes } ) ), @@ -2705,8 +2705,8 @@ t_volatile_offload_mode(_Config) -> ?TEST_RESOURCE, #{name => test_resource}, DefaultOpts#{ - queue_mode => volatile_offload, - queue_seg_bytes => 2 * MaxQueueBytes + buffer_mode => volatile_offload, + buffer_seg_bytes => 2 * MaxBufferBytes } ) ), @@ -2715,7 +2715,7 @@ t_volatile_offload_mode(_Config) -> ok end, fun(Trace) -> - HalfMaxQueueBytes = MaxQueueBytes div 2, + HalfMaxBufferBytes = MaxBufferBytes div 2, ?assertMatch( [ #{ @@ -2729,7 +2729,7 @@ t_volatile_offload_mode(_Config) -> max_total_bytes := MaxTotalBytes, %% uses the specified value since it's smaller %% than max bytes. - seg_bytes := HalfMaxQueueBytes, + seg_bytes := HalfMaxBufferBytes, offload := {true, volatile} }, #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl index 78db8352a..26c6de04d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl @@ -63,7 +63,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl index 0b611c142..56671c586 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl @@ -61,7 +61,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl index e6a3d1a58..ba1fd0c70 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl @@ -56,7 +56,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index f3ed44247..7914c77e2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -57,7 +57,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl index 958bc3449..a5dcb19e6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl @@ -59,7 +59,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl index 78fd527d3..28b94a1a4 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl @@ -56,7 +56,7 @@ values(post) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }; values(put) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl index 49a5ed0ce..e216299c2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl @@ -60,7 +60,7 @@ values(post) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }; values(put) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl index 7a958d45f..54406541d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl @@ -58,7 +58,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index 885a6c4dd..031a5b412 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -168,7 +168,7 @@ When disabled the messages are buffered in RAM only.""" } } - max_queue_bytes { + max_buffer_bytes { desc { en: """Maximum number of bytes to buffer for each buffer worker.""" zh: """每个缓存 worker 允许使用的最大字节数。""" @@ -179,7 +179,7 @@ When disabled the messages are buffered in RAM only.""" } } - queue_seg_bytes { + buffer_seg_bytes { desc { en: "Applicable when buffer mode is set to volatile_offload.\n" "This value is to specify the size of each on-disk buffer file." @@ -192,9 +192,9 @@ When disabled the messages are buffered in RAM only.""" } } - queue_mode { + buffer_mode { desc { - en: "Queue operation mode.\n" + en: "Buffer operation mode.\n" "memory_only: Buffer all messages in memory." "volatile_offload: Buffer message in memory first, when up to certain limit" " (see buffer_seg_bytes config for more information), then start offloading messages to disk" diff --git a/scripts/test/influx/influx-bridge.conf b/scripts/test/influx/influx-bridge.conf index 0416e42b6..0574ac38a 100644 --- a/scripts/test/influx/influx-bridge.conf +++ b/scripts/test/influx/influx-bridge.conf @@ -11,7 +11,7 @@ bridges { batch_size = 100 batch_time = "10ms" health_check_interval = "15s" - max_queue_bytes = "1GB" + max_buffer_bytes = "1GB" query_mode = "sync" request_timeout = "15s" start_after_created = "true" From 24df1045dec62fdeb777fe6be8a4de785c974a4d Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 16:40:25 +0200 Subject: [PATCH 26/32] fix: test not updated after rebase --- apps/emqx_utils/test/emqx_utils_SUITE.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_utils/test/emqx_utils_SUITE.erl b/apps/emqx_utils/test/emqx_utils_SUITE.erl index 598f05a21..99516b0eb 100644 --- a/apps/emqx_utils/test/emqx_utils_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_SUITE.erl @@ -147,7 +147,10 @@ t_check(_) -> [self() ! {msg, I} || I <- lists:seq(1, 5)], ?assertEqual(ok, emqx_utils:check_oom(Policy)), [self() ! {msg, I} || I <- lists:seq(1, 6)], - ?assertEqual({shutdown, message_queue_too_long}, emqx_utils:check_oom(Policy)). + ?assertEqual( + {shutdown, #{reason => message_queue_too_long, value => 11, max => 10}}, + emqx_utils:check_oom(Policy) + ). drain() -> drain([]). From 16c49b2cc185f703b15590d443382b642d74e6af Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 17:21:38 +0200 Subject: [PATCH 27/32] fix: undo wrong thinking about returned types from decode --- apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index 672ddc73c..ab673b225 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -60,19 +60,19 @@ set_special_configs(emqx_authz) -> set_special_configs(_App) -> ok. -t_clean_cahce(_) -> +t_clean_cache(_) -> {ok, C} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), {ok, _} = emqtt:connect(C), {ok, _, _} = emqtt:subscribe(C, <<"a/b/c">>, 0), ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0), {ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(2, maps:size(emqx_utils_json:decode(Result3))), + ?assertEqual(2, length(emqx_utils_json:decode(Result3))), request(delete, uri(["authorization", "cache"])), {ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(0, maps:size(emqx_utils_json:decode(Result4))), + ?assertEqual(0, length(emqx_utils_json:decode(Result4))), ok. From 3373a63137a7c0443831985919fef131c2d7d6b4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 13:21:30 -0300 Subject: [PATCH 28/32] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- rel/i18n/emqx_resource_schema.hocon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index 031a5b412..f4a9982bc 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -201,8 +201,8 @@ When disabled the messages are buffered in RAM only.""" zh: "队列操作模式。\n" "memory_only: 所有的消息都缓存在内存里。" "volatile_offload: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" - "（配置项 buffer_seg_bytes 该限制）后，" - " 消息会缓存到磁盘上" + "（配置项 buffer_seg_bytes 指定该限制）后，" + " 消息会开始缓存到磁盘上。" } label { en: "Buffer Mode" From f0c13e01345f367553fddb1874677fbf0e906993 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 18:43:53 +0200 Subject: [PATCH 29/32] fix: stale ref to emqx_map_lib --- lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl index 6e6d7bf03..68bf7a057 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -489,7 +489,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(sqlserver_bridge_type, Config), Name = ?config(sqlserver_name, Config), SSConfig0 = ?config(sqlserver_config, Config), - SSConfig = emqx_map_lib:deep_merge(SSConfig0, Overrides), + SSConfig = emqx_utils_maps:deep_merge(SSConfig0, Overrides), emqx_bridge:create(BridgeType, Name, SSConfig). delete_bridge(Config) -> From a20797160e11336a765f308fe3b2024fc548373f Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 18:45:25 +0200 Subject: [PATCH 30/32] style: remove unnecessary ifdef(TEST) --- apps/emqx_utils/test/emqx_utils_binary_tests.erl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apps/emqx_utils/test/emqx_utils_binary_tests.erl b/apps/emqx_utils/test/emqx_utils_binary_tests.erl index 3709fb014..79851dca5 100644 --- a/apps/emqx_utils/test/emqx_utils_binary_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_binary_tests.erl @@ -26,8 +26,6 @@ -include_lib("eunit/include/eunit.hrl"). --ifdef(TEST). - trim1_test_() -> [ ?_assertEqual(trim(<<>>), <<>>), @@ -213,5 +211,3 @@ optimize_patterns_test_() -> optimize_patterns([<<"test">>, <<"t">>, <<"t">>]) ) ]. - --endif. From 9ccfa643cecfb98e826a86a6175f260b0c7efbbc Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 18:51:51 +0200 Subject: [PATCH 31/32] style: fix wording Co-authored-by: Thales Macedo Garitezi --- apps/emqx_utils/README.md | 6 +++--- apps/emqx_utils/src/emqx_utils.app.src | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_utils/README.md b/apps/emqx_utils/README.md index 046b66321..f8c386f3d 100644 --- a/apps/emqx_utils/README.md +++ b/apps/emqx_utils/README.md @@ -1,10 +1,10 @@ -# emqx_utils - Erlang utility library for EMQ X +# Erlang utility library for EMQX ## Overview -`emqx_utils` is a collection of utility functions for EMQ X, organized into +`emqx_utils` is a collection of utility functions for EMQX, organized into several modules. It provides various functionalities to make it easier to work -with EMQ X, such as binary manipulations, maps, JSON en- and decoding, ets table +with EMQX, such as binary manipulations, maps, JSON en- and decoding, ets table handling, data conversions, and more. ## Features diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index b0dedbd72..eb6371411 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -1,6 +1,6 @@ %% -*- mode: erlang -*- {application, emqx_utils, [ - {description, "An OTP application"}, + {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! {vsn, "5.0.0"}, {modules, [ From 89fa07c9be64465c50d6762b9849513d758b2093 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 18:53:01 +0200 Subject: [PATCH 32/32] fix: emqx_utils is just a library, so 'load' is enough Co-authored-by: Thales Macedo Garitezi --- mix.exs | 2 +- rebar.config.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 299f23ce6..454c398a2 100644 --- a/mix.exs +++ b/mix.exs @@ -303,7 +303,7 @@ defmodule EMQXUmbrella.MixProject do tools: :load, covertool: :load, system_monitor: :load, - emqx_utils: :permanent, + emqx_utils: :load, emqx_http_lib: :permanent, emqx_resource: :permanent, emqx_connector: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index b6b6a7c18..7c00622c2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -391,7 +391,7 @@ relx_apps(ReleaseType, Edition) -> {covertool, load}, % started by emqx_machine {system_monitor, load}, - emqx_utils, + {emqx_utils, load}, emqx_http_lib, emqx_resource, emqx_connector,