From 0e1d27c836a760ba97140af57697076b3c7fe5d9 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 30 Jan 2024 21:52:25 +0100 Subject: [PATCH] fix(gateway): avoid scanning modules of known gateway apps Since the set of known gateways is already predefined in `emqx_gateway_utils` it makes no sense to scan over all modules just to find out the same set of modules. Cutting out this scanning saves few seconds per each `emqx_conf` application startup, which is especially noticeable when running tests. --- apps/emqx_gateway/src/emqx_gateway_utils.erl | 120 +++++------------- apps/emqx_gateway/test/emqx_gateway_SUITE.erl | 5 +- .../test/emqx_gateway_api_SUITE.erl | 25 ++-- .../src/emqx_jt808_schema.erl | 5 +- .../test/emqx_jt808_SUITE.erl | 3 +- 5 files changed, 59 insertions(+), 99 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 47e1f7583..33f9e7de8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -20,6 +20,17 @@ -include("emqx_gateway.hrl"). -include_lib("emqx/include/logger.hrl"). +-define(GATEWAYS, [ + emqx_gateway_coap, + emqx_gateway_exproto, + emqx_gateway_gbt32960, + emqx_gateway_jt808, + emqx_gateway_lwm2m, + emqx_gateway_mqttsn, + emqx_gateway_ocpp, + emqx_gateway_stomp +]). + -export([ childspec/2, childspec/3, @@ -679,32 +690,28 @@ default_subopts() -> -spec find_gateway_definitions() -> list(gateway_def()). find_gateway_definitions() -> - ensure_gateway_loaded(), - lists:flatten( - lists:map( - fun(App) -> - gateways(find_attrs(App, gateway)) - end, - ignore_lib_apps(application:loaded_applications()) - ) + lists:flatmap( + fun(App) -> + lists:flatmap(fun gateways/1, find_attrs(App, gateway)) + end, + ?GATEWAYS ). -spec find_gateway_definition(atom()) -> {ok, map()} | {error, term()}. find_gateway_definition(Name) -> - ensure_gateway_loaded(), - find_gateway_definition(Name, ignore_lib_apps(application:loaded_applications())). + find_gateway_definition(Name, ?GATEWAYS). -dialyzer({no_match, [find_gateway_definition/2]}). find_gateway_definition(Name, [App | T]) -> Attrs = find_attrs(App, gateway), - SearchFun = fun({_App, _Mod, #{name := GwName}}) -> + SearchFun = fun(#{name := GwName}) -> GwName =:= Name end, case lists:search(SearchFun, Attrs) of - {value, {_App, _Mod, Defination}} -> - case check_gateway_edition(Defination) of + {value, Definition} -> + case check_gateway_edition(Definition) of true -> - {ok, Defination}; + {ok, Definition}; _ -> {error, invalid_edition} end; @@ -715,23 +722,18 @@ find_gateway_definition(_Name, []) -> {error, not_found}. -dialyzer({no_match, [gateways/1]}). -gateways([]) -> - []; -gateways([ - {_App, _Mod, - Defination = - #{ - name := Name, - callback_module := CbMod, - config_schema_module := SchemaMod - }} - | More -]) when is_atom(Name), is_atom(CbMod), is_atom(SchemaMod) -> - case check_gateway_edition(Defination) of +gateways( + Definition = #{ + name := Name, + callback_module := CbMod, + config_schema_module := SchemaMod + } +) when is_atom(Name), is_atom(CbMod), is_atom(SchemaMod) -> + case check_gateway_edition(Definition) of true -> - [Defination | gateways(More)]; + [Definition]; _ -> - gateways(More) + [] end. -if(?EMQX_RELEASE_EDITION == ee). @@ -742,12 +744,10 @@ check_gateway_edition(Defination) -> ce == maps:get(edition, Defination, ce). -endif. -find_attrs(App, Def) -> +find_attrs(AppMod, Def) -> [ - {App, Mod, Attr} - || {ok, Modules} <- [application:get_key(App, modules)], - Mod <- Modules, - {Name, Attrs} <- module_attributes(Mod), + Attr + || {Name, Attrs} <- module_attributes(AppMod), Name =:= Def, Attr <- Attrs ]. @@ -759,43 +759,6 @@ module_attributes(Module) -> error:undef -> [] end. -ignore_lib_apps(Apps) -> - LibApps = [ - kernel, - stdlib, - sasl, - appmon, - eldap, - erts, - syntax_tools, - ssl, - crypto, - mnesia, - os_mon, - inets, - goldrush, - gproc, - runtime_tools, - snmp, - otp_mibs, - public_key, - asn1, - ssh, - hipe, - common_test, - observer, - webtool, - xmerl, - tools, - test_server, - compiler, - debugger, - eunit, - et, - wx - ], - [AppName || {AppName, _, _} <- Apps, not lists:member(AppName, LibApps)]. - -spec plus_max_connections(non_neg_integer() | infinity, non_neg_integer() | infinity) -> pos_integer() | infinity. plus_max_connections(_, infinity) -> @@ -805,20 +768,5 @@ plus_max_connections(infinity, _) -> plus_max_connections(A, B) when is_integer(A) andalso is_integer(B) -> A + B. -%% we need to load all gateway applications before generate doc from cli -ensure_gateway_loaded() -> - lists:foreach( - fun application:load/1, - [ - emqx_gateway_exproto, - emqx_gateway_stomp, - emqx_gateway_coap, - emqx_gateway_lwm2m, - emqx_gateway_mqttsn, - emqx_gateway_gbt32960, - emqx_gateway_ocpp - ] - ). - random_clientid(GwName) when is_atom(GwName) -> iolist_to_binary([atom_to_list(GwName), "-", emqx_utils:gen_id()]). diff --git a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl index 2574db644..78993a5df 100644 --- a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl @@ -74,7 +74,10 @@ end_per_testcase(_TestCase, _Config) -> %%-------------------------------------------------------------------- t_registered_gateway(_) -> - [{coap, #{cbkmod := emqx_gateway_coap}} | _] = emqx_gateway:registered_gateway(). + ?assertMatch( + [{coap, #{cbkmod := emqx_gateway_coap}} | _], + lists:sort(emqx_gateway:registered_gateway()) + ). t_load_unload_list_lookup(_) -> {ok, _} = emqx_gateway:load(?GWNAME, #{idle_timeout => 1000}), diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 41409693a..a98624014 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -45,17 +45,24 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> - application:load(emqx), - emqx_gateway_test_utils:load_all_gateway_apps(), - emqx_config:delete_override_conf_files(), - emqx_config:erase(gateway), - emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), - emqx_mgmt_api_test_util:init_suite([grpc, emqx_conf, emqx_auth, emqx_auth_mnesia, emqx_gateway]), - Conf. + Apps = emqx_cth_suite:start( + [ + emqx_conf, + emqx_auth, + emqx_auth_mnesia, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}, + {emqx_gateway, ?CONF_DEFAULT} + | emqx_gateway_test_utils:all_gateway_apps() + ], + #{work_dir => emqx_cth_suite:work_dir(Conf)} + ), + _ = emqx_common_test_http:create_default_app(), + [{suite_apps, Apps} | Conf]. end_per_suite(Conf) -> - emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_auth_mnesia, emqx_auth, emqx_conf, grpc]), - Conf. + _ = emqx_common_test_http:delete_default_app(), + ok = emqx_cth_suite:stop(proplists:get_value(suite_apps, Conf)). init_per_testcase(t_gateway_fail, Config) -> meck:expect( diff --git a/apps/emqx_gateway_jt808/src/emqx_jt808_schema.erl b/apps/emqx_gateway_jt808/src/emqx_jt808_schema.erl index 3e0c31f5d..427c7bdfe 100644 --- a/apps/emqx_gateway_jt808/src/emqx_jt808_schema.erl +++ b/apps/emqx_gateway_jt808/src/emqx_jt808_schema.erl @@ -8,10 +8,13 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). --export([fields/1, desc/1]). +-behaviour(hocon_schema). +-export([namespace/0, fields/1, desc/1]). -define(NOT_EMPTY(MSG), emqx_resource_validator:not_empty(MSG)). +namespace() -> gateway. + fields(jt808) -> [ {frame, sc(ref(jt808_frame))}, diff --git a/apps/emqx_gateway_jt808/test/emqx_jt808_SUITE.erl b/apps/emqx_gateway_jt808/test/emqx_jt808_SUITE.erl index 20aae1acb..af2ffd071 100644 --- a/apps/emqx_gateway_jt808/test/emqx_jt808_SUITE.erl +++ b/apps/emqx_gateway_jt808/test/emqx_jt808_SUITE.erl @@ -101,10 +101,9 @@ end_per_testcase(_Case, Config) -> ok. boot_apps(Case, JT808Conf, Config) -> - application:load(emqx_gateway_jt808), Apps = emqx_cth_suite:start( [ - cowboy, + emqx, {emqx_conf, JT808Conf}, emqx_gateway ],