Merge pull request #12507 from keynslug/fix/gw-app-no-scan

fix(gateway): avoid scanning modules of known gateway apps

apps/emqx_gateway/src/emqx_gateway_utils.erl

@@ -20,6 +20,17 @@
 -include("emqx_gateway.hrl").
 -include_lib("emqx/include/logger.hrl").
 
+-define(GATEWAYS, [
+    emqx_gateway_coap,
+    emqx_gateway_exproto,
+    emqx_gateway_gbt32960,
+    emqx_gateway_jt808,
+    emqx_gateway_lwm2m,
+    emqx_gateway_mqttsn,
+    emqx_gateway_ocpp,
+    emqx_gateway_stomp
+]).
+
 -export([
     childspec/2,
     childspec/3,
@@ -679,32 +690,28 @@ default_subopts() ->
 
 -spec find_gateway_definitions() -> list(gateway_def()).
 find_gateway_definitions() ->
-    ensure_gateway_loaded(),
+    lists:flatmap(
-    lists:flatten(
-        lists:map(
         fun(App) ->
-                gateways(find_attrs(App, gateway))
+            lists:flatmap(fun gateways/1, find_attrs(App, gateway))
         end,
-            ignore_lib_apps(application:loaded_applications())
+        ?GATEWAYS
-        )
     ).
 
 -spec find_gateway_definition(atom()) -> {ok, map()} | {error, term()}.
 find_gateway_definition(Name) ->
-    ensure_gateway_loaded(),
+    find_gateway_definition(Name, ?GATEWAYS).
-    find_gateway_definition(Name, ignore_lib_apps(application:loaded_applications())).
 
 -dialyzer({no_match, [find_gateway_definition/2]}).
 find_gateway_definition(Name, [App | T]) ->
     Attrs = find_attrs(App, gateway),
-    SearchFun = fun({_App, _Mod, #{name := GwName}}) ->
+    SearchFun = fun(#{name := GwName}) ->
         GwName =:= Name
     end,
     case lists:search(SearchFun, Attrs) of
-        {value, {_App, _Mod, Defination}} ->
+        {value, Definition} ->
-            case check_gateway_edition(Defination) of
+            case check_gateway_edition(Definition) of
                 true ->
-                    {ok, Defination};
+                    {ok, Definition};
                 _ ->
                     {error, invalid_edition}
             end;
@@ -715,23 +722,18 @@ find_gateway_definition(_Name, []) ->
     {error, not_found}.
 
 -dialyzer({no_match, [gateways/1]}).
-gateways([]) ->
+gateways(
-    [];
+    Definition = #{
-gateways([
-    {_App, _Mod,
-        Defination =
-            #{
         name := Name,
         callback_module := CbMod,
         config_schema_module := SchemaMod
-            }}
+    }
-    | More
+) when is_atom(Name), is_atom(CbMod), is_atom(SchemaMod) ->
-]) when is_atom(Name), is_atom(CbMod), is_atom(SchemaMod) ->
+    case check_gateway_edition(Definition) of
-    case check_gateway_edition(Defination) of
         true ->
-            [Defination | gateways(More)];
+            [Definition];
         _ ->
-            gateways(More)
+            []
     end.
 
 -if(?EMQX_RELEASE_EDITION == ee).
@@ -742,12 +744,10 @@ check_gateway_edition(Defination) ->
     ce == maps:get(edition, Defination, ce).
 -endif.
 
-find_attrs(App, Def) ->
+find_attrs(AppMod, Def) ->
     [
-        {App, Mod, Attr}
+        Attr
-     || {ok, Modules} <- [application:get_key(App, modules)],
+     || {Name, Attrs} <- module_attributes(AppMod),
-        Mod <- Modules,
-        {Name, Attrs} <- module_attributes(Mod),
         Name =:= Def,
         Attr <- Attrs
     ].
@@ -759,43 +759,6 @@ module_attributes(Module) ->
         error:undef -> []
     end.
 
-ignore_lib_apps(Apps) ->
-    LibApps = [
-        kernel,
-        stdlib,
-        sasl,
-        appmon,
-        eldap,
-        erts,
-        syntax_tools,
-        ssl,
-        crypto,
-        mnesia,
-        os_mon,
-        inets,
-        goldrush,
-        gproc,
-        runtime_tools,
-        snmp,
-        otp_mibs,
-        public_key,
-        asn1,
-        ssh,
-        hipe,
-        common_test,
-        observer,
-        webtool,
-        xmerl,
-        tools,
-        test_server,
-        compiler,
-        debugger,
-        eunit,
-        et,
-        wx
-    ],
-    [AppName || {AppName, _, _} <- Apps, not lists:member(AppName, LibApps)].
-
 -spec plus_max_connections(non_neg_integer() | infinity, non_neg_integer() | infinity) ->
     pos_integer() | infinity.
 plus_max_connections(_, infinity) ->
@@ -805,20 +768,5 @@ plus_max_connections(infinity, _) ->
 plus_max_connections(A, B) when is_integer(A) andalso is_integer(B) ->
     A + B.
 
-%% we need to load all gateway applications before generate doc from cli
-ensure_gateway_loaded() ->
-    lists:foreach(
-        fun application:load/1,
-        [
-            emqx_gateway_exproto,
-            emqx_gateway_stomp,
-            emqx_gateway_coap,
-            emqx_gateway_lwm2m,
-            emqx_gateway_mqttsn,
-            emqx_gateway_gbt32960,
-            emqx_gateway_ocpp
-        ]
-    ).
-
 random_clientid(GwName) when is_atom(GwName) ->
     iolist_to_binary([atom_to_list(GwName), "-", emqx_utils:gen_id()]).

apps/emqx_gateway/test/emqx_gateway_SUITE.erl

@@ -74,7 +74,10 @@ end_per_testcase(_TestCase, _Config) ->
 %%--------------------------------------------------------------------
 
 t_registered_gateway(_) ->
-    [{coap, #{cbkmod := emqx_gateway_coap}} | _] = emqx_gateway:registered_gateway().
+    ?assertMatch(
+        [{coap, #{cbkmod := emqx_gateway_coap}} | _],
+        lists:sort(emqx_gateway:registered_gateway())
+    ).
 
 t_load_unload_list_lookup(_) ->
     {ok, _} = emqx_gateway:load(?GWNAME, #{idle_timeout => 1000}),

apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl

@@ -45,17 +45,24 @@
 all() -> emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Conf) ->
-    application:load(emqx),
+    Apps = emqx_cth_suite:start(
-    emqx_gateway_test_utils:load_all_gateway_apps(),
+        [
-    emqx_config:delete_override_conf_files(),
+            emqx_conf,
-    emqx_config:erase(gateway),
+            emqx_auth,
-    emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT),
+            emqx_auth_mnesia,
-    emqx_mgmt_api_test_util:init_suite([grpc, emqx_conf, emqx_auth, emqx_auth_mnesia, emqx_gateway]),
+            emqx_management,
-    Conf.
+            {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"},
+            {emqx_gateway, ?CONF_DEFAULT}
+            | emqx_gateway_test_utils:all_gateway_apps()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Conf)}
+    ),
+    _ = emqx_common_test_http:create_default_app(),
+    [{suite_apps, Apps} | Conf].
 
 end_per_suite(Conf) ->
-    emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_auth_mnesia, emqx_auth, emqx_conf, grpc]),
+    _ = emqx_common_test_http:delete_default_app(),
-    Conf.
+    ok = emqx_cth_suite:stop(proplists:get_value(suite_apps, Conf)).
 
 init_per_testcase(t_gateway_fail, Config) ->
     meck:expect(

apps/emqx_gateway_jt808/src/emqx_jt808_schema.erl

@@ -8,10 +8,13 @@
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("typerefl/include/types.hrl").
 
--export([fields/1, desc/1]).
+-behaviour(hocon_schema).
+-export([namespace/0, fields/1, desc/1]).
 
 -define(NOT_EMPTY(MSG), emqx_resource_validator:not_empty(MSG)).
 
+namespace() -> gateway.
+
 fields(jt808) ->
     [
         {frame, sc(ref(jt808_frame))},
@@ -51,9 +54,8 @@ fields(jt808_proto) ->
     [
         {auth,
             sc(
-                hoconsc:union([
+                hoconsc:union([ref(anonymous_true), ref(anonymous_false)]),
-                    ref(anonymous_true), ref(anonymous_false)
+                #{desc => ?DESC(jt808_auth)}
-                ])
             )},
         {up_topic, fun up_topic/1},
         {dn_topic, fun dn_topic/1}
@@ -61,12 +63,18 @@ fields(jt808_proto) ->
 fields(anonymous_true) ->
     [
         {allow_anonymous,
-            sc(hoconsc:union([true]), #{desc => ?DESC(allow_anonymous), required => true})}
+            sc(
+                hoconsc:union([true]),
+                #{desc => ?DESC(jt808_allow_anonymous), required => true}
+            )}
     ] ++ fields_reg_auth_required(false);
 fields(anonymous_false) ->
     [
         {allow_anonymous,
-            sc(hoconsc:union([false]), #{desc => ?DESC(allow_anonymous), required => true})}
+            sc(
+                hoconsc:union([false]),
+                #{desc => ?DESC(jt808_allow_anonymous), required => true}
+            )}
     ] ++ fields_reg_auth_required(true).
 
 fields_reg_auth_required(Required) ->
@@ -102,14 +110,14 @@ jt808_frame_max_length(_) ->
     undefined.
 
 up_topic(type) -> binary();
-up_topic(desc) -> ?DESC(?FUNCTION_NAME);
+up_topic(desc) -> ?DESC(jt808_up_topic);
 up_topic(default) -> ?DEFAULT_UP_TOPIC;
 up_topic(validator) -> [?NOT_EMPTY("the value of the field 'up_topic' cannot be empty")];
 up_topic(required) -> true;
 up_topic(_) -> undefined.
 
 dn_topic(type) -> binary();
-dn_topic(desc) -> ?DESC(?FUNCTION_NAME);
+dn_topic(desc) -> ?DESC(jt808_dn_topic);
 dn_topic(default) -> ?DEFAULT_DN_TOPIC;
 dn_topic(validator) -> [?NOT_EMPTY("the value of the field 'dn_topic' cannot be empty")];
 dn_topic(required) -> true;
@@ -121,6 +129,10 @@ desc(jt808_frame) ->
     "Limits for the JT/T 808 frames.";
 desc(jt808_proto) ->
     "The JT/T 808 protocol options.";
+desc(anonymous_false) ->
+    ?DESC(jt808_allow_anonymous);
+desc(anonymous_true) ->
+    ?DESC(jt808_allow_anonymous);
 desc(_) ->
     undefined.
 

apps/emqx_gateway_jt808/test/emqx_jt808_SUITE.erl

@@ -101,10 +101,9 @@ end_per_testcase(_Case, Config) ->
     ok.
 
 boot_apps(Case, JT808Conf, Config) ->
-    application:load(emqx_gateway_jt808),
     Apps = emqx_cth_suite:start(
         [
-            cowboy,
+            emqx,
             {emqx_conf, JT808Conf},
             emqx_gateway
         ],

rel/i18n/emqx_jt808_schema.hocon

@@ -3,19 +3,22 @@ emqx_jt808_schema {
 jt808_frame_max_length.desc:
 """The maximum length of the JT/T 808 frame."""
 
+jt808_auth.desc:
+"""Authentication settings of the JT/T 808 Gateway."""
+
 jt808_allow_anonymous.desc:
 """Allow anonymous access to the JT/T 808 Gateway."""
 
-registry_url.desc
+registry_url.desc:
 """The JT/T 808 device registry central URL."""
 
-authentication_url.desc
+authentication_url.desc:
 """The JT/T 808 device authentication central URL."""
 
-jt808_up_topic.desc
+jt808_up_topic.desc:
 """The topic of the JT/T 808 protocol upstream message."""
 
-jt808_dn_topic.desc
+jt808_dn_topic.desc:
 """The topic of the JT/T 808 protocol downstream message."""
 
 retry_interval.desc: