refactor(schema): implement new hocon_schema callbacks

apps/emqx/rebar.config

@@ -28,7 +28,7 @@
        [{deps,
            [ meck
            , {bbmustache,"1.10.0"}
-           , {emqx_ct_helpers, {git,"https://github.com/emqx/emqx-ct-helpers.git", {branch,"hocon"}}}
+           , {emqx_ct_helpers, {git,"https://github.com/emqx/emqx-ct-helpers.git", {tag,"2.1.0"}}}
            , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}}
            ]},
          {extra_src_dirs, [{"test",[recursive]}]}

apps/emqx/src/emqx_config.erl

@@ -298,10 +298,11 @@ read_override_conf() ->
 
 -spec save_schema_mod_and_names(module()) -> ok.
 save_schema_mod_and_names(SchemaMod) ->
-    RootNames = SchemaMod:structs(),
+    RootNames = hocon_schema:root_names(SchemaMod),
     OldMods = get_schema_mod(),
     OldNames = get_root_names(),
-    NewMods = maps:from_list([{root_bin(Name), SchemaMod} || Name <- RootNames]),
+    %% map from root name to schema module name
+    NewMods = maps:from_list([{Name, SchemaMod} || Name <- RootNames]),
     persistent_term:put(?PERSIS_SCHEMA_MODS, #{
         mods => maps:merge(OldMods, NewMods),
         names => lists:usort(OldNames ++ RootNames)
@@ -442,6 +443,3 @@ conf_key(?CONF, RootName) ->
     atom(RootName);
 conf_key(?RAW_CONF, RootName) ->
     bin(RootName).
-
-root_bin({array, Bin}) -> bin(Bin);
-root_bin(Bin) -> bin(Bin).

apps/emqx/src/emqx_schema.erl

@@ -65,12 +65,13 @@
                 cipher/0,
                 comma_separated_atoms/0]).
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 -export([t/1, t/3, t/4, ref/1]).
 -export([conf_get/2, conf_get/3, keys/2, filter/1]).
 -export([ssl/1]).
 
-structs() -> ["zones", "mqtt", "flapping_detect", "force_shutdown", "force_gc",
+roots() ->
+    ["zones", "mqtt", "flapping_detect", "force_shutdown", "force_gc",
      "conn_congestion", "rate_limit", "quota", "listeners", "broker", "plugins",
      "stats", "sysmon", "alarm", "authorization"].
 

apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl

@@ -2,11 +2,11 @@
 
 -include_lib("typerefl/include/types.hrl").
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 
 -behaviour(hocon_schema).
 
-structs() -> ["emqx_hocon_plugin"].
+roots() -> ["emqx_hocon_plugin"].
 
 fields("emqx_hocon_plugin") ->
     [{name, fun name/1}].

apps/emqx_authn/src/emqx_authn_schema.erl

@@ -21,7 +21,7 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1
         ]).
 
@@ -32,7 +32,7 @@
 -export([ authenticators/1
         ]).
 
-structs() -> [ "authentication" ].
+roots() -> [ "authentication" ].
 
 fields("authentication") ->
     [ {enable, fun enable/1}

apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl

@@ -21,7 +21,7 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1
         ]).
 
@@ -74,7 +74,7 @@ mnesia(copy) ->
 %% Hocon Schema
 %%------------------------------------------------------------------------------
 
-structs() -> [config].
+roots() -> [config].
 
 fields(config) ->
     [ {name,            fun emqx_authn_schema:authenticator_name/1}

apps/emqx_authn/src/simple_authn/emqx_authn_http.erl

@@ -22,7 +22,7 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1
         , validations/0
         ]).
@@ -37,13 +37,11 @@
 %% Hocon Schema
 %%------------------------------------------------------------------------------
 
-structs() -> [""].
+roots() ->
-
-fields("") ->
     [ {config, {union, [ hoconsc:t(get)
                        , hoconsc:t(post)
                        ]}}
-    ];
+    ].
 
 fields(get) ->
     [ {method,          #{type => get,

apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl

@@ -20,7 +20,7 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1
         ]).
 
@@ -34,14 +34,12 @@
 %% Hocon Schema
 %%------------------------------------------------------------------------------
 
-structs() -> [""].
+roots() ->
-
-fields("") ->
     [ {config, {union, [ hoconsc:t('hmac-based')
                        , hoconsc:t('public-key')
                        , hoconsc:t('jwks')
                        ]}}
-    ];
+    ].
 
 fields('hmac-based') ->
     [ {use_jwks,              {enum, [false]}}

apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl

@@ -21,7 +21,7 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0, fields/1 ]).
+-export([ roots/0, fields/1 ]).
 
 -export([ create/1
         , update/2
@@ -79,7 +79,7 @@ mnesia(copy) ->
 %% Hocon Schema
 %%------------------------------------------------------------------------------
 
-structs() -> [config].
+roots() -> [config].
 
 fields(config) ->
     [ {name,                    fun emqx_authn_schema:authenticator_name/1}

apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl

@@ -22,7 +22,7 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1
         ]).
 
@@ -36,14 +36,12 @@
 %% Hocon Schema
 %%------------------------------------------------------------------------------
 
-structs() -> [""].
+roots() ->
-
-fields("") ->
     [ {config, {union, [ hoconsc:t(standalone)
                        , hoconsc:t('replica-set')
                        , hoconsc:t('sharded-cluster')
                        ]}}
-    ];
+    ].
 
 fields(standalone) ->
     common_fields() ++ emqx_connector_mongo:fields(single);

apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl

@@ -22,7 +22,7 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1
         ]).
 
@@ -36,7 +36,7 @@
 %% Hocon Schema
 %%------------------------------------------------------------------------------
 
-structs() -> [config].
+roots() -> [config].
 
 fields(config) ->
     [ {name,                    fun emqx_authn_schema:authenticator_name/1}

apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl

@@ -23,7 +23,7 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0, fields/1 ]).
+-export([ roots/0, fields/1 ]).
 
 -export([ create/1
         , update/2
@@ -35,7 +35,7 @@
 %% Hocon Schema
 %%------------------------------------------------------------------------------
 
-structs() -> [config].
+roots() -> [config].
 
 fields(config) ->
     [ {name,                    fun emqx_authn_schema:authenticator_name/1}

apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl

@@ -22,7 +22,7 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1
         ]).
 
@@ -36,14 +36,12 @@
 %% Hocon Schema
 %%------------------------------------------------------------------------------
 
-structs() -> [""].
+roots() ->
-
-fields("") ->
     [ {config, {union, [ hoconsc:t(standalone)
                        , hoconsc:t(cluster)
                        , hoconsc:t(sentinel)
                        ]}}
-    ];
+    ].
 
 fields(standalone) ->
     common_fields() ++ emqx_connector_redis:fields(single);

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -13,11 +13,11 @@
 -type permission() :: allow | deny.
 -type url() :: emqx_http_lib:uri_map().
 
--export([ structs/0
+-export([ roots/0
         , fields/1
         ]).
 
-structs() -> ["authorization"].
+roots() -> ["authorization"].
 
 fields("authorization") ->
     [ {sources, sources()}
@@ -180,4 +180,4 @@ connector_fields(DB) ->
     [ {type, #{type => DB}}
     , {enable, #{type => boolean(),
                  default => true}}
-    ] ++ Mod:fields("").
+    ] ++ Mod:roots().

apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl

@@ -19,10 +19,10 @@
 
 -include_lib("typerefl/include/types.hrl").
 
--export([ structs/0
+-export([ roots/0
         , fields/1]).
 
-structs() ->
+roots() ->
     ["auto_subscribe"].
 
 fields("auto_subscribe") ->

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl

@@ -20,10 +20,12 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1]).
 
-structs() -> [{array, "bridge_mqtt"}].
+roots() -> [array("bridge_mqtt")].
+
+array(Name) -> {Name, hoconsc:array(hoconsc:ref(Name))}.
 
 fields("bridge_mqtt") ->
     [ {name, emqx_schema:t(string(), undefined, true)}

apps/emqx_connector/src/emqx_connector_http.erl

@@ -32,7 +32,7 @@
 -reflect_type([url/0]).
 -typerefl_from_string({url/0, emqx_http_lib, uri_parse}).
 
--export([ structs/0
+-export([ roots/0
         , fields/1
         , validations/0]).
 
@@ -47,10 +47,8 @@
 
 %%=====================================================================
 %% Hocon schema
-structs() -> [""].
+roots() ->
-
+    [{config, #{type => hoconsc:ref(?MODULE, config)}}].
-fields("") ->
-    [{config, #{type => hoconsc:ref(?MODULE, config)}}];
 
 fields(config) ->
     [ {base_url,          fun base_url/1}

apps/emqx_connector/src/emqx_connector_ldap.erl

@@ -19,7 +19,7 @@
 -include_lib("typerefl/include/types.hrl").
 -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 
 %% callbacks of behaviour emqx_resource
 -export([ on_start/2
@@ -35,11 +35,11 @@
 
 -export([search/4]).
 %%=====================================================================
-structs() -> [""].
+roots() ->
+    ldap_fields() ++ emqx_connector_schema_lib:ssl_fields().
 
-fields("") ->
+%% this schema has no sub-structs
-    ldap_fields() ++
+fields(_) -> [].
-    emqx_connector_schema_lib:ssl_fields().
 
 on_jsonify(Config) ->
     Config.

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -33,19 +33,18 @@
 
 -export([connect/1]).
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 
 -export([mongo_query/5]).
 %%=====================================================================
-structs() -> [""].
+roots() ->
-
-fields("") ->
     [ {config, #{type => hoconsc:union(
                           [ hoconsc:ref(?MODULE, single)
                           , hoconsc:ref(?MODULE, rs)
                           , hoconsc:ref(?MODULE, sharded)
                           ])}}
-    ];
+    ].
+
 fields(single) ->
     [ {mongo_type, #{type => single,
                      default => single}}

apps/emqx_connector/src/emqx_connector_mysql.erl

@@ -28,16 +28,14 @@
 
 -export([connect/1]).
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 
 -export([do_health_check/1]).
 
 %%=====================================================================
 %% Hocon schema
-structs() -> [""].
+roots() ->
-
+    [{config, #{type => hoconsc:ref(?MODULE, config)}}].
-fields("") ->
-    [{config, #{type => hoconsc:ref(?MODULE, config)}}];
 
 fields(config) ->
     emqx_connector_schema_lib:relational_db_fields() ++

apps/emqx_connector/src/emqx_connector_pgsql.erl

@@ -18,7 +18,7 @@
 -include_lib("typerefl/include/types.hrl").
 -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 
 %% callbacks of behaviour emqx_resource
 -export([ on_start/2
@@ -35,10 +35,9 @@
 -export([do_health_check/1]).
 
 %%=====================================================================
-structs() -> [""].
 
-fields("") ->
+roots() ->
-    [{config, #{type => hoconsc:ref(?MODULE, config)}}];
+    [{config, #{type => hoconsc:ref(?MODULE, config)}}].
 
 fields(config) ->
     emqx_connector_schema_lib:relational_db_fields() ++

apps/emqx_connector/src/emqx_connector_redis.erl

@@ -23,7 +23,7 @@
 -reflect_type([server/0]).
 -typerefl_from_string({server/0, emqx_connector_schema_lib, to_ip_port}).
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 
 %% callbacks of behaviour emqx_resource
 -export([ on_start/2
@@ -40,16 +40,15 @@
 -export([cmd/3]).
 
 %%=====================================================================
-structs() -> [""].
+roots() ->
-
-fields("") ->
     [ {config, #{type => hoconsc:union(
                   [ hoconsc:ref(?MODULE, cluster)
                   , hoconsc:ref(?MODULE, single)
                   , hoconsc:ref(?MODULE, sentinel)
                   ])}
       }
-    ];
+    ].
+
 fields(single) ->
     [ {server, #{type => server()}}
     , {redis_type, #{type => hoconsc:enum([single]),

apps/emqx_connector/src/emqx_connector_schema_lib.erl

@@ -51,9 +51,9 @@
               , servers/0
              ]).
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 
-structs() -> [ssl_on, ssl_off].
+roots() -> [ssl_on, ssl_off].
 
 fields(ssl_on) ->
     [ {enable, #{type => true}}

apps/emqx_dashboard/src/emqx_dashboard_schema.erl

@@ -17,10 +17,10 @@
 
 -include_lib("typerefl/include/types.hrl").
 
--export([ structs/0
+-export([ roots/0
         , fields/1]).
 
-structs() -> ["emqx_dashboard"].
+roots() -> ["emqx_dashboard"].
 
 fields("emqx_dashboard") ->
     [ {listeners, hoconsc:array(hoconsc:union([hoconsc:ref(?MODULE, "http"),

apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl

@@ -1,6 +1,6 @@
 -module(emqx_data_bridge_schema).
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 
 %%======================================================================================
 %% Hocon Schema Definitions
@@ -8,7 +8,7 @@
 -define(TYPES, [mysql, pgsql, mongo, redis, ldap]).
 -define(BRIDGES, [hoconsc:ref(?MODULE, T) || T <- ?TYPES]).
 
-structs() -> ["emqx_data_bridge"].
+roots() -> ["emqx_data_bridge"].
 
 fields("emqx_data_bridge") ->
     [{bridges, #{type => hoconsc:array(hoconsc:union(?BRIDGES)),
@@ -23,4 +23,4 @@ fields(ldap)  -> connector_fields(ldap).
 connector_fields(DB) ->
     Mod = list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])),
     [{name, hoconsc:t(typerefl:binary())},
-     {type, #{type => DB}}] ++ Mod:fields("").
+     {type, #{type => DB}}] ++ Mod:roots().

apps/emqx_exhook/src/emqx_exhook_schema.erl

@@ -32,11 +32,11 @@
 
 -reflect_type([duration/0]).
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 
 -export([t/1, t/3, t/4, ref/1]).
 
-structs() -> [exhook].
+roots() -> [exhook].
 
 fields(exhook) ->
     [ {request_failed_action, t(union([deny, ignore]), undefined, deny)}

apps/emqx_gateway/src/emqx_gateway_schema.erl

@@ -43,14 +43,10 @@
               , ip_port/0
               ]).
 
--export([structs/0 , fields/1]).
+-export([roots/0 , fields/1]).
-
 -export([t/1, t/3, t/4, ref/1]).
 
-%%--------------------------------------------------------------------
+roots() -> [gateway].
-%% Structs
-
-structs() -> [gateway].
 
 fields(gateway) ->
     [{stomp, t(ref(stomp_structs))},

apps/emqx_machine/src/emqx_machine_schema.erl

@@ -34,7 +34,7 @@
                 file/0,
                 cipher/0]).
 
--export([structs/0, fields/1, translations/0, translation/1]).
+-export([roots/0, fields/1, translations/0, translation/1]).
 -export([t/1, t/3, t/4, ref/1]).
 -export([conf_get/2, conf_get/3, keys/2, filter/1]).
 
@@ -59,9 +59,9 @@
         ]).
 
 %% TODO: add a test case to ensure the list elements are unique
-structs() ->
+roots() ->
     ["cluster", "node", "rpc", "log"]
-    ++ lists:flatmap(fun(Mod) -> Mod:structs() end, ?MERGED_CONFIGS).
+    ++ lists:flatmap(fun(Mod) -> Mod:roots() end, ?MERGED_CONFIGS).
 
 fields("cluster") ->
     [ {"name", t(atom(), "ekka.cluster_name", emqxcl)}
@@ -215,8 +215,7 @@ fields(Name) ->
 find_field(Name, []) ->
     error({unknown_config_struct_field, Name});
 find_field(Name, [SchemaModule | Rest]) ->
-    case lists:member(Name, SchemaModule:structs()) orelse
+    case lists:member(bin(Name), hocon_schema:root_names(SchemaModule)) of
-        lists:keymember(Name, 2, SchemaModule:structs()) of
         true -> SchemaModule:fields(Name);
         false -> find_field(Name, Rest)
     end.
@@ -475,3 +474,7 @@ to_atom(Str) when is_list(Str) ->
     list_to_atom(Str);
 to_atom(Bin) when is_binary(Bin) ->
     binary_to_atom(Bin, utf8).
+
+bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
+bin(Bin) when is_binary(Bin) -> Bin;
+bin(L) when is_list(L) -> iolist_to_binary(L).

apps/emqx_management/src/emqx_management_schema.erl

@@ -19,9 +19,9 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1]).
 
-structs() -> [].
+roots() -> [].
 
 fields(_) -> [].

apps/emqx_modules/src/emqx_modules_schema.erl

@@ -20,16 +20,16 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1]).
 
-structs() ->
+roots() ->
     ["delayed",
      "recon",
      "telemetry",
      "event_message",
-     {array, "rewrite"},
+     array("rewrite"),
-     {array, "topic_metrics"}].
+     array("topic_metrics")].
 
 fields(Name) when Name =:= "recon";
                   Name =:= "telemetry" ->
@@ -61,3 +61,4 @@ fields("event_message") ->
 fields("topic_metrics") ->
     [{topic, emqx_schema:t(binary())}].
 
+array(Name) -> {Name, hoconsc:array(hoconsc:ref(Name))}.

apps/emqx_prometheus/src/emqx_prometheus_schema.erl

@@ -19,10 +19,10 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1]).
 
-structs() -> ["prometheus"].
+roots() -> ["prometheus"].
 
 fields("prometheus") ->
     [ {push_gateway_server, emqx_schema:t(string())}

apps/emqx_retainer/src/emqx_retainer_schema.erl

@@ -2,11 +2,11 @@
 
 -include_lib("typerefl/include/types.hrl").
 
--export([structs/0, fields/1]).
+-export([roots/0, fields/1]).
 
 -define(TYPE(Type), hoconsc:t(Type)).
 
-structs() -> ["emqx_retainer"].
+roots() -> ["emqx_retainer"].
 
 fields("emqx_retainer") ->
     [ {enable, t(boolean(), false)}

apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl

@@ -20,10 +20,10 @@
 
 -behaviour(hocon_schema).
 
--export([ structs/0
+-export([ roots/0
         , fields/1]).
 
-structs() -> ["emqx_rule_engine"].
+roots() -> ["emqx_rule_engine"].
 
 fields("emqx_rule_engine") ->
     [{ignore_sys_message, emqx_schema:t(boolean(), undefined, true)}].

apps/emqx_statsd/src/emqx_statsd_schema.erl

@@ -6,12 +6,12 @@
 
 -export([to_ip_port/1]).
 
--export([ structs/0
+-export([ roots/0
         , fields/1]).
 
 -typerefl_from_string({ip_port/0, emqx_statsd_schema, to_ip_port}).
 
-structs() -> ["statsd"].
+roots() -> ["statsd"].
 
 fields("statsd") ->
     [ {enable, emqx_schema:t(boolean(), undefined, false)}

rebar.config.erl

@@ -127,8 +127,7 @@ test_plugins() ->
 
 test_deps() ->
     [ {bbmustache, "1.10.0"}
-    %, {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "2.0.0"}}}
+    , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "2.1.0"}}}
-    , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {branch, "hocon"}}}
     , meck
     ].