feat(plugins): Support load plugins of hocon configuration format

apps/emqx/rebar.config

@@ -28,6 +28,7 @@
    {test,
        [{deps,
            [ meck
+           , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.6.0"}}}
            , {bbmustache,"1.10.0"}
            , {emqx_ct_helpers, {git,"https://github.com/zmstone/emqx-ct-helpers", {branch,"hocon"}}}
            , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}}

apps/emqx/src/emqx_plugins.erl

@@ -175,7 +175,7 @@ load_ext_plugin(PluginDir) ->
               end,
     ok = load_plugin_app(AppName, Ebin),
     try
-        ok = load_plugin_conf(AppName, PluginDir)
+        ok = generate_configs(AppName)
     catch
         throw : {conf_file_not_found, ConfFile} ->
             %% this is maybe a dependency of an external plugin
@@ -199,21 +199,6 @@ load_plugin_app(AppName, Ebin) ->
         {error, {already_loaded, _}} -> ok
     end.
 
-load_plugin_conf(AppName, PluginDir) ->
-    Priv = filename:join([PluginDir, "priv"]),
-    Etc  = filename:join([PluginDir, "etc"]),
-    ConfFile = filename:join([Etc, atom_to_list(AppName) ++ ".conf"]),
-    Conf = case filelib:is_file(ConfFile) of
-               true -> cuttlefish_conf:file(ConfFile);
-               false -> throw({conf_file_not_found, ConfFile})
-           end,
-    Schema = filelib:wildcard(filename:join([Priv, "*.schema"])),
-    ?LOG(debug, "loading_extra_plugin_config conf=~s, schema=~s", [ConfFile, Schema]),
-    AppsEnv = cuttlefish_generator:map(cuttlefish_schema:files(Schema), Conf),
-    lists:foreach(fun({AppName1, Envs}) ->
-        [application:set_env(AppName1, Par, Val) || {Par, Val} <- Envs]
-    end, AppsEnv).
-
 ensure_file(File) ->
     case filelib:is_file(File) of false -> write_loaded([]); true -> ok end.
 
@@ -246,41 +231,6 @@ load_plugins(Names, Persistent) ->
                       load_plugin(Plugin#plugin.name, Persistent)
                   end, NeedToLoad).
 
-generate_configs(App) ->
-    ConfigFile = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".config",
-    case filelib:is_file(ConfigFile) of
-        true ->
-            {ok, [Configs]} = file:consult(ConfigFile),
-            Configs;
-        false ->
-            do_generate_configs(App)
-    end.
-
-do_generate_configs(App) ->
-    Name1 = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf",
-    Name2 = filename:join([code:lib_dir(App), "etc", App]) ++ ".conf",
-    ConfFile = case {filelib:is_file(Name1), filelib:is_file(Name2)} of
-                   {true, _} -> Name1;
-                   {false, true} -> Name2;
-                   {false, false} -> error({config_not_found, [Name1, Name2]})
-               end,
-    SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema",
-    case filelib:is_file(SchemaFile) of
-        true ->
-            Schema = cuttlefish_schema:files([SchemaFile]),
-            Conf = cuttlefish_conf:file(ConfFile),
-            cuttlefish_generator:map(Schema, Conf, undefined, fun ?MODULE:funlog/2);
-        false ->
-            error({schema_not_found, SchemaFile})
-    end.
-
-apply_configs([]) ->
-    ok;
-apply_configs([{App, Config} | More]) ->
-    lists:foreach(fun({Key, _}) -> application:unset_env(App, Key) end, application:get_all_env(App)),
-    lists:foreach(fun({Key, Val}) -> application:set_env(App, Key, Val) end, Config),
-    apply_configs(More).
-
 %% Stop plugins
 stop_plugins(Names) ->
     _ = [stop_app(App) || App <- Names],
@@ -296,8 +246,7 @@ plugin(AppName, Type) ->
 
 load_plugin(Name, Persistent) ->
     try
-        Configs = ?MODULE:generate_configs(Name),
+        ok = ?MODULE:generate_configs(Name),
-        ?MODULE:apply_configs(Configs),
         case load_app(Name) of
             ok ->
                 start_app(Name, fun(App) -> plugin_loaded(App, Persistent) end);
@@ -416,3 +365,65 @@ plugin_type(_) -> feature.
 
 funlog(Key, Value) ->
     ?LOG(info, "~s = ~p", [string:join(Key, "."), Value]).
+
+generate_configs(App) ->
+    PluginConfDir = filename:join([code:lib_dir(App), "etc"]),
+    PluginSchemaDir = filename:join([code:lib_dir(App), "priv"]),
+    ConfigFile = filename:join([PluginConfDir, App]) ++ ".config",
+    case filelib:is_file(ConfigFile) of
+        true ->
+            {ok, [Configs]} = file:consult(ConfigFile),
+            apply_configs(Configs);
+        false ->
+            SchemaFile = filename:join([PluginSchemaDir, App]) ++ ".schema",
+            case filelib:is_file(SchemaFile) of
+                true ->
+                    AppsEnv = do_generate_configs(App),
+                    apply_configs(AppsEnv);
+                false ->
+                    SchemaMod = lists:concat([App, "_schema"]),
+                    ConfName = filename:join([PluginConfDir, App]) ++ ".conf",
+                    SchemaFile1 = filename:join([code:lib_dir(App), "ebin", SchemaMod]) ++ ".beam",
+                    do_generate_hocon_configs(App, ConfName, SchemaFile1)
+            end
+    end.
+
+do_generate_configs(App) ->
+    Name1 = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf",
+    Name2 = filename:join([code:lib_dir(App), "etc", App]) ++ ".conf",
+    ConfFile = case {filelib:is_file(Name1), filelib:is_file(Name2)} of
+                   {true, _} -> Name1;
+                   {false, true} -> Name2;
+                   {false, false} -> error({config_not_found, [Name1, Name2]})
+               end,
+    SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema",
+    case filelib:is_file(SchemaFile) of
+        true ->
+            Schema = cuttlefish_schema:files([SchemaFile]),
+            Conf = cuttlefish_conf:file(ConfFile),
+            cuttlefish_generator:map(Schema, Conf, undefined, fun ?MODULE:funlog/2);
+        false ->
+            error({schema_not_found, SchemaFile})
+    end.
+
+do_generate_hocon_configs(App, ConfName, SchemaFile) ->
+    SchemaMod = lists:concat([App, "_schema"]),
+    case {filelib:is_file(ConfName), filelib:is_file(SchemaFile)} of
+        {true, true} ->
+            {ok, RawConfig} = hocon:load(ConfName),
+            Config = hocon_schema:check_plain(list_to_atom(SchemaMod), RawConfig, #{atom_key => true}),
+            emqx_config_handler:update_config(emqx_config_handler, Config);
+        {true, false} ->
+            error({schema_not_found, [SchemaFile]});
+        {false, true} ->
+            error({config_not_found, [SchemaFile]});
+        {false, false} ->
+            error({conf_and_schema_not_found, [ConfName, SchemaFile]})
+    end.
+
+apply_configs([]) ->
+    ok;
+apply_configs([{App, Config} | More]) ->
+    lists:foreach(fun({Key, _}) -> application:unset_env(App, Key) end, application:get_all_env(App)),
+    lists:foreach(fun({Key, Val}) -> application:set_env(App, Key, Val) end, Config),
+    apply_configs(More).

apps/emqx/test/emqx_plugins_SUITE.erl

@@ -30,11 +30,16 @@ init_per_suite(Config) ->
 
     DataPath = proplists:get_value(data_dir, Config),
     AppPath = filename:join([DataPath, "emqx_mini_plugin"]),
+    HoconPath = filename:join([DataPath, "emqx_hocon_plugin"]),
     Cmd = lists:flatten(io_lib:format("cd ~s && make", [AppPath])),
+    CmdPath = lists:flatten(io_lib:format("cd ~s && make", [HoconPath])),
 
     ct:pal("Executing ~s~n", [Cmd]),
     ct:pal("~n ~s~n", [os:cmd(Cmd)]),
 
+    ct:pal("Executing ~s~n", [CmdPath]),
+    ct:pal("~n ~s~n", [os:cmd(CmdPath)]),
+
     put(loaded_file, filename:join([DataPath, "loaded_plugins"])),
     emqx_ct_helpers:boot_modules([]),
     emqx_ct_helpers:start_apps([], fun(_) -> set_special_cfg(DataPath) end),
@@ -55,6 +60,7 @@ t_load(_) ->
 
     ?assertEqual({error, not_found}, emqx_plugins:load(not_existed_plugin)),
     ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_mini_plugin)),
+    ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_hocon_plugin)),
 
     application:set_env(emqx, expand_plugins_dir, undefined),
     application:set_env(emqx, plugins_loaded_file, undefined),
@@ -78,7 +84,8 @@ t_list(_) ->
     ?assertMatch([{plugin, _, _, _, _, _, _, _} | _ ], emqx_plugins:list()).
 
 t_find_plugin(_) ->
-    ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_mini_plugin)).
+    ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_mini_plugin)),
+    ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_hocon_plugin)).
 
 t_plugin_type(_) ->
     ?assertEqual(auth, emqx_plugins:plugin_type(auth)),
@@ -92,11 +99,15 @@ t_with_loaded_file(_) ->
 
 t_plugin_loaded(_) ->
     ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, false)),
-    ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, true)).
+    ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, true)),
+    ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_hocon_plugin, false)),
+    ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_hocon_plugin, true)).
 
 t_plugin_unloaded(_) ->
     ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, false)),
-    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, true)).
+    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, true)),
+    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_hocon_plugin, false)),
+    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_hocon_plugin, true)).
 
 t_plugin(_) ->
     try
@@ -105,7 +116,8 @@ t_plugin(_) ->
         _Error:Reason:_Stacktrace ->
             ?assertEqual({plugin_not_found,not_existed_plugin}, Reason)
     end,
-    ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_mini_plugin, undefined)).
+    ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_mini_plugin, undefined)),
+    ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_hocon_plugin, undefined)).
 
 t_filter_plugins(_) ->
     ?assertEqual([name1, name2], emqx_plugins:filter_plugins([name1, {name2,true}, {name3, false}])).

apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/Makefile

@@ -0,0 +1,26 @@
+## shallow clone for speed
+
+REBAR_GIT_CLONE_OPTIONS += --depth 1
+export REBAR_GIT_CLONE_OPTIONS
+
+REBAR = rebar3
+all: compile
+
+compile:
+	$(REBAR) compile
+	cp -r _build/default/lib/emqx_hocon_plugin/ebin ./
+
+clean: distclean
+
+ct: compile
+	$(REBAR) as test ct -v
+
+eunit: compile
+	$(REBAR) as test eunit
+
+xref:
+	$(REBAR) xref
+
+distclean:
+	@rm -rf _build
+	@rm -f ebin/ data/app.*.config data/vm.*.args rebar.lock

apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/etc/emqx_hocon_plugin.conf

@@ -0,0 +1,3 @@
+emqx_hocon_plugin: {
+    name: test
+}

apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/rebar.config

@@ -0,0 +1,23 @@
+{deps, [{hocon, {git, "https://github.com/emqx/hocon", {tag, "0.6.0"}}}]}.
+
+{edoc_opts, [{preprocess, true}]}.
+{erl_opts, [warn_unused_vars,
+            warn_shadow_vars,
+            warn_unused_import,
+            warn_obsolete_guard,
+            debug_info,
+            {parse_transform}]}.
+
+{xref_checks, [undefined_function_calls, undefined_functions,
+               locals_not_used, deprecated_function_calls,
+               warnings_as_errors, deprecated_functions]}.
+{cover_enabled, true}.
+{cover_opts, [verbose]}.
+{cover_export_enabled, true}.
+
+{profiles,
+    [{test, [
+        {deps, [ {emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.1.4"}}}
+               ]}
+    ]}
+]}.

apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src

@@ -0,0 +1,15 @@
+{application, emqx_hocon_plugin,
+ [{description, "An EMQ X plugin for hocon testcase"},
+  {vsn, "0.1"},
+  {modules, []},
+  {registered, []},
+  {mod, {emqx_hocon_plugin_app, []}},
+  {applications,
+   [kernel,
+    stdlib,
+    typerefl
+   ]},
+  {env,[]},
+  {licenses, ["Apache 2.0"]},
+  {links, []}
+ ]}.

apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_app.erl

@@ -0,0 +1,42 @@
+%%%-------------------------------------------------------------------
+%% @doc emqx_mini_plugin public API
+%% @end
+%%%-------------------------------------------------------------------
+
+-module(emqx_hocon_plugin_app).
+
+-behaviour(application).
+-behaviour(supervisor).
+
+-emqx_plugin(?MODULE).
+
+%% Application APIs
+-export([ start/2
+        , stop/1
+        ]).
+
+%% Supervisor callback
+-export([init/1]).
+
+
+%% -- Application
+
+start(_StartType, _StartArgs) ->
+    {ok, Sup} = start_link(),
+    {ok, Sup}.
+
+stop(_State) ->
+    ok.
+
+%% --- Supervisor
+
+start_link() ->
+    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+init([]) ->
+    SupFlags = #{strategy => one_for_all,
+                 intensity => 0,
+                 period => 1},
+    ChildSpecs = [],
+    {ok, {SupFlags, ChildSpecs}}.
+

apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl

@@ -0,0 +1,15 @@
+-module(emqx_hocon_plugin_schema).
+
+-include_lib("typerefl/include/types.hrl").
+
+-export([structs/0, fields/1]).
+
+-behaviour(hocon_schema).
+
+structs() -> ["emqx_hocon_plugin"].
+
+fields("emqx_hocon_plugin") ->
+    [{name, fun name/1}].
+
+name(type) -> binary();
+name(_) -> undefined.