From ffcbcaed3c8742a86e6d0809cde175f4176addd3 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 19 Jun 2021 12:02:01 +0800 Subject: [PATCH] feat(plugins): Support load plugins of hocon configuration format --- apps/emqx/rebar.config | 1 + apps/emqx/src/emqx_plugins.erl | 117 ++++++++++-------- apps/emqx/test/emqx_plugins_SUITE.erl | 20 ++- .../emqx_hocon_plugin/Makefile | 26 ++++ .../etc/emqx_hocon_plugin.conf | 3 + .../emqx_hocon_plugin/rebar.config | 23 ++++ .../src/emqx_hocon_plugin.app.src | 15 +++ .../src/emqx_hocon_plugin_app.erl | 42 +++++++ .../src/emqx_hocon_plugin_schema.erl | 15 +++ 9 files changed, 205 insertions(+), 57 deletions(-) create mode 100644 apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/Makefile create mode 100644 apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/etc/emqx_hocon_plugin.conf create mode 100644 apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/rebar.config create mode 100644 apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src create mode 100644 apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_app.erl create mode 100644 apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 693e96c74..17ba58d37 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -28,6 +28,7 @@ {test, [{deps, [ meck + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.6.0"}}} , {bbmustache,"1.10.0"} , {emqx_ct_helpers, {git,"https://github.com/zmstone/emqx-ct-helpers", {branch,"hocon"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}} diff --git a/apps/emqx/src/emqx_plugins.erl b/apps/emqx/src/emqx_plugins.erl index 6ef3d2d21..7df1f368c 100644 --- a/apps/emqx/src/emqx_plugins.erl +++ b/apps/emqx/src/emqx_plugins.erl @@ -175,7 +175,7 @@ load_ext_plugin(PluginDir) -> end, ok = load_plugin_app(AppName, Ebin), try - ok = load_plugin_conf(AppName, PluginDir) + ok = generate_configs(AppName) catch throw : {conf_file_not_found, ConfFile} -> %% this is maybe a dependency of an external plugin @@ -199,21 +199,6 @@ load_plugin_app(AppName, Ebin) -> {error, {already_loaded, _}} -> ok end. -load_plugin_conf(AppName, PluginDir) -> - Priv = filename:join([PluginDir, "priv"]), - Etc = filename:join([PluginDir, "etc"]), - ConfFile = filename:join([Etc, atom_to_list(AppName) ++ ".conf"]), - Conf = case filelib:is_file(ConfFile) of - true -> cuttlefish_conf:file(ConfFile); - false -> throw({conf_file_not_found, ConfFile}) - end, - Schema = filelib:wildcard(filename:join([Priv, "*.schema"])), - ?LOG(debug, "loading_extra_plugin_config conf=~s, schema=~s", [ConfFile, Schema]), - AppsEnv = cuttlefish_generator:map(cuttlefish_schema:files(Schema), Conf), - lists:foreach(fun({AppName1, Envs}) -> - [application:set_env(AppName1, Par, Val) || {Par, Val} <- Envs] - end, AppsEnv). - ensure_file(File) -> case filelib:is_file(File) of false -> write_loaded([]); true -> ok end. @@ -246,41 +231,6 @@ load_plugins(Names, Persistent) -> load_plugin(Plugin#plugin.name, Persistent) end, NeedToLoad). -generate_configs(App) -> - ConfigFile = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".config", - case filelib:is_file(ConfigFile) of - true -> - {ok, [Configs]} = file:consult(ConfigFile), - Configs; - false -> - do_generate_configs(App) - end. - -do_generate_configs(App) -> - Name1 = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf", - Name2 = filename:join([code:lib_dir(App), "etc", App]) ++ ".conf", - ConfFile = case {filelib:is_file(Name1), filelib:is_file(Name2)} of - {true, _} -> Name1; - {false, true} -> Name2; - {false, false} -> error({config_not_found, [Name1, Name2]}) - end, - SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema", - case filelib:is_file(SchemaFile) of - true -> - Schema = cuttlefish_schema:files([SchemaFile]), - Conf = cuttlefish_conf:file(ConfFile), - cuttlefish_generator:map(Schema, Conf, undefined, fun ?MODULE:funlog/2); - false -> - error({schema_not_found, SchemaFile}) - end. - -apply_configs([]) -> - ok; -apply_configs([{App, Config} | More]) -> - lists:foreach(fun({Key, _}) -> application:unset_env(App, Key) end, application:get_all_env(App)), - lists:foreach(fun({Key, Val}) -> application:set_env(App, Key, Val) end, Config), - apply_configs(More). - %% Stop plugins stop_plugins(Names) -> _ = [stop_app(App) || App <- Names], @@ -296,8 +246,7 @@ plugin(AppName, Type) -> load_plugin(Name, Persistent) -> try - Configs = ?MODULE:generate_configs(Name), - ?MODULE:apply_configs(Configs), + ok = ?MODULE:generate_configs(Name), case load_app(Name) of ok -> start_app(Name, fun(App) -> plugin_loaded(App, Persistent) end); @@ -416,3 +365,65 @@ plugin_type(_) -> feature. funlog(Key, Value) -> ?LOG(info, "~s = ~p", [string:join(Key, "."), Value]). + +generate_configs(App) -> + PluginConfDir = filename:join([code:lib_dir(App), "etc"]), + PluginSchemaDir = filename:join([code:lib_dir(App), "priv"]), + ConfigFile = filename:join([PluginConfDir, App]) ++ ".config", + case filelib:is_file(ConfigFile) of + true -> + {ok, [Configs]} = file:consult(ConfigFile), + apply_configs(Configs); + false -> + SchemaFile = filename:join([PluginSchemaDir, App]) ++ ".schema", + case filelib:is_file(SchemaFile) of + true -> + AppsEnv = do_generate_configs(App), + apply_configs(AppsEnv); + false -> + SchemaMod = lists:concat([App, "_schema"]), + ConfName = filename:join([PluginConfDir, App]) ++ ".conf", + SchemaFile1 = filename:join([code:lib_dir(App), "ebin", SchemaMod]) ++ ".beam", + do_generate_hocon_configs(App, ConfName, SchemaFile1) + end + end. + +do_generate_configs(App) -> + Name1 = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf", + Name2 = filename:join([code:lib_dir(App), "etc", App]) ++ ".conf", + ConfFile = case {filelib:is_file(Name1), filelib:is_file(Name2)} of + {true, _} -> Name1; + {false, true} -> Name2; + {false, false} -> error({config_not_found, [Name1, Name2]}) + end, + SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema", + case filelib:is_file(SchemaFile) of + true -> + Schema = cuttlefish_schema:files([SchemaFile]), + Conf = cuttlefish_conf:file(ConfFile), + cuttlefish_generator:map(Schema, Conf, undefined, fun ?MODULE:funlog/2); + false -> + error({schema_not_found, SchemaFile}) + end. + +do_generate_hocon_configs(App, ConfName, SchemaFile) -> + SchemaMod = lists:concat([App, "_schema"]), + case {filelib:is_file(ConfName), filelib:is_file(SchemaFile)} of + {true, true} -> + {ok, RawConfig} = hocon:load(ConfName), + Config = hocon_schema:check_plain(list_to_atom(SchemaMod), RawConfig, #{atom_key => true}), + emqx_config_handler:update_config(emqx_config_handler, Config); + {true, false} -> + error({schema_not_found, [SchemaFile]}); + {false, true} -> + error({config_not_found, [SchemaFile]}); + {false, false} -> + error({conf_and_schema_not_found, [ConfName, SchemaFile]}) + end. + +apply_configs([]) -> + ok; +apply_configs([{App, Config} | More]) -> + lists:foreach(fun({Key, _}) -> application:unset_env(App, Key) end, application:get_all_env(App)), + lists:foreach(fun({Key, Val}) -> application:set_env(App, Key, Val) end, Config), + apply_configs(More). diff --git a/apps/emqx/test/emqx_plugins_SUITE.erl b/apps/emqx/test/emqx_plugins_SUITE.erl index 6d8847f43..6a76cb9d2 100644 --- a/apps/emqx/test/emqx_plugins_SUITE.erl +++ b/apps/emqx/test/emqx_plugins_SUITE.erl @@ -30,11 +30,16 @@ init_per_suite(Config) -> DataPath = proplists:get_value(data_dir, Config), AppPath = filename:join([DataPath, "emqx_mini_plugin"]), + HoconPath = filename:join([DataPath, "emqx_hocon_plugin"]), Cmd = lists:flatten(io_lib:format("cd ~s && make", [AppPath])), + CmdPath = lists:flatten(io_lib:format("cd ~s && make", [HoconPath])), ct:pal("Executing ~s~n", [Cmd]), ct:pal("~n ~s~n", [os:cmd(Cmd)]), + ct:pal("Executing ~s~n", [CmdPath]), + ct:pal("~n ~s~n", [os:cmd(CmdPath)]), + put(loaded_file, filename:join([DataPath, "loaded_plugins"])), emqx_ct_helpers:boot_modules([]), emqx_ct_helpers:start_apps([], fun(_) -> set_special_cfg(DataPath) end), @@ -55,6 +60,7 @@ t_load(_) -> ?assertEqual({error, not_found}, emqx_plugins:load(not_existed_plugin)), ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_mini_plugin)), + ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_hocon_plugin)), application:set_env(emqx, expand_plugins_dir, undefined), application:set_env(emqx, plugins_loaded_file, undefined), @@ -78,7 +84,8 @@ t_list(_) -> ?assertMatch([{plugin, _, _, _, _, _, _, _} | _ ], emqx_plugins:list()). t_find_plugin(_) -> - ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_mini_plugin)). + ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_mini_plugin)), + ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_hocon_plugin)). t_plugin_type(_) -> ?assertEqual(auth, emqx_plugins:plugin_type(auth)), @@ -92,11 +99,15 @@ t_with_loaded_file(_) -> t_plugin_loaded(_) -> ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, true)). + ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, true)), + ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_hocon_plugin, false)), + ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_hocon_plugin, true)). t_plugin_unloaded(_) -> ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, true)). + ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, true)), + ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_hocon_plugin, false)), + ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_hocon_plugin, true)). t_plugin(_) -> try @@ -105,7 +116,8 @@ t_plugin(_) -> _Error:Reason:_Stacktrace -> ?assertEqual({plugin_not_found,not_existed_plugin}, Reason) end, - ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_mini_plugin, undefined)). + ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_mini_plugin, undefined)), + ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_hocon_plugin, undefined)). t_filter_plugins(_) -> ?assertEqual([name1, name2], emqx_plugins:filter_plugins([name1, {name2,true}, {name3, false}])). diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/Makefile b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/Makefile new file mode 100644 index 000000000..e0ed250e1 --- /dev/null +++ b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/Makefile @@ -0,0 +1,26 @@ +## shallow clone for speed + +REBAR_GIT_CLONE_OPTIONS += --depth 1 +export REBAR_GIT_CLONE_OPTIONS + +REBAR = rebar3 +all: compile + +compile: + $(REBAR) compile + cp -r _build/default/lib/emqx_hocon_plugin/ebin ./ + +clean: distclean + +ct: compile + $(REBAR) as test ct -v + +eunit: compile + $(REBAR) as test eunit + +xref: + $(REBAR) xref + +distclean: + @rm -rf _build + @rm -f ebin/ data/app.*.config data/vm.*.args rebar.lock diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/etc/emqx_hocon_plugin.conf b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/etc/emqx_hocon_plugin.conf new file mode 100644 index 000000000..e1f7bc5b9 --- /dev/null +++ b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/etc/emqx_hocon_plugin.conf @@ -0,0 +1,3 @@ +emqx_hocon_plugin: { + name: test +} diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/rebar.config b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/rebar.config new file mode 100644 index 000000000..888a03bc4 --- /dev/null +++ b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/rebar.config @@ -0,0 +1,23 @@ +{deps, [{hocon, {git, "https://github.com/emqx/hocon", {tag, "0.6.0"}}}]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {parse_transform}]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{profiles, + [{test, [ + {deps, [ {emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.1.4"}}} + ]} + ]} +]}. diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src new file mode 100644 index 000000000..d09f01b1c --- /dev/null +++ b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src @@ -0,0 +1,15 @@ +{application, emqx_hocon_plugin, + [{description, "An EMQ X plugin for hocon testcase"}, + {vsn, "0.1"}, + {modules, []}, + {registered, []}, + {mod, {emqx_hocon_plugin_app, []}}, + {applications, + [kernel, + stdlib, + typerefl + ]}, + {env,[]}, + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_app.erl b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_app.erl new file mode 100644 index 000000000..04191aac8 --- /dev/null +++ b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_app.erl @@ -0,0 +1,42 @@ +%%%------------------------------------------------------------------- +%% @doc emqx_mini_plugin public API +%% @end +%%%------------------------------------------------------------------- + +-module(emqx_hocon_plugin_app). + +-behaviour(application). +-behaviour(supervisor). + +-emqx_plugin(?MODULE). + +%% Application APIs +-export([ start/2 + , stop/1 + ]). + +%% Supervisor callback +-export([init/1]). + + +%% -- Application + +start(_StartType, _StartArgs) -> + {ok, Sup} = start_link(), + {ok, Sup}. + +stop(_State) -> + ok. + +%% --- Supervisor + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + SupFlags = #{strategy => one_for_all, + intensity => 0, + period => 1}, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl new file mode 100644 index 000000000..fab74b5e8 --- /dev/null +++ b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin_schema.erl @@ -0,0 +1,15 @@ +-module(emqx_hocon_plugin_schema). + +-include_lib("typerefl/include/types.hrl"). + +-export([structs/0, fields/1]). + +-behaviour(hocon_schema). + +structs() -> ["emqx_hocon_plugin"]. + +fields("emqx_hocon_plugin") -> + [{name, fun name/1}]. + +name(type) -> binary(); +name(_) -> undefined.