From 6436217e075a591450c7d51f7779d23eb4655750 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 11:40:09 +0200 Subject: [PATCH 1/5] chore: fix wrong plugins dir --- rebar.config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.erl b/rebar.config.erl index b5223013c..c6c9e3409 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -194,7 +194,7 @@ overlay_vars_pkg(bin) -> , {platform_etc_dir, "etc"} , {platform_lib_dir, "lib"} , {platform_log_dir, "log"} - , {platform_plugins_dir, "plugins"} + , {platform_plugins_dir, "etc/plugins"} , {runner_root_dir, "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)"} , {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"} , {runner_etc_dir, "$RUNNER_ROOT_DIR/etc"} From faecde9ce196500b6e55e4971f9fe4f61e00ec42 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 13:31:10 +0200 Subject: [PATCH 2/5] fix(emqx_plugins): allow loading conf for plugin app dir Prior to this change, plugin config files are only allowed to be placed in the collective config dir etc/plugins. In order to support external plugin's drop-in deployment, this commit made emqx_plugins module to read conf file in application's etc dir --- src/emqx_plugins.erl | 103 +++++++++++------- test/emqx_plugins_SUITE.erl | 18 ++- .../emqx_mini_plugin/Makefile | 13 +-- .../emqx_mini_plugin/rebar.config | 4 +- 4 files changed, 75 insertions(+), 63 deletions(-) diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index bf8de4a59..2616c1f70 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -61,7 +61,7 @@ init() -> %% @doc Load all plugins when the broker started. -spec(load() -> ok | ignore | {error, term()}). load() -> - load_expand_plugins(), + ok = load_ext_plugins(emqx:get_env(expand_plugins_dir)), case emqx:get_env(plugins_loaded_file) of undefined -> ignore; %% No plugins available File -> @@ -148,46 +148,61 @@ init_config(CfgFile) -> [application:set_env(App, Par, Val) || {Par, Val} <- Envs] end, AppsEnv). -load_expand_plugins() -> - case emqx:get_env(expand_plugins_dir) of - undefined -> ok; - ExpandPluginsDir -> - Plugins = filelib:wildcard("*", ExpandPluginsDir), - lists:foreach(fun(Plugin) -> - PluginDir = filename:join(ExpandPluginsDir, Plugin), +%% load external plugins which are placed in etc/plugins dir +load_ext_plugins(undefined) -> ok; +load_ext_plugins(Dir) -> + lists:foreach( + fun(Plugin) -> + PluginDir = filename:join(Dir, Plugin), case filelib:is_dir(PluginDir) of - true -> load_expand_plugin(PluginDir); + true -> load_ext_plugin(PluginDir); false -> ok end - end, Plugins) - end. + end, filelib:wildcard("*", Dir)). -load_expand_plugin(PluginDir) -> - init_expand_plugin_config(PluginDir), +load_ext_plugin(PluginDir) -> + ?LOG(debug, "loading_extra_plugin: ~s", [PluginDir]), Ebin = filename:join([PluginDir, "ebin"]), + AppFile = filename:join([Ebin, "*.app"]), + AppName = case filelib:wildcard(AppFile) of + [App] -> + list_to_atom(filename:basename(App, ".app")); + [] -> + ?LOG(alert, "plugin_app_file_not_found: ~s", [AppFile]), + error({plugin_app_file_not_found, AppFile}) + end, + ok = load_plugin_app(AppName, Ebin), + ok = load_plugin_conf(AppName, PluginDir). + +load_plugin_app(AppName, Ebin) -> _ = code:add_patha(Ebin), Modules = filelib:wildcard(filename:join([Ebin, "*.beam"])), - lists:foreach(fun(Mod) -> - Module = list_to_atom(filename:basename(Mod, ".beam")), - code:load_file(Module) - end, Modules), - case filelib:wildcard(Ebin ++ "/*.app") of - [App|_] -> application:load(list_to_atom(filename:basename(App, ".app"))); - _ -> ?LOG(alert, "Plugin not found."), - {error, load_app_fail} + lists:foreach( + fun(BeamFile) -> + Module = list_to_atom(filename:basename(BeamFile, ".beam")), + case code:ensure_loaded(Module) of + {module, Module} -> ok; + {error, Reason} -> error({failed_to_load_plugin_beam, BeamFile, Reason}) + end + end, Modules), + case application:load(AppName) of + ok -> ok; + {error, {already_loaded, _}} -> ok end. -init_expand_plugin_config(PluginDir) -> - Priv = PluginDir ++ "/priv", - Etc = PluginDir ++ "/etc", - Schema = filelib:wildcard(Priv ++ "/*.schema"), - Conf = case filelib:wildcard(Etc ++ "/*.conf") of - [] -> []; - [Conf1] -> cuttlefish_conf:file(Conf1) - end, +load_plugin_conf(AppName, PluginDir) -> + Priv = filename:join([PluginDir, "priv"]), + Etc = filename:join([PluginDir, "etc"]), + Schema = filelib:wildcard(filename:join([Priv, "*.schema"])), + ConfFile = filename:join([Etc, atom_to_list(AppName) ++ ".conf"]), + Conf = case filelib:is_file(ConfFile) of + true -> cuttlefish_conf:file(ConfFile); + false -> error({conf_file_not_found, ConfFile}) + end, + ?LOG(debug, "loading_extra_plugin_config conf=~s, schema=~s", [ConfFile, Schema]), AppsEnv = cuttlefish_generator:map(cuttlefish_schema:files(Schema), Conf), - lists:foreach(fun({AppName, Envs}) -> - [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs] + lists:foreach(fun({AppName1, Envs}) -> + [application:set_env(AppName1, Par, Val) || {Par, Val} <- Envs] end, AppsEnv). ensure_file(File) -> @@ -223,19 +238,31 @@ load_plugins(Names, Persistent) -> generate_configs(App) -> ConfigFile = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".config", - ConfFile = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf", - SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema", - case {filelib:is_file(ConfigFile), filelib:is_file(ConfFile) andalso filelib:is_file(SchemaFile)} of - {true, _} -> + case filelib:is_file(ConfigFile) of + true -> {ok, [Configs]} = file:consult(ConfigFile), Configs; - {_, true} -> + false -> + do_generate_configs(App) + end. + +do_generate_configs(App) -> + Name1 = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf", + Name2 = filename:join([code:lib_dir(App), "etc", App]) ++ ".conf", + ConfFile = case {filelib:is_file(Name1), filelib:is_file(Name2)} of + {true, _} -> Name1; + {false, true} -> Name2; + {false, false} -> error({config_not_found, [Name1, Name2]}) + end, + SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema", + case filelib:is_file(SchemaFile) of + true -> Schema = cuttlefish_schema:files([SchemaFile]), Conf = cuttlefish_conf:file(ConfFile), LogFun = fun(Key, Value) -> ?LOG(info, "~s = ~p", [string:join(Key, "."), Value]) end, cuttlefish_generator:map(Schema, Conf, undefined, LogFun); - {false, false} -> - error({config_not_found, {ConfigFile, ConfFile, SchemaFile}}) + false -> + error({schema_not_found, SchemaFile}) end. apply_configs([]) -> diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index 401cf87dd..f62fea5e7 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -30,24 +30,20 @@ init_per_suite(Config) -> DataPath = proplists:get_value(data_dir, Config), AppPath = filename:join([DataPath, "emqx_mini_plugin"]), - Cmd = lists:flatten(io_lib:format("cd ~s && make && cp -r etc _build/default/lib/emqx_mini_plugin/", [AppPath])), + Cmd = lists:flatten(io_lib:format("cd ~s && make", [AppPath])), ct:pal("Executing ~s~n", [Cmd]), ct:pal("~n ~s~n", [os:cmd(Cmd)]), - code:add_path(filename:join([AppPath, "_build", "default", "lib", "emqx_mini_plugin", "ebin"])), - put(loaded_file, filename:join([DataPath, "loaded_plugins"])), emqx_ct_helpers:boot_modules([]), - emqx_ct_helpers:start_apps([], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([], fun(_) -> set_sepecial_cfg(DataPath) end), Config. - -set_sepecial_cfg(_) -> - ExpandPath = filename:dirname(code:lib_dir(emqx_mini_plugin)), +set_sepecial_cfg(PluginsDir) -> application:set_env(emqx, plugins_loaded_file, get(loaded_file)), - application:set_env(emqx, expand_plugins_dir, ExpandPath), + application:set_env(emqx, expand_plugins_dir, PluginsDir), ok. end_per_suite(_Config) -> @@ -58,7 +54,6 @@ t_load(_) -> ?assertEqual(ok, emqx_plugins:unload()), ?assertEqual({error, not_found}, emqx_plugins:load(not_existed_plugin)), - ?assertEqual({error, parse_config_file_failed}, emqx_plugins:load(emqx_mini_plugin)), ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_mini_plugin)), application:set_env(emqx, expand_plugins_dir, undefined), @@ -75,8 +70,9 @@ t_init_config(_) -> file:delete(ConfFile), ?assertEqual({ok,test}, application:get_env(emqx_mini_plugin, mininame)). -t_load_expand_plugin(_) -> - ?assertEqual({error, load_app_fail}, emqx_plugins:load_expand_plugin("./not_existed_path/")). +t_load_ext_plugin(_) -> + ?assertError({plugin_app_file_not_found, _}, + emqx_plugins:load_ext_plugin("./not_existed_path/")). t_list(_) -> ?assertMatch([{plugin, _, _, _, _, _, _, _} | _ ], emqx_plugins:list()). diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile b/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile index ad02951a3..fd38ff640 100644 --- a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile +++ b/test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile @@ -8,6 +8,7 @@ all: compile compile: $(REBAR) compile + cp -r _build/default/lib/emqx_mini_plugin/ebin ./ clean: distclean @@ -22,14 +23,4 @@ xref: distclean: @rm -rf _build - @rm -f data/app.*.config data/vm.*.args rebar.lock - -CUTTLEFISH_SCRIPT = _build/default/lib/cuttlefish/cuttlefish - -$(CUTTLEFISH_SCRIPT): - @${REBAR} get-deps - @if [ ! -f cuttlefish ]; then make -C _build/default/lib/cuttlefish; fi - -app.config: $(CUTTLEFISH_SCRIPT) etc/emqx_mini_plugin.conf - $(verbose) $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/emqx_mini_plugin.conf -i priv/emqx_mini_plugin.schema -d data - + @rm -f ebin/ data/app.*.config data/vm.*.args rebar.lock diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config b/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config index c690b88b1..4c49da1dc 100644 --- a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config +++ b/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config @@ -1,5 +1,4 @@ -{deps, - []}. +{deps, []}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, @@ -19,7 +18,6 @@ {profiles, [{test, [ {deps, [ {emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.1.4"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} ]} ]} ]}. From 8443aa5e21c7c4fa1009c6faa1da9c12fce05c8b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 15:15:50 +0200 Subject: [PATCH 3/5] chore(typo): sepecial -> special --- apps/emqx_coap/test/emqx_coap_SUITE.erl | 6 +++--- apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl | 6 +++--- apps/emqx_exproto/test/emqx_exproto_SUITE.erl | 6 +++--- lib-ce/emqx_modules/test/emqx_modules_SUITE.erl | 4 ++-- test/emqx_plugins_SUITE.erl | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apps/emqx_coap/test/emqx_coap_SUITE.erl b/apps/emqx_coap/test/emqx_coap_SUITE.erl index ed59b309c..444bcc064 100644 --- a/apps/emqx_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_SUITE.erl @@ -28,16 +28,16 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_coap], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1), Config. -set_sepecial_cfg(emqx_coap) -> +set_special_cfg(emqx_coap) -> Opts = application:get_env(emqx_coap, dtls_opts,[]), Opts2 = [{keyfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/key.pem")}, {certfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/cert.pem")}], application:set_env(emqx_coap, dtls_opts, emqx_misc:merge_opts(Opts, Opts2)), application:set_env(emqx_coap, enable_stats, true); -set_sepecial_cfg(_) -> +set_special_cfg(_) -> ok. end_per_suite(Config) -> diff --git a/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl b/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl index 7f49ece7b..1aaf6cb69 100644 --- a/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl @@ -28,12 +28,12 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_coap], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1), Config. -set_sepecial_cfg(emqx_coap) -> +set_special_cfg(emqx_coap) -> application:set_env(emqx_coap, enable_stats, true); -set_sepecial_cfg(_) -> +set_special_cfg(_) -> ok. end_per_suite(Config) -> diff --git a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl index f1fdfa9f8..70484c30e 100644 --- a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl @@ -55,7 +55,7 @@ metrics() -> init_per_group(GrpName, Cfg) -> put(grpname, GrpName), Svrs = emqx_exproto_echo_svr:start(), - emqx_ct_helpers:start_apps([emqx_exproto], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([emqx_exproto], fun set_special_cfg/1), emqx_logger:set_log_level(debug), [{servers, Svrs}, {listener_type, GrpName} | Cfg]. @@ -63,7 +63,7 @@ end_per_group(_, Cfg) -> emqx_ct_helpers:stop_apps([emqx_exproto]), emqx_exproto_echo_svr:stop(proplists:get_value(servers, Cfg)). -set_sepecial_cfg(emqx_exproto) -> +set_special_cfg(emqx_exproto) -> LisType = get(grpname), Listeners = application:get_env(emqx_exproto, listeners, []), SockOpts = socketopts(LisType), @@ -77,7 +77,7 @@ set_sepecial_cfg(emqx_exproto) -> NListeners = [{Proto, LisType, LisOn, UpgradeOpts(Opts)} || {Proto, _Type, LisOn, Opts} <- Listeners], application:set_env(emqx_exproto, listeners, NListeners); -set_sepecial_cfg(emqx) -> +set_special_cfg(emqx) -> application:set_env(emqx, allow_anonymous, true), application:set_env(emqx, enable_acl_cache, false), ok. diff --git a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl index 48bd0bd5b..dc76e8eb7 100644 --- a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl @@ -32,11 +32,11 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_management, emqx_modules], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([emqx_management, emqx_modules], fun set_special_cfg/1), emqx_ct_http:create_default_app(), Config. -set_sepecial_cfg(_) -> +set_special_cfg(_) -> application:set_env(emqx, modules_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_modules")), ok. diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index f62fea5e7..6d8847f43 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -37,11 +37,11 @@ init_per_suite(Config) -> put(loaded_file, filename:join([DataPath, "loaded_plugins"])), emqx_ct_helpers:boot_modules([]), - emqx_ct_helpers:start_apps([], fun(_) -> set_sepecial_cfg(DataPath) end), + emqx_ct_helpers:start_apps([], fun(_) -> set_special_cfg(DataPath) end), Config. -set_sepecial_cfg(PluginsDir) -> +set_special_cfg(PluginsDir) -> application:set_env(emqx, plugins_loaded_file, get(loaded_file)), application:set_env(emqx, expand_plugins_dir, PluginsDir), ok. From 4ec10461604ef05580d594c9241ab810491ce27a Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 15:47:16 +0200 Subject: [PATCH 4/5] fix: bad test case --- apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 2 +- src/emqx_plugins.erl | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index 1ba958cc5..bf3727ff5 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -322,7 +322,7 @@ t_plugins(_) -> auth_header_()), [Plugin3] = filter(get(<<"data">>, Plugins3), <<"name">>, <<"emqx_auth_mnesia">>), ?assertEqual(<<"emqx_auth_mnesia">>, maps:get(<<"name">>, Plugin3)), - ?assertEqual(false, maps:get(<<"active">>, Plugin3)), + ?assertEqual(true, maps:get(<<"active">>, Plugin3)), {ok, _} = request_api(put, api_path(["nodes", diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 2616c1f70..f05da760e 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -225,10 +225,11 @@ filter_plugins(Names) -> end, Names). load_plugins(Names, Persistent) -> - Plugins = list(), NotFound = Names -- names(Plugins), + Plugins = list(), + NotFound = Names -- names(Plugins), case NotFound of [] -> ok; - NotFound -> ?LOG(alert, "Cannot find plugins: ~p", [NotFound]) + NotFound -> ?LOG(alert, "cannot_find_plugins: ~p", [NotFound]) end, NeedToLoad = Names -- NotFound -- names(started_app), lists:foreach(fun(Name) -> From 623ffc78612e3619168807bf1798516048003612 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 20 May 2021 16:28:02 +0200 Subject: [PATCH 5/5] fix: add emqx_plugins to appup --- src/emqx.appup.src | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 4c649e68d..3bf40272c 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -7,7 +7,8 @@ {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, {load_module, emqx_channel, brutal_purge, soft_purge, []}, - {load_module, emqx_app, brutal_purge, soft_purge, []} + {load_module, emqx_app, brutal_purge, soft_purge, []}, + {load_module, emqx_plugins, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, @@ -19,6 +20,7 @@ {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, {load_module, emqx_channel, brutal_purge, soft_purge, []}, {load_module, emqx_app, brutal_purge, soft_purge, []}, + {load_module, emqx_plugins, brutal_purge, soft_purge, []}, %% {load_module, emqx_metrics, brutal_purge, soft_purge, []}, {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}} @@ -32,7 +34,8 @@ {load_module, emqx_congestion, brutal_purge, soft_purge, []}, {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, {load_module, emqx_channel, brutal_purge, soft_purge, []}, - {load_module, emqx_app, brutal_purge, soft_purge, []} + {load_module, emqx_app, brutal_purge, soft_purge, []}, + {load_module, emqx_plugins, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, @@ -44,6 +47,7 @@ {load_module, emqx_node_dump, brutal_purge, soft_purge, []}, {load_module, emqx_channel, brutal_purge, soft_purge, []}, {load_module, emqx_app, brutal_purge, soft_purge, []}, + {load_module, emqx_plugins, brutal_purge, soft_purge, []}, %% Just load the module. We don't need to change the 'messages.retained' %% and 'messages.retained' counter type. {load_module, emqx_metrics, brutal_purge, soft_purge, []},