Merge pull request #4837 from zmstone/external-plugin-lib

External plugin lib

apps/emqx_coap/test/emqx_coap_SUITE.erl

@@ -28,16 +28,16 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_coap], fun set_sepecial_cfg/1),
+    emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1),
     Config.
 
-set_sepecial_cfg(emqx_coap) ->
+set_special_cfg(emqx_coap) ->
     Opts = application:get_env(emqx_coap, dtls_opts,[]),
     Opts2 = [{keyfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/key.pem")},
              {certfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/cert.pem")}],
     application:set_env(emqx_coap, dtls_opts, emqx_misc:merge_opts(Opts, Opts2)),
     application:set_env(emqx_coap, enable_stats, true);
-set_sepecial_cfg(_) ->
+set_special_cfg(_) ->
     ok.
 
 end_per_suite(Config) ->

apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl

@@ -28,12 +28,12 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_coap], fun set_sepecial_cfg/1),
+    emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1),
     Config.
 
-set_sepecial_cfg(emqx_coap) ->
+set_special_cfg(emqx_coap) ->
     application:set_env(emqx_coap, enable_stats, true);
-set_sepecial_cfg(_) ->
+set_special_cfg(_) ->
     ok.
 
 end_per_suite(Config) ->

apps/emqx_exproto/test/emqx_exproto_SUITE.erl

@@ -55,7 +55,7 @@ metrics() ->
 init_per_group(GrpName, Cfg) ->
     put(grpname, GrpName),
     Svrs = emqx_exproto_echo_svr:start(),
-    emqx_ct_helpers:start_apps([emqx_exproto], fun set_sepecial_cfg/1),
+    emqx_ct_helpers:start_apps([emqx_exproto], fun set_special_cfg/1),
     emqx_logger:set_log_level(debug),
     [{servers, Svrs}, {listener_type, GrpName} | Cfg].
 
@@ -63,7 +63,7 @@ end_per_group(_, Cfg) ->
     emqx_ct_helpers:stop_apps([emqx_exproto]),
     emqx_exproto_echo_svr:stop(proplists:get_value(servers, Cfg)).
 
-set_sepecial_cfg(emqx_exproto) ->
+set_special_cfg(emqx_exproto) ->
     LisType = get(grpname),
     Listeners = application:get_env(emqx_exproto, listeners, []),
     SockOpts = socketopts(LisType),
@@ -77,7 +77,7 @@ set_sepecial_cfg(emqx_exproto) ->
     NListeners = [{Proto, LisType, LisOn, UpgradeOpts(Opts)}
                   || {Proto, _Type, LisOn, Opts} <- Listeners],
     application:set_env(emqx_exproto, listeners, NListeners);
-set_sepecial_cfg(emqx) ->
+set_special_cfg(emqx) ->
     application:set_env(emqx, allow_anonymous, true),
     application:set_env(emqx, enable_acl_cache, false),
     ok.

apps/emqx_management/test/emqx_mgmt_api_SUITE.erl

@@ -322,7 +322,7 @@ t_plugins(_) ->
                                  auth_header_()),
     [Plugin3] = filter(get(<<"data">>, Plugins3), <<"name">>, <<"emqx_auth_mnesia">>),
     ?assertEqual(<<"emqx_auth_mnesia">>, maps:get(<<"name">>, Plugin3)),
-    ?assertEqual(false, maps:get(<<"active">>, Plugin3)),
+    ?assertEqual(true, maps:get(<<"active">>, Plugin3)),
 
     {ok, _} = request_api(put,
                           api_path(["nodes",

lib-ce/emqx_modules/test/emqx_modules_SUITE.erl

@@ -32,11 +32,11 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_management, emqx_modules], fun set_sepecial_cfg/1),
+    emqx_ct_helpers:start_apps([emqx_management, emqx_modules], fun set_special_cfg/1),
     emqx_ct_http:create_default_app(),
     Config.
 
-set_sepecial_cfg(_) ->
+set_special_cfg(_) ->
     application:set_env(emqx, modules_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_modules")),
     ok.
 

rebar.config.erl

@@ -194,7 +194,7 @@ overlay_vars_pkg(bin) ->
     , {platform_etc_dir, "etc"}
     , {platform_lib_dir, "lib"}
     , {platform_log_dir, "log"}
-    , {platform_plugins_dir,  "plugins"}
+    , {platform_plugins_dir,  "etc/plugins"}
     , {runner_root_dir, "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)"}
     , {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"}
     , {runner_etc_dir, "$RUNNER_ROOT_DIR/etc"}

src/emqx.appup.src

@@ -7,7 +7,8 @@
      {load_module, emqx_congestion, brutal_purge, soft_purge, []},
      {load_module, emqx_node_dump, brutal_purge, soft_purge, []},
      {load_module, emqx_channel, brutal_purge, soft_purge, []},
-     {load_module, emqx_app, brutal_purge, soft_purge, []}
+     {load_module, emqx_app, brutal_purge, soft_purge, []},
+     {load_module, emqx_plugins, brutal_purge, soft_purge, []}
    ]},
    {"4.3.0", [
      {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []},
@@ -19,6 +20,7 @@
      {load_module, emqx_node_dump, brutal_purge, soft_purge, []},
      {load_module, emqx_channel, brutal_purge, soft_purge, []},
      {load_module, emqx_app, brutal_purge, soft_purge, []},
+     {load_module, emqx_plugins, brutal_purge, soft_purge, []},
      %%
      {load_module, emqx_metrics, brutal_purge, soft_purge, []},
      {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}}
@@ -32,7 +34,8 @@
      {load_module, emqx_congestion, brutal_purge, soft_purge, []},
      {load_module, emqx_node_dump, brutal_purge, soft_purge, []},
      {load_module, emqx_channel, brutal_purge, soft_purge, []},
-     {load_module, emqx_app, brutal_purge, soft_purge, []}
+     {load_module, emqx_app, brutal_purge, soft_purge, []},
+     {load_module, emqx_plugins, brutal_purge, soft_purge, []}
    ]},
    {"4.3.0", [
      {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []},
@@ -44,6 +47,7 @@
      {load_module, emqx_node_dump, brutal_purge, soft_purge, []},
      {load_module, emqx_channel, brutal_purge, soft_purge, []},
      {load_module, emqx_app, brutal_purge, soft_purge, []},
+     {load_module, emqx_plugins, brutal_purge, soft_purge, []},
      %% Just load the module. We don't need to change the 'messages.retained'
      %% and 'messages.retained' counter type.
      {load_module, emqx_metrics, brutal_purge, soft_purge, []},

src/emqx_plugins.erl

@@ -61,7 +61,7 @@ init() ->
 %% @doc Load all plugins when the broker started.
 -spec(load() -> ok | ignore | {error, term()}).
 load() ->
-    load_expand_plugins(),
+    ok = load_ext_plugins(emqx:get_env(expand_plugins_dir)),
     case emqx:get_env(plugins_loaded_file) of
         undefined -> ignore; %% No plugins available
         File ->
@@ -148,46 +148,61 @@ init_config(CfgFile) ->
                       [application:set_env(App, Par, Val) || {Par, Val} <- Envs]
                   end, AppsEnv).
 
-load_expand_plugins() ->
+%% load external plugins which are placed in etc/plugins dir
-    case emqx:get_env(expand_plugins_dir) of
+load_ext_plugins(undefined) -> ok;
-        undefined -> ok;
+load_ext_plugins(Dir) ->
-        ExpandPluginsDir ->
+    lists:foreach(
-            Plugins = filelib:wildcard("*", ExpandPluginsDir),
+        fun(Plugin) ->
-            lists:foreach(fun(Plugin) ->
+                PluginDir = filename:join(Dir, Plugin),
-                PluginDir = filename:join(ExpandPluginsDir, Plugin),
                 case filelib:is_dir(PluginDir) of
-                    true  -> load_expand_plugin(PluginDir);
+                    true  -> load_ext_plugin(PluginDir);
                     false -> ok
                 end
-            end, Plugins)
+        end, filelib:wildcard("*", Dir)).
-    end.
 
-load_expand_plugin(PluginDir) ->
+load_ext_plugin(PluginDir) ->
-    init_expand_plugin_config(PluginDir),
+    ?LOG(debug, "loading_extra_plugin: ~s", [PluginDir]),
     Ebin = filename:join([PluginDir, "ebin"]),
+    AppFile = filename:join([Ebin, "*.app"]),
+    AppName = case filelib:wildcard(AppFile) of
+                  [App] ->
+                      list_to_atom(filename:basename(App, ".app"));
+                  [] ->
+                      ?LOG(alert, "plugin_app_file_not_found: ~s", [AppFile]),
+                      error({plugin_app_file_not_found, AppFile})
+              end,
+    ok = load_plugin_app(AppName, Ebin),
+    ok = load_plugin_conf(AppName, PluginDir).
+
+load_plugin_app(AppName, Ebin) ->
     _ = code:add_patha(Ebin),
     Modules = filelib:wildcard(filename:join([Ebin, "*.beam"])),
-    lists:foreach(fun(Mod) ->
+    lists:foreach(
-        Module = list_to_atom(filename:basename(Mod, ".beam")),
+        fun(BeamFile) ->
-        code:load_file(Module)
+                Module = list_to_atom(filename:basename(BeamFile, ".beam")),
+                case code:ensure_loaded(Module) of
+                    {module, Module} -> ok;
+                    {error, Reason} -> error({failed_to_load_plugin_beam, BeamFile, Reason})
+                end
         end, Modules),
-    case filelib:wildcard(Ebin ++ "/*.app") of
+    case application:load(AppName) of
-        [App|_] -> application:load(list_to_atom(filename:basename(App, ".app")));
+        ok -> ok;
-        _ -> ?LOG(alert, "Plugin not found."),
+        {error, {already_loaded, _}} -> ok
-             {error, load_app_fail}
     end.
 
-init_expand_plugin_config(PluginDir) ->
+load_plugin_conf(AppName, PluginDir) ->
-    Priv = PluginDir ++ "/priv",
+    Priv = filename:join([PluginDir, "priv"]),
-    Etc  = PluginDir ++ "/etc",
+    Etc  = filename:join([PluginDir, "etc"]),
-    Schema = filelib:wildcard(Priv ++ "/*.schema"),
+    Schema = filelib:wildcard(filename:join([Priv, "*.schema"])),
-    Conf = case filelib:wildcard(Etc ++ "/*.conf") of
+    ConfFile = filename:join([Etc, atom_to_list(AppName) ++ ".conf"]),
-        [] -> [];
+    Conf = case filelib:is_file(ConfFile) of
-        [Conf1] -> cuttlefish_conf:file(Conf1)
+               true -> cuttlefish_conf:file(ConfFile);
+               false -> error({conf_file_not_found, ConfFile})
            end,
+    ?LOG(debug, "loading_extra_plugin_config conf=~s, schema=~s", [ConfFile, Schema]),
     AppsEnv = cuttlefish_generator:map(cuttlefish_schema:files(Schema), Conf),
-    lists:foreach(fun({AppName, Envs}) ->
+    lists:foreach(fun({AppName1, Envs}) ->
-        [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs]
+        [application:set_env(AppName1, Par, Val) || {Par, Val} <- Envs]
     end, AppsEnv).
 
 ensure_file(File) ->
@@ -210,10 +225,11 @@ filter_plugins(Names) ->
                     end, Names).
 
 load_plugins(Names, Persistent) ->
-    Plugins = list(), NotFound = Names -- names(Plugins),
+    Plugins = list(),
+    NotFound = Names -- names(Plugins),
     case NotFound of
         []       -> ok;
-        NotFound -> ?LOG(alert, "Cannot find plugins: ~p", [NotFound])
+        NotFound -> ?LOG(alert, "cannot_find_plugins: ~p", [NotFound])
     end,
     NeedToLoad = Names -- NotFound -- names(started_app),
     lists:foreach(fun(Name) ->
@@ -223,19 +239,31 @@ load_plugins(Names, Persistent) ->
 
 generate_configs(App) ->
     ConfigFile = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".config",
-    ConfFile = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf",
+    case filelib:is_file(ConfigFile) of
-    SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema",
+        true ->
-    case {filelib:is_file(ConfigFile), filelib:is_file(ConfFile) andalso filelib:is_file(SchemaFile)} of
-        {true, _} ->
             {ok, [Configs]} = file:consult(ConfigFile),
             Configs;
-        {_, true} ->
+        false ->
+            do_generate_configs(App)
+    end.
+
+do_generate_configs(App) ->
+    Name1 = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf",
+    Name2 = filename:join([code:lib_dir(App), "etc", App]) ++ ".conf",
+    ConfFile = case {filelib:is_file(Name1), filelib:is_file(Name2)} of
+                   {true, _} -> Name1;
+                   {false, true} -> Name2;
+                   {false, false} -> error({config_not_found, [Name1, Name2]})
+               end,
+    SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema",
+    case filelib:is_file(SchemaFile) of
+        true ->
             Schema = cuttlefish_schema:files([SchemaFile]),
             Conf = cuttlefish_conf:file(ConfFile),
             LogFun = fun(Key, Value) -> ?LOG(info, "~s = ~p", [string:join(Key, "."), Value]) end,
             cuttlefish_generator:map(Schema, Conf, undefined, LogFun);
-        {false, false} ->
+        false ->
-            error({config_not_found, {ConfigFile, ConfFile, SchemaFile}})
+            error({schema_not_found, SchemaFile})
     end.
 
 apply_configs([]) ->

test/emqx_plugins_SUITE.erl

@@ -30,24 +30,20 @@ init_per_suite(Config) ->
 
     DataPath = proplists:get_value(data_dir, Config),
     AppPath = filename:join([DataPath, "emqx_mini_plugin"]),
-    Cmd = lists:flatten(io_lib:format("cd ~s && make && cp -r etc _build/default/lib/emqx_mini_plugin/", [AppPath])),
+    Cmd = lists:flatten(io_lib:format("cd ~s && make", [AppPath])),
 
     ct:pal("Executing ~s~n", [Cmd]),
     ct:pal("~n ~s~n", [os:cmd(Cmd)]),
 
-    code:add_path(filename:join([AppPath, "_build", "default", "lib", "emqx_mini_plugin", "ebin"])),
-
     put(loaded_file, filename:join([DataPath, "loaded_plugins"])),
     emqx_ct_helpers:boot_modules([]),
-    emqx_ct_helpers:start_apps([], fun set_sepecial_cfg/1),
+    emqx_ct_helpers:start_apps([], fun(_) -> set_special_cfg(DataPath) end),
 
     Config.
 
-set_sepecial_cfg(_) ->
+set_special_cfg(PluginsDir) ->
-    ExpandPath = filename:dirname(code:lib_dir(emqx_mini_plugin)),
-
     application:set_env(emqx, plugins_loaded_file, get(loaded_file)),
-    application:set_env(emqx, expand_plugins_dir, ExpandPath),
+    application:set_env(emqx, expand_plugins_dir, PluginsDir),
     ok.
 
 end_per_suite(_Config) ->
@@ -58,7 +54,6 @@ t_load(_) ->
     ?assertEqual(ok, emqx_plugins:unload()),
 
     ?assertEqual({error, not_found}, emqx_plugins:load(not_existed_plugin)),
-    ?assertEqual({error, parse_config_file_failed}, emqx_plugins:load(emqx_mini_plugin)),
     ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_mini_plugin)),
 
     application:set_env(emqx, expand_plugins_dir, undefined),
@@ -75,8 +70,9 @@ t_init_config(_) ->
     file:delete(ConfFile),
     ?assertEqual({ok,test}, application:get_env(emqx_mini_plugin, mininame)).
 
-t_load_expand_plugin(_) ->
+t_load_ext_plugin(_) ->
-    ?assertEqual({error, load_app_fail}, emqx_plugins:load_expand_plugin("./not_existed_path/")).
+    ?assertError({plugin_app_file_not_found, _},
+                 emqx_plugins:load_ext_plugin("./not_existed_path/")).
 
 t_list(_) ->
     ?assertMatch([{plugin, _, _, _, _, _, _, _} | _ ], emqx_plugins:list()).

test/emqx_plugins_SUITE_data/emqx_mini_plugin/Makefile

@@ -8,6 +8,7 @@ all: compile
 
 compile:
 	$(REBAR) compile
+	cp -r _build/default/lib/emqx_mini_plugin/ebin ./
 
 clean: distclean
 
@@ -22,14 +23,4 @@ xref:
 
 distclean:
 	@rm -rf _build
-	@rm -f data/app.*.config data/vm.*.args rebar.lock
+	@rm -f ebin/ data/app.*.config data/vm.*.args rebar.lock
-
-CUTTLEFISH_SCRIPT = _build/default/lib/cuttlefish/cuttlefish
-
-$(CUTTLEFISH_SCRIPT):
-	@${REBAR} get-deps
-	@if [ ! -f cuttlefish ]; then make -C _build/default/lib/cuttlefish; fi
-
-app.config: $(CUTTLEFISH_SCRIPT) etc/emqx_mini_plugin.conf
-	$(verbose) $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/emqx_mini_plugin.conf -i priv/emqx_mini_plugin.schema -d data
-

test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config

@@ -1,5 +1,4 @@
-{deps,
+{deps, []}.
-    []}.
 
 {edoc_opts, [{preprocess, true}]}.
 {erl_opts, [warn_unused_vars,
@@ -19,7 +18,6 @@
 {profiles,
     [{test, [
         {deps, [ {emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.1.4"}}}
-               , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}}
                ]}
     ]}
 ]}.