diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index e2d3213ee..81f91450e 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -29,6 +29,9 @@ File format: * Make sure ehttpc delete useless pool always succeed. * Update mongodb driver to fix potential process leak. * Dashboard admin password persists after leaving/joining the cluster +* Silence grep/sed warnings in docker-entrypoint.sh. [#7520] +* Generate `loaded_modules` and `loaded_plugins` files with default + values when no such files exists. [#7520] ## v4.3.13 diff --git a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl index 5a8d99d0e..265072b97 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl @@ -44,6 +44,7 @@ groups() -> init_per_suite(Config) -> emqx_ct_helpers:start_apps([emqx]), + catch emqx_rule_metrics:stop(), {ok, _} = emqx_rule_metrics:start_link(), Config. diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index ecbec57df..8d82318d7 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -105,13 +105,13 @@ fill_tuples() { local file=$1 local elements=${*:2} for var in $elements; do - if grep -qE "\{\s*$var\s*,\s*(true|false)\s*\}\s*\." "$file"; then - sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file" - elif grep -q "$var\s*\." "$file"; then + if grep -qE "\{\s*$var\s*,\s*(true|false)\s*\}\s*\." "$file" 2>/dev/null; then + sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" 2>/dev/null > tmpfile && cat tmpfile > "$file" + elif grep -q "$var\s*\." "$file" 2>/dev/null; then # backward compatible. - sed -r "s/($var)\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file" + sed -r "s/($var)\s*\./{\1, true}./1" "$file" > tmpfile 2>/dev/null && cat tmpfile > "$file" else - sed '$a'\\ "$file" > tmpfile && cat tmpfile > "$file" + sed '$a'\\ "$file" 2>/dev/null > tmpfile && cat tmpfile > "$file" echo "{$var, true}." >> "$file" fi done diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index fbcc6fc68..1ff39b0f8 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.4.2"}, + {vsn, "4.4.3"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index 7e79063f8..51382cdfa 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,21 +1,27 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.1", - [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, + [{"4.4.2", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, + [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.1", - [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, + [{"4.4.2", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, + {"4.4.1", + [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.4.0", - [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, + [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.erl b/lib-ce/emqx_modules/src/emqx_modules.erl index 3de9c6ba3..c5565d7d6 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.erl +++ b/lib-ce/emqx_modules/src/emqx_modules.erl @@ -43,6 +43,7 @@ load() -> case emqx:get_env(modules_loaded_file) of undefined -> ok; File -> + ensure_loaded_modules_file(File), load_modules(File) end. @@ -58,6 +59,31 @@ load(ModuleName) -> emqx_modules:load_module(ModuleName, true) end. +%% @doc Creates a `loaded_modules' file with default values if one +%% doesn't exist. +-spec ensure_loaded_modules_file(file:filename()) -> ok. +ensure_loaded_modules_file(Filepath) -> + case filelib:is_regular(Filepath) of + true -> + ok; + false -> + do_ensure_loaded_modules_file(Filepath) + end. + +do_ensure_loaded_modules_file(Filepath) -> + DefaultModules = [emqx_mod_acl_internal, emqx_mod_presence], + Res = file:write_file(Filepath, + [io_lib:format("{~p, true}.~n", [Mod]) + || Mod <- DefaultModules]), + case Res of + ok -> + ok; + {error, Reason} -> + ?LOG(error, "Could not write default loaded_modules file ~p ; Error: ~p", + [Filepath, Reason]), + ok + end. + %% @doc Unload all the extended modules. -spec(unload() -> ok). unload() -> @@ -175,8 +201,10 @@ write_loaded(false) -> ok. %%-------------------------------------------------------------------- %% @doc Modules Command +%%-------------------------------------------------------------------- + cli(["list"]) -> - lists:foreach(fun({Name, Active}) -> + lists:foreach(fun({Name, Active}) -> emqx_ctl:print("Module(~s, description=~s, active=~s)~n", [Name, Name:description(), Active]) end, emqx_modules:list()); diff --git a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl index 175b24bba..a149d9991 100644 --- a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl @@ -20,6 +20,7 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -define(CONTENT_TYPE, "application/x-www-form-urlencoded"). @@ -44,6 +45,32 @@ end_per_suite(_Config) -> emqx_ct_http:delete_default_app(), emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]). +init_per_testcase(t_ensure_default_loaded_modules_file, Config) -> + LoadedModulesFilepath = application:get_env(emqx, modules_loaded_file), + ok = application:stop(emqx_modules), + TmpFilepath = filename:join(["/", "tmp", "loaded_modules_tmp"]), + case file:delete(TmpFilepath) of + ok -> ok; + {error, enoent} -> ok + end, + application:set_env(emqx, modules_loaded_file, TmpFilepath), + [ {loaded_modules_filepath, LoadedModulesFilepath} + , {tmp_filepath, TmpFilepath} + | Config]; +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(t_ensure_default_loaded_modules_file, Config) -> + LoadedModulesFilepath = ?config(loaded_modules_filepath, Config), + TmpFilepath = ?config(tmp_filepath, Config), + file:delete(TmpFilepath), + ok = application:stop(emqx_modules), + application:set_env(emqx, modules_loaded_file, LoadedModulesFilepath), + ok = application:start(emqx_modules), + ok; +end_per_testcase(_TestCase, _Config) -> + ok. + t_load(_) -> ?assertEqual(ok, emqx_modules:unload()), ?assertEqual(ok, emqx_modules:load()), @@ -52,6 +79,19 @@ t_load(_) -> ?assertEqual(ignore, emqx_modules:reload(emqx_mod_rewrite)), ?assertEqual(ok, emqx_modules:reload(emqx_mod_acl_internal)). +t_ensure_default_loaded_modules_file(_Config) -> + ok = application:start(emqx_modules), + ?assertEqual( + [ {emqx_mod_acl_internal,true} + , {emqx_mod_delayed,false} + , {emqx_mod_presence,true} + , {emqx_mod_rewrite,false} + , {emqx_mod_subscription,false} + , {emqx_mod_topic_metrics,false} + ], + lists:sort(emqx_modules:list())), + ok. + t_list(_) -> ?assertMatch([{_, _} | _ ], emqx_modules:list()). diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 88003d6df..6c25d9f5c 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.4.2", - [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -61,7 +62,8 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.2", - [{load_module,emqx_frame,brutal_purge,soft_purge,[]}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 7c4633e38..3aa0287da 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -215,7 +215,21 @@ load_plugin_conf(AppName, PluginDir) -> end, AppsEnv). ensure_file(File) -> - case filelib:is_file(File) of false -> write_loaded([]); true -> ok end. + case filelib:is_file(File) of + false -> + DefaultPlugins = [ {emqx_management, true} + , {emqx_dashboard, true} + , {emqx_modules, false} + , {emqx_recon, true} + , {emqx_retainer, true} + , {emqx_telemetry, true} + , {emqx_rule_engine, true} + , {emqx_bridge_mqtt, false} + ], + write_loaded(DefaultPlugins); + true -> + ok + end. with_loaded_file(File, SuccFun) -> case read_loaded(File) of diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index 6d8847f43..395318d43 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -21,11 +21,11 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - %% Compile extra plugin code DataPath = proplists:get_value(data_dir, Config), @@ -47,7 +47,35 @@ set_special_cfg(PluginsDir) -> ok. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([]). + emqx_ct_helpers:stop_apps([]), + file:delete(get(loaded_file)). + +init_per_testcase(t_ensure_default_loaded_plugins_file, Config) -> + {ok, LoadedPluginsFilepath} = application:get_env(emqx, plugins_loaded_file), + TmpFilepath = filename:join(["/", "tmp", "loaded_plugins_tmp"]), + case file:delete(TmpFilepath) of + ok -> ok; + {error, enoent} -> ok + end, + application:set_env(emqx, plugins_loaded_file, TmpFilepath), + [ {loaded_plugins_filepath, LoadedPluginsFilepath} + , {tmp_filepath, TmpFilepath} + | Config]; +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(t_ensure_default_loaded_plugins_file, Config) -> + LoadedPluginsFilepath = ?config(loaded_plugins_filepath, Config), + TmpFilepath = ?config(tmp_filepath, Config), + file:delete(TmpFilepath), + emqx_plugins:unload(), + application:set_env(emqx, plugins_loaded_file, LoadedPluginsFilepath), + %% need to purge the plugin to avoid inter-testcase dependencies. + code:purge(emqx_mini_plugin_app), + ok; +end_per_testcase(_TestCase, _Config) -> + emqx_plugins:unload(), + ok. t_load(_) -> ?assertEqual(ok, emqx_plugins:load()), @@ -61,6 +89,25 @@ t_load(_) -> ?assertEqual(ignore, emqx_plugins:load()), ?assertEqual(ignore, emqx_plugins:unload()). +t_ensure_default_loaded_plugins_file(Config) -> + %% this will trigger it to write the default plugins to the + %% inexistent file; but it won't truly load them in this test + %% because there are no config files in `expand_plugins_dir'. + TmpFilepath = ?config(tmp_filepath, Config), + ok = emqx_plugins:load(), + {ok, Contents} = file:consult(TmpFilepath), + ?assertEqual( + [ {emqx_bridge_mqtt, false} + , {emqx_dashboard, true} + , {emqx_management, true} + , {emqx_modules, false} + , {emqx_recon, true} + , {emqx_retainer, true} + , {emqx_rule_engine, true} + , {emqx_telemetry, true} + ], + lists:sort(Contents)), + ok. t_init_config(_) -> ConfFile = "emqx_mini_plugin.config",