Merge remote-tracking branch 'ce/main-v4.3' into merge-v43-to-v44

CHANGES-4.3.md

@@ -29,6 +29,9 @@ File format:
 * Make sure ehttpc delete useless pool always succeed.
 * Update mongodb driver to fix potential process leak.
 * Dashboard admin password persists after leaving/joining the cluster
+* Silence grep/sed warnings in docker-entrypoint.sh. [#7520]
+* Generate `loaded_modules` and `loaded_plugins` files with default
+  values when no such files exists. [#7520]
 
 ## v4.3.13
 

apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl

@@ -44,6 +44,7 @@ groups() ->
 
 init_per_suite(Config) ->
     emqx_ct_helpers:start_apps([emqx]),
+    catch emqx_rule_metrics:stop(),
     {ok, _} = emqx_rule_metrics:start_link(),
     Config.
 

deploy/docker/docker-entrypoint.sh

@@ -105,13 +105,13 @@ fill_tuples() {
     local file=$1
     local elements=${*:2}
     for var in $elements; do
-        if grep -qE "\{\s*$var\s*,\s*(true|false)\s*\}\s*\." "$file"; then
+        if grep -qE "\{\s*$var\s*,\s*(true|false)\s*\}\s*\." "$file" 2>/dev/null; then
-            sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file" 
+            sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" 2>/dev/null > tmpfile && cat tmpfile > "$file"
-        elif grep -q "$var\s*\." "$file"; then
+        elif grep -q "$var\s*\." "$file" 2>/dev/null; then
             # backward compatible.
-            sed -r "s/($var)\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file"
+            sed -r "s/($var)\s*\./{\1, true}./1" "$file" > tmpfile 2>/dev/null && cat tmpfile > "$file"
         else
-            sed '$a'\\ "$file" > tmpfile && cat tmpfile > "$file"
+            sed '$a'\\ "$file" 2>/dev/null > tmpfile && cat tmpfile > "$file"
             echo "{$var, true}." >> "$file"
         fi
     done

lib-ce/emqx_modules/src/emqx_modules.app.src

@@ -1,6 +1,6 @@
 {application, emqx_modules,
  [{description, "EMQ X Module Management"},
-  {vsn, "4.4.2"},
+  {vsn, "4.4.3"},
   {modules, []},
   {applications, [kernel,stdlib]},
   {mod, {emqx_modules_app, []}},

lib-ce/emqx_modules/src/emqx_modules.appup.src

@@ -1,21 +1,27 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.4.1",
+  [{"4.4.2", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
-    [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
+   {"4.4.1",
+    [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
    {"4.4.0",
-    [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.4.1",
+  [{"4.4.2", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
-    [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
+   {"4.4.1",
+    [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]},
    {"4.4.0",
-    [{load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_modules,brutal_purge,soft_purge,[]},
+     {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_presence,brutal_purge,soft_purge,[]},
      {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]},

lib-ce/emqx_modules/src/emqx_modules.erl

@@ -43,6 +43,7 @@ load() ->
     case emqx:get_env(modules_loaded_file) of
         undefined -> ok;
         File ->
+            ensure_loaded_modules_file(File),
             load_modules(File)
     end.
 
@@ -58,6 +59,31 @@ load(ModuleName) ->
             emqx_modules:load_module(ModuleName, true)
     end.
 
+%% @doc Creates a `loaded_modules' file with default values if one
+%% doesn't exist.
+-spec ensure_loaded_modules_file(file:filename()) -> ok.
+ensure_loaded_modules_file(Filepath) ->
+    case filelib:is_regular(Filepath) of
+        true ->
+            ok;
+        false ->
+            do_ensure_loaded_modules_file(Filepath)
+    end.
+
+do_ensure_loaded_modules_file(Filepath) ->
+    DefaultModules = [emqx_mod_acl_internal, emqx_mod_presence],
+    Res = file:write_file(Filepath,
+                          [io_lib:format("{~p, true}.~n", [Mod])
+                           || Mod <- DefaultModules]),
+    case Res of
+        ok ->
+            ok;
+        {error, Reason} ->
+            ?LOG(error, "Could not write default loaded_modules file ~p ; Error: ~p",
+                 [Filepath, Reason]),
+            ok
+    end.
+
 %% @doc Unload all the extended modules.
 -spec(unload() -> ok).
 unload() ->
@@ -175,8 +201,10 @@ write_loaded(false) -> ok.
 
 %%--------------------------------------------------------------------
 %% @doc Modules Command
+%%--------------------------------------------------------------------
+
 cli(["list"]) ->
-    lists:foreach(fun({Name, Active}) -> 
+    lists:foreach(fun({Name, Active}) ->
                     emqx_ctl:print("Module(~s, description=~s, active=~s)~n",
                         [Name, Name:description(), Active])
                   end, emqx_modules:list());

lib-ce/emqx_modules/test/emqx_modules_SUITE.erl

@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
 
 -define(CONTENT_TYPE, "application/x-www-form-urlencoded").
 
@@ -44,6 +45,32 @@ end_per_suite(_Config) ->
     emqx_ct_http:delete_default_app(),
     emqx_ct_helpers:stop_apps([emqx_modules, emqx_management]).
 
+init_per_testcase(t_ensure_default_loaded_modules_file, Config) ->
+    LoadedModulesFilepath = application:get_env(emqx, modules_loaded_file),
+    ok = application:stop(emqx_modules),
+    TmpFilepath = filename:join(["/", "tmp", "loaded_modules_tmp"]),
+    case file:delete(TmpFilepath) of
+        ok -> ok;
+        {error, enoent} -> ok
+    end,
+    application:set_env(emqx, modules_loaded_file, TmpFilepath),
+    [ {loaded_modules_filepath, LoadedModulesFilepath}
+    , {tmp_filepath, TmpFilepath}
+    | Config];
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(t_ensure_default_loaded_modules_file, Config) ->
+    LoadedModulesFilepath = ?config(loaded_modules_filepath, Config),
+    TmpFilepath = ?config(tmp_filepath, Config),
+    file:delete(TmpFilepath),
+    ok = application:stop(emqx_modules),
+    application:set_env(emqx, modules_loaded_file, LoadedModulesFilepath),
+    ok = application:start(emqx_modules),
+    ok;
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
 t_load(_) ->
     ?assertEqual(ok, emqx_modules:unload()),
     ?assertEqual(ok, emqx_modules:load()),
@@ -52,6 +79,19 @@ t_load(_) ->
     ?assertEqual(ignore, emqx_modules:reload(emqx_mod_rewrite)),
     ?assertEqual(ok, emqx_modules:reload(emqx_mod_acl_internal)).
 
+t_ensure_default_loaded_modules_file(_Config) ->
+    ok = application:start(emqx_modules),
+    ?assertEqual(
+       [ {emqx_mod_acl_internal,true}
+       , {emqx_mod_delayed,false}
+       , {emqx_mod_presence,true}
+       , {emqx_mod_rewrite,false}
+       , {emqx_mod_subscription,false}
+       , {emqx_mod_topic_metrics,false}
+       ],
+       lists:sort(emqx_modules:list())),
+    ok.
+
 t_list(_) ->
     ?assertMatch([{_, _} | _ ], emqx_modules:list()).
 

src/emqx.appup.src

@@ -2,7 +2,8 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"4.4.2",
-    [{load_module,emqx_frame,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_frame,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_hooks,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
@@ -61,7 +62,8 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.4.2",
-    [{load_module,emqx_frame,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_frame,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_hooks,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},

src/emqx_plugins.erl

@@ -215,7 +215,21 @@ load_plugin_conf(AppName, PluginDir) ->
     end, AppsEnv).
 
 ensure_file(File) ->
-    case filelib:is_file(File) of false -> write_loaded([]); true -> ok end.
+    case filelib:is_file(File) of
+        false ->
+            DefaultPlugins = [ {emqx_management, true}
+                             , {emqx_dashboard, true}
+                             , {emqx_modules, false}
+                             , {emqx_recon, true}
+                             , {emqx_retainer, true}
+                             , {emqx_telemetry, true}
+                             , {emqx_rule_engine, true}
+                             , {emqx_bridge_mqtt, false}
+                             ],
+            write_loaded(DefaultPlugins);
+        true ->
+            ok
+    end.
 
 with_loaded_file(File, SuccFun) ->
     case read_loaded(File) of

test/emqx_plugins_SUITE.erl

@@ -21,11 +21,11 @@
 
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
 
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-
     %% Compile extra plugin code
 
     DataPath = proplists:get_value(data_dir, Config),
@@ -47,7 +47,35 @@ set_special_cfg(PluginsDir) ->
     ok.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([]),
+    file:delete(get(loaded_file)).
+
+init_per_testcase(t_ensure_default_loaded_plugins_file, Config) ->
+    {ok, LoadedPluginsFilepath} = application:get_env(emqx, plugins_loaded_file),
+    TmpFilepath = filename:join(["/", "tmp", "loaded_plugins_tmp"]),
+    case file:delete(TmpFilepath) of
+        ok -> ok;
+        {error, enoent} -> ok
+    end,
+    application:set_env(emqx, plugins_loaded_file, TmpFilepath),
+    [ {loaded_plugins_filepath, LoadedPluginsFilepath}
+    , {tmp_filepath, TmpFilepath}
+    | Config];
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(t_ensure_default_loaded_plugins_file, Config) ->
+    LoadedPluginsFilepath = ?config(loaded_plugins_filepath, Config),
+    TmpFilepath = ?config(tmp_filepath, Config),
+    file:delete(TmpFilepath),
+    emqx_plugins:unload(),
+    application:set_env(emqx, plugins_loaded_file, LoadedPluginsFilepath),
+    %% need to purge the plugin to avoid inter-testcase dependencies.
+    code:purge(emqx_mini_plugin_app),
+    ok;
+end_per_testcase(_TestCase, _Config) ->
+    emqx_plugins:unload(),
+    ok.
 
 t_load(_) ->
     ?assertEqual(ok, emqx_plugins:load()),
@@ -61,6 +89,25 @@ t_load(_) ->
     ?assertEqual(ignore, emqx_plugins:load()),
     ?assertEqual(ignore, emqx_plugins:unload()).
 
+t_ensure_default_loaded_plugins_file(Config) ->
+    %% this will trigger it to write the default plugins to the
+    %% inexistent file; but it won't truly load them in this test
+    %% because there are no config files in `expand_plugins_dir'.
+    TmpFilepath = ?config(tmp_filepath, Config),
+    ok = emqx_plugins:load(),
+    {ok, Contents} = file:consult(TmpFilepath),
+    ?assertEqual(
+       [ {emqx_bridge_mqtt, false}
+       , {emqx_dashboard, true}
+       , {emqx_management, true}
+       , {emqx_modules, false}
+       , {emqx_recon, true}
+       , {emqx_retainer, true}
+       , {emqx_rule_engine, true}
+       , {emqx_telemetry, true}
+       ],
+       lists:sort(Contents)),
+    ok.
 
 t_init_config(_) ->
     ConfFile = "emqx_mini_plugin.config",