From 5c66b6f04dce31cce719cc96dffd05398a6cb479 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Fri, 6 May 2022 15:54:54 +0800
Subject: [PATCH 01/11] feat: generate dashboard's dispatch asynchronously

---
 apps/emqx/src/emqx_listeners.erl              |  8 +++-
 apps/emqx/test/emqx_common_test_helpers.erl   | 13 +++++-
 apps/emqx_dashboard/src/emqx_dashboard.erl    |  8 ++--
 .../emqx_dashboard/src/emqx_dashboard_app.erl |  9 ++--
 ...config.erl => emqx_dashboard_listener.erl} | 44 ++++++++++++++++---
 .../emqx_dashboard/src/emqx_dashboard_sup.erl |  6 +--
 6 files changed, 66 insertions(+), 22 deletions(-)
 rename apps/emqx_dashboard/src/{emqx_dashboard_config.erl => emqx_dashboard_listener.erl} (80%)

diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index d4004cbcf..2807608d1 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -209,7 +209,7 @@ start_listener(Type, ListenerName, #{bind := Bind} = Conf) ->
             Msg = lists:flatten(
                 io_lib:format(
                     "~ts(~ts) : ~p",
-                    [ListenerId, BindStr, element(1, Reason)]
+                    [ListenerId, BindStr, filter_stacktrace(Reason)]
                 )
             ),
             {error, {failed_to_start, Msg}}
@@ -514,7 +514,8 @@ foreach_listeners(Do) ->
             {ok, #{type := Type, name := Name}} = parse_listener_id(Id),
             case Do(Type, Name, LConf) of
                 {error, {failed_to_start, _} = Reason} -> error(Reason);
-                _ -> ok
+                {error, {already_started, _}} -> ok;
+                ok -> ok
             end
         end,
         list()
@@ -568,3 +569,6 @@ convert_certs(CertsDir, Conf) ->
 clear_certs(CertsDir, Conf) ->
     OldSSL = maps:get(<<"ssl">>, Conf, undefined),
     emqx_tls_lib:delete_ssl_files(CertsDir, undefined, OldSSL).
+
+filter_stacktrace({Reason, _Stacktrace}) -> Reason;
+filter_stacktrace(Reason) -> Reason.
diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index 2169f42ba..386c0ea33 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -193,8 +193,11 @@ start_app(App, Schema, ConfigFile, SpecAppConfig) ->
     copy_certs(App, RenderedConfigFile),
     SpecAppConfig(App),
     case application:ensure_all_started(App) of
-        {ok, _} -> ok;
-        {error, Reason} -> error({failed_to_start_app, App, Reason})
+        {ok, _} ->
+            ok = ensure_dashboard_listeners_started(App),
+            ok;
+        {error, Reason} ->
+            error({failed_to_start_app, App, Reason})
     end.
 
 render_config_file(ConfigFile, Vars0) ->
@@ -494,3 +497,9 @@ start_ekka() ->
             application:set_env(mria, db_backend, mnesia),
             ekka:start()
     end.
+
+ensure_dashboard_listeners_started(emqx_dashboard) ->
+    ok = gen_server:call(emqx_dashboard_listener, sync),
+    ok;
+ensure_dashboard_listeners_started(_App) ->
+    ok.
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl
index b76858d4b..6c50ca0c7 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard.erl
@@ -22,7 +22,8 @@
     start_listeners/0,
     start_listeners/1,
     stop_listeners/1,
-    stop_listeners/0
+    stop_listeners/0,
+    list_listeners/0
 ]).
 
 -export([
@@ -54,7 +55,6 @@ stop_listeners() ->
 
 start_listeners(Listeners) ->
     {ok, _} = application:ensure_all_started(minirest),
-    init_i18n(),
     Authorization = {?MODULE, authorize},
     GlobalSpec = #{
         openapi => "3.0.0",
@@ -101,7 +101,6 @@ start_listeners(Listeners) ->
             [],
             listeners(Listeners)
         ),
-    clear_i18n(),
     case Res of
         [] -> ok;
         _ -> {error, Res}
@@ -164,6 +163,9 @@ listeners(Listeners) ->
         maps:to_list(Listeners)
     ).
 
+list_listeners() ->
+    listeners(listeners()).
+
 ip_port(Opts) -> ip_port(maps:take(bind, Opts), Opts).
 
 ip_port(error, Opts) -> {Opts#{port => 18083}, 18083};
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_app.erl b/apps/emqx_dashboard/src/emqx_dashboard_app.erl
index 094d1cc67..08bfe1d21 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_app.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_app.erl
@@ -26,19 +26,18 @@
 -include("emqx_dashboard.hrl").
 
 start(_StartType, _StartArgs) ->
-    {ok, Sup} = emqx_dashboard_sup:start_link(),
     ok = mria_rlog:wait_for_shards([?DASHBOARD_SHARD], infinity),
+    {ok, Sup} = emqx_dashboard_sup:start_link(),
     case emqx_dashboard:start_listeners() of
         ok ->
             emqx_dashboard_cli:load(),
-            {ok, _Result} = emqx_dashboard_admin:add_default_user(),
-            ok = emqx_dashboard_config:add_handler(),
+            {ok, _} = emqx_dashboard_admin:add_default_user(),
             {ok, Sup};
         {error, Reason} ->
             {error, Reason}
     end.
 
 stop(_State) ->
-    ok = emqx_dashboard_config:remove_handler(),
+    ok = emqx_dashboard:stop_listeners(),
     emqx_dashboard_cli:unload(),
-    emqx_dashboard:stop_listeners().
+    ok.
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_config.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
similarity index 80%
rename from apps/emqx_dashboard/src/emqx_dashboard_config.erl
rename to apps/emqx_dashboard/src/emqx_dashboard_listener.erl
index e6e374d23..ad86b732e 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_config.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
@@ -13,7 +13,7 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--module(emqx_dashboard_config).
+-module(emqx_dashboard_listener).
 
 -include_lib("emqx/include/logger.hrl").
 -behaviour(emqx_config_handler).
@@ -26,33 +26,63 @@
 
 -export([start_link/0]).
 
--export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
+-export([
+    init/1,
+    handle_continue/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    terminate/2,
+    code_change/3
+]).
 
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
 init([]) ->
-    {ok, #{}, hibernate}.
+    erlang:process_flag(trap_exit, true),
+    ok = add_handler(),
+    {ok, #{}, {continue, regenerate_dispatch}}.
+
+handle_continue(regenerate_dispatch, State) ->
+    regenerate_minirest_dispatch(),
+    {noreply, State, hibernate}.
 
 handle_call(_Request, _From, State) ->
-    {reply, ok, State}.
+    {reply, ok, State, hibernate}.
 
 handle_cast(_Request, State) ->
-    {noreply, State}.
+    {noreply, State, hibernate}.
 
 handle_info({update_listeners, OldListeners, NewListeners}, State) ->
     ok = emqx_dashboard:stop_listeners(OldListeners),
     ok = emqx_dashboard:start_listeners(NewListeners),
-    {noreply, State};
+    regenerate_minirest_dispatch(),
+    {noreply, State, hibernate};
 handle_info(_Info, State) ->
-    {noreply, State}.
+    {noreply, State, hibernate}.
 
 terminate(_Reason, _State) ->
+    ok = remove_handler(),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
+%% generate dispatch is very slow.
+regenerate_minirest_dispatch() ->
+    try
+        emqx_dashboard:init_i18n(),
+        lists:foreach(
+            fun(Listener) ->
+                minirest:regenerate_dispatch(element(1, Listener))
+            end,
+            emqx_dashboard:list_listeners()
+        )
+    catch
+        _:_ -> emqx_dashboard:clear_i18n()
+    end.
+
 add_handler() ->
     Roots = emqx_dashboard_schema:roots(),
     ok = emqx_config_handler:add_handler(Roots, ?MODULE),
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_sup.erl b/apps/emqx_dashboard/src/emqx_dashboard_sup.erl
index 6b9e2adcb..89231fadf 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_sup.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_sup.erl
@@ -29,8 +29,8 @@ start_link() ->
 
 init([]) ->
     {ok,
-        {{one_for_one, 10, 100}, [
+        {{one_for_one, 5, 100}, [
+            ?CHILD(emqx_dashboard_listener),
             ?CHILD(emqx_dashboard_token),
-            ?CHILD(emqx_dashboard_monitor),
-            ?CHILD(emqx_dashboard_config)
+            ?CHILD(emqx_dashboard_monitor)
         ]}}.

From 23a01e393c6a3146a3dd027586fbbc1ea0a6e572 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Fri, 6 May 2022 22:15:55 +0800
Subject: [PATCH 02/11] chore: bump minirest to 1.3.0

---
 apps/emqx_dashboard/src/emqx_dashboard_listener.erl | 2 +-
 mix.exs                                             | 2 +-
 rebar.config                                        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
index ad86b732e..585f81b64 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
@@ -75,7 +75,7 @@ regenerate_minirest_dispatch() ->
         emqx_dashboard:init_i18n(),
         lists:foreach(
             fun(Listener) ->
-                minirest:regenerate_dispatch(element(1, Listener))
+                minirest:update_dispatch(element(1, Listener))
             end,
             emqx_dashboard:list_listeners()
         )
diff --git a/mix.exs b/mix.exs
index 26cf84ca0..30ae36a99 100644
--- a/mix.exs
+++ b/mix.exs
@@ -57,7 +57,7 @@ defmodule EMQXUmbrella.MixProject do
       {:mria, github: "emqx/mria", tag: "0.2.4", override: true},
       {:ekka, github: "emqx/ekka", tag: "0.12.5", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
-      {:minirest, github: "emqx/minirest", tag: "1.2.13", override: true},
+      {:minirest, github: "emqx/minirest", tag: "1.3.0", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.2"},
       {:replayq, "0.3.4", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
diff --git a/rebar.config b/rebar.config
index 91fe427ad..7fd9f2a5e 100644
--- a/rebar.config
+++ b/rebar.config
@@ -56,7 +56,7 @@
     , {mria, {git, "https://github.com/emqx/mria", {tag, "0.2.4"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.5"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
-    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.13"}}}
+    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.0"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
     , {replayq, "0.3.4"}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}

From e6e66b032ebf95b952ccc302b6836123931419e9 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Fri, 6 May 2022 22:30:24 +0800
Subject: [PATCH 03/11] fix: mgmt_cli_SUITE ct failed

---
 .../test/emqx_mgmt_cli_SUITE.erl              | 21 +------------------
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl
index 4445dba94..43a1b55e1 100644
--- a/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl
@@ -35,26 +35,7 @@ end_per_suite(_) ->
     ok.
 
 set_special_configs(emqx_dashboard) ->
-    Config = #{
-        default_username => <<"admin">>,
-        default_password => <<"public">>,
-        listeners =>
-            #{
-                http =>
-                    #{
-                        backlog => 512,
-                        bind => 18083,
-                        enable => true,
-                        inet6 => false,
-                        ipv6_v6only => false,
-                        max_connections => 512,
-                        num_acceptors => 4,
-                        send_timeout => 5000
-                    }
-            }
-    },
-    emqx_config:put([dashboard], Config),
-    ok;
+    emqx_dashboard_api_test_helpers:set_default_config();
 set_special_configs(_App) ->
     ok.
 

From ec297965f6827cc1bc10327894d6aef7e87e78ae Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Fri, 6 May 2022 23:15:13 +0800
Subject: [PATCH 04/11] fix: move clear_i18n to after clause

---
 apps/emqx_dashboard/src/emqx_dashboard_listener.erl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
index 585f81b64..14379a463 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
@@ -80,7 +80,9 @@ regenerate_minirest_dispatch() ->
             emqx_dashboard:list_listeners()
         )
     catch
-        _:_ -> emqx_dashboard:clear_i18n()
+        _:_ -> ok
+    after
+        emqx_dashboard:clear_i18n()
     end.
 
 add_handler() ->

From b91e9e59baf748f6f7b64fe698f976fff35e6eea Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Sat, 7 May 2022 15:40:30 +0800
Subject: [PATCH 05/11] chore: bump minirest to 1.3.1(generate api_spec
 concurrency)

---
 apps/emqx/src/emqx_tls_lib.erl                | 33 ++++++++++++++++---
 .../src/emqx_dashboard_listener.erl           |  1 +
 mix.exs                                       |  2 +-
 rebar.config                                  |  2 +-
 4 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl
index 4250ff430..17aaad74e 100644
--- a/apps/emqx/src/emqx_tls_lib.erl
+++ b/apps/emqx/src/emqx_tls_lib.erl
@@ -27,6 +27,9 @@
     all_ciphers/0
 ]).
 
+-compile(export_all).
+-compile(nowarn_export_all).
+
 %% SSL files
 -export([
     ensure_ssl_files/2,
@@ -99,11 +102,20 @@ all_ciphers(['tlsv1.3']) ->
     ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
 all_ciphers(Versions) ->
     %% assert non-empty
-    [_ | _] = dedup(lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions])).
+    List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]),
+    dedup(List).
 
 %% @doc All Pre-selected TLS ciphers.
+%% ssl:cipher_suites(all, V, openssl) is too slow. so we cache default ciphers.
 default_ciphers() ->
-    selected_ciphers(available_versions()).
+    case persistent_term:get(default_ciphers, undefined) of
+        undefined ->
+            Default = selected_ciphers(available_versions()),
+            persistent_term:put(default_ciphers, Default),
+            Default;
+        Default ->
+            Default
+    end.
 
 %% @doc Pre-selected TLS ciphers for given versions..
 selected_ciphers(Vsns) ->
@@ -205,8 +217,21 @@ default_versions(_) ->
     lists:delete('tlsv1.3', proplists:get_value(available, ssl:versions())).
 
 %% Deduplicate a list without re-ordering the elements.
-dedup([]) -> [];
-dedup([H | T]) -> [H | dedup([I || I <- T, I =/= H])].
+dedup([]) ->
+    [];
+dedup(List) ->
+    lists:reverse(
+        lists:foldl(
+            fun(L, Acc) ->
+                case lists:member(L, Acc) of
+                    false -> [L | Acc];
+                    true -> Acc
+                end
+            end,
+            [],
+            List
+        )
+    ).
 
 %% parse comma separated tls version strings
 parse_versions(Versions) ->
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
index 14379a463..db06d42ab 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
@@ -21,6 +21,7 @@
 %% API
 -export([add_handler/0, remove_handler/0]).
 -export([pre_config_update/3, post_config_update/5]).
+-export([regenerate_minirest_dispatch/0]).
 
 -behaviour(gen_server).
 
diff --git a/mix.exs b/mix.exs
index 30ae36a99..fccc9641c 100644
--- a/mix.exs
+++ b/mix.exs
@@ -57,7 +57,7 @@ defmodule EMQXUmbrella.MixProject do
       {:mria, github: "emqx/mria", tag: "0.2.4", override: true},
       {:ekka, github: "emqx/ekka", tag: "0.12.5", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
-      {:minirest, github: "emqx/minirest", tag: "1.3.0", override: true},
+      {:minirest, github: "emqx/minirest", tag: "1.3.1", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.2"},
       {:replayq, "0.3.4", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
diff --git a/rebar.config b/rebar.config
index 7fd9f2a5e..6f9396b29 100644
--- a/rebar.config
+++ b/rebar.config
@@ -56,7 +56,7 @@
     , {mria, {git, "https://github.com/emqx/mria", {tag, "0.2.4"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.5"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
-    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.0"}}}
+    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.1"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
     , {replayq, "0.3.4"}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}

From 068421d0e14d50ef35c9f7fe3eb92311586a7513 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Sat, 7 May 2022 16:48:03 +0800
Subject: [PATCH 06/11] feat: don't dispatch requests until dispatch is ready

---
 apps/emqx/src/emqx_tls_lib.erl                | 28 ++++++++-----------
 apps/emqx_dashboard/src/emqx_dashboard.erl    |  2 +-
 .../src/emqx_dashboard_listener.erl           | 17 +++++++----
 .../src/emqx_dashboard_middleware.erl         | 20 +++++++++++++
 4 files changed, 44 insertions(+), 23 deletions(-)

diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl
index 17aaad74e..f59a67f4a 100644
--- a/apps/emqx/src/emqx_tls_lib.erl
+++ b/apps/emqx/src/emqx_tls_lib.erl
@@ -27,9 +27,6 @@
     all_ciphers/0
 ]).
 
--compile(export_all).
--compile(nowarn_export_all).
-
 %% SSL files
 -export([
     ensure_ssl_files/2,
@@ -219,19 +216,18 @@ default_versions(_) ->
 %% Deduplicate a list without re-ordering the elements.
 dedup([]) ->
     [];
-dedup(List) ->
-    lists:reverse(
-        lists:foldl(
-            fun(L, Acc) ->
-                case lists:member(L, Acc) of
-                    false -> [L | Acc];
-                    true -> Acc
-                end
-            end,
-            [],
-            List
-        )
-    ).
+dedup(List0) ->
+    List = lists:foldl(
+        fun(L, Acc) ->
+            case lists:member(L, Acc) of
+                false -> [L | Acc];
+                true -> Acc
+            end
+        end,
+        [],
+        List0
+    ),
+    lists:reverse(List).
 
 %% parse comma separated tls version strings
 parse_versions(Versions) ->
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl
index 6c50ca0c7..974d438be 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard.erl
@@ -81,7 +81,7 @@ start_listeners(Listeners) ->
         security => [#{'basicAuth' => []}, #{'bearerAuth' => []}],
         swagger_global_spec => GlobalSpec,
         dispatch => Dispatch,
-        middlewares => [cowboy_router, ?EMQX_MIDDLE, cowboy_handler]
+        middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler]
     },
     Res =
         lists:foldl(
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
index db06d42ab..31f309f9a 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
@@ -25,7 +25,7 @@
 
 -behaviour(gen_server).
 
--export([start_link/0]).
+-export([start_link/0, is_ready/0]).
 
 -export([
     init/1,
@@ -37,29 +37,34 @@
     code_change/3
 ]).
 
+is_ready() ->
+    ready =:= gen_server:call(?MODULE, get_state, 10000).
+
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
 init([]) ->
     erlang:process_flag(trap_exit, true),
     ok = add_handler(),
-    {ok, #{}, {continue, regenerate_dispatch}}.
+    {ok, undefined, {continue, regenerate_dispatch}}.
 
-handle_continue(regenerate_dispatch, State) ->
+handle_continue(regenerate_dispatch, _State) ->
     regenerate_minirest_dispatch(),
-    {noreply, State, hibernate}.
+    {noreply, ready, hibernate}.
 
+handle_call(get_state, _From, State) ->
+    {reply, State, State, hibernate};
 handle_call(_Request, _From, State) ->
     {reply, ok, State, hibernate}.
 
 handle_cast(_Request, State) ->
     {noreply, State, hibernate}.
 
-handle_info({update_listeners, OldListeners, NewListeners}, State) ->
+handle_info({update_listeners, OldListeners, NewListeners}, _State) ->
     ok = emqx_dashboard:stop_listeners(OldListeners),
     ok = emqx_dashboard:start_listeners(NewListeners),
     regenerate_minirest_dispatch(),
-    {noreply, State, hibernate};
+    {noreply, ready, hibernate};
 handle_info(_Info, State) ->
     {noreply, State, hibernate}.
 
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl
index 77b6dbdd0..b07ab21f4 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl
@@ -21,6 +21,7 @@
 -export([execute/2]).
 
 execute(Req, Env) ->
+    waiting_dispatch_ready(),
     CORS = emqx_conf:get([dashboard, cors], false),
     case CORS andalso cowboy_req:header(<<"origin">>, Req, undefined) of
         false ->
@@ -31,3 +32,22 @@ execute(Req, Env) ->
             Req2 = cowboy_req:set_resp_header(<<"Access-Control-Allow-Origin">>, <<"*">>, Req),
             {ok, Req2, Env}
     end.
+
+waiting_dispatch_ready() ->
+    waiting_dispatch_ready(5).
+
+waiting_dispatch_ready(0) ->
+    ok;
+waiting_dispatch_ready(Count) ->
+    case emqx_sys:uptime() < timer:minutes(1) of
+        true ->
+            case emqx_dashboard_listener:is_ready() of
+                true ->
+                    ok;
+                false ->
+                    timer:sleep(100),
+                    waiting_dispatch_ready(Count - 1)
+            end;
+        false ->
+            ok
+    end.

From 34ad395abec7382a9b80122885ca4210c684bfdd Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Mon, 9 May 2022 11:30:48 +0800
Subject: [PATCH 07/11] fix: don't router request until dispatch is ready

---
 .../src/emqx_dashboard_listener.erl           |  6 ++--
 .../src/emqx_dashboard_middleware.erl         | 36 ++++++++-----------
 2 files changed, 18 insertions(+), 24 deletions(-)

diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
index 31f309f9a..b649575ed 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
@@ -25,7 +25,7 @@
 
 -behaviour(gen_server).
 
--export([start_link/0, is_ready/0]).
+-export([start_link/0, is_ready/1]).
 
 -export([
     init/1,
@@ -37,8 +37,8 @@
     code_change/3
 ]).
 
-is_ready() ->
-    ready =:= gen_server:call(?MODULE, get_state, 10000).
+is_ready(Timeout) ->
+    ready =:= gen_server:call(?MODULE, get_state, Timeout).
 
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl
index b07ab21f4..a2cf9db6f 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_middleware.erl
@@ -21,33 +21,27 @@
 -export([execute/2]).
 
 execute(Req, Env) ->
-    waiting_dispatch_ready(),
+    case check_dispatch_ready(Env) of
+        true -> add_cors_flag(Req, Env);
+        false -> {stop, cowboy_req:reply(503, Req)}
+    end.
+
+add_cors_flag(Req, Env) ->
     CORS = emqx_conf:get([dashboard, cors], false),
-    case CORS andalso cowboy_req:header(<<"origin">>, Req, undefined) of
+    Origin = cowboy_req:header(<<"origin">>, Req, undefined),
+    case CORS andalso Origin =/= undefined of
         false ->
             {ok, Req, Env};
-        undefined ->
-            {ok, Req, Env};
-        _ ->
+        true ->
             Req2 = cowboy_req:set_resp_header(<<"Access-Control-Allow-Origin">>, <<"*">>, Req),
             {ok, Req2, Env}
     end.
 
-waiting_dispatch_ready() ->
-    waiting_dispatch_ready(5).
-
-waiting_dispatch_ready(0) ->
-    ok;
-waiting_dispatch_ready(Count) ->
-    case emqx_sys:uptime() < timer:minutes(1) of
-        true ->
-            case emqx_dashboard_listener:is_ready() of
-                true ->
-                    ok;
-                false ->
-                    timer:sleep(100),
-                    waiting_dispatch_ready(Count - 1)
-            end;
+check_dispatch_ready(Env) ->
+    case maps:is_key(options, Env) of
         false ->
-            ok
+            true;
+        true ->
+            %% dashboard should always ready, if not, is_ready/1 will block until ready.
+            emqx_dashboard_listener:is_ready(timer:seconds(15))
     end.

From 7c34c8a8b1fb51f32b90df146e2423585e4a5791 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Mon, 9 May 2022 23:03:52 +0800
Subject: [PATCH 08/11] fix: retry if init_disaptch failed

---
 apps/emqx_conf/src/emqx_conf_schema.erl       |  1 +
 .../src/emqx_dashboard_listener.erl           | 27 +++++++++++++------
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl
index 7ab815d9a..aedfe6a25 100644
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@@ -443,6 +443,7 @@ fields("node") ->
                 #{
                     mapping => "vm_args.-env ERL_CRASH_DUMP",
                     desc => ?DESC(node_crash_dump_file),
+                    default => "log/erl_crash.dump",
                     'readOnly' => true
                 }
             )},
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
index b649575ed..8bd5a4eb3 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl
@@ -38,7 +38,7 @@
 ]).
 
 is_ready(Timeout) ->
-    ready =:= gen_server:call(?MODULE, get_state, Timeout).
+    ready =:= gen_server:call(?MODULE, is_ready, Timeout).
 
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
@@ -49,10 +49,13 @@ init([]) ->
     {ok, undefined, {continue, regenerate_dispatch}}.
 
 handle_continue(regenerate_dispatch, _State) ->
-    regenerate_minirest_dispatch(),
-    {noreply, ready, hibernate}.
+    NewState = regenerate_minirest_dispatch(),
+    {noreply, NewState, hibernate}.
 
-handle_call(get_state, _From, State) ->
+handle_call(is_ready, _From, retry) ->
+    NewState = regenerate_minirest_dispatch(),
+    {reply, NewState, NewState, hibernate};
+handle_call(is_ready, _From, State) ->
     {reply, State, State, hibernate};
 handle_call(_Request, _From, State) ->
     {reply, ok, State, hibernate}.
@@ -63,8 +66,8 @@ handle_cast(_Request, State) ->
 handle_info({update_listeners, OldListeners, NewListeners}, _State) ->
     ok = emqx_dashboard:stop_listeners(OldListeners),
     ok = emqx_dashboard:start_listeners(NewListeners),
-    regenerate_minirest_dispatch(),
-    {noreply, ready, hibernate};
+    NewState = regenerate_minirest_dispatch(),
+    {noreply, NewState, hibernate};
 handle_info(_Info, State) ->
     {noreply, State, hibernate}.
 
@@ -84,9 +87,17 @@ regenerate_minirest_dispatch() ->
                 minirest:update_dispatch(element(1, Listener))
             end,
             emqx_dashboard:list_listeners()
-        )
+        ),
+        ready
     catch
-        _:_ -> ok
+        T:E:S ->
+            ?SLOG(error, #{
+                msg => "regenerate_minirest_dispatch_failed",
+                reason => E,
+                type => T,
+                stacktrace => S
+            }),
+            retry
     after
         emqx_dashboard:clear_i18n()
     end.

From 9c42c9216d49346bfbf509b1e71e8ad6226246f4 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Mon, 9 May 2022 23:52:39 +0800
Subject: [PATCH 09/11] fix: bump minirest to 1.3.2(pmap api_spec will crash if
 timeout)

---
 mix.exs      | 2 +-
 rebar.config | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index fccc9641c..e5a8edebb 100644
--- a/mix.exs
+++ b/mix.exs
@@ -57,7 +57,7 @@ defmodule EMQXUmbrella.MixProject do
       {:mria, github: "emqx/mria", tag: "0.2.4", override: true},
       {:ekka, github: "emqx/ekka", tag: "0.12.5", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
-      {:minirest, github: "emqx/minirest", tag: "1.3.1", override: true},
+      {:minirest, github: "emqx/minirest", tag: "1.3.2", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.2"},
       {:replayq, "0.3.4", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
diff --git a/rebar.config b/rebar.config
index 6f9396b29..529952878 100644
--- a/rebar.config
+++ b/rebar.config
@@ -56,7 +56,7 @@
     , {mria, {git, "https://github.com/emqx/mria", {tag, "0.2.4"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.5"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
-    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.1"}}}
+    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.2"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
     , {replayq, "0.3.4"}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}

From 83b4b658e2f6f97c6edef1a1ccc22086d2781b4b Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Tue, 10 May 2022 09:39:53 +0800
Subject: [PATCH 10/11] fix: assert not empty list

---
 apps/emqx/src/emqx_tls_lib.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl
index f59a67f4a..b08270df9 100644
--- a/apps/emqx/src/emqx_tls_lib.erl
+++ b/apps/emqx/src/emqx_tls_lib.erl
@@ -100,7 +100,7 @@ all_ciphers(['tlsv1.3']) ->
 all_ciphers(Versions) ->
     %% assert non-empty
     List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]),
-    dedup(List).
+    [_ | _] = dedup(List).
 
 %% @doc All Pre-selected TLS ciphers.
 %% ssl:cipher_suites(all, V, openssl) is too slow. so we cache default ciphers.

From 703a8d3effa48626d7057cd773abd81268d1848d Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Tue, 10 May 2022 15:47:16 +0800
Subject: [PATCH 11/11] fix: clear cert crash when config is undefined

---
 apps/emqx_connector/src/emqx_connector_ssl.erl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl
index e20596b6c..c08defc29 100644
--- a/apps/emqx_connector/src/emqx_connector_ssl.erl
+++ b/apps/emqx_connector/src/emqx_connector_ssl.erl
@@ -30,6 +30,8 @@ convert_certs(RltvDir, NewConfig) ->
             {error, {bad_ssl_config, Reason}}
     end.
 
+clear_certs(_RltvDir, undefined) ->
+    ok;
 clear_certs(RltvDir, Config) ->
     OldSSL = map_get_oneof([<<"ssl">>, ssl], Config, undefined),
     ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL).