feat: make the dashboard restart quicker

README-CN.md

@@ -1,3 +1,5 @@
+简体中文 | [English](./README.md) | [Русский](./README-RU.md)
+
 # EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)

README-RU.md

@@ -1,3 +1,5 @@
+Русский | [简体中文](./README-CN.md) | [English](./README.md)
+
 # Брокер EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)

README.md

@@ -1,3 +1,5 @@
+English | [简体中文](./README-CN.md) | [Русский](./README-RU.md)
+
 # EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)

apps/emqx/include/logger.hrl

@@ -119,7 +119,13 @@ end).
 -endif.
 
 %% print to 'user' group leader
--define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
+-define(ULOG(Fmt, Args),
--define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)).
+    io:format(user, "~ts " ++ Fmt, [emqx_utils_calendar:now_to_rfc3339(millisecond) | Args])
+).
+-define(ELOG(Fmt, Args),
+    io:format(standard_error, "~ts " ++ Fmt, [
+        emqx_utils_calendar:now_to_rfc3339(millisecond) | Args
+    ])
+).
 
 -endif.

apps/emqx_bridge/src/emqx_bridge_v2_api.erl

@@ -32,6 +32,7 @@
 %% Swagger specs from hocon schema
 -export([
     api_spec/0,
+    check_api_schema/2,
     paths/0,
     schema/1,
     namespace/0
@@ -96,7 +97,7 @@
 namespace() -> "actions_and_sources".
 
 api_spec() ->
-    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun check_api_schema/2}).
+    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun ?MODULE:check_api_schema/2}).
 
 paths() ->
     [

apps/emqx_conf/src/emqx_conf.erl

@@ -144,17 +144,27 @@ reset(Node, KeyPath, Opts) ->
 %% @doc Called from build script.
 %% TODO: move to a external escript after all refactoring is done
 dump_schema(Dir, SchemaModule) ->
-    %% TODO: Load all apps instead of only emqx_dashboard
+    %% Load all apps in ERL_LIBS
     %% as this will help schemas that searches for apps with
     %% relevant schema definitions
-    _ = application:load(emqx_dashboard),
+    lists:foreach(
+        fun(LibPath) ->
+            Lib = list_to_atom(lists:last(filename:split(LibPath))),
+            load(SchemaModule, Lib)
+        end,
+        string:lexemes(os:getenv("ERL_LIBS"), ":;")
+    ),
     ok = emqx_dashboard_desc_cache:init(),
     lists:foreach(
         fun(Lang) ->
             ok = gen_schema_json(Dir, SchemaModule, Lang)
         end,
         ["en", "zh"]
-    ).
+    ),
+    emqx_dashboard:save_dispatch_eterm(SchemaModule).
+
+load(emqx_enterprise_schema, emqx_telemetry) -> ignore;
+load(_, Lib) -> ok = application:load(Lib).
 
 %% for scripts/spellcheck.
 gen_schema_json(Dir, SchemaModule, Lang) ->

apps/emqx_dashboard/src/emqx_dashboard.erl

@@ -28,11 +28,14 @@
 %% Authorization
 -export([authorize/1]).
 
+-export([save_dispatch_eterm/1]).
+
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx/include/http_api.hrl").
 -include_lib("emqx/include/emqx_release.hrl").
 
 -define(EMQX_MIDDLE, emqx_dashboard_middleware).
+-define(DISPATCH_FILE, "dispatch.eterm").
 
 %%--------------------------------------------------------------------
 %% Start/Stop Listeners
@@ -46,6 +49,42 @@ stop_listeners() ->
 
 start_listeners(Listeners) ->
     {ok, _} = application:ensure_all_started(minirest),
+    SwaggerSupport = emqx:get_config([dashboard, swagger_support], true),
+    InitDispatch = dispatch(),
+    {OkListeners, ErrListeners} =
+        lists:foldl(
+            fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) ->
+                init_cache_dispatch(Name, InitDispatch),
+                Options = #{
+                    dispatch => InitDispatch,
+                    swagger_support => SwaggerSupport,
+                    protocol => Protocol,
+                    protocol_options => ProtoOpts
+                },
+                Minirest = minirest_option(Options),
+                case minirest:start(Name, RanchOptions, Minirest) of
+                    {ok, _} ->
+                        ?ULOG("Listener ~ts on ~ts started.~n", [
+                            Name, emqx_listeners:format_bind(Bind)
+                        ]),
+                        {[Name | OkAcc], ErrAcc};
+                    {error, _Reason} ->
+                        %% Don't record the reason because minirest already does(too much logs noise).
+                        {OkAcc, [Name | ErrAcc]}
+                end
+            end,
+            {[], []},
+            listeners(ensure_ssl_cert(Listeners))
+        ),
+    case ErrListeners of
+        [] ->
+            optvar:set(emqx_dashboard_listeners_ready, OkListeners),
+            ok;
+        _ ->
+            {error, ErrListeners}
+    end.
+
+minirest_option(Options) ->
     Authorization = {?MODULE, authorize},
     GlobalSpec = #{
         openapi => "3.0.0",
@@ -68,42 +107,33 @@ start_listeners(Listeners) ->
             }
         }
     },
-    BaseMinirest = #{
+    Base =
-        base_path => emqx_dashboard_swagger:base_path(),
+        #{
-        modules => minirest_api:find_api_modules(apps()),
+            base_path => emqx_dashboard_swagger:base_path(),
-        authorization => Authorization,
+            modules => minirest_api:find_api_modules(apps()),
-        log => audit_log_fun(),
+            authorization => Authorization,
-        security => [#{'basicAuth' => []}, #{'bearerAuth' => []}],
+            log => audit_log_fun(),
-        swagger_global_spec => GlobalSpec,
+            security => [#{'basicAuth' => []}, #{'bearerAuth' => []}],
-        dispatch => dispatch(),
+            swagger_global_spec => GlobalSpec,
-        middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler],
+            dispatch => static_dispatch(),
-        swagger_support => emqx:get_config([dashboard, swagger_support], true)
+            middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler],
-    },
+            swagger_support => true
-    {OkListeners, ErrListeners} =
+        },
-        lists:foldl(
+    maps:merge(Base, Options).
-            fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) ->
+
-                Minirest = BaseMinirest#{protocol => Protocol, protocol_options => ProtoOpts},
+%% save dispatch to priv dir.
-                case minirest:start(Name, RanchOptions, Minirest) of
+save_dispatch_eterm(SchemaMod) ->
-                    {ok, _} ->
+    Dir = code:priv_dir(emqx_dashboard),
-                        ?ULOG("Listener ~ts on ~ts started.~n", [
+    emqx_config:put([dashboard], #{i18n_lang => en, swagger_support => false}),
-                            Name, emqx_listeners:format_bind(Bind)
+    os:putenv("SCHEMA_MOD", atom_to_list(SchemaMod)),
-                        ]),
+    DispatchFile = filename:join([Dir, ?DISPATCH_FILE]),
-                        {[Name | OkAcc], ErrAcc};
+    io:format(user, "===< Generating: ~s~n", [DispatchFile]),
-                    {error, _Reason} ->
+    #{dispatch := Dispatch} = generate_dispatch(),
-                        %% Don't record the reason because minirest already does(too much logs noise).
+    IoData = io_lib:format("~p.~n", [Dispatch]),
-                        {OkAcc, [Name | ErrAcc]}
+    ok = file:write_file(DispatchFile, IoData),
-                end
+    {ok, [SaveDispatch]} = file:consult(DispatchFile),
-            end,
+    SaveDispatch =/= Dispatch andalso erlang:error("bad dashboard dispatch.eterm file generated"),
-            {[], []},
+    ok.
-            listeners(ensure_ssl_cert(Listeners))
-        ),
-    case ErrListeners of
-        [] ->
-            optvar:set(emqx_dashboard_listeners_ready, OkListeners),
-            ok;
-        _ ->
-            {error, ErrListeners}
-    end.
 
 stop_listeners(Listeners) ->
     optvar:unset(emqx_dashboard_listeners_ready),
@@ -127,6 +157,34 @@ wait_for_listeners() ->
 
 %%--------------------------------------------------------------------
 %% internal
+%%--------------------------------------------------------------------
+
+init_cache_dispatch(Name, Dispatch0) ->
+    Dispatch1 = [{_, _, Rules}] = trails:single_host_compile(Dispatch0),
+    FileName = filename:join(code:priv_dir(emqx_dashboard), ?DISPATCH_FILE),
+    Dispatch2 =
+        case file:consult(FileName) of
+            {ok, [[{Host, Path, CacheRules}]]} ->
+                Trails = trails:trails([{cowboy_swagger_handler, #{server => 'http:dashboard'}}]),
+                [{_, _, SwaggerRules}] = trails:single_host_compile(Trails),
+                [{Host, Path, CacheRules ++ SwaggerRules ++ Rules}];
+            {error, _} ->
+                Dispatch1
+        end,
+    persistent_term:put(Name, Dispatch2).
+
+generate_dispatch() ->
+    Options = #{
+        dispatch => [],
+        swagger_support => false,
+        protocol => http,
+        protocol_options => proto_opts(#{})
+    },
+    Minirest = minirest_option(Options),
+    minirest:generate_dispatch(Minirest).
+
+dispatch() ->
+    static_dispatch() ++ dynamic_dispatch().
 
 apps() ->
     [
@@ -287,9 +345,6 @@ ensure_ssl_cert(Listeners = #{https := Https0 = #{ssl_options := SslOpts}}) ->
 ensure_ssl_cert(Listeners) ->
     Listeners.
 
-dispatch() ->
-    static_dispatch() ++ dynamic_dispatch().
-
 static_dispatch() ->
     StaticFiles = ["/editor.worker.js", "/json.worker.js", "/version"],
     [

apps/emqx_dashboard/src/emqx_dashboard_middleware.erl

@@ -21,28 +21,14 @@
 -export([execute/2]).
 
 execute(Req, Env) ->
-    case check_dispatch_ready(Env) of
+    add_cors_flag(Req, Env).
-        true -> add_cors_flag(Req, Env);
-        false -> {stop, cowboy_req:reply(503, #{<<"retry-after">> => <<"15">>}, Req)}
-    end.
 
 add_cors_flag(Req, Env) ->
     CORS = emqx_conf:get([dashboard, cors], false),
-    Origin = cowboy_req:header(<<"origin">>, Req, undefined),
+    case CORS andalso cowboy_req:header(<<"origin">>, Req, undefined) =/= undefined of
-    case CORS andalso Origin =/= undefined of
         false ->
             {ok, Req, Env};
         true ->
             Req2 = cowboy_req:set_resp_header(<<"Access-Control-Allow-Origin">>, <<"*">>, Req),
             {ok, Req2, Env}
     end.
-
-check_dispatch_ready(Env) ->
-    case maps:is_key(options, Env) of
-        false ->
-            true;
-        true ->
-            %% dashboard should always ready, if not, is_ready/1 will block until ready.
-            %% if not ready, dashboard will return 503.
-            emqx_dashboard_listener:is_ready(timer:seconds(20))
-    end.

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -323,14 +323,7 @@ compose_filters(undefined, Filter2) ->
 compose_filters(Filter1, undefined) ->
     Filter1;
 compose_filters(Filter1, Filter2) ->
-    fun(Request, RequestMeta) ->
+    [Filter1, Filter2].
-        case Filter1(Request, RequestMeta) of
-            {ok, Request1} ->
-                Filter2(Request1, RequestMeta);
-            Response ->
-                Response
-        end
-    end.
 
 %%------------------------------------------------------------------------------
 %% Private functions

apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl

@@ -61,6 +61,7 @@ init_per_suite(Config) ->
     Apps = emqx_machine_boot:reboot_apps(),
     ct:pal("load apps:~p~n", [Apps]),
     lists:foreach(fun(App) -> application:load(App) end, Apps),
+    emqx_dashboard:save_dispatch_eterm(emqx_conf:schema_module()),
     SuiteApps = emqx_cth_suite:start(
         [
             emqx_conf,
@@ -87,6 +88,60 @@ t_overview(_) ->
      || Overview <- ?OVERVIEWS
     ].
 
+t_dashboard_restart(Config) ->
+    Name = 'http:dashboard',
+    t_overview(Config),
+    [{'_', [], Rules}] = Dispatch = persistent_term:get(Name),
+    %% complete dispatch has more than 150 rules.
+    ?assertNotMatch([{[], [], cowboy_static, _} | _], Rules),
+    ?assert(erlang:length(Rules) > 150),
+    CheckRules = fun(Tag) ->
+        io:format("zhongwen:~p~n", [Tag]),
+        [{'_', [], NewRules}] = persistent_term:get(Name),
+        ?assertEqual(length(NewRules), length(Rules), Tag),
+        ?assertEqual(lists:sort(NewRules), lists:sort(Rules), Tag)
+    end,
+    ?check_trace(
+        ?wait_async_action(
+            begin
+                ok = application:stop(emqx_dashboard),
+                ?assertEqual(Dispatch, persistent_term:get(Name)),
+                ok = application:start(emqx_dashboard),
+                %% After we restart the dashboard, the dispatch rules should be the same.
+                CheckRules(step_1)
+            end,
+            #{?snk_kind := regenerate_minirest_dispatch},
+            30_000
+        ),
+        fun(Trace) ->
+            ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)),
+            %% The dispatch is updated after being regenerated.
+            CheckRules(step_2)
+        end
+    ),
+    t_overview(Config),
+    ?check_trace(
+        ?wait_async_action(
+            begin
+                ok = application:stop(emqx_dashboard),
+                %% erase to mock the initial dashboard startup.
+                persistent_term:erase(Name),
+                ok = application:start(emqx_dashboard),
+                ct:sleep(800),
+                %% regenerate the dispatch rules again
+                CheckRules(step_3)
+            end,
+            #{?snk_kind := regenerate_minirest_dispatch},
+            30_000
+        ),
+        fun(Trace) ->
+            ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)),
+            CheckRules(step_4)
+        end
+    ),
+    t_overview(Config),
+    ok.
+
 t_admins_add_delete(_) ->
     mnesia:clear_table(?ADMIN),
     Desc = <<"simple description">>,
@@ -196,28 +251,41 @@ t_disable_swagger_json(_Config) ->
         {ok, {{"HTTP/1.1", 200, "OK"}, __, _}},
         httpc:request(get, {Url, []}, [], [{body_format, binary}])
     ),
-
     DashboardCfg = emqx:get_raw_config([dashboard]),
-    DashboardCfg2 = DashboardCfg#{<<"swagger_support">> => false},
-    emqx:update_config([dashboard], DashboardCfg2),
-    ?retry(
-        _Sleep = 1000,
-        _Attempts = 5,
-        ?assertMatch(
-            {ok, {{"HTTP/1.1", 404, "Not Found"}, _, _}},
-            httpc:request(get, {Url, []}, [], [{body_format, binary}])
-        )
-    ),
 
-    DashboardCfg3 = DashboardCfg#{<<"swagger_support">> => true},
+    ?check_trace(
-    emqx:update_config([dashboard], DashboardCfg3),
+        ?wait_async_action(
-    ?retry(
+            begin
-        _Sleep0 = 1000,
+                DashboardCfg2 = DashboardCfg#{<<"swagger_support">> => false},
-        _Attempts0 = 5,
+                emqx:update_config([dashboard], DashboardCfg2)
-        ?assertMatch(
+            end,
-            {ok, {{"HTTP/1.1", 200, "OK"}, __, _}},
+            #{?snk_kind := regenerate_minirest_dispatch},
-            httpc:request(get, {Url, []}, [], [{body_format, binary}])
+            30_000
-        )
+        ),
+        fun(Trace) ->
+            ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)),
+            ?assertMatch(
+                {ok, {{"HTTP/1.1", 404, "Not Found"}, _, _}},
+                httpc:request(get, {Url, []}, [], [{body_format, binary}])
+            )
+        end
+    ),
+    ?check_trace(
+        ?wait_async_action(
+            begin
+                DashboardCfg3 = DashboardCfg#{<<"swagger_support">> => true},
+                emqx:update_config([dashboard], DashboardCfg3)
+            end,
+            #{?snk_kind := regenerate_minirest_dispatch},
+            30_000
+        ),
+        fun(Trace) ->
+            ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)),
+            ?assertMatch(
+                {ok, {{"HTTP/1.1", 200, "OK"}, __, _}},
+                httpc:request(get, {Url, []}, [], [{body_format, binary}])
+            )
+        end
     ),
     ok.
 

apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src

@@ -1,6 +1,6 @@
 {application, emqx_dashboard_sso, [
     {description, "EMQX Dashboard Single Sign-On"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, [emqx_dashboard_sso_sup]},
     {applications, [
         kernel,

apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl

@@ -20,6 +20,7 @@
 
 -export([
     api_spec/0,
+    validate_xml_content_type/2,
     paths/0,
     schema/1,
     namespace/0
@@ -40,6 +41,9 @@ namespace() -> "dashboard_sso".
 api_spec() ->
     emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false, translate_body => false}).
 
+validate_xml_content_type(Params, Meta) ->
+    emqx_dashboard_swagger:validate_content_type(Params, Meta, <<"application/xml">>).
+
 paths() ->
     [
         "/sso/saml/acs",

changes/ce/feat-13242.en.md

@@ -0,0 +1 @@
+Significantly increased the startup speed of the EMQX management dashboard.

mix.exs

@@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do
       {:ekka, github: "emqx/ekka", tag: "0.19.3", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true},
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true},
-      {:minirest, github: "emqx/minirest", tag: "1.4.0", override: true},
+      {:minirest, github: "emqx/minirest", tag: "1.4.3", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true},
       {:replayq, github: "emqx/replayq", tag: "0.3.8", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},

rebar.config

@@ -86,7 +86,7 @@
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.3"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
     {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}},
-    {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.4.0"}}},
+    {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.4.3"}}},
     {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.7"}}},
     {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.8"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},