refactor: make schema dump and swagger spec work with split desc files

Makefile

@@ -239,7 +239,6 @@ $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt))))
 .PHONY:
 merge-config:
 	@$(SCRIPTS)/merge-config.escript
-	@$(SCRIPTS)/merge-i18n.escript
 
 ## elixir target is to create release packages using Elixir's Mix
 .PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir)

apps/emqx/rebar.config

@@ -29,7 +29,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.1"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},

apps/emqx_conf/src/emqx_conf.erl

@@ -25,9 +25,9 @@
 -export([update/3, update/4]).
 -export([remove/2, remove/3]).
 -export([reset/2, reset/3]).
--export([dump_schema/1, dump_schema/3]).
+-export([dump_schema/2]).
 -export([schema_module/0]).
--export([gen_example_conf/4]).
+-export([gen_example_conf/2]).
 
 %% for rpc
 -export([get_node_and_config/1]).
@@ -136,24 +136,21 @@ reset(Node, KeyPath, Opts) ->
     emqx_conf_proto_v2:reset(Node, KeyPath, Opts).
 
 %% @doc Called from build script.
--spec dump_schema(file:name_all()) -> ok.
+dump_schema(Dir, SchemaModule) ->
-dump_schema(Dir) ->
+    _ = application:load(emqx_dashboard),
-    I18nFile = emqx_dashboard:i18n_file(),
+    ok = emqx_dashboard_desc_cache:init(),
-    dump_schema(Dir, emqx_conf_schema, I18nFile).
-
-dump_schema(Dir, SchemaModule, I18nFile) ->
     lists:foreach(
         fun(Lang) ->
-            gen_config_md(Dir, I18nFile, SchemaModule, Lang),
+            ok = gen_config_md(Dir, SchemaModule, Lang),
-            gen_api_schema_json(Dir, I18nFile, Lang),
+            ok = gen_api_schema_json(Dir, Lang),
-            gen_example_conf(Dir, I18nFile, SchemaModule, Lang),
+            ok = gen_schema_json(Dir, SchemaModule, Lang)
-            gen_schema_json(Dir, I18nFile, SchemaModule, Lang)
         end,
         ["en", "zh"]
-    ).
+    ),
+    ok = gen_example_conf(Dir, SchemaModule).
 
 %% for scripts/spellcheck.
-gen_schema_json(Dir, I18nFile, SchemaModule, Lang) ->
+gen_schema_json(Dir, SchemaModule, Lang) ->
     SchemaJsonFile = filename:join([Dir, "schema-" ++ Lang ++ ".json"]),
     io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]),
     %% EMQX_SCHEMA_FULL_DUMP is quite a hidden API
@@ -164,40 +161,44 @@ gen_schema_json(Dir, I18nFile, SchemaModule, Lang) ->
             false -> ?IMPORTANCE_LOW
         end,
     io:format(user, "===< Including fields from importance level: ~p~n", [IncludeImportance]),
-    Opts = #{desc_file => I18nFile, lang => Lang, include_importance_up_from => IncludeImportance},
+    Opts = #{
+        include_importance_up_from => IncludeImportance,
+        desc_resolver => make_desc_resolver(Lang)
+    },
     JsonMap = hocon_schema_json:gen(SchemaModule, Opts),
     IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]),
     ok = file:write_file(SchemaJsonFile, IoData).
 
-gen_api_schema_json(Dir, I18nFile, Lang) ->
+gen_api_schema_json(Dir, Lang) ->
-    emqx_dashboard:init_i18n(I18nFile, list_to_binary(Lang)),
     gen_api_schema_json_hotconf(Dir, Lang),
-    gen_api_schema_json_bridge(Dir, Lang),
+    gen_api_schema_json_bridge(Dir, Lang).
-    emqx_dashboard:clear_i18n().
 
 gen_api_schema_json_hotconf(Dir, Lang) ->
     SchemaInfo = #{title => <<"EMQX Hot Conf API Schema">>, version => <<"0.1.0">>},
     File = schema_filename(Dir, "hot-config-schema-", Lang),
-    ok = do_gen_api_schema_json(File, emqx_mgmt_api_configs, SchemaInfo).
+    ok = do_gen_api_schema_json(File, emqx_mgmt_api_configs, SchemaInfo, Lang).
 
 gen_api_schema_json_bridge(Dir, Lang) ->
     SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>},
     File = schema_filename(Dir, "bridge-api-", Lang),
-    ok = do_gen_api_schema_json(File, emqx_bridge_api, SchemaInfo).
+    ok = do_gen_api_schema_json(File, emqx_bridge_api, SchemaInfo, Lang).
 
 schema_filename(Dir, Prefix, Lang) ->
     Filename = Prefix ++ Lang ++ ".json",
     filename:join([Dir, Filename]).
 
-gen_config_md(Dir, I18nFile, SchemaModule, Lang) ->
+%% TODO: remove it and also remove hocon_md.erl and friends.
+%% markdown generation from schema is a failure and we are moving to an interactive
+%% viewer like swagger UI.
+gen_config_md(Dir, SchemaModule, Lang) ->
     SchemaMdFile = filename:join([Dir, "config-" ++ Lang ++ ".md"]),
     io:format(user, "===< Generating: ~s~n", [SchemaMdFile]),
-    ok = gen_doc(SchemaMdFile, SchemaModule, I18nFile, Lang).
+    ok = gen_doc(SchemaMdFile, SchemaModule, Lang).
 
-gen_example_conf(Dir, I18nFile, SchemaModule, Lang) ->
+gen_example_conf(Dir, SchemaModule) ->
-    SchemaMdFile = filename:join([Dir, "emqx.conf." ++ Lang ++ ".example"]),
+    SchemaMdFile = filename:join([Dir, "emqx.conf.example"]),
     io:format(user, "===< Generating: ~s~n", [SchemaMdFile]),
-    ok = gen_example(SchemaMdFile, SchemaModule, I18nFile, Lang).
+    ok = gen_example(SchemaMdFile, SchemaModule).
 
 %% @doc return the root schema module.
 -spec schema_module() -> module().
@@ -211,35 +212,48 @@ schema_module() ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
--spec gen_doc(file:name_all(), module(), file:name_all(), string()) -> ok.
+%% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump.
-gen_doc(File, SchemaModule, I18nFile, Lang) ->
+make_desc_resolver(Lang) ->
+    fun
+        ({desc, Namespace, Id}) ->
+            emqx_dashboard_desc_cache:lookup(Lang, Namespace, Id, desc);
+        (Desc) ->
+            unicode:characters_to_binary(Desc)
+    end.
+
+-spec gen_doc(file:name_all(), module(), string()) -> ok.
+gen_doc(File, SchemaModule, Lang) ->
     Version = emqx_release:version(),
     Title =
         "# " ++ emqx_release:description() ++ " Configuration\n\n" ++
             "<!--" ++ Version ++ "-->",
     BodyFile = filename:join([rel, "emqx_conf.template." ++ Lang ++ ".md"]),
     {ok, Body} = file:read_file(BodyFile),
-    Opts = #{title => Title, body => Body, desc_file => I18nFile, lang => Lang},
+    Resolver = make_desc_resolver(Lang),
+    Opts = #{title => Title, body => Body, desc_resolver => Resolver},
     Doc = hocon_schema_md:gen(SchemaModule, Opts),
     file:write_file(File, Doc).
 
-gen_example(File, SchemaModule, I18nFile, Lang) ->
+gen_example(File, SchemaModule) ->
+    %% we do not generate description in example files
+    %% so there is no need for a desc_resolver
     Opts = #{
         title => <<"EMQX Configuration Example">>,
         body => <<"">>,
-        desc_file => I18nFile,
-        lang => Lang,
         include_importance_up_from => ?IMPORTANCE_MEDIUM
     },
     Example = hocon_schema_example:gen(SchemaModule, Opts),
     file:write_file(File, Example).
 
 %% Only gen hot_conf schema, not all configuration fields.
-do_gen_api_schema_json(File, SchemaMod, SchemaInfo) ->
+do_gen_api_schema_json(File, SchemaMod, SchemaInfo, Lang) ->
     io:format(user, "===< Generating: ~s~n", [File]),
     {ApiSpec0, Components0} = emqx_dashboard_swagger:spec(
         SchemaMod,
-        #{schema_converter => fun hocon_schema_to_spec/2}
+        #{
+            schema_converter => fun hocon_schema_to_spec/2,
+            i18n_lang => Lang
+        }
     ),
     ApiSpec = lists:foldl(
         fun({Path, Spec, _, _}, Acc) ->
@@ -278,13 +292,6 @@ do_gen_api_schema_json(File, SchemaMod, SchemaInfo) ->
     ),
     file:write_file(File, IoData).
 
--define(INIT_SCHEMA, #{
-    fields => #{},
-    translations => #{},
-    validations => [],
-    namespace => undefined
-}).
-
 -define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])).
 -define(TO_COMPONENTS_SCHEMA(_M_, _F_),
     iolist_to_binary([

apps/emqx_dashboard/src/emqx_dashboard.erl

@@ -16,22 +16,13 @@
 
 -module(emqx_dashboard).
 
--define(APP, ?MODULE).
-
 -export([
     start_listeners/0,
     start_listeners/1,
     stop_listeners/1,
     stop_listeners/0,
-    list_listeners/0
+    list_listeners/0,
-]).
+    wait_for_listeners/0
-
--export([
-    init_i18n/2,
-    init_i18n/0,
-    get_i18n/0,
-    i18n_file/0,
-    clear_i18n/0
 ]).
 
 %% Authorization
@@ -90,30 +81,34 @@ start_listeners(Listeners) ->
         dispatch => Dispatch,
         middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler]
     },
-    Res =
+    {OkListeners, ErrListeners} =
         lists:foldl(
-            fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, Acc) ->
+            fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) ->
                 Minirest = BaseMinirest#{protocol => Protocol, protocol_options => ProtoOpts},
                 case minirest:start(Name, RanchOptions, Minirest) of
                     {ok, _} ->
                         ?ULOG("Listener ~ts on ~ts started.~n", [
                             Name, emqx_listeners:format_bind(Bind)
                         ]),
-                        Acc;
+                        {[Name | OkAcc], ErrAcc};
                     {error, _Reason} ->
                         %% Don't record the reason because minirest already does(too much logs noise).
-                        [Name | Acc]
+                        {OkAcc, [Name | ErrAcc]}
                 end
             end,
-            [],
+            {[], []},
             listeners(Listeners)
         ),
-    case Res of
+    case ErrListeners of
-        [] -> ok;
+        [] ->
-        _ -> {error, Res}
+            optvar:set(emqx_dashboard_listeners_ready, OkListeners),
+            ok;
+        _ ->
+            {error, ErrListeners}
     end.
 
 stop_listeners(Listeners) ->
+    optvar:unset(emqx_dashboard_listeners_ready),
     [
         begin
             case minirest:stop(Name) of
@@ -129,23 +124,8 @@ stop_listeners(Listeners) ->
     ],
     ok.
 
-get_i18n() ->
+wait_for_listeners() ->
-    application:get_env(emqx_dashboard, i18n).
+    optvar:read(emqx_dashboard_listeners_ready).
-
-init_i18n(File, Lang) when is_atom(Lang) ->
-    init_i18n(File, atom_to_binary(Lang));
-init_i18n(File, Lang) when is_binary(Lang) ->
-    Cache = hocon_schema:new_desc_cache(File),
-    application:set_env(emqx_dashboard, i18n, #{lang => Lang, cache => Cache}).
-
-clear_i18n() ->
-    case application:get_env(emqx_dashboard, i18n) of
-        {ok, #{cache := Cache}} ->
-            hocon_schema:delete_desc_cache(Cache),
-            application:unset_env(emqx_dashboard, i18n);
-        undefined ->
-            ok
-    end.
 
 %%--------------------------------------------------------------------
 %% internal
@@ -187,11 +167,6 @@ ip_port(error, Opts) -> {Opts#{port => 18083}, 18083};
 ip_port({Port, Opts}, _) when is_integer(Port) -> {Opts#{port => Port}, Port};
 ip_port({{IP, Port}, Opts}, _) -> {Opts#{port => Port, ip => IP}, {IP, Port}}.
 
-init_i18n() ->
-    File = i18n_file(),
-    Lang = emqx_conf:get([dashboard, i18n_lang], en),
-    init_i18n(File, Lang).
-
 ranch_opts(Options) ->
     Keys = [
         handshake_timeout,
@@ -255,12 +230,6 @@ return_unauthorized(Code, Message) ->
         },
         #{code => Code, message => Message}}.
 
-i18n_file() ->
-    case application:get_env(emqx_dashboard, i18n_file) of
-        undefined -> filename:join([code:priv_dir(emqx_dashboard), "i18n.conf"]);
-        {ok, File} -> File
-    end.
-
 listeners() ->
     emqx_conf:get([dashboard, listeners], #{}).
 

apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl

@@ -0,0 +1,105 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc This module is used to cache the description of the configuration items.
+-module(emqx_dashboard_desc_cache).
+
+-export([init/0]).
+
+%% internal exports
+-export([load_desc/2, lookup/4, lookup/5]).
+
+-include_lib("emqx/include/logger.hrl").
+
+%% @doc Global ETS table to cache the description of the configuration items.
+%% The table is owned by the emqx_dashboard_sup the root supervisor of emqx_dashboard.
+%% The cache is initialized with the default language (English) and
+%% all the desc.<lang>.hocon files in the www/static directory (extracted from dashboard package).
+init() ->
+    ok = ensure_app_loaded(emqx_dashboard),
+    PrivDir = code:priv_dir(emqx_dashboard),
+    EngDesc = filename:join([PrivDir, "desc.en.hocon"]),
+    WwwStaticDir = filename:join([PrivDir, "www", "static"]),
+    OtherLangDesc0 = filelib:wildcard("desc.*.hocon", WwwStaticDir),
+    OtherLangDesc = lists:map(fun(F) -> filename:join([WwwStaticDir, F]) end, OtherLangDesc0),
+    Files = [EngDesc | OtherLangDesc],
+    ?MODULE = ets:new(?MODULE, [named_table, public, set, {read_concurrency, true}]),
+    ok = lists:foreach(fun(F) -> load_desc(?MODULE, F) end, Files).
+
+%% @doc Load the description of the configuration items from the file.
+%% Load is incremental, so it can be called multiple times.
+%% NOTE: no garbage collection is done, because stale entries are harmless.
+load_desc(EtsTab, File) ->
+    ?SLOG(info, #{msg => "loading desc", file => File}),
+    {ok, Descs} = hocon:load(File),
+    ["desc", Lang, "hocon"] = string:tokens(filename:basename(File), "."),
+    Insert = fun(Namespace, Id, Tag, Text) ->
+        Key = {bin(Lang), bin(Namespace), bin(Id), bin(Tag)},
+        true = ets:insert(EtsTab, {Key, bin(Text)}),
+        ok
+    end,
+    walk_ns(Insert, maps:to_list(Descs)).
+
+%% @doc Lookup the description of the configuration item from the global cache.
+lookup(Lang, Namespace, Id, Tag) ->
+    lookup(?MODULE, Lang, Namespace, Id, Tag).
+
+%% @doc Lookup the description of the configuration item from the given cache.
+lookup(EtsTab, Lang0, Namespace, Id, Tag) ->
+    Lang = bin(Lang0),
+    case ets:lookup(EtsTab, {Lang, bin(Namespace), bin(Id), bin(Tag)}) of
+        [{_, Desc}] ->
+            Desc;
+        [] when Lang =/= <<"en">> ->
+            %% fallback to English
+            lookup(EtsTab, <<"en">>, Namespace, Id, Tag);
+        _ ->
+            %% undefined but not <<>>
+            undefined
+    end.
+
+%% The desc files are of names like:
+%%   desc.en.hocon or desc.zh.hocon
+%% And with content like:
+%%   namespace.id.desc = "description"
+%%   namespace.id.label = "label"
+walk_ns(_Insert, []) ->
+    ok;
+walk_ns(Insert, [{Namespace, Ids} | Rest]) ->
+    walk_id(Insert, Namespace, maps:to_list(Ids)),
+    walk_ns(Insert, Rest).
+
+walk_id(_Insert, _Namespace, []) ->
+    ok;
+walk_id(Insert, Namespace, [{Id, Tags} | Rest]) ->
+    walk_tag(Insert, Namespace, Id, maps:to_list(Tags)),
+    walk_id(Insert, Namespace, Rest).
+
+walk_tag(_Insert, _Namespace, _Id, []) ->
+    ok;
+walk_tag(Insert, Namespace, Id, [{Tag, Text} | Rest]) ->
+    ok = Insert(Namespace, Id, Tag, Text),
+    walk_tag(Insert, Namespace, Id, Rest).
+
+bin(A) when is_atom(A) -> atom_to_binary(A, utf8);
+bin(B) when is_binary(B) -> B;
+bin(L) when is_list(L) -> list_to_binary(L).
+
+ensure_app_loaded(App) ->
+    case application:load(App) of
+        ok -> ok;
+        {error, {already_loaded, _}} -> ok
+    end.

apps/emqx_dashboard/src/emqx_dashboard_listener.erl

@@ -15,9 +15,11 @@
 %%--------------------------------------------------------------------
 -module(emqx_dashboard_listener).
 
--include_lib("emqx/include/logger.hrl").
 -behaviour(emqx_config_handler).
 
+-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
 %% API
 -export([add_handler/0, remove_handler/0]).
 -export([pre_config_update/3, post_config_update/5]).
@@ -54,12 +56,10 @@ init([]) ->
     {ok, undefined, {continue, regenerate_dispatch}}.
 
 handle_continue(regenerate_dispatch, _State) ->
-    NewState = regenerate_minirest_dispatch(),
+    %% initialize the swagger dispatches
-    {noreply, NewState, hibernate}.
+    ready = regenerate_minirest_dispatch(),
+    {noreply, ready, hibernate}.
 
-handle_call(is_ready, _From, retry) ->
-    NewState = regenerate_minirest_dispatch(),
-    {reply, NewState, NewState, hibernate};
 handle_call(is_ready, _From, State) ->
     {reply, State, State, hibernate};
 handle_call(_Request, _From, State) ->
@@ -68,6 +68,9 @@ handle_call(_Request, _From, State) ->
 handle_cast(_Request, State) ->
     {noreply, State, hibernate}.
 
+handle_info(i18n_lang_changed, _State) ->
+    NewState = regenerate_minirest_dispatch(),
+    {noreply, NewState, hibernate};
 handle_info({update_listeners, OldListeners, NewListeners}, _State) ->
     ok = emqx_dashboard:stop_listeners(OldListeners),
     ok = emqx_dashboard:start_listeners(NewListeners),
@@ -83,29 +86,26 @@ terminate(_Reason, _State) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
-%% generate dispatch is very slow.
+%% generate dispatch is very slow, takes about 1s.
 regenerate_minirest_dispatch() ->
-    try
+    %% optvar:read waits for the var to be set
-        emqx_dashboard:init_i18n(),
+    Names = emqx_dashboard:wait_for_listeners(),
-        lists:foreach(
+    {Time, ok} = timer:tc(fun() -> do_regenerate_minirest_dispatch(Names) end),
-            fun(Listener) ->
+    Lang = emqx:get_config([dashboard, i18n_lang]),
-                minirest:update_dispatch(element(1, Listener))
+    ?tp(info, regenerate_minirest_dispatch, #{
-            end,
+        elapsed => erlang:convert_time_unit(Time, microsecond, millisecond),
-            emqx_dashboard:list_listeners()
+        listeners => Names,
-        ),
+        i18n_lang => Lang
-        ready
+    }),
-    catch
+    ready.
-        T:E:S ->
+
-            ?SLOG(error, #{
+do_regenerate_minirest_dispatch(Names) ->
-                msg => "regenerate_minirest_dispatch_failed",
+    lists:foreach(
-                reason => E,
+        fun(Name) ->
-                type => T,
+            ok = minirest:update_dispatch(Name)
-                stacktrace => S
+        end,
-            }),
+        Names
-            retry
+    ).
-    after
-        emqx_dashboard:clear_i18n()
-    end.
 
 add_handler() ->
     Roots = emqx_dashboard_schema:roots(),
@@ -117,6 +117,12 @@ remove_handler() ->
     ok = emqx_config_handler:remove_handler(Roots),
     ok.
 
+pre_config_update(_Path, {change_i18n_lang, NewLang}, RawConf) ->
+    %% e.g. emqx_conf:update([dashboard], {change_i18n_lang, zh}, #{}).
+    %% TODO: check if there is such a language (all languages are cached in emqx_dashboard_desc_cache)
+    Update = #{<<"i18n_lang">> => NewLang},
+    NewConf = emqx_utils_maps:deep_merge(RawConf, Update),
+    {ok, NewConf};
 pre_config_update(_Path, UpdateConf0, RawConf) ->
     UpdateConf = remove_sensitive_data(UpdateConf0),
     NewConf = emqx_utils_maps:deep_merge(RawConf, UpdateConf),
@@ -139,6 +145,8 @@ remove_sensitive_data(Conf0) ->
             Conf1
     end.
 
+post_config_update(_, {change_i18n_lang, _}, _NewConf, _OldConf, _AppEnvs) ->
+    delay_job(i18n_lang_changed);
 post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) ->
     OldHttp = get_listener(http, OldConf),
     OldHttps = get_listener(https, OldConf),
@@ -148,7 +156,12 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) ->
     {StopHttps, StartHttps} = diff_listeners(https, OldHttps, NewHttps),
     Stop = maps:merge(StopHttp, StopHttps),
     Start = maps:merge(StartHttp, StartHttps),
-    _ = erlang:send_after(500, ?MODULE, {update_listeners, Stop, Start}),
+    delay_job({update_listeners, Stop, Start}).
+
+%% in post_config_update, the config is not yet persisted to persistent_term
+%% so we need to delegate the listener update to the gen_server a bit later
+delay_job(Msg) ->
+    _ = erlang:send_after(500, ?MODULE, Msg),
     ok.
 
 get_listener(Type, Conf) ->

apps/emqx_dashboard/src/emqx_dashboard_schema.erl

@@ -233,6 +233,8 @@ cors(required) -> false;
 cors(desc) -> ?DESC(cors);
 cors(_) -> undefined.
 
+%% TODO: change it to string type
+%% It will be up to the dashboard package which languagues to support
 i18n_lang(type) -> ?ENUM([en, zh]);
 i18n_lang(default) -> en;
 i18n_lang('readOnly') -> true;

apps/emqx_dashboard/src/emqx_dashboard_sup.erl

@@ -28,6 +28,8 @@ start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 init([]) ->
+    %% supervisor owns the cache table
+    ok = emqx_dashboard_desc_cache:init(),
     {ok,
         {{one_for_one, 5, 100}, [
             ?CHILD(emqx_dashboard_listener, brutal_kill),

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -84,7 +84,8 @@
 -type spec_opts() :: #{
     check_schema => boolean() | filter(),
     translate_body => boolean(),
-    schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map())
+    schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map()),
+    i18n_lang => atom()
 }.
 
 -type route_path() :: string() | binary().
@@ -333,11 +334,11 @@ check_request_body(#{body := Body}, Spec, _Module, _CheckFun, false) when is_map
 
 %% tags, description, summary, security, deprecated
 meta_to_spec(Meta, Module, Options) ->
-    {Params, Refs1} = parameters(maps:get(parameters, Meta, []), Module),
+    {Params, Refs1} = parameters(maps:get(parameters, Meta, []), Module, Options),
     {RequestBody, Refs2} = request_body(maps:get('requestBody', Meta, []), Module, Options),
     {Responses, Refs3} = responses(maps:get(responses, Meta, #{}), Module, Options),
     {
-        generate_method_desc(to_spec(Meta, Params, RequestBody, Responses)),
+        generate_method_desc(to_spec(Meta, Params, RequestBody, Responses), Options),
         lists:usort(Refs1 ++ Refs2 ++ Refs3)
     }.
 
@@ -348,13 +349,13 @@ to_spec(Meta, Params, RequestBody, Responses) ->
     Spec = to_spec(Meta, Params, [], Responses),
     maps:put('requestBody', RequestBody, Spec).
 
-generate_method_desc(Spec = #{desc := _Desc}) ->
+generate_method_desc(Spec = #{desc := _Desc}, Options) ->
-    Spec1 = trans_description(maps:remove(desc, Spec), Spec),
+    Spec1 = trans_description(maps:remove(desc, Spec), Spec, Options),
     trans_tags(Spec1);
-generate_method_desc(Spec = #{description := _Desc}) ->
+generate_method_desc(Spec = #{description := _Desc}, Options) ->
-    Spec1 = trans_description(Spec, Spec),
+    Spec1 = trans_description(Spec, Spec, Options),
     trans_tags(Spec1);
-generate_method_desc(Spec) ->
+generate_method_desc(Spec, _Options) ->
     trans_tags(Spec).
 
 trans_tags(Spec = #{tags := Tags}) ->
@@ -362,7 +363,7 @@ trans_tags(Spec = #{tags := Tags}) ->
 trans_tags(Spec) ->
     Spec.
 
-parameters(Params, Module) ->
+parameters(Params, Module, Options) ->
     {SpecList, AllRefs} =
         lists:foldl(
             fun(Param, {Acc, RefsAcc}) ->
@@ -388,7 +389,7 @@ parameters(Params, Module) ->
                             Type
                         ),
                         Spec1 = trans_required(Spec0, Required, In),
-                        Spec2 = trans_description(Spec1, Type),
+                        Spec2 = trans_description(Spec1, Type, Options),
                         {[Spec2 | Acc], Refs ++ RefsAcc}
                 end
             end,
@@ -432,38 +433,38 @@ trans_required(Spec, true, _) -> Spec#{required => true};
 trans_required(Spec, _, path) -> Spec#{required => true};
 trans_required(Spec, _, _) -> Spec.
 
-trans_desc(Init, Hocon, Func, Name) ->
+trans_desc(Init, Hocon, Func, Name, Options) ->
-    Spec0 = trans_description(Init, Hocon),
+    Spec0 = trans_description(Init, Hocon, Options),
     case Func =:= fun hocon_schema_to_spec/2 of
         true ->
             Spec0;
         false ->
-            Spec1 = trans_label(Spec0, Hocon, Name),
+            Spec1 = trans_label(Spec0, Hocon, Name, Options),
             case Spec1 of
                 #{description := _} -> Spec1;
                 _ -> Spec1#{description => <<Name/binary, " Description">>}
             end
     end.
 
-trans_description(Spec, Hocon) ->
+trans_description(Spec, Hocon, Options) ->
     Desc =
         case desc_struct(Hocon) of
             undefined -> undefined;
-            ?DESC(_, _) = Struct -> get_i18n(<<"desc">>, Struct, undefined);
+            ?DESC(_, _) = Struct -> get_i18n(<<"desc">>, Struct, undefined, Options);
-            Struct -> to_bin(Struct)
+            Text -> to_bin(Text)
         end,
     case Desc of
         undefined ->
             Spec;
         Desc ->
             Desc1 = binary:replace(Desc, [<<"\n">>], <<"<br/>">>, [global]),
-            maybe_add_summary_from_label(Spec#{description => Desc1}, Hocon)
+            maybe_add_summary_from_label(Spec#{description => Desc1}, Hocon, Options)
     end.
 
-maybe_add_summary_from_label(Spec, Hocon) ->
+maybe_add_summary_from_label(Spec, Hocon, Options) ->
     Label =
         case desc_struct(Hocon) of
-            ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, undefined);
+            ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, undefined, Options);
             _ -> undefined
         end,
     case Label of
@@ -471,29 +472,44 @@ maybe_add_summary_from_label(Spec, Hocon) ->
         _ -> Spec#{summary => Label}
     end.
 
-get_i18n(Key, Struct, Default) ->
+get_i18n(Tag, ?DESC(Namespace, Id), Default, Options) ->
-    {ok, #{cache := Cache, lang := Lang}} = emqx_dashboard:get_i18n(),
+    Lang = get_lang(Options),
-    Desc = hocon_schema:resolve_schema(Struct, Cache),
+    case emqx_dashboard_desc_cache:lookup(Lang, Namespace, Id, Tag) of
-    emqx_utils_maps:deep_get([Key, Lang], Desc, Default).
+        undefined ->
+            Default;
+        Text ->
+            Text
+    end.
 
-trans_label(Spec, Hocon, Default) ->
+%% So far i18n_lang in options is only used at build time.
+%% At runtime, it's still the global config which controls the language.
+get_lang(#{i18n_lang := Lang}) -> Lang;
+get_lang(_) -> emqx:get_config([dashboard, i18n_lang]).
+
+trans_label(Spec, Hocon, Default, Options) ->
     Label =
         case desc_struct(Hocon) of
-            ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, Default);
+            ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, Default, Options);
             _ -> Default
         end,
     Spec#{label => Label}.
 
 desc_struct(Hocon) ->
-    case hocon_schema:field_schema(Hocon, desc) of
+    R =
-        undefined ->
+        case hocon_schema:field_schema(Hocon, desc) of
-            case hocon_schema:field_schema(Hocon, description) of
+            undefined ->
-                undefined -> get_ref_desc(Hocon);
+                case hocon_schema:field_schema(Hocon, description) of
-                Struct1 -> Struct1
+                    undefined -> get_ref_desc(Hocon);
-            end;
+                    Struct1 -> Struct1
-        Struct ->
+                end;
-            Struct
+            Struct ->
-    end.
+                Struct
+        end,
+    ensure_bin(R).
+
+ensure_bin(undefined) -> undefined;
+ensure_bin(?DESC(_Namespace, _Id) = Desc) -> Desc;
+ensure_bin(Text) -> to_bin(Text).
 
 get_ref_desc(?R_REF(Mod, Name)) ->
     case erlang:function_exported(Mod, desc, 1) of
@@ -524,7 +540,7 @@ responses(Responses, Module, Options) ->
     {Spec, Refs}.
 
 response(Status, ?DESC(_Mod, _Id) = Schema, {Acc, RefsAcc, Module, Options}) ->
-    Desc = trans_description(#{}, #{desc => Schema}),
+    Desc = trans_description(#{}, #{desc => Schema}, Options),
     {Acc#{integer_to_binary(Status) => Desc}, RefsAcc, Module, Options};
 response(Status, Bin, {Acc, RefsAcc, Module, Options}) when is_binary(Bin) ->
     {Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module, Options};
@@ -553,7 +569,7 @@ response(Status, Schema, {Acc, RefsAcc, Module, Options}) ->
             Hocon = hocon_schema:field_schema(Schema, type),
             Examples = hocon_schema:field_schema(Schema, examples),
             {Spec, Refs} = hocon_schema_to_spec(Hocon, Module),
-            Init = trans_description(#{}, Schema),
+            Init = trans_description(#{}, Schema, Options),
             Content = content(Spec, Examples),
             {
                 Acc#{integer_to_binary(Status) => Init#{<<"content">> => Content}},
@@ -563,7 +579,7 @@ response(Status, Schema, {Acc, RefsAcc, Module, Options}) ->
             };
         false ->
             {Props, Refs} = parse_object(Schema, Module, Options),
-            Init = trans_description(#{}, Schema),
+            Init = trans_description(#{}, Schema, Options),
             Content = Init#{<<"content">> => content(Props)},
             {Acc#{integer_to_binary(Status) => Content}, Refs ++ RefsAcc, Module, Options}
     end.
@@ -590,7 +606,7 @@ components(Options, [{Module, Field} | Refs], SpecAcc, SubRefsAcc) ->
 %% parameters in ref only have one value, not array
 components(Options, [{Module, Field, parameter} | Refs], SpecAcc, SubRefsAcc) ->
     Props = hocon_schema_fields(Module, Field),
-    {[Param], SubRefs} = parameters(Props, Module),
+    {[Param], SubRefs} = parameters(Props, Module, Options),
     Namespace = namespace(Module),
     NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Param},
     components(Options, Refs, NewSpecAcc, SubRefs ++ SubRefsAcc).
@@ -869,7 +885,7 @@ parse_object_loop([{Name, Hocon} | Rest], Module, Options, Props, Required, Refs
             HoconType = hocon_schema:field_schema(Hocon, type),
             Init0 = init_prop([default | ?DEFAULT_FIELDS], #{}, Hocon),
             SchemaToSpec = schema_converter(Options),
-            Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin),
+            Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin, Options),
             {Prop, Refs1} = SchemaToSpec(HoconType, Module),
             NewRequiredAcc =
                 case is_required(Hocon) of

apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl

@@ -0,0 +1,51 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_dashboard_listener_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_mgmt_api_test_util:init_suite([emqx_conf]),
+    ok = change_i18n_lang(en),
+    Config.
+
+end_per_suite(_Config) ->
+    ok = change_i18n_lang(en),
+    emqx_mgmt_api_test_util:end_suite([emqx_conf]).
+
+t_change_i18n_lang(_Config) ->
+    ?check_trace(
+        begin
+            ok = change_i18n_lang(zh),
+            {ok, _} = ?block_until(#{?snk_kind := regenerate_minirest_dispatch}, 10_000),
+            ok
+        end,
+        fun(ok, Trace) ->
+            ?assertMatch([#{i18n_lang := zh}], ?of_kind(regenerate_minirest_dispatch, Trace))
+        end
+    ),
+    ok.
+
+change_i18n_lang(Lang) ->
+    {ok, _} = emqx_conf:update([dashboard], {change_i18n_lang, Lang}, #{}),
+    ok.

apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl

@@ -64,7 +64,6 @@ groups() ->
 
 init_per_suite(Config) ->
     emqx_mgmt_api_test_util:init_suite([emqx_conf]),
-    emqx_dashboard:init_i18n(),
     Config.
 
 end_per_suite(_Config) ->

apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl

@@ -33,7 +33,6 @@ init_per_suite(Config) ->
     mria:start(),
     application:load(emqx_dashboard),
     emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1),
-    emqx_dashboard:init_i18n(),
     Config.
 
 set_special_configs(emqx_dashboard) ->

apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl

@@ -33,7 +33,6 @@ all() -> emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
     emqx_mgmt_api_test_util:init_suite([emqx_conf]),
-    emqx_dashboard:init_i18n(),
     Config.
 
 end_per_suite(Config) ->

apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl

@@ -686,7 +686,6 @@ t_jq(_) ->
                 %% Got timeout as expected
                 got_timeout
         end,
-    _ConfigRootKey = emqx_rule_engine_schema:namespace(),
     ?assertThrow(
         {jq_exception, {timeout, _}},
         apply_func(jq, [TOProgram, <<"-2">>])

build

@@ -117,8 +117,7 @@ make_docs() {
     mkdir -p "$docdir" "$dashboard_www_static"
     # shellcheck disable=SC2086
     erl -noshell -pa $libs_dir1 $libs_dir2 $libs_dir3 -eval \
-        "I18nFile = filename:join([apps, emqx_dashboard, priv, 'i18n.conf']), \
+        "ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE), \
-         ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE, I18nFile), \
          halt(0)."
     cp "$docdir"/bridge-api-*.json "$dashboard_www_static"
     cp "$docdir"/hot-config-schema-*.json "$dashboard_www_static"

mix.exs

@@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true},
-      {:hocon, github: "emqx/hocon", tag: "0.38.1", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.39.1", override: true},
       {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true},
       {:esasl, github: "emqx/esasl", tag: "0.2.0"},
       {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

rebar.config

@@ -75,7 +75,7 @@
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.1"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
     , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}

rebar.config.erl

@@ -479,8 +479,7 @@ etc_overlay(ReleaseType, Edition) ->
     [
         {mkdir, "etc/"},
         {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"},
-        {copy, "_build/docgen/" ++ name(Edition) ++ "/emqx.conf.en.example",
+        {copy, "_build/docgen/" ++ name(Edition) ++ "/emqx.conf.example", "etc/emqx.conf.example"}
-            "etc/emqx.conf.example"}
     ] ++
         lists:map(
             fun

scripts/merge-config.escript

@@ -34,7 +34,10 @@ main(_) ->
             ok = file:write_file("apps/emqx_conf/etc/emqx-enterprise.conf.all", EnterpriseConf);
         false ->
             ok
-    end.
+    end,
+    merge_desc_files_per_lang("en"),
+    %% TODO: remove this when we have zh translation moved to dashboard package
+    merge_desc_files_per_lang("zh").
 
 is_enterprise() ->
     Profile = os:getenv("PROFILE", "emqx"),
@@ -96,3 +99,48 @@ try_enter_child(Dir, Files, Cfgs) ->
         true ->
             get_all_cfgs(filename:join([Dir, "src"]), Cfgs)
     end.
+
+%% Desc files merge is for now done locally in emqx.git repo for all languages.
+%% When zh and other languages are moved to a separate repo,
+%% we will only merge the en files.
+%% The file for other languages will be merged in the other repo,
+%% the built as a part of the dashboard package,
+%% finally got pulled at build time as a part of the dashboard package.
+merge_desc_files_per_lang(Lang) ->
+    BaseConf = <<"">>,
+    Cfgs0 = get_all_desc_files(Lang),
+    Conf = do_merge_desc_files_per_lang(BaseConf, Cfgs0),
+    OutputFile = case Lang of 
+        "en" ->
+            %% en desc will always be in the priv dir of emqx_dashboard
+             "apps/emqx_dashboard/priv/desc.en.hocon";
+        "zh" ->
+            %% so far we inject zh desc as if it's extracted from dashboard package
+            %% TODO: remove this when we have zh translation moved to dashboard package
+             "apps/emqx_dashboard/priv/www/static/desc.zh.hocon"
+    end,
+    ok = filelib:ensure_dir(OutputFile),
+    ok = file:write_file(OutputFile, Conf).
+
+do_merge_desc_files_per_lang(BaseConf, Cfgs) ->
+    lists:foldl(
+      fun(CfgFile, Acc) ->
+              case filelib:is_regular(CfgFile) of
+                  true ->
+                      {ok, Bin1} = file:read_file(CfgFile),
+                      [Acc, io_lib:nl(), Bin1];
+                  false -> Acc
+              end
+      end, BaseConf, Cfgs).
+
+get_all_desc_files(Lang) ->
+    Dir =
+        case Lang of
+            "en" ->
+                 filename:join(["rel", "i18n"]);
+            "zh" ->
+                %% TODO: remove this when we have zh translation moved to dashboard package
+                 filename:join(["rel", "i18n", "zh"])
+        end,
+    Files = filelib:wildcard("*.hocon", Dir),
+    lists:map(fun(Name) -> filename:join([Dir, Name]) end, Files).

scripts/merge-i18n.escript

@@ -1,35 +0,0 @@
-#!/usr/bin/env escript
-
--mode(compile).
-
-main(_) ->
-    main_per_lang("en"),
-    main_per_lang("zh").
-
-main_per_lang(Lang) ->
-    BaseConf = <<"">>,
-    Cfgs0 = get_all_files(Lang),
-    Conf = merge(BaseConf, Cfgs0),
-    OutputFile = "apps/emqx_dashboard/priv/i18n." ++ Lang ++ ".conf",
-    ok = filelib:ensure_dir(OutputFile),
-    ok = file:write_file(OutputFile, Conf).
-
-merge(BaseConf, Cfgs) ->
-    lists:foldl(
-      fun(CfgFile, Acc) ->
-              case filelib:is_regular(CfgFile) of
-                  true ->
-                      {ok, Bin1} = file:read_file(CfgFile),
-                      [Acc, io_lib:nl(), Bin1];
-                  false -> Acc
-              end
-      end, BaseConf, Cfgs).
-
-get_all_files(Lang) ->
-    Dir =
-        case Lang of
-            "en" -> filename:join(["rel", "i18n"]);
-            "zh" -> filename:join(["rel", "i18n", "zh"])
-        end,
-    Files = filelib:wildcard("*.hocon", Dir),
-    lists:map(fun(Name) -> filename:join([Dir, Name]) end, Files).

scripts/pre-compile.sh

@@ -20,5 +20,4 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
 
 ./scripts/get-dashboard.sh "$dashboard_version"
 ./scripts/merge-config.escript
-./scripts/merge-i18n.escript
 ./scripts/update-bom.sh "$PROFILE_STR" ./rel