diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl
index 140b008d1..0a8339ddd 100644
--- a/apps/emqx_conf/src/emqx_conf.erl
+++ b/apps/emqx_conf/src/emqx_conf.erl
@@ -28,7 +28,7 @@
-export([remove/2, remove/3]).
-export([tombstone/2]).
-export([reset/2, reset/3]).
--export([dump_schema/2, reformat_schema_dump/1]).
+-export([dump_schema/2, reformat_schema_dump/2]).
-export([schema_module/0]).
%% TODO: move to emqx_dashboard when we stop building api schema at build time
@@ -186,7 +186,7 @@ gen_schema_json(Dir, SchemaModule, Lang) ->
ok = gen_preformat_md_json_files(Dir, StructsJsonArray, Lang).
gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) ->
- NestedStruct = reformat_schema_dump(StructsJsonArray),
+ NestedStruct = reformat_schema_dump(StructsJsonArray, Lang),
%% write to files
NestedJsonFile = filename:join([Dir, "schema-v2-" ++ Lang ++ ".json"]),
io:format(user, "===< Generating: ~s~n", [NestedJsonFile]),
@@ -196,15 +196,17 @@ gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) ->
ok.
%% @doc This function is exported for scripts/schema-dump-reformat.escript
-reformat_schema_dump(StructsJsonArray0) ->
+reformat_schema_dump(StructsJsonArray0, Lang) ->
%% prepare
+ DescResolver = make_desc_resolver(Lang),
StructsJsonArray = deduplicate_by_full_name(StructsJsonArray0),
#{fields := RootFields} = hd(StructsJsonArray),
RootNames0 = lists:map(fun(#{name := RootName}) -> RootName end, RootFields),
RootNames = lists:map(fun to_bin/1, RootNames0),
%% reformat
[Root | FlatStructs0] = lists:map(
- fun(Struct) -> gen_flat_doc(RootNames, Struct) end, StructsJsonArray
+ fun(Struct) -> gen_flat_doc(RootNames, Struct, DescResolver) end,
+ StructsJsonArray
),
FlatStructs = [Root#{text => <<"root">>, hash => <<"root">>} | FlatStructs0],
gen_nested_doc(FlatStructs).
@@ -302,7 +304,7 @@ expand_ref(#{hash := FullName}, FindFn, Path) ->
%% generate flat docs for each struct.
%% using references to link to other structs.
-gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) ->
+gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S, DescResolver) ->
ShortName = short_name(FullName),
case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of
true ->
@@ -314,18 +316,19 @@ gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) ->
text => short_name(FullName),
hash => format_hash(FullName),
doc => maps:get(desc, S, <<"">>),
- fields => format_fields(Fields)
+ fields => format_fields(Fields, DescResolver)
}.
-format_fields([]) ->
- [];
-format_fields([Field | Fields]) ->
- [format_field(Field) | format_fields(Fields)].
+format_fields(Fields, DescResolver) ->
+ [format_field(F, DescResolver) || F <- Fields].
-format_field(#{name := Name, aliases := Aliases, type := Type} = F) ->
+format_field(#{name := Name, aliases := Aliases, type := Type} = F, DescResolver) ->
+ TypeDoc = format_type_desc(Type, DescResolver),
L = [
{text, Name},
{type, format_type(Type)},
+ %% TODO: Make it into a separate field.
+ %% {typedoc, format_type_desc(Type, DescResolver)},
{refs, format_refs(Type)},
{aliases,
case Aliases of
@@ -333,7 +336,7 @@ format_field(#{name := Name, aliases := Aliases, type := Type} = F) ->
_ -> Aliases
end},
{default, maps:get(hocon, maps:get(default, F, #{}), undefined)},
- {doc, maps:get(desc, F, undefined)}
+ {doc, join_format([maps:get(desc, F, undefined), TypeDoc])}
],
maps:from_list([{K, V} || {K, V} <- L, V =/= undefined]).
@@ -393,10 +396,26 @@ format_union_members([Member | Members], Acc) ->
NewAcc = [format_type(Member) | Acc],
format_union_members(Members, NewAcc).
+format_type_desc(#{kind := primitive, name := Name}, DescResolver) ->
+ format_primitive_type_desc(Name, DescResolver);
+format_type_desc(#{}, _DescResolver) ->
+ undefined.
+
format_primitive_type(TypeStr) ->
- Spec = emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr),
+ Spec = get_primitive_typespec(TypeStr),
to_bin(maps:get(type, Spec)).
+format_primitive_type_desc(TypeStr, DescResolver) ->
+ case get_primitive_typespec(TypeStr) of
+ #{desc := Desc} ->
+ DescResolver(Desc);
+ #{} ->
+ undefined
+ end.
+
+get_primitive_typespec(TypeStr) ->
+ emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr).
+
%% All types should have a namespace to avlid name clashing.
is_missing_namespace(ShortName, FullName, RootNames) ->
case lists:member(ShortName, RootNames) of
@@ -560,6 +579,14 @@ hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->
typename_to_spec(TypeStr, Module) ->
emqx_conf_schema_types:readable_dashboard(Module, TypeStr).
+join_format(Snippets) ->
+ case [S || S <- Snippets, S =/= undefined] of
+ [] ->
+ undefined;
+ NonEmpty ->
+ to_bin(lists:join("
", NonEmpty))
+ end.
+
to_bin(List) when is_list(List) -> iolist_to_binary(List);
to_bin(Boolean) when is_boolean(Boolean) -> Boolean;
to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
diff --git a/apps/emqx_conf/src/emqx_conf_schema_types.erl b/apps/emqx_conf/src/emqx_conf_schema_types.erl
index 239624a81..9d00b9650 100644
--- a/apps/emqx_conf/src/emqx_conf_schema_types.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema_types.erl
@@ -16,6 +16,8 @@
-module(emqx_conf_schema_types).
+-include_lib("hocon/include/hocon_types.hrl").
+
-export([readable/2]).
-export([readable_swagger/2, readable_dashboard/2, readable_docgen/2]).
@@ -165,37 +167,37 @@ readable("duration()") ->
#{
swagger => #{type => string, example => <<"12m">>},
dashboard => #{type => duration},
- docgen => #{type => "String", example => <<"12m">>}
+ docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)}
};
readable("duration_s()") ->
#{
swagger => #{type => string, example => <<"1h">>},
dashboard => #{type => duration},
- docgen => #{type => "String", example => <<"1h">>}
+ docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)}
};
readable("duration_ms()") ->
#{
swagger => #{type => string, example => <<"32s">>},
dashboard => #{type => duration},
- docgen => #{type => "String", example => <<"32s">>}
+ docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)}
};
readable("timeout_duration()") ->
#{
swagger => #{type => string, example => <<"12m">>},
dashboard => #{type => duration},
- docgen => #{type => "String", example => <<"12m">>}
+ docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)}
};
readable("timeout_duration_s()") ->
#{
swagger => #{type => string, example => <<"1h">>},
dashboard => #{type => duration},
- docgen => #{type => "String", example => <<"1h">>}
+ docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)}
};
readable("timeout_duration_ms()") ->
#{
swagger => #{type => string, example => <<"32s">>},
dashboard => #{type => duration},
- docgen => #{type => "String", example => <<"32s">>}
+ docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)}
};
readable("percent()") ->
#{
@@ -219,13 +221,13 @@ readable("bytesize()") ->
#{
swagger => #{type => string, example => <<"32MB">>},
dashboard => #{type => 'byteSize'},
- docgen => #{type => "String", example => <<"32MB">>}
+ docgen => #{type => "Bytesize", example => <<"32MB">>, desc => ?DESC(bytesize)}
};
readable("wordsize()") ->
#{
swagger => #{type => string, example => <<"1024KB">>},
dashboard => #{type => 'wordSize'},
- docgen => #{type => "String", example => <<"1024KB">>}
+ docgen => #{type => "Bytesize", example => <<"1024KB">>, desc => ?DESC(bytesize)}
};
readable("map(" ++ Map) ->
[$) | _MapArgs] = lists:reverse(Map),
@@ -287,7 +289,11 @@ readable("secret()") ->
#{
swagger => #{type => string, example => <<"R4ND0M/S∃CЯ∃T"/utf8>>},
dashboard => #{type => string},
- docgen => #{type => "String", example => <<"R4ND0M/S∃CЯ∃T"/utf8>>}
+ docgen => #{
+ type => "Secret",
+ example => <<"R4ND0M/S∃CЯ∃T"/utf8>>,
+ desc => ?DESC(secret)
+ }
};
readable(TypeStr0) ->
case string:split(TypeStr0, ":") of
diff --git a/rel/i18n/emqx_conf_schema_types.hocon b/rel/i18n/emqx_conf_schema_types.hocon
new file mode 100644
index 000000000..6b9dac9ea
--- /dev/null
+++ b/rel/i18n/emqx_conf_schema_types.hocon
@@ -0,0 +1,12 @@
+emqx_conf_schema_types {
+
+ duration.desc:
+ """A string that represents a time duration, for example: 10s
, 2.5m
, 1h30m
, 1W2D
, or 2345ms
, which is the smallest unit. When precision is specified, finer portions of the duration may be ignored: writing 1200ms
for Duration(s)
is equivalent to writing 1s
. The unit part is case-insensitive."""
+
+ bytesize.desc:
+ """A string that represents a number of bytes, for example: 10B
, 640kb
, 4MB
, 1GB
. Units are interpreted as powers of 1024, and the unit part is case-insensitive."""
+
+ secret.desc:
+ """A string holding some sensitive information, such as a password. When secret starts with file://
, the rest of the string is interpreted as a path to a file containing the secret itself: whole content of the file except any trailing whitespace characters is considered a secret value. Note: when clustered, all EMQX nodes should have the same file present before using file://
secrets."""
+
+}
diff --git a/scripts/schema-dump-reformat.escript b/scripts/schema-dump-reformat.escript
deleted file mode 100755
index 31cfdd7d9..000000000
--- a/scripts/schema-dump-reformat.escript
+++ /dev/null
@@ -1,132 +0,0 @@
-#!/usr/bin/env escript
-
-%% This script translates the hocon_schema_json's schema dump to a new format.
-%% It is used to convert older version EMQX's schema dumps to the new format
-%% after all files are upgraded to the new format, this script can be removed.
-
--mode(compile).
-
-main([Input]) ->
- ok = add_libs(),
- _ = atoms(),
- {ok, Data} = file:read_file(Input),
- Json = jsx:decode(Data),
- NewJson = reformat(Json),
- io:format("~s~n", [jsx:encode(NewJson)]);
-main(_) ->
- io:format("Usage: schema-dump-reformat.escript ~n"),
- halt(1).
-
-reformat(Json) ->
- emqx_conf:reformat_schema_dump(fix(Json)).
-
-%% fix old type specs to make them compatible with new type specs
-fix(#{
- <<"kind">> := <<"union">>,
- <<"members">> := [#{<<"name">> := <<"string()">>}, #{<<"name">> := <<"function()">>}]
-}) ->
- %% s3_exporter.secret_access_key
- #{
- kind => primitive,
- name => <<"string()">>
- };
-fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_conf_schema:log_level()">>}) ->
- #{
- kind => enum,
- symbols => [emergency, alert, critical, error, warning, notice, info, debug, none, all]
- };
-fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_connector_http:pool_type()">>}) ->
- #{kind => enum, symbols => [random, hash]};
-fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_bridge_http_connector:pool_type()">>}) ->
- #{kind => enum, symbols => [random, hash]};
-fix(Map) when is_map(Map) ->
- maps:from_list(fix(maps:to_list(Map)));
-fix(List) when is_list(List) ->
- lists:map(fun fix/1, List);
-fix({<<"kind">>, Kind}) ->
- {kind, binary_to_atom(Kind, utf8)};
-fix({<<"name">>, Type}) ->
- {name, fix_type(Type)};
-fix({K, V}) ->
- {binary_to_atom(K, utf8), fix(V)};
-fix(V) when is_number(V) ->
- V;
-fix(V) when is_atom(V) ->
- V;
-fix(V) when is_binary(V) ->
- V.
-
-%% ensure below ebin dirs are added to code path:
-%% _build/default/lib/*/ebin
-%% _build/emqx/lib/*/ebin
-%% _build/emqx-enterprise/lib/*/ebin
-add_libs() ->
- Profile = os:getenv("PROFILE"),
- case Profile of
- "emqx" ->
- ok;
- "emqx-enterprise" ->
- ok;
- _ ->
- io:format("PROFILE is not set~n"),
- halt(1)
- end,
- Dirs =
- filelib:wildcard("_build/default/lib/*/ebin") ++
- filelib:wildcard("_build/" ++ Profile ++ "/lib/*/ebin"),
- lists:foreach(fun add_lib/1, Dirs).
-
-add_lib(Dir) ->
- code:add_patha(Dir),
- Beams = filelib:wildcard(Dir ++ "/*.beam"),
- _ = spawn(fun() -> lists:foreach(fun load_beam/1, Beams) end),
- ok.
-
-load_beam(BeamFile) ->
- ModuleName = filename:basename(BeamFile, ".beam"),
- Module = list_to_atom(ModuleName),
- %% load the beams to make sure the atoms are existing
- code:ensure_loaded(Module),
- ok.
-
-fix_type(<<"[{string(), string()}]">>) ->
- <<"map()">>;
-fix_type(<<"[{binary(), binary()}]">>) ->
- <<"map()">>;
-fix_type(<<"emqx_limiter_schema:rate()">>) ->
- <<"string()">>;
-fix_type(<<"emqx_limiter_schema:burst_rate()">>) ->
- <<"string()">>;
-fix_type(<<"emqx_limiter_schema:capacity()">>) ->
- <<"string()">>;
-fix_type(<<"emqx_limiter_schema:initial()">>) ->
- <<"string()">>;
-fix_type(<<"emqx_limiter_schema:failure_strategy()">>) ->
- <<"string()">>;
-fix_type(<<"emqx_conf_schema:file()">>) ->
- <<"string()">>;
-fix_type(<<"#{term() => binary()}">>) ->
- <<"map()">>;
-fix_type(<<"[term()]">>) ->
- %% jwt claims
- <<"map()">>;
-fix_type(<<"emqx_ee_bridge_influxdb:write_syntax()">>) ->
- <<"string()">>;
-fix_type(<<"emqx_bridge_influxdb:write_syntax()">>) ->
- <<"string()">>;
-fix_type(<<"emqx_schema:mqtt_max_packet_size()">>) ->
- <<"non_neg_integer()">>;
-fix_type(<<"emqx_s3_schema:secret_access_key()">>) ->
- <<"string()">>;
-fix_type(Type) ->
- Type.
-
-%% ensure atoms are loaded
-%% these atoms are from older version of emqx
-atoms() ->
- [
- emqx_ee_connector_clickhouse,
- emqx_ee_bridge_gcp_pubsub,
- emqx_ee_bridge_influxdb,
- emqx_connector_http
- ].