From 85d7518a7d4065ee53b8b18ae08934c91d0b35bb Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 20 Dec 2023 22:33:12 +0100 Subject: [PATCH 1/4] feat(schema): provide type-level documentation snippets For stuff like `duration()`, `bytesize()` and `secret()` for now. --- apps/emqx_conf/src/emqx_conf.erl | 41 +++++++++++++------ apps/emqx_conf/src/emqx_conf_schema_types.erl | 24 +++++++---- rel/i18n/emqx_conf_schema_types.hocon | 12 ++++++ scripts/schema-dump-reformat.escript | 3 +- 4 files changed, 58 insertions(+), 22 deletions(-) create mode 100644 rel/i18n/emqx_conf_schema_types.hocon diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 140b008d1..4f845a166 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -28,7 +28,7 @@ -export([remove/2, remove/3]). -export([tombstone/2]). -export([reset/2, reset/3]). --export([dump_schema/2, reformat_schema_dump/1]). +-export([dump_schema/2, reformat_schema_dump/2]). -export([schema_module/0]). %% TODO: move to emqx_dashboard when we stop building api schema at build time @@ -186,7 +186,7 @@ gen_schema_json(Dir, SchemaModule, Lang) -> ok = gen_preformat_md_json_files(Dir, StructsJsonArray, Lang). gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) -> - NestedStruct = reformat_schema_dump(StructsJsonArray), + NestedStruct = reformat_schema_dump(StructsJsonArray, Lang), %% write to files NestedJsonFile = filename:join([Dir, "schema-v2-" ++ Lang ++ ".json"]), io:format(user, "===< Generating: ~s~n", [NestedJsonFile]), @@ -196,15 +196,17 @@ gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) -> ok. %% @doc This function is exported for scripts/schema-dump-reformat.escript -reformat_schema_dump(StructsJsonArray0) -> +reformat_schema_dump(StructsJsonArray0, Lang) -> %% prepare + DescResolver = make_desc_resolver(Lang), StructsJsonArray = deduplicate_by_full_name(StructsJsonArray0), #{fields := RootFields} = hd(StructsJsonArray), RootNames0 = lists:map(fun(#{name := RootName}) -> RootName end, RootFields), RootNames = lists:map(fun to_bin/1, RootNames0), %% reformat [Root | FlatStructs0] = lists:map( - fun(Struct) -> gen_flat_doc(RootNames, Struct) end, StructsJsonArray + fun(Struct) -> gen_flat_doc(RootNames, Struct, DescResolver) end, + StructsJsonArray ), FlatStructs = [Root#{text => <<"root">>, hash => <<"root">>} | FlatStructs0], gen_nested_doc(FlatStructs). @@ -302,7 +304,7 @@ expand_ref(#{hash := FullName}, FindFn, Path) -> %% generate flat docs for each struct. %% using references to link to other structs. -gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) -> +gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S, DescResolver) -> ShortName = short_name(FullName), case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of true -> @@ -314,18 +316,17 @@ gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) -> text => short_name(FullName), hash => format_hash(FullName), doc => maps:get(desc, S, <<"">>), - fields => format_fields(Fields) + fields => format_fields(Fields, DescResolver) }. -format_fields([]) -> - []; -format_fields([Field | Fields]) -> - [format_field(Field) | format_fields(Fields)]. +format_fields(Fields, DescResolver) -> + [format_field(F, DescResolver) || F <- Fields]. -format_field(#{name := Name, aliases := Aliases, type := Type} = F) -> +format_field(#{name := Name, aliases := Aliases, type := Type} = F, DescResolver) -> L = [ {text, Name}, {type, format_type(Type)}, + {typedoc, format_type_desc(Type, DescResolver)}, {refs, format_refs(Type)}, {aliases, case Aliases of @@ -393,10 +394,26 @@ format_union_members([Member | Members], Acc) -> NewAcc = [format_type(Member) | Acc], format_union_members(Members, NewAcc). +format_type_desc(#{kind := primitive, name := Name}, DescResolver) -> + format_primitive_type_desc(Name, DescResolver); +format_type_desc(#{}, _DescResolver) -> + undefined. + format_primitive_type(TypeStr) -> - Spec = emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr), + Spec = get_primitive_typespec(TypeStr), to_bin(maps:get(type, Spec)). +format_primitive_type_desc(TypeStr, DescResolver) -> + case get_primitive_typespec(TypeStr) of + #{desc := Desc} -> + DescResolver(Desc); + #{} -> + undefined + end. + +get_primitive_typespec(TypeStr) -> + emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr). + %% All types should have a namespace to avlid name clashing. is_missing_namespace(ShortName, FullName, RootNames) -> case lists:member(ShortName, RootNames) of diff --git a/apps/emqx_conf/src/emqx_conf_schema_types.erl b/apps/emqx_conf/src/emqx_conf_schema_types.erl index 239624a81..9d00b9650 100644 --- a/apps/emqx_conf/src/emqx_conf_schema_types.erl +++ b/apps/emqx_conf/src/emqx_conf_schema_types.erl @@ -16,6 +16,8 @@ -module(emqx_conf_schema_types). +-include_lib("hocon/include/hocon_types.hrl"). + -export([readable/2]). -export([readable_swagger/2, readable_dashboard/2, readable_docgen/2]). @@ -165,37 +167,37 @@ readable("duration()") -> #{ swagger => #{type => string, example => <<"12m">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"12m">>} + docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)} }; readable("duration_s()") -> #{ swagger => #{type => string, example => <<"1h">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"1h">>} + docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)} }; readable("duration_ms()") -> #{ swagger => #{type => string, example => <<"32s">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"32s">>} + docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)} }; readable("timeout_duration()") -> #{ swagger => #{type => string, example => <<"12m">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"12m">>} + docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)} }; readable("timeout_duration_s()") -> #{ swagger => #{type => string, example => <<"1h">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"1h">>} + docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)} }; readable("timeout_duration_ms()") -> #{ swagger => #{type => string, example => <<"32s">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"32s">>} + docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)} }; readable("percent()") -> #{ @@ -219,13 +221,13 @@ readable("bytesize()") -> #{ swagger => #{type => string, example => <<"32MB">>}, dashboard => #{type => 'byteSize'}, - docgen => #{type => "String", example => <<"32MB">>} + docgen => #{type => "Bytesize", example => <<"32MB">>, desc => ?DESC(bytesize)} }; readable("wordsize()") -> #{ swagger => #{type => string, example => <<"1024KB">>}, dashboard => #{type => 'wordSize'}, - docgen => #{type => "String", example => <<"1024KB">>} + docgen => #{type => "Bytesize", example => <<"1024KB">>, desc => ?DESC(bytesize)} }; readable("map(" ++ Map) -> [$) | _MapArgs] = lists:reverse(Map), @@ -287,7 +289,11 @@ readable("secret()") -> #{ swagger => #{type => string, example => <<"R4ND0M/S∃CЯ∃T"/utf8>>}, dashboard => #{type => string}, - docgen => #{type => "String", example => <<"R4ND0M/S∃CЯ∃T"/utf8>>} + docgen => #{ + type => "Secret", + example => <<"R4ND0M/S∃CЯ∃T"/utf8>>, + desc => ?DESC(secret) + } }; readable(TypeStr0) -> case string:split(TypeStr0, ":") of diff --git a/rel/i18n/emqx_conf_schema_types.hocon b/rel/i18n/emqx_conf_schema_types.hocon new file mode 100644 index 000000000..4955d3173 --- /dev/null +++ b/rel/i18n/emqx_conf_schema_types.hocon @@ -0,0 +1,12 @@ +emqx_conf_schema_types { + + duration.desc: + """A string that represents a time duration, for example: 10s, 2.5m, 1h30m, 1W2D, or 2345ms, which is the smallest unit. When precision is specified, finer portions of the duration may be ignored: writing 1200ms for Duration(s) is equivalent to writing 1s. It doesn't matter if units are in upper or lower case.""" + + bytesize.desc: + """A string that represents a number of bytes, for example: 10B, 640kb, 4MB, 1GB. Units are interpreted as powers of 1024, and the unit part is case-insensitive.""" + + secret.desc: + """A string holding some sensitive information, such as a password. When secret starts with file://, the rest of the string is interpreted as a path to a file containing the secret itself: whole content of the file except any trailing whitespace characters is considered a secret value.""" + +} diff --git a/scripts/schema-dump-reformat.escript b/scripts/schema-dump-reformat.escript index 31cfdd7d9..a3e059b70 100755 --- a/scripts/schema-dump-reformat.escript +++ b/scripts/schema-dump-reformat.escript @@ -18,7 +18,8 @@ main(_) -> halt(1). reformat(Json) -> - emqx_conf:reformat_schema_dump(fix(Json)). + %% NOTE: Assuming schema would contain no types needing localized typedocs. + emqx_conf:reformat_schema_dump(fix(Json), _Lang = "en"). %% fix old type specs to make them compatible with new type specs fix(#{ From da49909ac45e0f2946d18eedb5e806324b3a6655 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 21 Dec 2023 13:42:30 +0100 Subject: [PATCH 2/4] chore(typedoc): refine some descriptions Co-authored-by: Zaiming (Stone) Shi --- rel/i18n/emqx_conf_schema_types.hocon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/i18n/emqx_conf_schema_types.hocon b/rel/i18n/emqx_conf_schema_types.hocon index 4955d3173..6b9dac9ea 100644 --- a/rel/i18n/emqx_conf_schema_types.hocon +++ b/rel/i18n/emqx_conf_schema_types.hocon @@ -1,12 +1,12 @@ emqx_conf_schema_types { duration.desc: - """A string that represents a time duration, for example: 10s, 2.5m, 1h30m, 1W2D, or 2345ms, which is the smallest unit. When precision is specified, finer portions of the duration may be ignored: writing 1200ms for Duration(s) is equivalent to writing 1s. It doesn't matter if units are in upper or lower case.""" + """A string that represents a time duration, for example: 10s, 2.5m, 1h30m, 1W2D, or 2345ms, which is the smallest unit. When precision is specified, finer portions of the duration may be ignored: writing 1200ms for Duration(s) is equivalent to writing 1s. The unit part is case-insensitive.""" bytesize.desc: """A string that represents a number of bytes, for example: 10B, 640kb, 4MB, 1GB. Units are interpreted as powers of 1024, and the unit part is case-insensitive.""" secret.desc: - """A string holding some sensitive information, such as a password. When secret starts with file://, the rest of the string is interpreted as a path to a file containing the secret itself: whole content of the file except any trailing whitespace characters is considered a secret value.""" + """A string holding some sensitive information, such as a password. When secret starts with file://, the rest of the string is interpreted as a path to a file containing the secret itself: whole content of the file except any trailing whitespace characters is considered a secret value. Note: when clustered, all EMQX nodes should have the same file present before using file:// secrets.""" } From 1290f1794abe49c6e7ca2449c3f01ec4edb835e3 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 21 Dec 2023 13:44:25 +0100 Subject: [PATCH 3/4] chore: drop `schema-dump-reformat.escript` It should not be needed anymore. --- scripts/schema-dump-reformat.escript | 133 --------------------------- 1 file changed, 133 deletions(-) delete mode 100755 scripts/schema-dump-reformat.escript diff --git a/scripts/schema-dump-reformat.escript b/scripts/schema-dump-reformat.escript deleted file mode 100755 index a3e059b70..000000000 --- a/scripts/schema-dump-reformat.escript +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env escript - -%% This script translates the hocon_schema_json's schema dump to a new format. -%% It is used to convert older version EMQX's schema dumps to the new format -%% after all files are upgraded to the new format, this script can be removed. - --mode(compile). - -main([Input]) -> - ok = add_libs(), - _ = atoms(), - {ok, Data} = file:read_file(Input), - Json = jsx:decode(Data), - NewJson = reformat(Json), - io:format("~s~n", [jsx:encode(NewJson)]); -main(_) -> - io:format("Usage: schema-dump-reformat.escript ~n"), - halt(1). - -reformat(Json) -> - %% NOTE: Assuming schema would contain no types needing localized typedocs. - emqx_conf:reformat_schema_dump(fix(Json), _Lang = "en"). - -%% fix old type specs to make them compatible with new type specs -fix(#{ - <<"kind">> := <<"union">>, - <<"members">> := [#{<<"name">> := <<"string()">>}, #{<<"name">> := <<"function()">>}] -}) -> - %% s3_exporter.secret_access_key - #{ - kind => primitive, - name => <<"string()">> - }; -fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_conf_schema:log_level()">>}) -> - #{ - kind => enum, - symbols => [emergency, alert, critical, error, warning, notice, info, debug, none, all] - }; -fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_connector_http:pool_type()">>}) -> - #{kind => enum, symbols => [random, hash]}; -fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_bridge_http_connector:pool_type()">>}) -> - #{kind => enum, symbols => [random, hash]}; -fix(Map) when is_map(Map) -> - maps:from_list(fix(maps:to_list(Map))); -fix(List) when is_list(List) -> - lists:map(fun fix/1, List); -fix({<<"kind">>, Kind}) -> - {kind, binary_to_atom(Kind, utf8)}; -fix({<<"name">>, Type}) -> - {name, fix_type(Type)}; -fix({K, V}) -> - {binary_to_atom(K, utf8), fix(V)}; -fix(V) when is_number(V) -> - V; -fix(V) when is_atom(V) -> - V; -fix(V) when is_binary(V) -> - V. - -%% ensure below ebin dirs are added to code path: -%% _build/default/lib/*/ebin -%% _build/emqx/lib/*/ebin -%% _build/emqx-enterprise/lib/*/ebin -add_libs() -> - Profile = os:getenv("PROFILE"), - case Profile of - "emqx" -> - ok; - "emqx-enterprise" -> - ok; - _ -> - io:format("PROFILE is not set~n"), - halt(1) - end, - Dirs = - filelib:wildcard("_build/default/lib/*/ebin") ++ - filelib:wildcard("_build/" ++ Profile ++ "/lib/*/ebin"), - lists:foreach(fun add_lib/1, Dirs). - -add_lib(Dir) -> - code:add_patha(Dir), - Beams = filelib:wildcard(Dir ++ "/*.beam"), - _ = spawn(fun() -> lists:foreach(fun load_beam/1, Beams) end), - ok. - -load_beam(BeamFile) -> - ModuleName = filename:basename(BeamFile, ".beam"), - Module = list_to_atom(ModuleName), - %% load the beams to make sure the atoms are existing - code:ensure_loaded(Module), - ok. - -fix_type(<<"[{string(), string()}]">>) -> - <<"map()">>; -fix_type(<<"[{binary(), binary()}]">>) -> - <<"map()">>; -fix_type(<<"emqx_limiter_schema:rate()">>) -> - <<"string()">>; -fix_type(<<"emqx_limiter_schema:burst_rate()">>) -> - <<"string()">>; -fix_type(<<"emqx_limiter_schema:capacity()">>) -> - <<"string()">>; -fix_type(<<"emqx_limiter_schema:initial()">>) -> - <<"string()">>; -fix_type(<<"emqx_limiter_schema:failure_strategy()">>) -> - <<"string()">>; -fix_type(<<"emqx_conf_schema:file()">>) -> - <<"string()">>; -fix_type(<<"#{term() => binary()}">>) -> - <<"map()">>; -fix_type(<<"[term()]">>) -> - %% jwt claims - <<"map()">>; -fix_type(<<"emqx_ee_bridge_influxdb:write_syntax()">>) -> - <<"string()">>; -fix_type(<<"emqx_bridge_influxdb:write_syntax()">>) -> - <<"string()">>; -fix_type(<<"emqx_schema:mqtt_max_packet_size()">>) -> - <<"non_neg_integer()">>; -fix_type(<<"emqx_s3_schema:secret_access_key()">>) -> - <<"string()">>; -fix_type(Type) -> - Type. - -%% ensure atoms are loaded -%% these atoms are from older version of emqx -atoms() -> - [ - emqx_ee_connector_clickhouse, - emqx_ee_bridge_gcp_pubsub, - emqx_ee_bridge_influxdb, - emqx_connector_http - ]. From aa3f8d6735cae1d9876627fd6e973ae0fb2a8f08 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 21 Dec 2023 17:13:37 +0100 Subject: [PATCH 4/4] fix(typedoc): meld it into the field doc in the meantime --- apps/emqx_conf/src/emqx_conf.erl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 4f845a166..0a8339ddd 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -323,10 +323,12 @@ format_fields(Fields, DescResolver) -> [format_field(F, DescResolver) || F <- Fields]. format_field(#{name := Name, aliases := Aliases, type := Type} = F, DescResolver) -> + TypeDoc = format_type_desc(Type, DescResolver), L = [ {text, Name}, {type, format_type(Type)}, - {typedoc, format_type_desc(Type, DescResolver)}, + %% TODO: Make it into a separate field. + %% {typedoc, format_type_desc(Type, DescResolver)}, {refs, format_refs(Type)}, {aliases, case Aliases of @@ -334,7 +336,7 @@ format_field(#{name := Name, aliases := Aliases, type := Type} = F, DescResolver _ -> Aliases end}, {default, maps:get(hocon, maps:get(default, F, #{}), undefined)}, - {doc, maps:get(desc, F, undefined)} + {doc, join_format([maps:get(desc, F, undefined), TypeDoc])} ], maps:from_list([{K, V} || {K, V} <- L, V =/= undefined]). @@ -577,6 +579,14 @@ hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> typename_to_spec(TypeStr, Module) -> emqx_conf_schema_types:readable_dashboard(Module, TypeStr). +join_format(Snippets) -> + case [S || S <- Snippets, S =/= undefined] of + [] -> + undefined; + NonEmpty -> + to_bin(lists:join("
", NonEmpty)) + end. + to_bin(List) when is_list(List) -> iolist_to_binary(List); to_bin(Boolean) when is_boolean(Boolean) -> Boolean; to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);