feat(schema): Add v2 scheam JSON dump

apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl

@@ -67,7 +67,7 @@
 -define(SALT_ROUNDS_MIN, 5).
 -define(SALT_ROUNDS_MAX, 10).
 
-namespace() -> "authn-hash".
+namespace() -> "authn_hash".
 roots() -> [pbkdf2, bcrypt, bcrypt_rw, bcrypt_rw_api, simple].
 
 fields(bcrypt_rw) ->

apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl

@@ -54,7 +54,7 @@
 %%=====================================================================
 %% Hocon schema
 
-namespace() -> "connector-http".
+namespace() -> "connector_http".
 
 roots() ->
     fields(config).

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge_mqtt, [
     {description, "EMQX MQTT Broker Bridge"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl

@@ -36,7 +36,7 @@
 
 -define(MQTT_HOST_OPTS, #{default_port => 1883}).
 
-namespace() -> "connector-mqtt".
+namespace() -> "connector_mqtt".
 
 roots() ->
     fields("config").

apps/emqx_conf/src/emqx_conf.erl

@@ -28,7 +28,7 @@
 -export([remove/2, remove/3]).
 -export([tombstone/2]).
 -export([reset/2, reset/3]).
--export([dump_schema/2]).
+-export([dump_schema/2, reformat_schema_dump/1]).
 -export([schema_module/0]).
 
 %% TODO: move to emqx_dashboard when we stop building api schema at build time
@@ -180,9 +180,263 @@ gen_schema_json(Dir, SchemaModule, Lang) ->
         include_importance_up_from => IncludeImportance,
         desc_resolver => make_desc_resolver(Lang)
     },
-    JsonMap = hocon_schema_json:gen(SchemaModule, Opts),
+    StructsJsonArray = hocon_schema_json:gen(SchemaModule, Opts),
-    IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]),
+    IoData = emqx_utils_json:encode(StructsJsonArray, [pretty, force_utf8]),
-    ok = file:write_file(SchemaJsonFile, IoData).
+    ok = file:write_file(SchemaJsonFile, IoData),
+    ok = gen_preformat_md_json_files(Dir, StructsJsonArray, Lang).
+
+gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) ->
+    NestedStruct = reformat_schema_dump(StructsJsonArray),
+    %% write to files
+    NestedJsonFile = filename:join([Dir, "schmea-v2-" ++ Lang ++ ".json"]),
+    io:format(user, "===< Generating: ~s~n", [NestedJsonFile]),
+    ok = file:write_file(
+        NestedJsonFile, emqx_utils_json:encode(NestedStruct, [pretty, force_utf8])
+    ),
+    ok.
+
+%% @doc This function is exported for scripts/schema-dump-reformat.escript
+reformat_schema_dump(StructsJsonArray0) ->
+    %% prepare
+    StructsJsonArray = deduplicate_by_full_name(StructsJsonArray0),
+    #{fields := RootFields} = hd(StructsJsonArray),
+    RootNames0 = lists:map(fun(#{name := RootName}) -> RootName end, RootFields),
+    RootNames = lists:map(fun to_bin/1, RootNames0),
+    %% reformat
+    [Root | FlatStructs0] = lists:map(
+        fun(Struct) -> gen_flat_doc(RootNames, Struct) end, StructsJsonArray
+    ),
+    FlatStructs = [Root#{text => <<"root">>, hash => <<"root">>} | FlatStructs0],
+    gen_nested_doc(FlatStructs).
+
+deduplicate_by_full_name(Structs) ->
+    deduplicate_by_full_name(Structs, #{}, []).
+
+deduplicate_by_full_name([], _Seen, Acc) ->
+    lists:reverse(Acc);
+deduplicate_by_full_name([#{full_name := FullName} = H | T], Seen, Acc) ->
+    case maps:get(FullName, Seen, false) of
+        false ->
+            deduplicate_by_full_name(T, Seen#{FullName => H}, [H | Acc]);
+        H ->
+            %% Name clash, but identical, ignore
+            deduplicate_by_full_name(T, Seen, Acc);
+        _Different ->
+            %% ADD NAMESPACE!
+            throw({duplicate_full_name, FullName})
+    end.
+
+%% Ggenerate nested docs from root struct.
+%% Due to the fact that the same struct can be referenced by multiple fields,
+%% we need to generate a unique nested doc for each reference.
+%% The unique path to each type and is of the below format:
+%% - A a path starts either with 'T-' or 'V-'. T stands for type, V stands for value.
+%% - A path is a list of strings delimited by '-'.
+%%   - The letter S is used to separate struct name from field name.
+%%   - Field names are however NOT denoted by a leading 'F-'.
+%% For example:
+%% - T-root: the root struct;
+%% - T-foo-S-footype: the struct named "footype" in the foo field of root struct;
+%% - V-foo-S-footype-bar: the field named "bar" in the struct named "footype" in the foo field of root struct
+gen_nested_doc(Structs) ->
+    KeyByFullName = lists:foldl(
+        fun(#{hash := FullName} = Struct, Acc) ->
+            maps:put(FullName, Struct, Acc)
+        end,
+        #{},
+        Structs
+    ),
+    FindFn = fun(Hash) -> maps:get(Hash, KeyByFullName) end,
+    gen_nested_doc(hd(Structs), FindFn, []).
+
+gen_nested_doc(#{fields := Fields} = Struct, FindFn, Path) ->
+    TypeAnchor = make_type_anchor(Path),
+    ValueAnchor = fun(FieldName) -> make_value_anchor(Path, FieldName) end,
+    NewFields = lists:map(
+        fun(#{text := Name} = Field) ->
+            NewField = expand_field(Field, FindFn, Path),
+            NewField#{hash => ValueAnchor(Name)}
+        end,
+        Fields
+    ),
+    Struct#{
+        fields => NewFields,
+        hash => TypeAnchor
+    }.
+
+%% Make anchor for type.
+%% Start with "T-" to distinguish from value anchor.
+make_type_anchor([]) ->
+    <<"T-root">>;
+make_type_anchor(Path) ->
+    to_bin(["T-", lists:join("-", lists:reverse(Path))]).
+
+%% Value anchor is used to link to the field's struct.
+%% Start with "V-" to distinguish from type anchor.
+make_value_anchor(Path, FieldName) ->
+    to_bin(["V-", join_path_hash(Path, FieldName)]).
+
+%% Make a globally unique "hash" (the http anchor) for each struct field.
+join_path_hash([], Name) ->
+    Name;
+join_path_hash(Path, Name) ->
+    to_bin(lists:join("-", lists:reverse([Name | Path]))).
+
+%% Expand field's struct reference to nested doc.
+expand_field(#{text := Name, refs := References} = Field, FindFn, Path) ->
+    %% Add struct type name in path to make it unique.
+    NewReferences = lists:map(
+        fun(#{text := StructName} = Ref) ->
+            expand_ref(Ref, FindFn, [StructName, "S", Name | Path])
+        end,
+        References
+    ),
+    Field#{refs => NewReferences};
+expand_field(Field, _FindFn, _Path) ->
+    %% No reference, no need to expand.
+    Field.
+
+expand_ref(#{hash := FullName}, FindFn, Path) ->
+    Struct = FindFn(FullName),
+    gen_nested_doc(Struct, FindFn, Path).
+
+%% generate flat docs for each struct.
+%% using references to link to other structs.
+gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) ->
+    ShortName = short_name(FullName),
+    case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of
+        true ->
+            io:format(standard_error, "WARN: no_namespace_for: ~s~n", [FullName]);
+        false ->
+            ok
+    end,
+    #{
+        text => short_name(FullName),
+        hash => format_hash(FullName),
+        doc => maps:get(desc, S, <<"">>),
+        fields => format_fields(Fields)
+    }.
+
+format_fields([]) ->
+    [];
+format_fields([Field | Fields]) ->
+    [format_field(Field) | format_fields(Fields)].
+
+format_field(#{name := Name, aliases := Aliases, type := Type} = F) ->
+    L = [
+        {text, Name},
+        {type, format_type(Type)},
+        {refs, format_refs(Type)},
+        {aliases,
+            case Aliases of
+                [] -> undefined;
+                _ -> Aliases
+            end},
+        {default, maps:get(hocon, maps:get(default, F, #{}), undefined)},
+        {doc, maps:get(desc, F, undefined)}
+    ],
+    maps:from_list([{K, V} || {K, V} <- L, V =/= undefined]).
+
+format_refs(Type) ->
+    References = find_refs(Type),
+    case lists:map(fun format_ref/1, References) of
+        [] -> undefined;
+        L -> L
+    end.
+
+format_ref(FullName) ->
+    #{text => short_name(FullName), hash => format_hash(FullName)}.
+
+find_refs(Type) ->
+    lists:reverse(find_refs(Type, [])).
+
+%% go deep into union, array, and map to find references
+find_refs(#{kind := union, members := Members}, Acc) ->
+    lists:foldl(fun find_refs/2, Acc, Members);
+find_refs(#{kind := array, elements := Elements}, Acc) ->
+    find_refs(Elements, Acc);
+find_refs(#{kind := map, values := Values}, Acc) ->
+    find_refs(Values, Acc);
+find_refs(#{kind := struct, name := FullName}, Acc) ->
+    [FullName | Acc];
+find_refs(_, Acc) ->
+    Acc.
+
+format_type(#{kind := primitive, name := Name}) ->
+    format_primitive_type(Name);
+format_type(#{kind := singleton, name := Name}) ->
+    to_bin(["String(\"", to_bin(Name), "\")"]);
+format_type(#{kind := enum, symbols := Symbols}) ->
+    CommaSep = lists:join(",", lists:map(fun(S) -> to_bin(S) end, Symbols)),
+    to_bin(["Enum(", CommaSep, ")"]);
+format_type(#{kind := array, elements := ElementsType}) ->
+    to_bin(["Array(", format_type(ElementsType), ")"]);
+format_type(#{kind := union, members := MemberTypes} = U) ->
+    DN = maps:get(display_name, U, undefined),
+    case DN of
+        undefined ->
+            to_bin(["OneOf(", format_union_members(MemberTypes), ")"]);
+        Name ->
+            format_primitive_type(Name)
+    end;
+format_type(#{kind := struct, name := FullName}) ->
+    to_bin(["Struct(", short_name(FullName), ")"]);
+format_type(#{kind := map, name := Name, values := ValuesType}) ->
+    to_bin(["Map($", Name, "->", format_type(ValuesType), ")"]).
+
+format_union_members(Members) ->
+    format_union_members(Members, []).
+
+format_union_members([], Acc) ->
+    lists:join(",", lists:reverse(Acc));
+format_union_members([Member | Members], Acc) ->
+    NewAcc = [format_type(Member) | Acc],
+    format_union_members(Members, NewAcc).
+
+format_primitive_type(TypeStr) ->
+    Spec = emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr),
+    to_bin(maps:get(type, Spec)).
+
+%% All types should have a namespace to avlid name clashing.
+is_missing_namespace(ShortName, FullName, RootNames) ->
+    case lists:member(ShortName, RootNames) of
+        true ->
+            false;
+        false ->
+            ShortName =:= FullName
+    end.
+
+%% Returns short name from full name, fullname delemited by colon(:).
+short_name(FullName) ->
+    case string:split(FullName, ":") of
+        [_, Name] -> to_bin(Name);
+        _ -> to_bin(FullName)
+    end.
+
+%% Returns the hash-anchor from full name, fullname delemited by colon(:).
+format_hash(FullName) ->
+    case string:split(FullName, ":") of
+        [Namespace, Name] ->
+            ok = warn_bad_namespace(Namespace),
+            iolist_to_binary([Namespace, "__", Name]);
+        _ ->
+            iolist_to_binary(FullName)
+    end.
+
+%% namespace should only have letters, numbers, and underscores.
+warn_bad_namespace(Namespace) ->
+    case re:run(Namespace, "^[a-zA-Z0-9_]+$", [{capture, none}]) of
+        nomatch ->
+            case erlang:get({bad_namespace, Namespace}) of
+                true ->
+                    ok;
+                _ ->
+                    erlang:put({bad_namespace, Namespace}, true),
+                    io:format(standard_error, "WARN: bad_namespace: ~s~n", [Namespace])
+            end;
+        _ ->
+            ok
+    end.
 
 %% TODO: move this function to emqx_dashboard when we stop generating this JSON at build time.
 hotconf_schema_json() ->
@@ -306,12 +560,7 @@ hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->
 typename_to_spec(TypeStr, Module) ->
     emqx_conf_schema_types:readable_dashboard(Module, TypeStr).
 
-to_bin(List) when is_list(List) ->
+to_bin(List) when is_list(List) -> iolist_to_binary(List);
-    case io_lib:printable_list(List) of
-        true -> unicode:characters_to_binary(List);
-        false -> List
-    end;
 to_bin(Boolean) when is_boolean(Boolean) -> Boolean;
 to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
-to_bin(X) ->
+to_bin(X) -> X.
-    X.

apps/emqx_psk/src/emqx_psk.app.src

@@ -2,7 +2,7 @@
 {application, emqx_psk, [
     {description, "EMQX PSK"},
     % strict semver, bump manually!
-    {vsn, "5.0.4"},
+    {vsn, "5.0.5"},
     {modules, []},
     {registered, [emqx_psk_sup]},
     {applications, [kernel, stdlib]},

apps/emqx_psk/src/emqx_psk_schema.erl

@@ -28,7 +28,7 @@
     fields/1
 ]).
 
-namespace() -> "authn-psk".
+namespace() -> "psk".
 
 roots() -> ["psk_authentication"].