From db33bc616ae1b942fd73b23c788c98a5d2af6105 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 8 Nov 2023 23:31:29 +0100 Subject: [PATCH] feat(schema): Add v2 scheam JSON dump --- .../emqx_authn_password_hashing.erl | 2 +- .../src/emqx_bridge_http_connector.erl | 2 +- .../src/emqx_bridge_mqtt.app.src | 2 +- .../src/emqx_bridge_mqtt_connector_schema.erl | 2 +- apps/emqx_conf/src/emqx_conf.erl | 271 +++++++++++++++++- apps/emqx_psk/src/emqx_psk.app.src | 2 +- apps/emqx_psk/src/emqx_psk_schema.erl | 2 +- 7 files changed, 266 insertions(+), 17 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl index 756f39d06..16af4fd23 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl @@ -67,7 +67,7 @@ -define(SALT_ROUNDS_MIN, 5). -define(SALT_ROUNDS_MAX, 10). -namespace() -> "authn-hash". +namespace() -> "authn_hash". roots() -> [pbkdf2, bcrypt, bcrypt_rw, bcrypt_rw_api, simple]. fields(bcrypt_rw) -> diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 743ab97fe..5a5e790e5 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -54,7 +54,7 @@ %%===================================================================== %% Hocon schema -namespace() -> "connector-http". +namespace() -> "connector_http". roots() -> fields(config). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index e39c4df69..cbef0dda8 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 1dc3ca5f8..f671bec71 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -36,7 +36,7 @@ -define(MQTT_HOST_OPTS, #{default_port => 1883}). -namespace() -> "connector-mqtt". +namespace() -> "connector_mqtt". roots() -> fields("config"). diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 7ff06b0ef..0925141de 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -28,7 +28,7 @@ -export([remove/2, remove/3]). -export([tombstone/2]). -export([reset/2, reset/3]). --export([dump_schema/2]). +-export([dump_schema/2, reformat_schema_dump/1]). -export([schema_module/0]). %% TODO: move to emqx_dashboard when we stop building api schema at build time @@ -180,9 +180,263 @@ gen_schema_json(Dir, SchemaModule, Lang) -> include_importance_up_from => IncludeImportance, desc_resolver => make_desc_resolver(Lang) }, - JsonMap = hocon_schema_json:gen(SchemaModule, Opts), - IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]), - ok = file:write_file(SchemaJsonFile, IoData). + StructsJsonArray = hocon_schema_json:gen(SchemaModule, Opts), + IoData = emqx_utils_json:encode(StructsJsonArray, [pretty, force_utf8]), + ok = file:write_file(SchemaJsonFile, IoData), + ok = gen_preformat_md_json_files(Dir, StructsJsonArray, Lang). + +gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) -> + NestedStruct = reformat_schema_dump(StructsJsonArray), + %% write to files + NestedJsonFile = filename:join([Dir, "schmea-v2-" ++ Lang ++ ".json"]), + io:format(user, "===< Generating: ~s~n", [NestedJsonFile]), + ok = file:write_file( + NestedJsonFile, emqx_utils_json:encode(NestedStruct, [pretty, force_utf8]) + ), + ok. + +%% @doc This function is exported for scripts/schema-dump-reformat.escript +reformat_schema_dump(StructsJsonArray0) -> + %% prepare + StructsJsonArray = deduplicate_by_full_name(StructsJsonArray0), + #{fields := RootFields} = hd(StructsJsonArray), + RootNames0 = lists:map(fun(#{name := RootName}) -> RootName end, RootFields), + RootNames = lists:map(fun to_bin/1, RootNames0), + %% reformat + [Root | FlatStructs0] = lists:map( + fun(Struct) -> gen_flat_doc(RootNames, Struct) end, StructsJsonArray + ), + FlatStructs = [Root#{text => <<"root">>, hash => <<"root">>} | FlatStructs0], + gen_nested_doc(FlatStructs). + +deduplicate_by_full_name(Structs) -> + deduplicate_by_full_name(Structs, #{}, []). + +deduplicate_by_full_name([], _Seen, Acc) -> + lists:reverse(Acc); +deduplicate_by_full_name([#{full_name := FullName} = H | T], Seen, Acc) -> + case maps:get(FullName, Seen, false) of + false -> + deduplicate_by_full_name(T, Seen#{FullName => H}, [H | Acc]); + H -> + %% Name clash, but identical, ignore + deduplicate_by_full_name(T, Seen, Acc); + _Different -> + %% ADD NAMESPACE! + throw({duplicate_full_name, FullName}) + end. + +%% Ggenerate nested docs from root struct. +%% Due to the fact that the same struct can be referenced by multiple fields, +%% we need to generate a unique nested doc for each reference. +%% The unique path to each type and is of the below format: +%% - A a path starts either with 'T-' or 'V-'. T stands for type, V stands for value. +%% - A path is a list of strings delimited by '-'. +%% - The letter S is used to separate struct name from field name. +%% - Field names are however NOT denoted by a leading 'F-'. +%% For example: +%% - T-root: the root struct; +%% - T-foo-S-footype: the struct named "footype" in the foo field of root struct; +%% - V-foo-S-footype-bar: the field named "bar" in the struct named "footype" in the foo field of root struct +gen_nested_doc(Structs) -> + KeyByFullName = lists:foldl( + fun(#{hash := FullName} = Struct, Acc) -> + maps:put(FullName, Struct, Acc) + end, + #{}, + Structs + ), + FindFn = fun(Hash) -> maps:get(Hash, KeyByFullName) end, + gen_nested_doc(hd(Structs), FindFn, []). + +gen_nested_doc(#{fields := Fields} = Struct, FindFn, Path) -> + TypeAnchor = make_type_anchor(Path), + ValueAnchor = fun(FieldName) -> make_value_anchor(Path, FieldName) end, + NewFields = lists:map( + fun(#{text := Name} = Field) -> + NewField = expand_field(Field, FindFn, Path), + NewField#{hash => ValueAnchor(Name)} + end, + Fields + ), + Struct#{ + fields => NewFields, + hash => TypeAnchor + }. + +%% Make anchor for type. +%% Start with "T-" to distinguish from value anchor. +make_type_anchor([]) -> + <<"T-root">>; +make_type_anchor(Path) -> + to_bin(["T-", lists:join("-", lists:reverse(Path))]). + +%% Value anchor is used to link to the field's struct. +%% Start with "V-" to distinguish from type anchor. +make_value_anchor(Path, FieldName) -> + to_bin(["V-", join_path_hash(Path, FieldName)]). + +%% Make a globally unique "hash" (the http anchor) for each struct field. +join_path_hash([], Name) -> + Name; +join_path_hash(Path, Name) -> + to_bin(lists:join("-", lists:reverse([Name | Path]))). + +%% Expand field's struct reference to nested doc. +expand_field(#{text := Name, refs := References} = Field, FindFn, Path) -> + %% Add struct type name in path to make it unique. + NewReferences = lists:map( + fun(#{text := StructName} = Ref) -> + expand_ref(Ref, FindFn, [StructName, "S", Name | Path]) + end, + References + ), + Field#{refs => NewReferences}; +expand_field(Field, _FindFn, _Path) -> + %% No reference, no need to expand. + Field. + +expand_ref(#{hash := FullName}, FindFn, Path) -> + Struct = FindFn(FullName), + gen_nested_doc(Struct, FindFn, Path). + +%% generate flat docs for each struct. +%% using references to link to other structs. +gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) -> + ShortName = short_name(FullName), + case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of + true -> + io:format(standard_error, "WARN: no_namespace_for: ~s~n", [FullName]); + false -> + ok + end, + #{ + text => short_name(FullName), + hash => format_hash(FullName), + doc => maps:get(desc, S, <<"">>), + fields => format_fields(Fields) + }. + +format_fields([]) -> + []; +format_fields([Field | Fields]) -> + [format_field(Field) | format_fields(Fields)]. + +format_field(#{name := Name, aliases := Aliases, type := Type} = F) -> + L = [ + {text, Name}, + {type, format_type(Type)}, + {refs, format_refs(Type)}, + {aliases, + case Aliases of + [] -> undefined; + _ -> Aliases + end}, + {default, maps:get(hocon, maps:get(default, F, #{}), undefined)}, + {doc, maps:get(desc, F, undefined)} + ], + maps:from_list([{K, V} || {K, V} <- L, V =/= undefined]). + +format_refs(Type) -> + References = find_refs(Type), + case lists:map(fun format_ref/1, References) of + [] -> undefined; + L -> L + end. + +format_ref(FullName) -> + #{text => short_name(FullName), hash => format_hash(FullName)}. + +find_refs(Type) -> + lists:reverse(find_refs(Type, [])). + +%% go deep into union, array, and map to find references +find_refs(#{kind := union, members := Members}, Acc) -> + lists:foldl(fun find_refs/2, Acc, Members); +find_refs(#{kind := array, elements := Elements}, Acc) -> + find_refs(Elements, Acc); +find_refs(#{kind := map, values := Values}, Acc) -> + find_refs(Values, Acc); +find_refs(#{kind := struct, name := FullName}, Acc) -> + [FullName | Acc]; +find_refs(_, Acc) -> + Acc. + +format_type(#{kind := primitive, name := Name}) -> + format_primitive_type(Name); +format_type(#{kind := singleton, name := Name}) -> + to_bin(["String(\"", to_bin(Name), "\")"]); +format_type(#{kind := enum, symbols := Symbols}) -> + CommaSep = lists:join(",", lists:map(fun(S) -> to_bin(S) end, Symbols)), + to_bin(["Enum(", CommaSep, ")"]); +format_type(#{kind := array, elements := ElementsType}) -> + to_bin(["Array(", format_type(ElementsType), ")"]); +format_type(#{kind := union, members := MemberTypes} = U) -> + DN = maps:get(display_name, U, undefined), + case DN of + undefined -> + to_bin(["OneOf(", format_union_members(MemberTypes), ")"]); + Name -> + format_primitive_type(Name) + end; +format_type(#{kind := struct, name := FullName}) -> + to_bin(["Struct(", short_name(FullName), ")"]); +format_type(#{kind := map, name := Name, values := ValuesType}) -> + to_bin(["Map($", Name, "->", format_type(ValuesType), ")"]). + +format_union_members(Members) -> + format_union_members(Members, []). + +format_union_members([], Acc) -> + lists:join(",", lists:reverse(Acc)); +format_union_members([Member | Members], Acc) -> + NewAcc = [format_type(Member) | Acc], + format_union_members(Members, NewAcc). + +format_primitive_type(TypeStr) -> + Spec = emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr), + to_bin(maps:get(type, Spec)). + +%% All types should have a namespace to avlid name clashing. +is_missing_namespace(ShortName, FullName, RootNames) -> + case lists:member(ShortName, RootNames) of + true -> + false; + false -> + ShortName =:= FullName + end. + +%% Returns short name from full name, fullname delemited by colon(:). +short_name(FullName) -> + case string:split(FullName, ":") of + [_, Name] -> to_bin(Name); + _ -> to_bin(FullName) + end. + +%% Returns the hash-anchor from full name, fullname delemited by colon(:). +format_hash(FullName) -> + case string:split(FullName, ":") of + [Namespace, Name] -> + ok = warn_bad_namespace(Namespace), + iolist_to_binary([Namespace, "__", Name]); + _ -> + iolist_to_binary(FullName) + end. + +%% namespace should only have letters, numbers, and underscores. +warn_bad_namespace(Namespace) -> + case re:run(Namespace, "^[a-zA-Z0-9_]+$", [{capture, none}]) of + nomatch -> + case erlang:get({bad_namespace, Namespace}) of + true -> + ok; + _ -> + erlang:put({bad_namespace, Namespace}, true), + io:format(standard_error, "WARN: bad_namespace: ~s~n", [Namespace]) + end; + _ -> + ok + end. %% TODO: move this function to emqx_dashboard when we stop generating this JSON at build time. hotconf_schema_json() -> @@ -306,12 +560,7 @@ hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> typename_to_spec(TypeStr, Module) -> emqx_conf_schema_types:readable_dashboard(Module, TypeStr). -to_bin(List) when is_list(List) -> - case io_lib:printable_list(List) of - true -> unicode:characters_to_binary(List); - false -> List - end; +to_bin(List) when is_list(List) -> iolist_to_binary(List); to_bin(Boolean) when is_boolean(Boolean) -> Boolean; to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -to_bin(X) -> - X. +to_bin(X) -> X. diff --git a/apps/emqx_psk/src/emqx_psk.app.src b/apps/emqx_psk/src/emqx_psk.app.src index be24112e4..abd862613 100644 --- a/apps/emqx_psk/src/emqx_psk.app.src +++ b/apps/emqx_psk/src/emqx_psk.app.src @@ -2,7 +2,7 @@ {application, emqx_psk, [ {description, "EMQX PSK"}, % strict semver, bump manually! - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {modules, []}, {registered, [emqx_psk_sup]}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index e6c922c1e..0a6e5d298 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -28,7 +28,7 @@ fields/1 ]). -namespace() -> "authn-psk". +namespace() -> "psk". roots() -> ["psk_authentication"].