refactor(schema): keep type converters close

This commit is contained in:
Zaiming (Stone) Shi 2023-11-13 12:50:34 +01:00
parent 518b02fc70
commit 4c5d64abc2
6 changed files with 343 additions and 259 deletions

View File

@ -101,7 +101,7 @@ fields(connector_config) ->
)},
{service_account_json,
sc(
typerefl:alias("map", ?MODULE:service_account_json()),
?MODULE:service_account_json(),
#{
required => true,
validator => fun ?MODULE:service_account_json_validator/1,

View File

@ -305,121 +305,8 @@ hocon_schema_to_spec(?UNION(Types, _DisplayName), LocalModule) ->
hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->
{#{type => enum, symbols => [Atom]}, []}.
typename_to_spec("boolean()", _Mod) ->
#{type => boolean};
typename_to_spec("binary()", _Mod) ->
#{type => string};
typename_to_spec("float()", _Mod) ->
#{type => number};
typename_to_spec("integer()", _Mod) ->
#{type => number};
typename_to_spec("pos_integer()", _Mod) ->
#{type => integer};
typename_to_spec("non_neg_integer()", _Mod) ->
#{type => number, minimum => 0};
typename_to_spec("number()", _Mod) ->
#{type => number};
typename_to_spec("string()", _Mod) ->
#{type => string};
typename_to_spec("atom()", _Mod) ->
#{type => string};
typename_to_spec("duration()", _Mod) ->
#{type => duration};
typename_to_spec("timeout_duration()", _Mod) ->
#{type => duration};
typename_to_spec("duration_s()", _Mod) ->
#{type => duration};
typename_to_spec("timeout_duration_s()", _Mod) ->
#{type => duration};
typename_to_spec("duration_ms()", _Mod) ->
#{type => duration};
typename_to_spec("timeout_duration_ms()", _Mod) ->
#{type => duration};
typename_to_spec("percent()", _Mod) ->
#{type => percent};
typename_to_spec("ip_port()", _Mod) ->
#{type => ip_port};
typename_to_spec("url()", _Mod) ->
#{type => url};
typename_to_spec("bytesize()", _Mod) ->
#{type => 'byteSize'};
typename_to_spec("wordsize()", _Mod) ->
#{type => 'byteSize'};
typename_to_spec("qos()", _Mod) ->
#{type => enum, symbols => [0, 1, 2]};
typename_to_spec("comma_separated_list()", _Mod) ->
#{type => comma_separated_string};
typename_to_spec("comma_separated_atoms()", _Mod) ->
#{type => comma_separated_string};
typename_to_spec("map(" ++ Map, _Mod) ->
[$) | _MapArgs] = lists:reverse(Map),
#{type => object};
typename_to_spec("port_number()", _Mod) ->
#{type => integer};
typename_to_spec(Name, Mod) ->
Spec = range(Name),
Spec1 = remote_module_type(Spec, Name, Mod),
Spec2 = typerefl_array(Spec1, Name, Mod),
Spec3 = integer(Spec2, Name),
default_type(Mod, Name, Spec3).
default_type(Mod, Name, nomatch) ->
error({unknown_type, Mod, Name});
default_type(_Mod, _Name, Type) ->
Type.
range(Name) ->
case string:split(Name, "..") of
%% 1..10 1..inf -inf..10
[MinStr, MaxStr] ->
Schema = #{type => number},
Schema1 = add_integer_prop(Schema, minimum, MinStr),
add_integer_prop(Schema1, maximum, MaxStr);
_ ->
nomatch
end.
%% Module:Type
remote_module_type(nomatch, Name, Mod) ->
case string:split(Name, ":") of
[_Module, Type] -> typename_to_spec(Type, Mod);
_ -> nomatch
end;
remote_module_type(Spec, _Name, _Mod) ->
Spec.
%% [string()] or [integer()] or [xxx].
typerefl_array(nomatch, Name, Mod) ->
case string:trim(Name, leading, "[") of
Name ->
nomatch;
Name1 ->
case string:trim(Name1, trailing, "]") of
Name1 ->
notmatch;
Name2 ->
Schema = typename_to_spec(Name2, Mod),
#{type => array, items => Schema}
end
end;
typerefl_array(Spec, _Name, _Mod) ->
Spec.
%% integer(1)
integer(nomatch, Name) ->
case string:to_integer(Name) of
{Int, []} -> #{type => enum, symbols => [Int], default => Int};
_ -> nomatch
end;
integer(Spec, _Name) ->
Spec.
add_integer_prop(Schema, Key, Value) ->
case string:to_integer(Value) of
{error, no_integer} -> Schema;
{Int, []} when Key =:= minimum -> Schema#{Key => Int};
{Int, []} -> Schema#{Key => Int}
end.
typename_to_spec(TypeStr, Module) ->
emqx_conf_schema_types:readable_dashboard(Module, TypeStr).
to_bin(List) when is_list(List) ->
case io_lib:printable_list(List) of

View File

@ -0,0 +1,335 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_conf_schema_types).
-export([readable/2]).
-export([readable_swagger/2, readable_dashboard/2, readable_docgen/2]).
%% Takes a typerefl name or hocon schema's display name and returns
%% a map of different flavors of more readable type specs.
%% - swagger: for swagger spec
%% - dashboard: to facilitate the dashboard UI rendering
%% - docgen: for documenation generation
readable(Module, TypeStr) when is_binary(TypeStr) ->
readable(Module, binary_to_list(TypeStr));
readable(Module, TypeStr) when is_list(TypeStr) ->
try
%% Module is ignored so far as all types are distinguished by their names
readable(TypeStr)
catch
throw:unknown_type ->
fail(#{reason => unknown_type, type => TypeStr, module => Module})
end.
readable_swagger(Module, TypeStr) ->
get_readable(Module, TypeStr, swagger).
readable_dashboard(Module, TypeStr) ->
get_readable(Module, TypeStr, dashboard).
readable_docgen(Module, TypeStr) ->
get_readable(Module, TypeStr, docgen).
get_readable(Module, TypeStr, Flavor) ->
Map = readable(Module, TypeStr),
case maps:get(Flavor, Map, undefined) of
undefined -> fail(#{reason => unknown_type, module => Module, type => TypeStr});
Value -> Value
end.
%% Fail the build or test. Production code should never get here.
-spec fail(_) -> no_return().
fail(Reason) ->
io:format(standard_error, "ERROR: ~p~n", [Reason]),
error(Reason).
readable("boolean()") ->
#{
swagger => #{type => boolean},
dashboard => #{type => boolean},
docgen => #{type => "Boolean"}
};
readable("binary()") ->
#{
swagger => #{type => string},
dashboard => #{type => string},
docgen => #{type => "String"}
};
readable("float()") ->
#{
swagger => #{type => number},
dashboard => #{type => number},
docgen => #{type => "Float"}
};
readable("integer()") ->
#{
swagger => #{type => integer},
dashboard => #{type => integer},
docgen => #{type => "Integer"}
};
readable("non_neg_integer()") ->
#{
swagger => #{type => integer, minimum => 0},
dashboard => #{type => integer, minimum => 0},
docgen => #{type => "Integer(0..+inf)"}
};
readable("pos_integer()") ->
#{
swagger => #{type => integer, minimum => 1},
dashboard => #{type => integer, minimum => 1},
docgen => #{type => "Integer(1..+inf)"}
};
readable("number()") ->
#{
swagger => #{type => number},
dashboard => #{type => number},
docgen => #{type => "Number"}
};
readable("string()") ->
#{
swagger => #{type => string},
dashboard => #{type => string},
docgen => #{type => "String"}
};
readable("atom()") ->
#{
swagger => #{type => string},
dashboard => #{type => string},
docgen => #{type => "String"}
};
readable("epoch_second()") ->
%% only for swagger
#{
swagger => #{
<<"oneOf">> => [
#{type => integer, example => 1640995200, description => <<"epoch-second">>},
#{
type => string,
example => <<"2022-01-01T00:00:00.000Z">>,
format => <<"date-time">>
}
]
}
};
readable("epoch_millisecond()") ->
%% only for swagger
#{
swagger => #{
<<"oneOf">> => [
#{
type => integer,
example => 1640995200000,
description => <<"epoch-millisecond">>
},
#{
type => string,
example => <<"2022-01-01T00:00:00.000Z">>,
format => <<"date-time">>
}
]
}
};
readable("duration()") ->
#{
swagger => #{type => string, example => <<"12m">>},
dashboard => #{type => duration},
docgen => #{type => "String", example => <<"12m">>}
};
readable("duration_s()") ->
#{
swagger => #{type => string, example => <<"1h">>},
dashboard => #{type => duration},
docgen => #{type => "String", example => <<"1h">>}
};
readable("duration_ms()") ->
#{
swagger => #{type => string, example => <<"32s">>},
dashboard => #{type => duration},
docgen => #{type => "String", example => <<"32s">>}
};
readable("timeout_duration()") ->
#{
swagger => #{type => string, example => <<"12m">>},
dashboard => #{type => duration},
docgen => #{type => "String", example => <<"12m">>}
};
readable("timeout_duration_s()") ->
#{
swagger => #{type => string, example => <<"1h">>},
dashboard => #{type => duration},
docgen => #{type => "String", example => <<"1h">>}
};
readable("timeout_duration_ms()") ->
#{
swagger => #{type => string, example => <<"32s">>},
dashboard => #{type => duration},
docgen => #{type => "String", example => <<"32s">>}
};
readable("percent()") ->
#{
swagger => #{type => string, example => <<"12%">>},
dashboard => #{type => percent},
docgen => #{type => "String", example => <<"12%">>}
};
readable("ip_port()") ->
#{
swagger => #{type => string, example => <<"127.0.0.1:80">>},
dashboard => #{type => ip_port},
docgen => #{type => "String", example => <<"127.0.0.1:80">>}
};
readable("url()") ->
#{
swagger => #{type => string, example => <<"http://127.0.0.1">>},
dashboard => #{type => url},
docgen => #{type => "String", example => <<"http://127.0.0.1">>}
};
readable("bytesize()") ->
#{
swagger => #{type => string, example => <<"32MB">>},
dashboard => #{type => 'byteSize'},
docgen => #{type => "String", example => <<"32MB">>}
};
readable("wordsize()") ->
#{
swagger => #{type => string, example => <<"1024KB">>},
dashboard => #{type => 'wordSize'},
docgen => #{type => "String", example => <<"1024KB">>}
};
readable("map(" ++ Map) ->
[$) | _MapArgs] = lists:reverse(Map),
%% TODO: for docgen, parse map args. e.g. Map(String,String)
#{
swagger => #{type => object, example => #{}},
dashboard => #{type => object},
docgen => #{type => "Map", example => #{}}
};
readable("qos()") ->
#{
swagger => #{type => integer, minimum => 0, maximum => 2, example => 0},
dashboard => #{type => enum, symbols => [0, 1, 2]},
docgen => #{type => "Integer(0..2)", example => 0}
};
readable("comma_separated_list()") ->
#{
swagger => #{type => string, example => <<"item1,item2">>},
dashboard => #{type => comma_separated_string},
docgen => #{type => "String", example => <<"item1,item2">>}
};
readable("comma_separated_binary()") ->
#{
swagger => #{type => string, example => <<"item1,item2">>},
dashboard => #{type => comma_separated_string},
docgen => #{type => "String", example => <<"item1,item2">>}
};
readable("comma_separated_atoms()") ->
#{
swagger => #{type => string, example => <<"item1,item2">>},
dashboard => #{type => comma_separated_string},
docgen => #{type => "String", example => <<"item1,item2">>}
};
readable("service_account_json()") ->
%% This is a bit special,
%% service_account_josn in swagger spec is an object
%% the same in documenation.
%% However, dashboard wish it to be a string
%% TODO:
%% - Change type definition to stirng().
%% - Convert the embedded object to a escaped JSON string.
%% - Delete this function clause once the above is done.
#{
swagger => #{type => object},
dashboard => #{type => string},
docgen => #{type => "Map"}
};
readable("json_binary()") ->
#{
swagger => #{type => string, example => <<"{\"a\": [1,true]}">>},
dashboard => #{type => object},
docgen => #{type => "String", example => <<"{\"a\": [1,true]}">>}
};
readable("port_number()") ->
Result = try_range("1..65535"),
true = is_map(Result),
Result;
readable(TypeStr0) ->
case string:split(TypeStr0, ":") of
[ModuleStr, TypeStr] ->
Module = list_to_existing_atom(ModuleStr),
readable(Module, TypeStr);
_ ->
parse(TypeStr0)
end.
parse(TypeStr) ->
try_parse(TypeStr, [
fun try_typerefl_array/1,
fun try_range/1
]).
try_parse(_TypeStr, []) ->
throw(unknown_type);
try_parse(TypeStr, [ParseFun | More]) ->
case ParseFun(TypeStr) of
nomatch ->
try_parse(TypeStr, More);
Result ->
Result
end.
%% [string()] or [integer()] or [xxx] or [xxx,...]
try_typerefl_array(Name) ->
case string:trim(Name, leading, "[") of
Name ->
nomatch;
Name1 ->
case string:trim(Name1, trailing, ",.]") of
Name1 ->
notmatch;
Name2 ->
Flavors = readable(Name2),
DocgenSpec = maps:get(docgen, Flavors),
DocgenType = maps:get(type, DocgenSpec),
#{
swagger => #{type => array, items => maps:get(swagger, Flavors)},
dashboard => #{type => array, items => maps:get(dashboard, Flavors)},
docgen => #{type => "Array(" ++ DocgenType ++ ")"}
}
end
end.
try_range(Name) ->
case string:split(Name, "..") of
%% 1..10 1..inf -inf..10
[MinStr, MaxStr] ->
Schema0 = #{type => integer},
Schema1 = add_integer_prop(Schema0, minimum, MinStr),
Schema = add_integer_prop(Schema1, maximum, MaxStr),
#{
swagger => Schema,
dashboard => Schema,
docgen => #{type => "Integer(" ++ MinStr ++ ".." ++ MaxStr ++ ")"}
};
_ ->
nomatch
end.
add_integer_prop(Schema, Key, Value) ->
case string:to_integer(Value) of
{error, no_integer} -> Schema;
{Int, []} when Key =:= minimum -> Schema#{Key => Int};
{Int, []} -> Schema#{Key => Int}
end.

View File

@ -20,13 +20,13 @@
-include_lib("hocon/include/hoconsc.hrl").
-export([
pool_size/1,
relational_db_fields/0,
ssl_fields/0,
prepare_statement_fields/0
]).
-export([
pool_size/1,
database/1,
username/1,
password/1,
@ -35,13 +35,11 @@
]).
-type database() :: binary().
-type pool_size() :: pos_integer().
-type username() :: binary().
-type password() :: binary().
-reflect_type([
database/0,
pool_size/0,
username/0,
password/0
]).

View File

@ -799,140 +799,8 @@ hocon_schema_to_spec(?UNION(Types, _DisplayName), LocalModule) ->
hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->
{#{type => string, enum => [Atom]}, []}.
typename_to_spec("boolean()", _Mod) ->
#{type => boolean};
typename_to_spec("binary()", _Mod) ->
#{type => string};
typename_to_spec("float()", _Mod) ->
#{type => number};
typename_to_spec("integer()", _Mod) ->
#{type => integer};
typename_to_spec("non_neg_integer()", _Mod) ->
#{type => integer, minimum => 0};
typename_to_spec("pos_integer()", _Mod) ->
#{type => integer, minimum => 1};
typename_to_spec("number()", _Mod) ->
#{type => number};
typename_to_spec("string()", _Mod) ->
#{type => string};
typename_to_spec("atom()", _Mod) ->
#{type => string};
typename_to_spec("epoch_second()", _Mod) ->
#{
<<"oneOf">> => [
#{type => integer, example => 1640995200, description => <<"epoch-second">>},
#{type => string, example => <<"2022-01-01T00:00:00.000Z">>, format => <<"date-time">>}
]
};
typename_to_spec("epoch_millisecond()", _Mod) ->
#{
<<"oneOf">> => [
#{type => integer, example => 1640995200000, description => <<"epoch-millisecond">>},
#{type => string, example => <<"2022-01-01T00:00:00.000Z">>, format => <<"date-time">>}
]
};
typename_to_spec("duration()", _Mod) ->
#{type => string, example => <<"12m">>};
typename_to_spec("duration_s()", _Mod) ->
#{type => string, example => <<"1h">>};
typename_to_spec("duration_ms()", _Mod) ->
#{type => string, example => <<"32s">>};
typename_to_spec("timeout_duration()", _Mod) ->
#{type => string, example => <<"12m">>};
typename_to_spec("timeout_duration_s()", _Mod) ->
#{type => string, example => <<"1h">>};
typename_to_spec("timeout_duration_ms()", _Mod) ->
#{type => string, example => <<"32s">>};
typename_to_spec("percent()", _Mod) ->
#{type => number, example => <<"12%">>};
typename_to_spec("ip_port()", _Mod) ->
#{type => string, example => <<"127.0.0.1:80">>};
typename_to_spec("url()", _Mod) ->
#{type => string, example => <<"http://127.0.0.1">>};
typename_to_spec("bytesize()", _Mod) ->
#{type => string, example => <<"32MB">>};
typename_to_spec("wordsize()", _Mod) ->
#{type => string, example => <<"1024KB">>};
typename_to_spec("map(" ++ Map, _Mod) ->
[$) | _MapArgs] = lists:reverse(Map),
#{type => object, example => #{}};
typename_to_spec("qos()", _Mod) ->
#{type => integer, minimum => 0, maximum => 2, example => 0};
typename_to_spec("comma_separated_list()", _Mod) ->
#{type => string, example => <<"item1,item2">>};
typename_to_spec("comma_separated_binary()", _Mod) ->
#{type => string, example => <<"item1,item2">>};
typename_to_spec("comma_separated_atoms()", _Mod) ->
#{type => string, example => <<"item1,item2">>};
typename_to_spec("json_binary()", _Mod) ->
#{type => string, example => <<"{\"a\": [1,true]}">>};
typename_to_spec("port_number()", _Mod) ->
range("1..65535");
typename_to_spec(Name, Mod) ->
try_convert_to_spec(Name, Mod, [
fun try_remote_module_type/2,
fun try_typerefl_array/2,
fun try_range/2,
fun try_integer/2
]).
range(Name) ->
#{} = try_range(Name, undefined).
try_convert_to_spec(Name, Mod, []) ->
throw({error, #{msg => <<"Unsupported Type">>, type => Name, module => Mod}});
try_convert_to_spec(Name, Mod, [Converter | Rest]) ->
case Converter(Name, Mod) of
nomatch -> try_convert_to_spec(Name, Mod, Rest);
Spec -> Spec
end.
try_range(Name, _Mod) ->
case string:split(Name, "..") of
%% 1..10 1..inf -inf..10
[MinStr, MaxStr] ->
Schema = #{type => integer},
Schema1 = add_integer_prop(Schema, minimum, MinStr),
add_integer_prop(Schema1, maximum, MaxStr);
_ ->
nomatch
end.
%% Module:Type
try_remote_module_type(Name, Mod) ->
case string:split(Name, ":") of
[_Module, Type] -> typename_to_spec(Type, Mod);
_ -> nomatch
end.
%% [string()] or [integer()] or [xxx] or [xxx,...]
try_typerefl_array(Name, Mod) ->
case string:trim(Name, leading, "[") of
Name ->
nomatch;
Name1 ->
case string:trim(Name1, trailing, ",.]") of
Name1 ->
notmatch;
Name2 ->
Schema = typename_to_spec(Name2, Mod),
#{type => array, items => Schema}
end
end.
%% integer(1)
try_integer(Name, _Mod) ->
case string:to_integer(Name) of
{Int, []} -> #{type => integer, enum => [Int], default => Int};
_ -> nomatch
end.
add_integer_prop(Schema, Key, Value) ->
case string:to_integer(Value) of
{error, no_integer} -> Schema;
{Int, []} when Key =:= minimum -> Schema#{Key => Int};
{Int, []} -> Schema#{Key => Int}
end.
typename_to_spec(TypeStr, Module) ->
emqx_conf_schema_types:readable_swagger(Module, TypeStr).
to_bin(List) when is_list(List) ->
case io_lib:printable_list(List) of

View File

@ -375,9 +375,6 @@ t_complex_type(_Config) ->
all
],
type := string
}},
{<<"fix_integer">>, #{
default := 100, enum := [100], type := integer
}}
],
Properties
@ -413,7 +410,7 @@ t_ref_array_with_key(_Config) ->
{<<"percent_ex">>, #{
description => <<"percent example">>,
example => <<"12%">>,
type => number
type => string
}},
{<<"duration_ms_ex">>, #{
description => <<"duration ms example">>,
@ -659,8 +656,7 @@ schema("/ref/complex_type") ->
{maps, hoconsc:mk(map(), #{})},
{comma_separated_list, hoconsc:mk(emqx_schema:comma_separated_list(), #{})},
{comma_separated_atoms, hoconsc:mk(emqx_schema:comma_separated_atoms(), #{})},
{log_level, hoconsc:mk(emqx_conf_schema:log_level(), #{})},
{fix_integer, hoconsc:mk(typerefl:integer(100), #{})}
{log_level, hoconsc:mk(emqx_conf_schema:log_level(), #{})}
]
}
}