yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #12559 from zmstone/0221-refactor-use-atom-fileds 

refactor: use atoms for root config fields
This commit is contained in:
Zaiming (Stone) Shi 2024-02-23 14:38:19 +01:00 committed by GitHub
parent 15f919e60f 88b1d9ba88
commit 5af01c041b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 119 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/emqx/src/emqx_config.erl
View File

@ -960,7 +960,7 @@ is_zone_root(Name) ->
-spec zone_roots() -> [atom()]. -spec zone_roots() -> [atom()].
zone_roots() -> zone_roots() ->
lists:map(fun list_to_atom/1, emqx_zone_schema:roots()). emqx_zone_schema:roots().
%%% %%%
%%% @doc During init, ensure order of puts that zone is put after the other global defaults. %%% @doc During init, ensure order of puts that zone is put after the other global defaults.

146
apps/emqx/src/emqx_schema.erl
View File

@ -139,6 +139,8 @@
get_tombstone_map_value_type/1 get_tombstone_map_value_type/1
]). ]).
-export([listeners/0]).
-behaviour(hocon_schema). -behaviour(hocon_schema).
-reflect_type([ -reflect_type([
@ -193,12 +195,12 @@ roots() ->
roots(high) -> roots(high) ->
[ [
{"listeners", {listeners,
sc( sc(
ref("listeners"), ref("listeners"),
#{importance => ?IMPORTANCE_HIGH} #{importance => ?IMPORTANCE_HIGH}
)}, )},
{"mqtt", {mqtt,
sc( sc(
ref("mqtt"), ref("mqtt"),
#{ #{
@ -207,9 +209,9 @@ roots(high) ->
importance => ?IMPORTANCE_MEDIUM importance => ?IMPORTANCE_MEDIUM
} }
)}, )},
{"zones", {zones,
sc( sc(
map("name", ref("zone")), map(name, ref("zone")),
#{ #{
desc => ?DESC(zones), desc => ?DESC(zones),
importance => ?IMPORTANCE_HIDDEN importance => ?IMPORTANCE_HIDDEN
@ -221,7 +223,7 @@ roots(high) ->
[ [
%% NOTE: authorization schema here is only to keep emqx app pure %% NOTE: authorization schema here is only to keep emqx app pure
%% the full schema for EMQX node is injected in emqx_conf_schema. %% the full schema for EMQX node is injected in emqx_conf_schema.
{?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, {?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_ATOM,
sc( sc(
ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME), ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME),
#{importance => ?IMPORTANCE_HIDDEN} #{importance => ?IMPORTANCE_HIDDEN}
@ -238,17 +240,17 @@ roots(medium) ->
importance => ?IMPORTANCE_HIDDEN importance => ?IMPORTANCE_HIDDEN
} }
)}, )},
{"sys_topics", {sys_topics,
sc( sc(
ref("sys_topics"), ref("sys_topics"),
#{desc => ?DESC(sys_topics)} #{desc => ?DESC(sys_topics)}
)}, )},
{"force_shutdown", {force_shutdown,
sc( sc(
ref("force_shutdown"), ref("force_shutdown"),
#{} #{}
)}, )},
{"overload_protection", {overload_protection,
sc( sc(
ref("overload_protection"), ref("overload_protection"),
#{importance => ?IMPORTANCE_HIDDEN} #{importance => ?IMPORTANCE_HIDDEN}
@ -256,36 +258,36 @@ roots(medium) ->
]; ];
roots(low) -> roots(low) ->
[ [
{"force_gc", {force_gc,
sc( sc(
ref("force_gc"), ref("force_gc"),
#{} #{}
)}, )},
{"conn_congestion", {conn_congestion,
sc( sc(
ref("conn_congestion"), ref("conn_congestion"),
#{ #{
importance => ?IMPORTANCE_HIDDEN importance => ?IMPORTANCE_HIDDEN
} }
)}, )},
{"stats", {stats,
sc( sc(
ref("stats"), ref("stats"),
#{ #{
importance => ?IMPORTANCE_HIDDEN importance => ?IMPORTANCE_HIDDEN
} }
)}, )},
{"sysmon", {sysmon,
sc( sc(
ref("sysmon"), ref("sysmon"),
#{} #{}
)}, )},
{"alarm", {alarm,
sc( sc(
ref("alarm"), ref("alarm"),
#{} #{}
)}, )},
{"flapping_detect", {flapping_detect,
sc( sc(
ref("flapping_detect"), ref("flapping_detect"),
#{ #{
@ -293,7 +295,7 @@ roots(low) ->
converter => fun flapping_detect_converter/2 converter => fun flapping_detect_converter/2
} }
)}, )},
{"persistent_session_store", {persistent_session_store,
sc( sc(
ref("persistent_session_store"), ref("persistent_session_store"),
#{ #{
@ -303,19 +305,19 @@ roots(low) ->
importance => ?IMPORTANCE_HIDDEN importance => ?IMPORTANCE_HIDDEN
} }
)}, )},
{"session_persistence", {session_persistence,
sc( sc(
ref("session_persistence"), ref("session_persistence"),
#{ #{
importance => ?IMPORTANCE_HIDDEN importance => ?IMPORTANCE_HIDDEN
} }
)}, )},
{"trace", {trace,
sc( sc(
ref("trace"), ref("trace"),
#{importance => ?IMPORTANCE_HIDDEN} #{importance => ?IMPORTANCE_HIDDEN}
)}, )},
{"crl_cache", {crl_cache,
sc( sc(
ref("crl_cache"), ref("crl_cache"),
#{importance => ?IMPORTANCE_HIDDEN} #{importance => ?IMPORTANCE_HIDDEN}
@ -441,7 +443,7 @@ fields("stats") ->
]; ];
fields("authorization") -> fields("authorization") ->
authz_fields(); authz_fields();
fields(authz_cache) -> fields("authz_cache") ->
[ [
{enable, {enable,
sc( sc(
@ -630,55 +632,7 @@ fields("force_gc") ->
)} )}
]; ];
fields("listeners") -> fields("listeners") ->
[ listeners();
{"tcp",
sc(
tombstone_map(name, ref("mqtt_tcp_listener")),
#{
desc => ?DESC(fields_listeners_tcp),
converter => fun(X, _) ->
ensure_default_listener(X, tcp)
end,
required => {false, recursively}
}
)},
{"ssl",
sc(
tombstone_map(name, ref("mqtt_ssl_listener")),
#{
desc => ?DESC(fields_listeners_ssl),
converter => fun(X, _) -> ensure_default_listener(X, ssl) end,
required => {false, recursively}
}
)},
{"ws",
sc(
tombstone_map(name, ref("mqtt_ws_listener")),
#{
desc => ?DESC(fields_listeners_ws),
converter => fun(X, _) -> ensure_default_listener(X, ws) end,
required => {false, recursively}
}
)},
{"wss",
sc(
tombstone_map(name, ref("mqtt_wss_listener")),
#{
desc => ?DESC(fields_listeners_wss),
converter => fun(X, _) -> ensure_default_listener(X, wss) end,
required => {false, recursively}
}
)},
{"quic",
sc(
tombstone_map(name, ref("mqtt_quic_listener")),
#{
desc => ?DESC(fields_listeners_quic),
converter => fun keep_default_tombstone/2,
required => {false, recursively}
}
)}
];
fields("crl_cache") -> fields("crl_cache") ->
%% Note: we make the refresh interval and HTTP timeout global (not %% Note: we make the refresh interval and HTTP timeout global (not
%% per-listener) because multiple SSL listeners might point to the %% per-listener) because multiple SSL listeners might point to the
@ -2082,7 +2036,7 @@ desc("authorization") ->
"Settings for client authorization."; "Settings for client authorization.";
desc("mqtt") -> desc("mqtt") ->
"Global MQTT configuration."; "Global MQTT configuration.";
desc(authz_cache) -> desc("authz_cache") ->
"Settings for the authorization cache."; "Settings for the authorization cache.";
desc("zone") -> desc("zone") ->
"A `Zone` defines a set of configuration items (such as the maximum number of connections)" "A `Zone` defines a set of configuration items (such as the maximum number of connections)"
@ -2644,7 +2598,7 @@ authz_fields() ->
)}, )},
{"cache", {"cache",
sc( sc(
ref(?MODULE, authz_cache), ref(?MODULE, "authz_cache"),
#{} #{}
)} )}
]. ].
@ -3592,6 +3546,7 @@ flapping_detect_converter(Conf = #{<<"window_time">> := <<"disable">>}, _Opts) -
Conf#{<<"window_time">> => ?DEFAULT_WINDOW_TIME, <<"enable">> => false}; Conf#{<<"window_time">> => ?DEFAULT_WINDOW_TIME, <<"enable">> => false};
flapping_detect_converter(Conf, _Opts) -> flapping_detect_converter(Conf, _Opts) ->
Conf. Conf.
mqtt_general() -> mqtt_general() ->
[ [
{"idle_timeout", {"idle_timeout",
@ -3923,3 +3878,54 @@ ensure_unicode_path(Path, _) when is_list(Path) ->
Path; Path;
ensure_unicode_path(Path, _) -> ensure_unicode_path(Path, _) ->
throw({"not_string", Path}). throw({"not_string", Path}).
listeners() ->
[
{"tcp",
sc(
tombstone_map(name, ref("mqtt_tcp_listener")),
#{
desc => ?DESC(fields_listeners_tcp),
converter => fun(X, _) ->
ensure_default_listener(X, tcp)
end,
required => {false, recursively}
}
)},
{"ssl",
sc(
tombstone_map(name, ref("mqtt_ssl_listener")),
#{
desc => ?DESC(fields_listeners_ssl),
converter => fun(X, _) -> ensure_default_listener(X, ssl) end,
required => {false, recursively}
}
)},
{"ws",
sc(
tombstone_map(name, ref("mqtt_ws_listener")),
#{
desc => ?DESC(fields_listeners_ws),
converter => fun(X, _) -> ensure_default_listener(X, ws) end,
required => {false, recursively}
}
)},
{"wss",
sc(
tombstone_map(name, ref("mqtt_wss_listener")),
#{
desc => ?DESC(fields_listeners_wss),
converter => fun(X, _) -> ensure_default_listener(X, wss) end,
required => {false, recursively}
}
)},
{"quic",
sc(
tombstone_map(name, ref("mqtt_quic_listener")),
#{
desc => ?DESC(fields_listeners_quic),
converter => fun keep_default_tombstone/2,
required => {false, recursively}
}
)}
].

33
apps/emqx/src/emqx_zone_schema.erl
View File

@ -23,17 +23,17 @@
namespace() -> zone. namespace() -> zone.
%% this schema module is not used at root level. %% Zone values are never checked as root level.
%% roots are added only for document generation. %% We need roots defined here because it's used to generate config API schema.
roots() -> roots() ->
[ [
"mqtt", mqtt,
"stats", stats,
"flapping_detect", flapping_detect,
"force_shutdown", force_shutdown,
"conn_congestion", conn_congestion,
"force_gc", force_gc,
"overload_protection" overload_protection
]. ].
zones_without_default() -> zones_without_default() ->
@ -43,22 +43,25 @@ zones_without_default() ->
fun(F) -> fun(F) ->
case lists:member(F, Hidden) of case lists:member(F, Hidden) of
true -> true ->
{F, ?HOCON(?R_REF(F), #{importance => ?IMPORTANCE_HIDDEN})}; {F,
?HOCON(?R_REF(?MODULE, atom_to_list(F)), #{importance => ?IMPORTANCE_HIDDEN})};
false -> false ->
{F, ?HOCON(?R_REF(F), #{})} {F, ?HOCON(?R_REF(?MODULE, atom_to_list(F)), #{})}
end end
end, end,
Fields Fields
). ).
global_zone_with_default() -> global_zone_with_default() ->
lists:map(fun(F) -> {F, ?HOCON(?R_REF(emqx_schema, F), #{})} end, roots() -- hidden()). lists:map(
fun(F) -> {F, ?HOCON(?R_REF(emqx_schema, atom_to_list(F)), #{})} end, roots() -- hidden()
).
hidden() -> hidden() ->
[ [
"stats", stats,
"overload_protection", overload_protection,
"conn_congestion" conn_congestion
]. ].
%% zone schemas are clones from the same name from root level %% zone schemas are clones from the same name from root level

11
apps/emqx/test/emqx_message_SUITE.erl
View File

@ -144,12 +144,13 @@ t_undefined_headers(_) ->
t_is_expired_1(_) -> t_is_expired_1(_) ->
test_msg_expired_property(?MODULE). test_msg_expired_property(?MODULE).
make_zone_default_conf() ->
maps:from_list([{Root, #{}} || Root <- emqx_zone_schema:roots()]).
t_is_expired_2(_) -> t_is_expired_2(_) ->
%% if the 'Message-Expiry-Interval' property is set, the message_expiry_interval should be ignored %% if the 'Message-Expiry-Interval' property is set, the message_expiry_interval should be ignored
try try
emqx_config:put( emqx_config:put(make_zone_default_conf()),
maps:from_list([{list_to_atom(Root), #{}} || Root <- emqx_zone_schema:roots()])
),
emqx_config:put_zone_conf(?MODULE, [mqtt, message_expiry_interval], timer:seconds(10)), emqx_config:put_zone_conf(?MODULE, [mqtt, message_expiry_interval], timer:seconds(10)),
test_msg_expired_property(?MODULE) test_msg_expired_property(?MODULE)
after after
@ -158,9 +159,7 @@ t_is_expired_2(_) ->
t_is_expired_3(_) -> t_is_expired_3(_) ->
try try
emqx_config:put( emqx_config:put(make_zone_default_conf()),
maps:from_list([{list_to_atom(Root), #{}} || Root <- emqx_zone_schema:roots()])
),
emqx_config:put_zone_conf(?MODULE, [mqtt, message_expiry_interval], 100), emqx_config:put_zone_conf(?MODULE, [mqtt, message_expiry_interval], 100),
Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>),
?assertNot(emqx_message:is_expired(Msg, ?MODULE)), ?assertNot(emqx_message:is_expired(Msg, ?MODULE)),

3
apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl
View File

@ -72,6 +72,7 @@ roots() -> [].
namespace() -> undefined. namespace() -> undefined.
%% "authorization"
fields(?CONF_NS) -> fields(?CONF_NS) ->
emqx_schema:authz_fields() ++ authz_fields(); emqx_schema:authz_fields() ++ authz_fields();
fields("metrics_status_fields") -> fields("metrics_status_fields") ->
@ -127,7 +128,7 @@ injected_fields(AuthzSchemaMods) ->
persistent_term:put(?AUTHZ_MODS_PT_KEY, AuthzSchemaMods), persistent_term:put(?AUTHZ_MODS_PT_KEY, AuthzSchemaMods),
#{ #{
'roots.high' => [ 'roots.high' => [
{?CONF_NS, ?HOCON(?R_REF(?CONF_NS), #{desc => ?DESC(?CONF_NS)})} {?CONF_NS_ATOM, ?HOCON(?R_REF(?CONF_NS), #{desc => ?DESC(?CONF_NS)})}
] ]
}. }.

16
apps/emqx_conf/src/emqx_conf_schema.erl
View File

@ -99,19 +99,19 @@ roots() ->
ok = emqx_schema_hooks:inject_from_modules(?INJECTING_CONFIGS), ok = emqx_schema_hooks:inject_from_modules(?INJECTING_CONFIGS),
emqx_schema_high_prio_roots() ++ emqx_schema_high_prio_roots() ++
[ [
{"node", {node,
sc( sc(
?R_REF("node"), ?R_REF("node"),
#{ #{
translate_to => ["emqx"] translate_to => ["emqx"]
} }
)}, )},
{"cluster", {cluster,
sc( sc(
?R_REF("cluster"), ?R_REF("cluster"),
#{translate_to => ["ekka"]} #{translate_to => ["ekka"]}
)}, )},
{"log", {log,
sc( sc(
?R_REF("log"), ?R_REF("log"),
#{ #{
@ -119,7 +119,7 @@ roots() ->
importance => ?IMPORTANCE_HIGH importance => ?IMPORTANCE_HIGH
} }
)}, )},
{"rpc", {rpc,
sc( sc(
?R_REF("rpc"), ?R_REF("rpc"),
#{ #{
@ -1421,19 +1421,19 @@ roots(Module) ->
lists:map(fun({_BinName, Root}) -> Root end, hocon_schema:roots(Module)). lists:map(fun({_BinName, Root}) -> Root end, hocon_schema:roots(Module)).
%% Like authentication schema, authorization schema is incomplete in emqx_schema %% Like authentication schema, authorization schema is incomplete in emqx_schema
%% module, this function replaces the root field "authorization" with a new schema %% module, this function replaces the root field 'authorization' with a new schema
emqx_schema_high_prio_roots() -> emqx_schema_high_prio_roots() ->
Roots = emqx_schema:roots(high), Roots = emqx_schema:roots(high),
Authz = Authz =
{"authorization", {authorization,
sc( sc(
?R_REF("authorization"), ?R_REF("authorization"),
#{ #{
desc => ?DESC(authorization), desc => ?DESC("authorization"),
importance => ?IMPORTANCE_HIGH importance => ?IMPORTANCE_HIGH
} }
)}, )},
lists:keyreplace("authorization", 1, Roots, Authz). lists:keyreplace(authorization, 1, Roots, Authz).
validate_time_offset(Offset) -> validate_time_offset(Offset) ->
ValidTimeOffset = "^([\\-\\+][0-1][0-9]:[0-6][0-9]|system|utc)$", ValidTimeOffset = "^([\\-\\+][0-1][0-9]:[0-6][0-9]|system|utc)$",

2
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -234,7 +234,7 @@ file_schema(FileName) ->
}. }.
gen_api_schema_json_iodata(SchemaMod, SchemaInfo, Converter) -> gen_api_schema_json_iodata(SchemaMod, SchemaInfo, Converter) ->
{ApiSpec0, Components0} = emqx_dashboard_swagger:spec( {ApiSpec0, Components0} = spec(
SchemaMod, SchemaMod,
#{ #{
schema_converter => Converter, schema_converter => Converter,

2
apps/emqx_enterprise/src/emqx_enterprise.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_enterprise, [ {application, emqx_enterprise, [
{description, "EMQX Enterprise Edition"}, {description, "EMQX Enterprise Edition"},
{vsn, "0.1.6"}, {vsn, "0.1.7"},
{registered, []}, {registered, []},
{applications, [ {applications, [
kernel, kernel,

4
apps/emqx_enterprise/src/emqx_enterprise_schema.erl
View File

@ -144,8 +144,8 @@ ee_delegate(Method, [], Name) ->
redefine_roots(Roots) -> redefine_roots(Roots) ->
Overrides = [ Overrides = [
{"node", #{type => hoconsc:ref(?MODULE, "node")}}, {node, #{type => hoconsc:ref(?MODULE, "node")}},
{"log", #{type => hoconsc:ref(?MODULE, "log")}} {log, #{type => hoconsc:ref(?MODULE, "log")}}
], ],
override(Roots, Overrides). override(Roots, Overrides).

2
apps/emqx_management/src/emqx_mgmt_api_configs.erl
View File

@ -502,7 +502,7 @@ conf_path(Req) ->
string:lexemes(Path, "/ "). string:lexemes(Path, "/ ").
global_zone_roots() -> global_zone_roots() ->
lists:map(fun({K, _}) -> list_to_binary(K) end, global_zone_schema()). lists:map(fun({K, _}) -> atom_to_binary(K) end, global_zone_schema()).
global_zone_schema() -> global_zone_schema() ->
emqx_zone_schema:global_zone_with_default(). emqx_zone_schema:global_zone_with_default().

7
apps/emqx_management/src/emqx_mgmt_api_listeners.erl
View File

@ -318,13 +318,10 @@ create_listener_schema(Opts) ->
). ).
listeners_type() -> listeners_type() ->
lists:map( lists:map(fun({Type, _}) -> list_to_existing_atom(Type) end, emqx_schema:listeners()).
fun({Type, _}) -> list_to_existing_atom(Type) end,
hocon_schema:fields(emqx_schema, "listeners")
).
listeners_info(Opts) -> listeners_info(Opts) ->
Listeners = hocon_schema:fields(emqx_schema, "listeners"), Listeners = emqx_schema:listeners(),
lists:map( lists:map(
fun({ListenerType, Schema}) -> fun({ListenerType, Schema}) ->
Type = emqx_schema:get_tombstone_map_value_type(Schema), Type = emqx_schema:get_tombstone_map_value_type(Schema),

2
apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl
View File

@ -138,7 +138,7 @@ t_log(_Config) ->
t_global_zone(_Config) -> t_global_zone(_Config) ->
{ok, Zones} = get_global_zone(), {ok, Zones} = get_global_zone(),
ZonesKeys = lists:map( ZonesKeys = lists:map(
fun({K, _}) -> list_to_binary(K) end, emqx_zone_schema:global_zone_with_default() fun({K, _}) -> atom_to_binary(K) end, emqx_zone_schema:global_zone_with_default()
), ),
?assertEqual(lists:usort(ZonesKeys), lists:usort(maps:keys(Zones))), ?assertEqual(lists:usort(ZonesKeys), lists:usort(maps:keys(Zones))),
?assertEqual( ?assertEqual(

2
apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl
View File

@ -516,7 +516,7 @@ cert_file(Name) ->
data_file(filename:join(["certs", Name])). data_file(filename:join(["certs", Name])).
default_listeners_hocon_text() -> default_listeners_hocon_text() ->
Sc = #{roots => emqx_schema:fields("listeners")}, Sc = #{roots => emqx_schema:listeners()},
Listeners = hocon_tconf:make_serializable(Sc, #{}, #{}), Listeners = hocon_tconf:make_serializable(Sc, #{}, #{}),
Config = #{<<"listeners">> => Listeners}, Config = #{<<"listeners">> => Listeners},
hocon_pp:do(Config, #{}). hocon_pp:do(Config, #{}).