diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index bb3a588a9..a8462ad82 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -15,7 +15,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.15.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 01989c5a1..ed3f64d0e 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -69,33 +69,82 @@ cipher/0, comma_separated_atoms/0]). --export([namespace/0, roots/0, fields/1]). +-export([namespace/0, roots/0, roots/1, fields/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([ssl/1]). namespace() -> undefined. roots() -> - ["zones", - "mqtt", - "flapping_detect", - "force_shutdown", - "force_gc", - "conn_congestion", - "rate_limit", - "quota", - {"listeners", + %% TODO change config importance to a field metadata + roots(high) ++ roots(medium) ++ roots(low). + +roots(high) -> + [ {"listeners", sc(ref("listeners"), - #{ desc => "MQTT listeners identified by their protocol type and assigned names. " - "The listeners enabled by default are named with 'default'"}) - }, - "broker", - "plugins", - "stats", - "sysmon", - "alarm", - "authorization", - {"authentication", sc(hoconsc:lazy(hoconsc:array(map())), #{})} + #{ desc => "MQTT listeners identified by their protocol type and assigned names" + }) + } + , {"zones", + sc(map("name", ref("zone")), + #{ desc => "A zone is a set of configs grouped by the zone name.
" + "For flexible configuration mapping, the name " + "can be set to a listener's zone config.
" + "NOTE: A builtin zone named default is auto created " + "and can not be deleted." + })} + , {"mqtt", + sc(ref("mqtt"), + #{ desc => "Global MQTT configuration.
" + "The configs here work as default values which can be overriden " + "in zone configs" + })} + , {"authentication", + sc(hoconsc:lazy(hoconsc:array(map())), + #{ desc => "Default authentication configs for all MQTT listeners.
" + "For per-listener overrides see authentication " + "in listener configs" + })} + , {"authorization", + sc(ref("authorization"), + #{})} + ]; +roots(medium) -> + [ {"broker", + sc(ref("broker"), + #{})} + , {"rate_limit", + sc(ref("rate_limit"), + #{})} + , {"force_shutdown", + sc(ref("force_shutdown"), + #{})} + ]; +roots(low) -> + [ {"force_gc", + sc(ref("force_gc"), + #{})} + , {"conn_congestion", + sc(ref("conn_congestion"), + #{})} + , {"quota", + sc(ref("quota"), + #{})} + , {"plugins", %% TODO: move to emqx_machine_schema + sc(ref("plugins"), + #{})} + , {"stats", + sc(ref("stats"), + #{})} + , {"sysmon", + sc(ref("sysmon"), + #{})} + , {"alarm", + sc(ref("alarm"), + #{})} + , {"flapping_detect", + sc(ref("flapping_detect"), + #{})} ]. fields("stats") -> @@ -117,8 +166,7 @@ fields("authorization") -> , {"cache", sc(ref(?MODULE, "cache"), #{ - }) - } + })} ]; fields("cache") -> @@ -270,14 +318,7 @@ fields("mqtt") -> })} ]; -fields("zones") -> - [ {"$name", - sc(ref("zone_settings"), - #{ - } - )}]; - -fields("zone_settings") -> +fields("zone") -> Fields = ["mqtt", "stats", "flapping_detect", "force_shutdown", "conn_congestion", "rate_limit", "quota", "force_gc"], [{F, ref(emqx_zone_schema, F)} || F <- Fields]; @@ -375,48 +416,37 @@ fields("force_gc") -> fields("listeners") -> [ {"tcp", - sc(ref("tcp_listeners"), + sc(map(name, ref("mqtt_tcp_listener")), #{ desc => "TCP listeners" + , nullable => {true, recursive} }) } , {"ssl", - sc(ref("ssl_listeners"), + sc(map(name, ref("mqtt_ssl_listener")), #{ desc => "SSL listeners" + , nullable => {true, recursive} }) } , {"ws", - sc(ref("ws_listeners"), + sc(map(name, ref("mqtt_ws_listener")), #{ desc => "HTTP websocket listeners" + , nullable => {true, recursive} }) } , {"wss", - sc(ref("wss_listeners"), + sc(map(name, ref("mqtt_wss_listener")), #{ desc => "HTTPS websocket listeners" + , nullable => {true, recursive} }) } , {"quic", - sc(ref("quic_listeners"), + sc(map(name, ref("mqtt_quic_listener")), #{ desc => "QUIC listeners" + , nullable => {true, recursive} }) } ]; -fields("tcp_listeners") -> - [ {"$name", ref("mqtt_tcp_listener")} - ]; -fields("ssl_listeners") -> - [ {"$name", ref("mqtt_ssl_listener")} - ]; -fields("ws_listeners") -> - [ {"$name", ref("mqtt_ws_listener")} - ]; -fields("wss_listeners") -> - [ {"$name", ref("mqtt_wss_listener")} - ]; -fields("quic_listeners") -> - [ {"$name", ref("mqtt_quic_listener")} - ]; - fields("mqtt_tcp_listener") -> [ {"tcp", sc(ref("tcp_opts"), @@ -1011,6 +1041,8 @@ ceiling(X) -> sc(Type, Meta) -> hoconsc:mk(Type, Meta). +map(Name, Type) -> hoconsc:map(Name, Type). + ref(Field) -> hoconsc:ref(?MODULE, Field). ref(Module, Field) -> hoconsc:ref(Module, Field). diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl index e3c6d8ee9..fe2d3947d 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl @@ -14,13 +14,12 @@ fields("emqx_data_bridge") -> [{bridges, #{type => hoconsc:array(hoconsc:union(?BRIDGES)), default => []}}]; -fields(mysql) -> connector_fields(mysql); -fields(pgsql) -> connector_fields(pgsql); -fields(mongo) -> connector_fields(mongo); -fields(redis) -> connector_fields(redis); -fields(ldap) -> connector_fields(ldap). +fields(mysql) -> connector_fields(emqx_connector_mysql, mysql); +fields(pgsql) -> connector_fields(emqx_connector_pgsql, pgsql); +fields(mongo) -> connector_fields(emqx_connector_mongo, mongo); +fields(redis) -> connector_fields(emqx_connector_redis, redis); +fields(ldap) -> connector_fields(emqx_connector_ldap, ldap). -connector_fields(DB) -> - Mod = list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])), +connector_fields(ConnectModule, DB) -> [{name, hoconsc:mk(typerefl:binary())}, - {type, #{type => DB}}] ++ Mod:roots(). + {type, #{type => DB}}] ++ ConnectModule:roots(). diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 7fb945ba0..3811d56c6 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -50,16 +50,16 @@ namespace() -> gateway. roots() -> [gateway]. fields(gateway) -> - [{stomp, sc(ref(stomp_structs))}, - {mqttsn, sc(ref(mqttsn_structs))}, - {coap, sc(ref(coap_structs))}, - {lwm2m, sc(ref(lwm2m_structs))}, - {exproto, sc(ref(exproto_structs))} + [{stomp, sc(ref(stomp))}, + {mqttsn, sc(ref(mqttsn))}, + {coap, sc(ref(coap))}, + {lwm2m, sc(ref(lwm2m))}, + {exproto, sc(ref(exproto))} ]; -fields(stomp_structs) -> +fields(stomp) -> [ {frame, sc(ref(stomp_frame))} - , {listeners, sc(ref(tcp_listener_group))} + , {listeners, sc(ref(tcp_listeners))} ] ++ gateway_common_options(); fields(stomp_frame) -> @@ -68,12 +68,12 @@ fields(stomp_frame) -> , {max_body_length, sc(integer(), 8192)} ]; -fields(mqttsn_structs) -> +fields(mqttsn) -> [ {gateway_id, sc(integer())} , {broadcast, sc(boolean())} , {enable_qos3, sc(boolean())} , {predefined, hoconsc:array(ref(mqttsn_predefined))} - , {listeners, sc(ref(udp_listener_group))} + , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); fields(mqttsn_predefined) -> @@ -81,34 +81,34 @@ fields(mqttsn_predefined) -> , {topic, sc(binary())} ]; -fields(coap_structs) -> +fields(coap) -> [ {heartbeat, sc(duration(), <<"30s">>)} , {connection_required, sc(boolean(), false)} - , {notify_type, sc(union([non, con, qos]), qos)} - , {subscribe_qos, sc(union([qos0, qos1, qos2, coap]), coap)} - , {publish_qos, sc(union([qos0, qos1, qos2, coap]), coap)} - , {listeners, sc(ref(udp_listener_group))} + , {notify_type, sc(hoconsc:union([non, con, qos]), qos)} + , {subscribe_qos, sc(hoconsc:union([qos0, qos1, qos2, coap]), coap)} + , {publish_qos, sc(hoconsc:union([qos0, qos1, qos2, coap]), coap)} + , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); -fields(lwm2m_structs) -> +fields(lwm2m) -> [ {xml_dir, sc(binary())} , {lifetime_min, sc(duration())} , {lifetime_max, sc(duration())} , {qmode_time_windonw, sc(integer())} , {auto_observe, sc(boolean())} - , {update_msg_publish_condition, sc(union([always, contains_object_list]))} + , {update_msg_publish_condition, sc(hoconsc:union([always, contains_object_list]))} , {translators, sc(ref(translators))} - , {listeners, sc(ref(udp_listener_group))} + , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); -fields(exproto_structs) -> +fields(exproto) -> [ {server, sc(ref(exproto_grpc_server))} , {handler, sc(ref(exproto_grpc_handler))} - , {listeners, sc(ref(udp_tcp_listener_group))} + , {listeners, sc(ref(udp_tcp_listeners))} ] ++ gateway_common_options(); fields(exproto_grpc_server) -> - [ {bind, sc(union(ip_port(), integer()))} + [ {bind, sc(hoconsc:union([ip_port(), integer()]))} %% TODO: ssl options ]; @@ -136,62 +136,45 @@ fields(translator) -> , {qos, sc(range(0, 2))} ]; -fields(udp_listener_group) -> - [ {udp, sc(ref(udp_listener))} - , {dtls, sc(ref(dtls_listener))} +fields(udp_listeners) -> + [ {udp, sc(map(name, ref(udp_listener)))} + , {dtls, sc(map(name, ref(dtls_listener)))} ]; -fields(tcp_listener_group) -> - [ {tcp, sc(ref(tcp_listener))} - , {ssl, sc(ref(ssl_listener))} +fields(tcp_listeners) -> + [ {tcp, sc(map(name, ref(tcp_listener)))} + , {ssl, sc(map(name, ref(ssl_listener)))} ]; -fields(udp_tcp_listener_group) -> - [ {udp, sc(ref(udp_listener))} - , {dtls, sc(ref(dtls_listener))} - , {tcp, sc(ref(tcp_listener))} - , {ssl, sc(ref(ssl_listener))} +fields(udp_tcp_listeners) -> + [ {udp, sc(map(name, ref(udp_listener)))} + , {dtls, sc(map(name, ref(dtls_listener)))} + , {tcp, sc(map(name, ref(tcp_listener)))} + , {ssl, sc(map(name, ref(ssl_listener)))} ]; fields(tcp_listener) -> - [ {"$name", sc(ref(tcp_listener_settings))}]; - -fields(ssl_listener) -> - [ {"$name", sc(ref(ssl_listener_settings))}]; - -fields(udp_listener) -> - [ {"$name", sc(ref(udp_listener_settings))}]; - -fields(dtls_listener) -> - [ {"$name", sc(ref(dtls_listener_settings))}]; - -fields(tcp_listener_settings) -> [ %% some special confs for tcp listener - ] ++ tcp_opts() - ++ proxy_protocol_opts() - ++ common_listener_opts(); + ] ++ + tcp_opts() ++ + proxy_protocol_opts() ++ + common_listener_opts(); -fields(ssl_listener_settings) -> - [ - %% some special confs for ssl listener - ] ++ tcp_opts() - ++ ssl_opts() - ++ proxy_protocol_opts() - ++ common_listener_opts(); +fields(ssl_listener) -> + fields(tcp_listener) ++ + ssl_opts(); -fields(udp_listener_settings) -> +fields(udp_listener) -> [ %% some special confs for udp listener - ] ++ udp_opts() - ++ common_listener_opts(); + ] ++ + udp_opts() ++ + common_listener_opts(); -fields(dtls_listener_settings) -> - [ - %% some special confs for dtls listener - ] ++ udp_opts() - ++ dtls_opts() - ++ common_listener_opts(); +fields(dtls_listener) -> + fields(udp_listener) ++ + dtls_opts(); fields(udp_opts) -> [ {active_n, sc(integer(), 100)} @@ -218,11 +201,7 @@ fields(dtls_listener_ssl_opts) -> lists:keyreplace("versions", 1, Base, {"versions", DtlsVers}), {"ciphers", Ciphers} ) - ); - -fields(ExtraField) -> - Mod = list_to_atom(ExtraField++"_schema"), - Mod:fields(ExtraField). + ). default_ciphers() -> ["ECDHE-ECDSA-AES256-GCM-SHA384", @@ -286,16 +265,16 @@ common_listener_opts() -> ]. tcp_opts() -> - [{tcp, sc(ref(emqx_schema, "tcp_opts"), #{})}]. + [{tcp, sc_meta(ref(emqx_schema, "tcp_opts"), #{})}]. udp_opts() -> - [{udp, sc(ref(udp_opts), #{})}]. + [{udp, sc_meta(ref(udp_opts), #{})}]. ssl_opts() -> - [{ssl, sc(ref(emqx_schema, "listener_ssl_opts"), #{})}]. + [{ssl, sc_meta(ref(emqx_schema, "listener_ssl_opts"), #{})}]. dtls_opts() -> - [{dtls, sc(ref(dtls_listener_ssl_opts), #{})}]. + [{dtls, sc_meta(ref(dtls_listener_ssl_opts), #{})}]. proxy_protocol_opts() -> [ {proxy_protocol, sc(boolean())} @@ -308,18 +287,20 @@ default_dtls_vsns() -> dtls_vsn(<<"dtlsv1.2">>) -> 'dtlsv1.2'; dtls_vsn(<<"dtlsv1">>) -> 'dtlsv1'. -%%-------------------------------------------------------------------- -%% Helpers - -%% types - -sc(Type) -> #{type => Type}. +sc(Type) -> + sc_meta(Type, #{}). sc(Type, Default) -> - hoconsc:mk(Type, #{default => Default}). + sc_meta(Type, #{default => Default}). -ref(Field) -> - hoconsc:ref(?MODULE, Field). +sc_meta(Type, Meta) -> + hoconsc:mk(Type, Meta). + +map(Name, Type) -> + hoconsc:map(Name, Type). + +ref(StructName) -> + ref(?MODULE, StructName). ref(Mod, Field) -> hoconsc:ref(Mod, Field). diff --git a/apps/emqx_machine/src/emqx_machine_schema.erl b/apps/emqx_machine/src/emqx_machine_schema.erl index 614609875..e4f7a8d1a 100644 --- a/apps/emqx_machine/src/emqx_machine_schema.erl +++ b/apps/emqx_machine/src/emqx_machine_schema.erl @@ -42,8 +42,7 @@ %% The list can not be made a dynamic read at run-time as it is used %% by nodetool to generate app.