yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5685 from zmstone/refactor-hocon-map-type-replace-wildcard 

refactor: hocon map type replace wildcard
This commit is contained in:
Zaiming (Stone) Shi 2021-09-10 09:25:13 +02:00 committed by GitHub
parent 07069898b1 9430f70dd8
commit a525e8fb9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 247 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/emqx/rebar.config
View File

@ -15,7 +15,7 @@
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.15.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.0"}}}
, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}}

134
apps/emqx/src/emqx_schema.erl
View File

@ -69,33 +69,82 @@
cipher/0, cipher/0,
comma_separated_atoms/0]). comma_separated_atoms/0]).
-export([namespace/0, roots/0, fields/1]). -export([namespace/0, roots/0, roots/1, fields/1]).
-export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]).
-export([ssl/1]). -export([ssl/1]).
namespace() -> undefined. namespace() -> undefined.
roots() -> roots() ->
["zones", %% TODO change config importance to a field metadata
"mqtt", roots(high) ++ roots(medium) ++ roots(low).
"flapping_detect",
"force_shutdown", roots(high) ->
"force_gc", [ {"listeners",
"conn_congestion",
"rate_limit",
"quota",
{"listeners",
sc(ref("listeners"), sc(ref("listeners"),
#{ desc => "MQTT listeners identified by their protocol type and assigned names. " #{ desc => "MQTT listeners identified by their protocol type and assigned names"
"The listeners enabled by default are named with 'default'"}) })
}, }
"broker", , {"zones",
"plugins", sc(map("name", ref("zone")),
"stats", #{ desc => "A zone is a set of configs grouped by the zone <code>name</code>. <br>"
"sysmon", "For flexible configuration mapping, the <code>name</code> "
"alarm", "can be set to a listener's <code>zone</code> config.<br>"
"authorization", "NOTE: A builtin zone named <code>default</code> is auto created "
{"authentication", sc(hoconsc:lazy(hoconsc:array(map())), #{})} "and can not be deleted."
})}
, {"mqtt",
sc(ref("mqtt"),
#{ desc => "Global MQTT configuration.<br>"
"The configs here work as default values which can be overriden "
"in <code>zone</code> configs"
})}
, {"authentication",
sc(hoconsc:lazy(hoconsc:array(map())),
#{ desc => "Default authentication configs for all MQTT listeners.<br>"
"For per-listener overrides see <code>authentication</code> "
"in listener configs"
})}
, {"authorization",
sc(ref("authorization"),
#{})}
];
roots(medium) ->
[ {"broker",
sc(ref("broker"),
#{})}
, {"rate_limit",
sc(ref("rate_limit"),
#{})}
, {"force_shutdown",
sc(ref("force_shutdown"),
#{})}
];
roots(low) ->
[ {"force_gc",
sc(ref("force_gc"),
#{})}
, {"conn_congestion",
sc(ref("conn_congestion"),
#{})}
, {"quota",
sc(ref("quota"),
#{})}
, {"plugins", %% TODO: move to emqx_machine_schema
sc(ref("plugins"),
#{})}
, {"stats",
sc(ref("stats"),
#{})}
, {"sysmon",
sc(ref("sysmon"),
#{})}
, {"alarm",
sc(ref("alarm"),
#{})}
, {"flapping_detect",
sc(ref("flapping_detect"),
#{})}
]. ].
fields("stats") -> fields("stats") ->
@ -117,8 +166,7 @@ fields("authorization") ->
, {"cache", , {"cache",
sc(ref(?MODULE, "cache"), sc(ref(?MODULE, "cache"),
#{ #{
}) })}
}
]; ];
fields("cache") -> fields("cache") ->
@ -270,14 +318,7 @@ fields("mqtt") ->
})} })}
]; ];
fields("zones") -> fields("zone") ->
[ {"$name",
sc(ref("zone_settings"),
#{
}
)}];
fields("zone_settings") ->
Fields = ["mqtt", "stats", "flapping_detect", "force_shutdown", Fields = ["mqtt", "stats", "flapping_detect", "force_shutdown",
"conn_congestion", "rate_limit", "quota", "force_gc"], "conn_congestion", "rate_limit", "quota", "force_gc"],
[{F, ref(emqx_zone_schema, F)} || F <- Fields]; [{F, ref(emqx_zone_schema, F)} || F <- Fields];
@ -375,48 +416,37 @@ fields("force_gc") ->
fields("listeners") -> fields("listeners") ->
[ {"tcp", [ {"tcp",
sc(ref("tcp_listeners"), sc(map(name, ref("mqtt_tcp_listener")),
#{ desc => "TCP listeners" #{ desc => "TCP listeners"
, nullable => {true, recursive}
}) })
} }
, {"ssl", , {"ssl",
sc(ref("ssl_listeners"), sc(map(name, ref("mqtt_ssl_listener")),
#{ desc => "SSL listeners" #{ desc => "SSL listeners"
, nullable => {true, recursive}
}) })
} }
, {"ws", , {"ws",
sc(ref("ws_listeners"), sc(map(name, ref("mqtt_ws_listener")),
#{ desc => "HTTP websocket listeners" #{ desc => "HTTP websocket listeners"
, nullable => {true, recursive}
}) })
} }
, {"wss", , {"wss",
sc(ref("wss_listeners"), sc(map(name, ref("mqtt_wss_listener")),
#{ desc => "HTTPS websocket listeners" #{ desc => "HTTPS websocket listeners"
, nullable => {true, recursive}
}) })
} }
, {"quic", , {"quic",
sc(ref("quic_listeners"), sc(map(name, ref("mqtt_quic_listener")),
#{ desc => "QUIC listeners" #{ desc => "QUIC listeners"
, nullable => {true, recursive}
}) })
} }
]; ];
fields("tcp_listeners") ->
[ {"$name", ref("mqtt_tcp_listener")}
];
fields("ssl_listeners") ->
[ {"$name", ref("mqtt_ssl_listener")}
];
fields("ws_listeners") ->
[ {"$name", ref("mqtt_ws_listener")}
];
fields("wss_listeners") ->
[ {"$name", ref("mqtt_wss_listener")}
];
fields("quic_listeners") ->
[ {"$name", ref("mqtt_quic_listener")}
];
fields("mqtt_tcp_listener") -> fields("mqtt_tcp_listener") ->
[ {"tcp", [ {"tcp",
sc(ref("tcp_opts"), sc(ref("tcp_opts"),
@ -1011,6 +1041,8 @@ ceiling(X) ->
sc(Type, Meta) -> hoconsc:mk(Type, Meta). sc(Type, Meta) -> hoconsc:mk(Type, Meta).
map(Name, Type) -> hoconsc:map(Name, Type).
ref(Field) -> hoconsc:ref(?MODULE, Field). ref(Field) -> hoconsc:ref(?MODULE, Field).
ref(Module, Field) -> hoconsc:ref(Module, Field). ref(Module, Field) -> hoconsc:ref(Module, Field).

15
apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl
View File

@ -14,13 +14,12 @@ fields("emqx_data_bridge") ->
[{bridges, #{type => hoconsc:array(hoconsc:union(?BRIDGES)), [{bridges, #{type => hoconsc:array(hoconsc:union(?BRIDGES)),
default => []}}]; default => []}}];
fields(mysql) -> connector_fields(mysql); fields(mysql) -> connector_fields(emqx_connector_mysql, mysql);
fields(pgsql) -> connector_fields(pgsql); fields(pgsql) -> connector_fields(emqx_connector_pgsql, pgsql);
fields(mongo) -> connector_fields(mongo); fields(mongo) -> connector_fields(emqx_connector_mongo, mongo);
fields(redis) -> connector_fields(redis); fields(redis) -> connector_fields(emqx_connector_redis, redis);
fields(ldap) -> connector_fields(ldap). fields(ldap) -> connector_fields(emqx_connector_ldap, ldap).
connector_fields(DB) -> connector_fields(ConnectModule, DB) ->
Mod = list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])),
[{name, hoconsc:mk(typerefl:binary())}, [{name, hoconsc:mk(typerefl:binary())},
{type, #{type => DB}}] ++ Mod:roots(). {type, #{type => DB}}] ++ ConnectModule:roots().

141
apps/emqx_gateway/src/emqx_gateway_schema.erl
View File

@ -50,16 +50,16 @@ namespace() -> gateway.
roots() -> [gateway]. roots() -> [gateway].
fields(gateway) -> fields(gateway) ->
[{stomp, sc(ref(stomp_structs))}, [{stomp, sc(ref(stomp))},
{mqttsn, sc(ref(mqttsn_structs))}, {mqttsn, sc(ref(mqttsn))},
{coap, sc(ref(coap_structs))}, {coap, sc(ref(coap))},
{lwm2m, sc(ref(lwm2m_structs))}, {lwm2m, sc(ref(lwm2m))},
{exproto, sc(ref(exproto_structs))} {exproto, sc(ref(exproto))}
]; ];
fields(stomp_structs) -> fields(stomp) ->
[ {frame, sc(ref(stomp_frame))} [ {frame, sc(ref(stomp_frame))}
, {listeners, sc(ref(tcp_listener_group))} , {listeners, sc(ref(tcp_listeners))}
] ++ gateway_common_options(); ] ++ gateway_common_options();
fields(stomp_frame) -> fields(stomp_frame) ->
@ -68,12 +68,12 @@ fields(stomp_frame) ->
, {max_body_length, sc(integer(), 8192)} , {max_body_length, sc(integer(), 8192)}
]; ];
fields(mqttsn_structs) -> fields(mqttsn) ->
[ {gateway_id, sc(integer())} [ {gateway_id, sc(integer())}
, {broadcast, sc(boolean())} , {broadcast, sc(boolean())}
, {enable_qos3, sc(boolean())} , {enable_qos3, sc(boolean())}
, {predefined, hoconsc:array(ref(mqttsn_predefined))} , {predefined, hoconsc:array(ref(mqttsn_predefined))}
, {listeners, sc(ref(udp_listener_group))} , {listeners, sc(ref(udp_listeners))}
] ++ gateway_common_options(); ] ++ gateway_common_options();
fields(mqttsn_predefined) -> fields(mqttsn_predefined) ->
@ -81,34 +81,34 @@ fields(mqttsn_predefined) ->
, {topic, sc(binary())} , {topic, sc(binary())}
]; ];
fields(coap_structs) -> fields(coap) ->
[ {heartbeat, sc(duration(), <<"30s">>)} [ {heartbeat, sc(duration(), <<"30s">>)}
, {connection_required, sc(boolean(), false)} , {connection_required, sc(boolean(), false)}
, {notify_type, sc(union([non, con, qos]), qos)} , {notify_type, sc(hoconsc:union([non, con, qos]), qos)}
, {subscribe_qos, sc(union([qos0, qos1, qos2, coap]), coap)} , {subscribe_qos, sc(hoconsc:union([qos0, qos1, qos2, coap]), coap)}
, {publish_qos, sc(union([qos0, qos1, qos2, coap]), coap)} , {publish_qos, sc(hoconsc:union([qos0, qos1, qos2, coap]), coap)}
, {listeners, sc(ref(udp_listener_group))} , {listeners, sc(ref(udp_listeners))}
] ++ gateway_common_options(); ] ++ gateway_common_options();
fields(lwm2m_structs) -> fields(lwm2m) ->
[ {xml_dir, sc(binary())} [ {xml_dir, sc(binary())}
, {lifetime_min, sc(duration())} , {lifetime_min, sc(duration())}
, {lifetime_max, sc(duration())} , {lifetime_max, sc(duration())}
, {qmode_time_windonw, sc(integer())} , {qmode_time_windonw, sc(integer())}
, {auto_observe, sc(boolean())} , {auto_observe, sc(boolean())}
, {update_msg_publish_condition, sc(union([always, contains_object_list]))} , {update_msg_publish_condition, sc(hoconsc:union([always, contains_object_list]))}
, {translators, sc(ref(translators))} , {translators, sc(ref(translators))}
, {listeners, sc(ref(udp_listener_group))} , {listeners, sc(ref(udp_listeners))}
] ++ gateway_common_options(); ] ++ gateway_common_options();
fields(exproto_structs) -> fields(exproto) ->
[ {server, sc(ref(exproto_grpc_server))} [ {server, sc(ref(exproto_grpc_server))}
, {handler, sc(ref(exproto_grpc_handler))} , {handler, sc(ref(exproto_grpc_handler))}
, {listeners, sc(ref(udp_tcp_listener_group))} , {listeners, sc(ref(udp_tcp_listeners))}
] ++ gateway_common_options(); ] ++ gateway_common_options();
fields(exproto_grpc_server) -> fields(exproto_grpc_server) ->
[ {bind, sc(union(ip_port(), integer()))} [ {bind, sc(hoconsc:union([ip_port(), integer()]))}
%% TODO: ssl options %% TODO: ssl options
]; ];
@ -136,62 +136,45 @@ fields(translator) ->
, {qos, sc(range(0, 2))} , {qos, sc(range(0, 2))}
]; ];
fields(udp_listener_group) -> fields(udp_listeners) ->
[ {udp, sc(ref(udp_listener))} [ {udp, sc(map(name, ref(udp_listener)))}
, {dtls, sc(ref(dtls_listener))} , {dtls, sc(map(name, ref(dtls_listener)))}
]; ];
fields(tcp_listener_group) -> fields(tcp_listeners) ->
[ {tcp, sc(ref(tcp_listener))} [ {tcp, sc(map(name, ref(tcp_listener)))}
, {ssl, sc(ref(ssl_listener))} , {ssl, sc(map(name, ref(ssl_listener)))}
]; ];
fields(udp_tcp_listener_group) -> fields(udp_tcp_listeners) ->
[ {udp, sc(ref(udp_listener))} [ {udp, sc(map(name, ref(udp_listener)))}
, {dtls, sc(ref(dtls_listener))} , {dtls, sc(map(name, ref(dtls_listener)))}
, {tcp, sc(ref(tcp_listener))} , {tcp, sc(map(name, ref(tcp_listener)))}
, {ssl, sc(ref(ssl_listener))} , {ssl, sc(map(name, ref(ssl_listener)))}
]; ];
fields(tcp_listener) -> fields(tcp_listener) ->
[ {"$name", sc(ref(tcp_listener_settings))}];
fields(ssl_listener) ->
[ {"$name", sc(ref(ssl_listener_settings))}];
fields(udp_listener) ->
[ {"$name", sc(ref(udp_listener_settings))}];
fields(dtls_listener) ->
[ {"$name", sc(ref(dtls_listener_settings))}];
fields(tcp_listener_settings) ->
[ [
%% some special confs for tcp listener %% some special confs for tcp listener
] ++ tcp_opts() ] ++
++ proxy_protocol_opts() tcp_opts() ++
++ common_listener_opts(); proxy_protocol_opts() ++
common_listener_opts();
fields(ssl_listener_settings) -> fields(ssl_listener) ->
[ fields(tcp_listener) ++
%% some special confs for ssl listener ssl_opts();
] ++ tcp_opts()
++ ssl_opts()
++ proxy_protocol_opts()
++ common_listener_opts();
fields(udp_listener_settings) -> fields(udp_listener) ->
[ [
%% some special confs for udp listener %% some special confs for udp listener
] ++ udp_opts() ] ++
++ common_listener_opts(); udp_opts() ++
common_listener_opts();
fields(dtls_listener_settings) -> fields(dtls_listener) ->
[ fields(udp_listener) ++
%% some special confs for dtls listener dtls_opts();
] ++ udp_opts()
++ dtls_opts()
++ common_listener_opts();
fields(udp_opts) -> fields(udp_opts) ->
[ {active_n, sc(integer(), 100)} [ {active_n, sc(integer(), 100)}
@ -218,11 +201,7 @@ fields(dtls_listener_ssl_opts) ->
lists:keyreplace("versions", 1, Base, {"versions", DtlsVers}), lists:keyreplace("versions", 1, Base, {"versions", DtlsVers}),
{"ciphers", Ciphers} {"ciphers", Ciphers}
) )
); ).
fields(ExtraField) ->
Mod = list_to_atom(ExtraField++"_schema"),
Mod:fields(ExtraField).
default_ciphers() -> default_ciphers() ->
["ECDHE-ECDSA-AES256-GCM-SHA384", ["ECDHE-ECDSA-AES256-GCM-SHA384",
@ -286,16 +265,16 @@ common_listener_opts() ->
]. ].
tcp_opts() -> tcp_opts() ->
[{tcp, sc(ref(emqx_schema, "tcp_opts"), #{})}]. [{tcp, sc_meta(ref(emqx_schema, "tcp_opts"), #{})}].
udp_opts() -> udp_opts() ->
[{udp, sc(ref(udp_opts), #{})}]. [{udp, sc_meta(ref(udp_opts), #{})}].
ssl_opts() -> ssl_opts() ->
[{ssl, sc(ref(emqx_schema, "listener_ssl_opts"), #{})}]. [{ssl, sc_meta(ref(emqx_schema, "listener_ssl_opts"), #{})}].
dtls_opts() -> dtls_opts() ->
[{dtls, sc(ref(dtls_listener_ssl_opts), #{})}]. [{dtls, sc_meta(ref(dtls_listener_ssl_opts), #{})}].
proxy_protocol_opts() -> proxy_protocol_opts() ->
[ {proxy_protocol, sc(boolean())} [ {proxy_protocol, sc(boolean())}
@ -308,18 +287,20 @@ default_dtls_vsns() ->
dtls_vsn(<<"dtlsv1.2">>) -> 'dtlsv1.2'; dtls_vsn(<<"dtlsv1.2">>) -> 'dtlsv1.2';
dtls_vsn(<<"dtlsv1">>) -> 'dtlsv1'. dtls_vsn(<<"dtlsv1">>) -> 'dtlsv1'.
%%-------------------------------------------------------------------- sc(Type) ->
%% Helpers sc_meta(Type, #{}).
%% types
sc(Type) -> #{type => Type}.
sc(Type, Default) -> sc(Type, Default) ->
hoconsc:mk(Type, #{default => Default}). sc_meta(Type, #{default => Default}).
ref(Field) -> sc_meta(Type, Meta) ->
hoconsc:ref(?MODULE, Field). hoconsc:mk(Type, Meta).
map(Name, Type) ->
hoconsc:map(Name, Type).
ref(StructName) ->
ref(?MODULE, StructName).
ref(Mod, Field) -> ref(Mod, Field) ->
hoconsc:ref(Mod, Field). hoconsc:ref(Mod, Field).

59
apps/emqx_machine/src/emqx_machine_schema.erl
View File

@ -42,8 +42,7 @@
%% The list can not be made a dynamic read at run-time as it is used %% The list can not be made a dynamic read at run-time as it is used
%% by nodetool to generate app.<time>.config before EMQ X is started %% by nodetool to generate app.<time>.config before EMQ X is started
-define(MERGED_CONFIGS, -define(MERGED_CONFIGS,
[ emqx_schema [ emqx_bridge_schema
, emqx_bridge_schema
, emqx_retainer_schema , emqx_retainer_schema
, emqx_statsd_schema , emqx_statsd_schema
, emqx_authz_schema , emqx_authz_schema
@ -59,13 +58,42 @@
namespace() -> undefined. namespace() -> undefined.
roots() -> roots() ->
%% This is a temp workaround to define part of authorization config %% authorization configs are merged in THIS schema's "authorization" fields
%% in emqx_schema and part of it in emqx_authz_schema but then lists:keydelete("authorization", 1, emqx_schema:roots(high)) ++
%% merged here in this module [ {"node",
%% The proper fix should be to make connection (channel, session) state sc(hoconsc:ref("node"),
%% extendable by e.g. allow hooks be stateful. #{ desc => "Node name, cookie, config & data directories "
["cluster", "node", "rpc", "log", "authorization"] ++ "and the Eralng virtual machine (beam) boot parameters."
lists:keydelete("authorization", 1, lists:flatmap(fun roots/1, ?MERGED_CONFIGS)). })}
, {"cluster",
sc(hoconsc:ref("cluster"),
#{ desc => "EMQ X nodes can form a cluster to scale up the total capacity.<br>"
"Here holds the configs to instruct how individual nodes "
"can discover each other, also the database replication "
"role of this node etc."
})}
, {"log",
sc(hoconsc:ref("log"),
#{ desc => "Configure logging backends (to console or to file), "
"and logging level for each logger backend."
})}
, {"rpc",
sc(hoconsc:ref("rpc"),
#{ desc => "EMQ X uses a library called <code>gen_rpc</code> for "
"inter-broker RPCs.<br>Most of the time the default config "
"should work, but in case you need to do performance "
"fine-turning or experiment a bit, this is where to look."
})}
, {"authorization",
sc(hoconsc:ref("authorization"),
#{ desc => "In EMQ X, MQTT client access control can be just a few "
"lines of text based rules, or delegated to an external "
"HTTP API, or base externa database query results."
})}
] ++
emqx_schema:roots(medium) ++
emqx_schema:roots(low) ++
lists:flatmap(fun roots/1, ?MERGED_CONFIGS).
fields("cluster") -> fields("cluster") ->
[ {"name", [ {"name",
@ -381,7 +409,7 @@ fields("rpc") ->
fields("log") -> fields("log") ->
[ {"console_handler", ref("console_handler")} [ {"console_handler", ref("console_handler")}
, {"file_handlers", , {"file_handlers",
sc(ref("file_handlers"), sc(map(name, ref("log_file_handler")),
#{})} #{})}
, {"error_logger", , {"error_logger",
sc(atom(), sc(atom(),
@ -396,12 +424,6 @@ fields("console_handler") ->
})} })}
] ++ log_handler_common_confs(); ] ++ log_handler_common_confs();
fields("file_handlers") ->
[ {"$name",
sc(ref("log_file_handler"),
#{})}
];
fields("log_file_handler") -> fields("log_file_handler") ->
[ {"file", [ {"file",
sc(file(), sc(file(),
@ -701,6 +723,8 @@ keys(Parent, Conf) ->
sc(Type, Meta) -> hoconsc:mk(Type, Meta). sc(Type, Meta) -> hoconsc:mk(Type, Meta).
map(Name, Type) -> hoconsc:map(Name, Type).
ref(Field) -> hoconsc:ref(?MODULE, Field). ref(Field) -> hoconsc:ref(?MODULE, Field).
options(static, Conf) -> options(static, Conf) ->
@ -742,5 +766,4 @@ to_atom(Bin) when is_binary(Bin) ->
binary_to_atom(Bin, utf8). binary_to_atom(Bin, utf8).
roots(Module) -> roots(Module) ->
lists:map(fun({_BinName, Root}) -> Root end, lists:map(fun({_BinName, Root}) -> Root end, hocon_schema:roots(Module)).
maps:to_list(hocon_schema:roots(Module))).

30
bin/emqx
View File

@ -14,6 +14,11 @@ ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
# shellcheck disable=SC1090 # shellcheck disable=SC1090
. "$ROOT_DIR"/releases/emqx_vars . "$ROOT_DIR"/releases/emqx_vars
# defined in emqx_vars
export RUNNER_ROOT_DIR
export RUNNER_ETC_DIR
export REL_VSN
RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME" RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME"
CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}" CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}"
REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN" REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN"
@ -133,7 +138,6 @@ check_user() {
fi fi
} }
# Make sure the user running this script is the owner and/or su to that user # Make sure the user running this script is the owner and/or su to that user
check_user "$@" check_user "$@"
ES=$? ES=$?
@ -198,18 +202,12 @@ relx_gen_id() {
# Control a node # Control a node
relx_nodetool() { relx_nodetool() {
command="$1"; shift command="$1"; shift
export RUNNER_ROOT_DIR
export REL_VSN
ERL_FLAGS="$ERL_FLAGS $EPMD_ARG" \ ERL_FLAGS="$ERL_FLAGS $EPMD_ARG" \
"$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \ "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \
-setcookie "$COOKIE" "$command" "$@" -setcookie "$COOKIE" "$command" "$@"
} }
call_hocon() { call_hocon() {
export RUNNER_ROOT_DIR
export RUNNER_ETC_DIR
export REL_VSN
"$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" hocon "$@" \ "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" hocon "$@" \
|| die "call_hocon_failed: $*" $? || die "call_hocon_failed: $*" $?
} }
@ -217,8 +215,6 @@ call_hocon() {
# Run an escript in the node's environment # Run an escript in the node's environment
relx_escript() { relx_escript() {
shift; scriptpath="$1"; shift shift; scriptpath="$1"; shift
export RUNNER_ROOT_DIR
"$ERTS_DIR/bin/escript" "$ROOTDIR/$scriptpath" "$@" "$ERTS_DIR/bin/escript" "$ROOTDIR/$scriptpath" "$@"
} }
@ -365,18 +361,14 @@ esac
## or long name (with '@') e.g. 'emqx@example.net' or 'emqx@127.0.0.1' ## or long name (with '@') e.g. 'emqx@example.net' or 'emqx@127.0.0.1'
NAME="${EMQX_NODE_NAME:-}" NAME="${EMQX_NODE_NAME:-}"
if [ -z "$NAME" ]; then if [ -z "$NAME" ]; then
if [ "$IS_BOOT_COMMAND" = 'no' ]; then if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
# for boot commands, inspect emqx.conf for node name
NAME="$(call_hocon -s $SCHEMA_MOD -c "$RUNNER_ETC_DIR"/emqx.conf get node.name | tr -d \")"
else
# for non-boot commands, inspect vm.<time>.args for node name # for non-boot commands, inspect vm.<time>.args for node name
# shellcheck disable=SC2012,SC2086 # shellcheck disable=SC2012,SC2086
LATEST_VM_ARGS="$(ls -t $CONFIGS_DIR/vm.*.args | head -1)" LATEST_VM_ARGS="$(ls -t $CONFIGS_DIR/vm.*.args | head -1)"
if [ -z "$LATEST_VM_ARGS" ]; then
echoerr "no_vm_arg_file_found_for $1 in $CONFIGS_DIR/"
exit 1
fi
NAME="$(grep -E '^-s?name' "$LATEST_VM_ARGS" | awk '{print $2}')" NAME="$(grep -E '^-s?name' "$LATEST_VM_ARGS" | awk '{print $2}')"
else
# for boot commands, inspect emqx.conf for node name
NAME="$(call_hocon -s $SCHEMA_MOD -c "$RUNNER_ETC_DIR"/emqx.conf get node.name | tr -d \")"
fi fi
fi fi
@ -402,10 +394,6 @@ if [ -z "$COOKIE" ]; then
else else
# shellcheck disable=SC2012,SC2086 # shellcheck disable=SC2012,SC2086
LATEST_VM_ARGS="$(ls -t $CONFIGS_DIR/vm.*.args | head -1)" LATEST_VM_ARGS="$(ls -t $CONFIGS_DIR/vm.*.args | head -1)"
if [ -z "$LATEST_VM_ARGS" ]; then
echoerr "no_vm_arg_file_found_for $1 in $CONFIGS_DIR/"
exit 1
fi
COOKIE="$(grep -E '^-setcookie' "$LATEST_VM_ARGS" | awk '{print $2}')" COOKIE="$(grep -E '^-setcookie' "$LATEST_VM_ARGS" | awk '{print $2}')"
fi fi
fi fi

51
bin/nodetool
View File

@ -117,40 +117,43 @@ do(Args) ->
io:format("~p\n", [Other]) io:format("~p\n", [Other])
end; end;
["eval" | ListOfArgs] -> ["eval" | ListOfArgs] ->
% shells may process args into more than one, and end up stripping Parsed = parse_eval_args(ListOfArgs),
% spaces, so this converts all of that to a single string to parse
String = binary_to_list(
list_to_binary(
join(ListOfArgs," ")
)
),
% then just as a convenience to users, if they forgot a trailing
% '.' add it for them.
Normalized =
case lists:reverse(String) of
[$. | _] -> String;
R -> lists:reverse([$. | R])
end,
% then scan and parse the string
{ok, Scanned, _} = erl_scan:string(Normalized),
{ok, Parsed } = erl_parse:parse_exprs(Scanned),
% and evaluate it on the remote node % and evaluate it on the remote node
case rpc:call(TargetNode, erl_eval, exprs, [Parsed, [] ]) of case rpc:call(TargetNode, erl_eval, exprs, [Parsed, [] ]) of
{value, Value, _} -> {value, Value, _} ->
io:format ("~p\n",[Value]); io:format ("~p~n",[Value]);
{badrpc, Reason} -> {badrpc, Reason} ->
io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), io:format("RPC to ~p failed: ~p~n", [TargetNode, Reason]),
halt(1) halt(1)
end; end;
Other -> Other ->
io:format("Other: ~p\n", [Other]), io:format("Other: ~p~n", [Other]),
io:format("Usage: nodetool {genconfig, chkconfig|getpid|ping|stop|rpc|rpc_infinity|rpcterms|eval [Terms]} [RPC]\n") io:format("Usage: nodetool chkconfig|getpid|ping|stop|rpc|rpc_infinity|rpcterms|eval|cold_eval [Terms] [RPC]\n")
end, end,
net_kernel:stop(). net_kernel:stop().
parse_eval_args(Args) ->
% shells may process args into more than one, and end up stripping
% spaces, so this converts all of that to a single string to parse
String = binary_to_list(
list_to_binary(
join(Args," ")
)
),
% then just as a convenience to users, if they forgot a trailing
% '.' add it for them.
Normalized =
case lists:reverse(String) of
[$. | _] -> String;
R -> lists:reverse([$. | R])
end,
% then scan and parse the string
{ok, Scanned, _} = erl_scan:string(Normalized),
{ok, Parsed } = erl_parse:parse_exprs(Scanned),
Parsed.
do_with_ret(Args, Name, Handler) -> do_with_ret(Args, Name, Handler) ->
{arity, Arity} = erlang:fun_info(Handler, arity), {arity, Arity} = erlang:fun_info(Handler, arity),
case take_args(Args, Name, Arity) of case take_args(Args, Name, Arity) of

18
build
View File

@ -62,13 +62,29 @@ log() {
echo "===< $msg" echo "===< $msg"
} }
docgen() {
local conf_doc_html libs_dir1 libs_dir2
conf_doc_html="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.html"
echo "===< Generating config document $conf_doc_html"
libs_dir1="$(find "_build/default/lib/" -maxdepth 2 -name ebin -type d)"
libs_dir2="$(find "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)"
# shellcheck disable=SC2086
erl -noshell -pa $libs_dir1 $libs_dir2 -eval "file:write_file('$conf_doc_html', hocon_schema_html:gen(emqx_machine_schema, \"EMQ X ${PKG_VSN}\")), halt(0)."
local conf_doc_markdown
conf_doc_markdown="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.md"
echo "===< Generating config document $conf_doc_markdown"
# shellcheck disable=SC2086
erl -noshell -pa $libs_dir1 $libs_dir2 -eval "file:write_file('$conf_doc_markdown', hocon_schema_doc:gen(emqx_machine_schema)), halt(0)."
}
make_rel() { make_rel() {
# shellcheck disable=SC1010 # shellcheck disable=SC1010
./rebar3 as "$PROFILE" do release,tar ./rebar3 as "$PROFILE" do release,tar
if [ "$(find "_build/$PROFILE/rel/emqx/lib/" -name 'gpb-*' -type d)" != "" ]; then if [ "$(find "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then
echo "gpb should not be included in the release" echo "gpb should not be included in the release"
exit 1 exit 1
fi fi
docgen
} }
## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup

2
rebar.config
View File

@ -60,7 +60,7 @@
, {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
, {getopt, "1.0.2"} , {getopt, "1.0.2"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.15.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.0"}}}
, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.0"}}}
, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}}