diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 67f452e1d..2a809f88d 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -521,12 +521,16 @@ merge_default(Options) -> integer() | {tuple(), integer()} | string() | binary() ) -> io_lib:chars(). format_bind(Port) when is_integer(Port) -> + %% **Note**: + %% 'For TCP, UDP and IP networks, if the host is empty or a literal + %% unspecified IP address, as in ":80", "0.0.0.0:80" or "[::]:80" for + %% TCP and UDP, "", "0.0.0.0" or "::" for IP, the local system is + %% assumed.' + %% + %% Quoted from: https://pkg.go.dev/net + %% Decided to use this format to display the bind for all interfaces and + %% IPv4/IPv6 support io_lib:format(":~w", [Port]); -%% Print only the port number when bound on all interfaces -format_bind({{0, 0, 0, 0}, Port}) -> - format_bind(Port); -format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, Port}) -> - format_bind(Port); format_bind({Addr, Port}) when is_list(Addr) -> io_lib:format("~ts:~w", [Addr, Port]); format_bind({Addr, Port}) when is_tuple(Addr), tuple_size(Addr) == 4 -> @@ -538,6 +542,8 @@ format_bind(Str) when is_list(Str) -> case emqx_schema:to_ip_port(Str) of {ok, {Ip, Port}} -> format_bind({Ip, Port}); + {ok, Port} -> + format_bind(Port); {error, _} -> format_bind(list_to_integer(Str)) end; diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 35550d4e2..bce1ca8f3 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -39,7 +39,8 @@ -type comma_separated_binary() :: [binary()]. -type comma_separated_atoms() :: [atom()]. -type bar_separated_list() :: list(). --type ip_port() :: tuple(). +-type ip_port() :: tuple() | integer(). +-type host_port() :: tuple(). -type cipher() :: map(). -typerefl_from_string({duration/0, emqx_schema, to_duration}). @@ -52,6 +53,7 @@ -typerefl_from_string({comma_separated_binary/0, emqx_schema, to_comma_separated_binary}). -typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}). -typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). +-typerefl_from_string({host_port/0, emqx_schema, to_host_port}). -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). @@ -78,6 +80,7 @@ to_comma_separated_binary/1, to_bar_separated_list/1, to_ip_port/1, + to_host_port/1, to_erl_cipher_suite/1, to_comma_separated_atoms/1 ]). @@ -96,6 +99,7 @@ comma_separated_binary/0, bar_separated_list/0, ip_port/0, + host_port/0, cipher/0, comma_separated_atoms/0 ]). @@ -2167,33 +2171,60 @@ to_bar_separated_list(Str) -> %% - :1883 %% - :::1883 to_ip_port(Str) -> - case split_ip_port(Str) of - {"", Port} -> - {ok, {{0, 0, 0, 0}, list_to_integer(Port)}}; - {Ip, Port} -> + to_host_port(Str, ip_addr). + +%% @doc support the following format: +%% - 127.0.0.1:1883 +%% - ::1:1883 +%% - [::1]:1883 +%% - :1883 +%% - :::1883 +%% - example.com:80 +to_host_port(Str) -> + to_host_port(Str, hostname). + +%% - example.com:80 +to_host_port(Str, IpOrHost) -> + case split_host_port(Str) of + {"", Port} when IpOrHost =:= ip_addr -> + %% this is a local address + {ok, list_to_integer(Port)}; + {"", _Port} -> + %% must specify host part when it's a remote endpoint + {error, bad_host_port}; + {MaybeIp, Port} -> PortVal = list_to_integer(Port), - case inet:parse_address(Ip) of - {ok, R} -> - {ok, {R, PortVal}}; - _ -> + case inet:parse_address(MaybeIp) of + {ok, IpTuple} -> + {ok, {IpTuple, PortVal}}; + _ when IpOrHost =:= hostname -> %% check is a rfc1035's hostname - case inet_parse:domain(Ip) of + case inet_parse:domain(MaybeIp) of true -> - {ok, {Ip, PortVal}}; + {ok, {MaybeIp, PortVal}}; _ -> - {error, Str} - end + {error, bad_hostname} + end; + _ -> + {error, bad_ip_port} end; _ -> - {error, Str} + {error, bad_ip_port} end. -split_ip_port(Str0) -> +split_host_port(Str0) -> Str = re:replace(Str0, " ", "", [{return, list}, global]), case lists:split(string:rchr(Str, $:), Str) of - %% no port + %% no colon {[], Str} -> - error; + try + %% if it's just a port number, then return as-is + _ = list_to_integer(Str), + {"", Str} + catch + _:_ -> + error + end; {IpPlusColon, PortString} -> IpStr0 = lists:droplast(IpPlusColon), case IpStr0 of diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 6ea4d043d..5eb216be5 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -148,6 +148,32 @@ t_wss_conn(_) -> {ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000), ok = ssl:close(Socket). +t_format_bind(_) -> + ?assertEqual( + ":1883", + lists:flatten(emqx_listeners:format_bind(1883)) + ), + ?assertEqual( + "0.0.0.0:1883", + lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0}, 1883})) + ), + ?assertEqual( + "[::]:1883", + lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, 1883})) + ), + ?assertEqual( + "127.0.0.1:1883", + lists:flatten(emqx_listeners:format_bind({{127, 0, 0, 1}, 1883})) + ), + ?assertEqual( + ":1883", + lists:flatten(emqx_listeners:format_bind("1883")) + ), + ?assertEqual( + ":1883", + lists:flatten(emqx_listeners:format_bind(":1883")) + ). + render_config_file() -> Path = local_path(["etc", "emqx.conf"]), {ok, Temp} = file:read_file(Path), diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index 9118ac226..fdda9ef44 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -175,3 +175,30 @@ ssl_opts_gc_after_handshake_test_not_rancher_listener_test() -> Checked ), ok. + +to_ip_port_test_() -> + Ip = fun emqx_schema:to_ip_port/1, + Host = fun(Str) -> + case Ip(Str) of + {ok, {_, _} = Res} -> + %% assert + {ok, Res} = emqx_schema:to_host_port(Str); + _ -> + emqx_schema:to_host_port(Str) + end + end, + [ + ?_assertEqual({ok, 80}, Ip("80")), + ?_assertEqual({error, bad_host_port}, Host("80")), + ?_assertEqual({ok, 80}, Ip(":80")), + ?_assertEqual({error, bad_host_port}, Host(":80")), + ?_assertEqual({error, bad_ip_port}, Ip("localhost:80")), + ?_assertEqual({ok, {"localhost", 80}}, Host("localhost:80")), + ?_assertEqual({ok, {"example.com", 80}}, Host("example.com:80")), + ?_assertEqual({ok, {{127, 0, 0, 1}, 80}}, Ip("127.0.0.1:80")), + ?_assertEqual({error, bad_ip_port}, Ip("$:1900")), + ?_assertEqual({error, bad_hostname}, Host("$:1900")), + ?_assertMatch({ok, {_, 1883}}, Ip("[::1]:1883")), + ?_assertMatch({ok, {_, 1883}}, Ip("::1:1883")), + ?_assertMatch({ok, {_, 1883}}, Ip(":::1883")) + ]. diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index f53e8f374..8616028ca 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -594,7 +594,7 @@ t_remote(_) -> try {ok, ClientPidLocal} = emqtt:connect(ConnPidLocal), - {ok, ClientPidRemote} = emqtt:connect(ConnPidRemote), + {ok, _ClientPidRemote} = emqtt:connect(ConnPidRemote), emqtt:subscribe(ConnPidRemote, {<<"$share/remote_group/", Topic/binary>>, 0}), diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 0734d47b8..06da66398 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "An OTP application"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 5b07c5003..a5e2be521 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -345,7 +345,7 @@ init_worker_options([], Acc) -> %% =================================================================== %% Schema funcs -server(type) -> emqx_schema:ip_port(); +server(type) -> emqx_schema:host_port(); server(required) -> true; server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(converter) -> fun to_server_raw/1; diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index d6963d04e..79a306b05 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -55,7 +55,7 @@ fields(config) -> emqx_connector_schema_lib:ssl_fields() ++ emqx_connector_schema_lib:prepare_statement_fields(). -server(type) -> emqx_schema:ip_port(); +server(type) -> emqx_schema:host_port(); server(required) -> true; server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(converter) -> fun to_server/1; diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 43ec6be8b..f642ba75c 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -58,7 +58,7 @@ fields(config) -> emqx_connector_schema_lib:ssl_fields() ++ emqx_connector_schema_lib:prepare_statement_fields(). -server(type) -> emqx_schema:ip_port(); +server(type) -> emqx_schema:host_port(); server(required) -> true; server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(converter) -> fun to_server/1; diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 67310dbac..95677b766 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -97,7 +97,7 @@ fields(sentinel) -> redis_fields() ++ emqx_connector_schema_lib:ssl_fields(). -server(type) -> emqx_schema:ip_port(); +server(type) -> emqx_schema:host_port(); server(required) -> true; server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; server(converter) -> fun to_server_raw/1; diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 3e683f5fb..6fa511d00 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -55,7 +55,7 @@ fields("connector") -> )}, {server, sc( - emqx_schema:ip_port(), + emqx_schema:host_port(), #{ required => true, desc => ?DESC("server") diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index adf974243..9d5f85c7b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.5"}, + {vsn, "5.0.6"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 52cbc4775..5674d273c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -656,8 +656,8 @@ typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>}; typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>}; -typename_to_spec("ip_ports()", _Mod) -> - #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>}; +typename_to_spec("host_port()", _Mod) -> + #{type => string, example => <<"example.host.domain:80">>}; typename_to_spec("url()", _Mod) -> #{type => string, example => <<"http://127.0.0.1">>}; typename_to_spec("connect_timeout()", Mod) -> diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 47245c0a2..98dbd6909 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, grpc, emqx, emqx_authn]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 8850fa462..e5d6dd52a 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -28,7 +28,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). --type ip_port() :: tuple(). +-type ip_port() :: tuple() | integer(). -type duration() :: non_neg_integer(). -type duration_s() :: non_neg_integer(). -type bytesize() :: pos_integer(). diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 21a972266..76b04204b 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_statsd, [ {description, "An OTP application"}, - {vsn, "5.0.1"}, + {vsn, "5.0.2"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, [ diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl index e8cc32f99..9efde5afc 100644 --- a/apps/emqx_statsd/src/emqx_statsd_schema.erl +++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl @@ -21,8 +21,6 @@ -behaviour(hocon_schema). --export([to_ip_port/1]). - -export([ namespace/0, roots/0, @@ -30,8 +28,6 @@ desc/1 ]). --typerefl_from_string({ip_port/0, emqx_statsd_schema, to_ip_port}). - namespace() -> "statsd". roots() -> ["statsd"]. @@ -55,7 +51,7 @@ fields("statsd") -> desc("statsd") -> ?DESC(statsd); desc(_) -> undefined. -server(type) -> emqx_schema:ip_port(); +server(type) -> emqx_schema:host_port(); server(required) -> true; server(default) -> "127.0.0.1:8125"; server(desc) -> ?DESC(?FUNCTION_NAME); @@ -72,14 +68,3 @@ flush_interval(required) -> true; flush_interval(default) -> "10s"; flush_interval(desc) -> ?DESC(?FUNCTION_NAME); flush_interval(_) -> undefined. - -to_ip_port(Str) -> - case string:tokens(Str, ":") of - [Ip, Port] -> - case inet:parse_address(Ip) of - {ok, R} -> {ok, {R, list_to_integer(Port)}}; - _ -> {error, Str} - end; - _ -> - {error, Str} - end. diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index 76d25680b..520ed4578 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -143,7 +143,19 @@ to even set complex values from environment variables. For example, this environment variable sets an array value. ``` -export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]" +export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]' +``` + +However this also means a string value should be quoted if it happen to contain special +characters such as `=` and `:`. + +For example, a string value `"localhost:1883"` would be +parsed into object (struct): `{"localhost": 1883}`. + +To keep it as a string, one should quote the value like below: + +``` +EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"' ``` ::: tip Tip diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index ac4c5ce39..86162a4a9 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -127,14 +127,24 @@ authentication.1.enable = true 例如 `node.name` 的重载变量名是 `EMQX_NODE__NAME`。 -环境变量的值,是解析成HOCON值的。所以这也使得环境变量可以用来传递复杂数据类型的值。 +环境变量的值,是按 HOCON 值解析的,这也使得环境变量可以用来传递复杂数据类型的值。 例如,下面这个环境变量传入一个数组类型的值。 ``` -export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]" +export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]' ``` +这也意味着有些带特殊字符(例如`:` 和 `=`),则需要用双引号对这个值包起来。 + +例如`localhost:1883` 会被解析成一个结构体 `{"localhost": 1883}`。 +想要把它当字符串使用时,就必需使用引号,如下: + +``` +EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"' +``` + + ::: tip Tip 未定义的根路径会被EMQX忽略,例如 `EMQX_UNKNOWN_ROOT__FOOBAR` 这个环境变量会被EMQX忽略, 因为 `UNKNOWN_ROOT` 不是预先定义好的根路径。