Merge pull request #9047 from HJianBo/remove-colon-port

Improve the `ip_port()` definition and `bind` print format

apps/emqx/src/emqx_listeners.erl

@@ -521,12 +521,16 @@ merge_default(Options) ->
     integer() | {tuple(), integer()} | string() | binary()
 ) -> io_lib:chars().
 format_bind(Port) when is_integer(Port) ->
+    %% **Note**:
+    %% 'For TCP, UDP and IP networks, if the host is empty or a literal
+    %% unspecified IP address, as in ":80", "0.0.0.0:80" or "[::]:80" for
+    %% TCP and UDP, "", "0.0.0.0" or "::" for IP, the local system is
+    %% assumed.'
+    %%
+    %% Quoted from: https://pkg.go.dev/net
+    %% Decided to use this format to display the bind for all interfaces and
+    %% IPv4/IPv6 support
     io_lib:format(":~w", [Port]);
-%% Print only the port number when bound on all interfaces
-format_bind({{0, 0, 0, 0}, Port}) ->
-    format_bind(Port);
-format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, Port}) ->
-    format_bind(Port);
 format_bind({Addr, Port}) when is_list(Addr) ->
     io_lib:format("~ts:~w", [Addr, Port]);
 format_bind({Addr, Port}) when is_tuple(Addr), tuple_size(Addr) == 4 ->
@@ -538,6 +542,8 @@ format_bind(Str) when is_list(Str) ->
     case emqx_schema:to_ip_port(Str) of
         {ok, {Ip, Port}} ->
             format_bind({Ip, Port});
+        {ok, Port} ->
+            format_bind(Port);
         {error, _} ->
             format_bind(list_to_integer(Str))
     end;

apps/emqx/src/emqx_schema.erl

@@ -39,7 +39,8 @@
 -type comma_separated_binary() :: [binary()].
 -type comma_separated_atoms() :: [atom()].
 -type bar_separated_list() :: list().
--type ip_port() :: tuple().
+-type ip_port() :: tuple() | integer().
+-type host_port() :: tuple().
 -type cipher() :: map().
 
 -typerefl_from_string({duration/0, emqx_schema, to_duration}).
@@ -52,6 +53,7 @@
 -typerefl_from_string({comma_separated_binary/0, emqx_schema, to_comma_separated_binary}).
 -typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}).
 -typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}).
+-typerefl_from_string({host_port/0, emqx_schema, to_host_port}).
 -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}).
 -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}).
 
@@ -78,6 +80,7 @@
     to_comma_separated_binary/1,
     to_bar_separated_list/1,
     to_ip_port/1,
+    to_host_port/1,
     to_erl_cipher_suite/1,
     to_comma_separated_atoms/1
 ]).
@@ -96,6 +99,7 @@
     comma_separated_binary/0,
     bar_separated_list/0,
     ip_port/0,
+    host_port/0,
     cipher/0,
     comma_separated_atoms/0
 ]).
@@ -2167,33 +2171,60 @@ to_bar_separated_list(Str) ->
 %%  - :1883
 %%  - :::1883
 to_ip_port(Str) ->
-    case split_ip_port(Str) of
+    to_host_port(Str, ip_addr).
-        {"", Port} ->
+
-            {ok, {{0, 0, 0, 0}, list_to_integer(Port)}};
+%% @doc support the following format:
-        {Ip, Port} ->
+%%  - 127.0.0.1:1883
+%%  - ::1:1883
+%%  - [::1]:1883
+%%  - :1883
+%%  - :::1883
+%%  - example.com:80
+to_host_port(Str) ->
+    to_host_port(Str, hostname).
+
+%%  - example.com:80
+to_host_port(Str, IpOrHost) ->
+    case split_host_port(Str) of
+        {"", Port} when IpOrHost =:= ip_addr ->
+            %% this is a local address
+            {ok, list_to_integer(Port)};
+        {"", _Port} ->
+            %% must specify host part when it's a remote endpoint
+            {error, bad_host_port};
+        {MaybeIp, Port} ->
             PortVal = list_to_integer(Port),
-            case inet:parse_address(Ip) of
+            case inet:parse_address(MaybeIp) of
-                {ok, R} ->
+                {ok, IpTuple} ->
-                    {ok, {R, PortVal}};
+                    {ok, {IpTuple, PortVal}};
-                _ ->
+                _ when IpOrHost =:= hostname ->
                     %% check is a rfc1035's hostname
-                    case inet_parse:domain(Ip) of
+                    case inet_parse:domain(MaybeIp) of
                         true ->
-                            {ok, {Ip, PortVal}};
+                            {ok, {MaybeIp, PortVal}};
                         _ ->
-                            {error, Str}
+                            {error, bad_hostname}
-                    end
                     end;
                 _ ->
-            {error, Str}
+                    {error, bad_ip_port}
+            end;
+        _ ->
+            {error, bad_ip_port}
     end.
 
-split_ip_port(Str0) ->
+split_host_port(Str0) ->
     Str = re:replace(Str0, " ", "", [{return, list}, global]),
     case lists:split(string:rchr(Str, $:), Str) of
-        %% no port
+        %% no colon
         {[], Str} ->
-            error;
+            try
+                %% if it's just a port number, then return as-is
+                _ = list_to_integer(Str),
+                {"", Str}
+            catch
+                _:_ ->
+                    error
+            end;
         {IpPlusColon, PortString} ->
             IpStr0 = lists:droplast(IpPlusColon),
             case IpStr0 of

apps/emqx/test/emqx_listeners_SUITE.erl

@@ -148,6 +148,32 @@ t_wss_conn(_) ->
     {ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000),
     ok = ssl:close(Socket).
 
+t_format_bind(_) ->
+    ?assertEqual(
+        ":1883",
+        lists:flatten(emqx_listeners:format_bind(1883))
+    ),
+    ?assertEqual(
+        "0.0.0.0:1883",
+        lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0}, 1883}))
+    ),
+    ?assertEqual(
+        "[::]:1883",
+        lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, 1883}))
+    ),
+    ?assertEqual(
+        "127.0.0.1:1883",
+        lists:flatten(emqx_listeners:format_bind({{127, 0, 0, 1}, 1883}))
+    ),
+    ?assertEqual(
+        ":1883",
+        lists:flatten(emqx_listeners:format_bind("1883"))
+    ),
+    ?assertEqual(
+        ":1883",
+        lists:flatten(emqx_listeners:format_bind(":1883"))
+    ).
+
 render_config_file() ->
     Path = local_path(["etc", "emqx.conf"]),
     {ok, Temp} = file:read_file(Path),

apps/emqx/test/emqx_schema_tests.erl

@@ -175,3 +175,30 @@ ssl_opts_gc_after_handshake_test_not_rancher_listener_test() ->
         Checked
     ),
     ok.
+
+to_ip_port_test_() ->
+    Ip = fun emqx_schema:to_ip_port/1,
+    Host = fun(Str) ->
+        case Ip(Str) of
+            {ok, {_, _} = Res} ->
+                %% assert
+                {ok, Res} = emqx_schema:to_host_port(Str);
+            _ ->
+                emqx_schema:to_host_port(Str)
+        end
+    end,
+    [
+        ?_assertEqual({ok, 80}, Ip("80")),
+        ?_assertEqual({error, bad_host_port}, Host("80")),
+        ?_assertEqual({ok, 80}, Ip(":80")),
+        ?_assertEqual({error, bad_host_port}, Host(":80")),
+        ?_assertEqual({error, bad_ip_port}, Ip("localhost:80")),
+        ?_assertEqual({ok, {"localhost", 80}}, Host("localhost:80")),
+        ?_assertEqual({ok, {"example.com", 80}}, Host("example.com:80")),
+        ?_assertEqual({ok, {{127, 0, 0, 1}, 80}}, Ip("127.0.0.1:80")),
+        ?_assertEqual({error, bad_ip_port}, Ip("$:1900")),
+        ?_assertEqual({error, bad_hostname}, Host("$:1900")),
+        ?_assertMatch({ok, {_, 1883}}, Ip("[::1]:1883")),
+        ?_assertMatch({ok, {_, 1883}}, Ip("::1:1883")),
+        ?_assertMatch({ok, {_, 1883}}, Ip(":::1883"))
+    ].

apps/emqx/test/emqx_shared_sub_SUITE.erl

@@ -594,7 +594,7 @@ t_remote(_) ->
 
     try
         {ok, ClientPidLocal} = emqtt:connect(ConnPidLocal),
-        {ok, ClientPidRemote} = emqtt:connect(ConnPidRemote),
+        {ok, _ClientPidRemote} = emqtt:connect(ConnPidRemote),
 
         emqtt:subscribe(ConnPidRemote, {<<"$share/remote_group/", Topic/binary>>, 0}),
 

apps/emqx_connector/src/emqx_connector.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "An OTP application"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -345,7 +345,7 @@ init_worker_options([], Acc) ->
 %% ===================================================================
 %% Schema funcs
 
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
 server(required) -> true;
 server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
 server(converter) -> fun to_server_raw/1;

apps/emqx_connector/src/emqx_connector_mysql.erl

@@ -55,7 +55,7 @@ fields(config) ->
         emqx_connector_schema_lib:ssl_fields() ++
         emqx_connector_schema_lib:prepare_statement_fields().
 
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
 server(required) -> true;
 server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
 server(converter) -> fun to_server/1;

apps/emqx_connector/src/emqx_connector_pgsql.erl

@@ -58,7 +58,7 @@ fields(config) ->
         emqx_connector_schema_lib:ssl_fields() ++
         emqx_connector_schema_lib:prepare_statement_fields().
 
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
 server(required) -> true;
 server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
 server(converter) -> fun to_server/1;

apps/emqx_connector/src/emqx_connector_redis.erl

@@ -97,7 +97,7 @@ fields(sentinel) ->
         redis_fields() ++
         emqx_connector_schema_lib:ssl_fields().
 
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
 server(required) -> true;
 server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
 server(converter) -> fun to_server_raw/1;

apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl

@@ -55,7 +55,7 @@ fields("connector") ->
             )},
         {server,
             sc(
-                emqx_schema:ip_port(),
+                emqx_schema:host_port(),
                 #{
                     required => true,
                     desc => ?DESC("server")

apps/emqx_dashboard/src/emqx_dashboard.app.src

@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.5"},
+    {vsn, "5.0.6"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx]},

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -656,8 +656,8 @@ typename_to_spec("file()", _Mod) ->
     #{type => string, example => <<"/path/to/file">>};
 typename_to_spec("ip_port()", _Mod) ->
     #{type => string, example => <<"127.0.0.1:80">>};
-typename_to_spec("ip_ports()", _Mod) ->
+typename_to_spec("host_port()", _Mod) ->
-    #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>};
+    #{type => string, example => <<"example.host.domain:80">>};
 typename_to_spec("url()", _Mod) ->
     #{type => string, example => <<"http://127.0.0.1">>};
 typename_to_spec("connect_timeout()", Mod) ->

apps/emqx_gateway/src/emqx_gateway.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway, [
     {description, "The Gateway management application"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {mod, {emqx_gateway_app, []}},
     {applications, [kernel, stdlib, grpc, emqx, emqx_authn]},

apps/emqx_gateway/src/emqx_gateway_schema.erl

@@ -28,7 +28,7 @@
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("typerefl/include/types.hrl").
 
--type ip_port() :: tuple().
+-type ip_port() :: tuple() | integer().
 -type duration() :: non_neg_integer().
 -type duration_s() :: non_neg_integer().
 -type bytesize() :: pos_integer().

apps/emqx_statsd/src/emqx_statsd.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_statsd, [
     {description, "An OTP application"},
-    {vsn, "5.0.1"},
+    {vsn, "5.0.2"},
     {registered, []},
     {mod, {emqx_statsd_app, []}},
     {applications, [

apps/emqx_statsd/src/emqx_statsd_schema.erl

@@ -21,8 +21,6 @@
 
 -behaviour(hocon_schema).
 
--export([to_ip_port/1]).
-
 -export([
     namespace/0,
     roots/0,
@@ -30,8 +28,6 @@
     desc/1
 ]).
 
--typerefl_from_string({ip_port/0, emqx_statsd_schema, to_ip_port}).
-
 namespace() -> "statsd".
 
 roots() -> ["statsd"].
@@ -55,7 +51,7 @@ fields("statsd") ->
 desc("statsd") -> ?DESC(statsd);
 desc(_) -> undefined.
 
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
 server(required) -> true;
 server(default) -> "127.0.0.1:8125";
 server(desc) -> ?DESC(?FUNCTION_NAME);
@@ -72,14 +68,3 @@ flush_interval(required) -> true;
 flush_interval(default) -> "10s";
 flush_interval(desc) -> ?DESC(?FUNCTION_NAME);
 flush_interval(_) -> undefined.
-
-to_ip_port(Str) ->
-    case string:tokens(Str, ":") of
-        [Ip, Port] ->
-            case inet:parse_address(Ip) of
-                {ok, R} -> {ok, {R, list_to_integer(Port)}};
-                _ -> {error, Str}
-            end;
-        _ ->
-            {error, Str}
-    end.

rel/emqx_conf.template.en.md

@@ -143,7 +143,19 @@ to even set complex values from environment variables.
 For example, this environment variable sets an array value.
 
 ```
-export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]"
+export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]'
+```
+
+However this also means a string value should be quoted if it happen to contain special
+characters such as `=` and `:`.
+
+For example, a string value `"localhost:1883"` would be 
+parsed into object (struct): `{"localhost": 1883}`.
+
+To keep it as a string, one should quote the value like below:
+
+```
+EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"'
 ```
 
 ::: tip Tip

rel/emqx_conf.template.zh.md

@@ -127,14 +127,24 @@ authentication.1.enable = true
 
 例如 `node.name` 的重载变量名是 `EMQX_NODE__NAME`。
 
-环境变量的值，是解析成HOCON值的。所以这也使得环境变量可以用来传递复杂数据类型的值。
+环境变量的值，是按 HOCON 值解析的，这也使得环境变量可以用来传递复杂数据类型的值。
 
 例如，下面这个环境变量传入一个数组类型的值。
 
 ```
-export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]"
+export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]'
 ```
 
+这也意味着有些带特殊字符（例如`:` 和 `=`），则需要用双引号对这个值包起来。
+
+例如`localhost:1883` 会被解析成一个结构体 `{"localhost": 1883}`。
+想要把它当字符串使用时，就必需使用引号，如下：
+
+```
+EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"'
+```
+
+
 ::: tip Tip
 未定义的根路径会被EMQX忽略，例如 `EMQX_UNKNOWN_ROOT__FOOBAR` 这个环境变量会被EMQX忽略，
 因为 `UNKNOWN_ROOT` 不是预先定义好的根路径。