Merge remote-tracking branch 'stone/0922-upgrade-ip-port-schema' into remove-colon-port

apps/emqx/src/emqx_schema.erl

@@ -40,6 +40,7 @@
 -type comma_separated_atoms() :: [atom()].
 -type bar_separated_list() :: list().
 -type ip_port() :: tuple() | integer().
+-type host_port() :: tuple().
 -type cipher() :: map().
 
 -typerefl_from_string({duration/0, emqx_schema, to_duration}).
@@ -52,6 +53,7 @@
 -typerefl_from_string({comma_separated_binary/0, emqx_schema, to_comma_separated_binary}).
 -typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}).
 -typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}).
+-typerefl_from_string({host_port/0, emqx_schema, to_host_port}).
 -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}).
 -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}).
 
@@ -78,6 +80,7 @@
     to_comma_separated_binary/1,
     to_bar_separated_list/1,
     to_ip_port/1,
+    to_host_port/1,
     to_erl_cipher_suite/1,
     to_comma_separated_atoms/1
 ]).
@@ -96,6 +99,7 @@
     comma_separated_binary/0,
     bar_separated_list/0,
     ip_port/0,
+    host_port/0,
     cipher/0,
     comma_separated_atoms/0
 ]).
@@ -2167,33 +2171,60 @@ to_bar_separated_list(Str) ->
 %%  - :1883
 %%  - :::1883
 to_ip_port(Str) ->
-    case split_ip_port(Str) of
+    to_host_port(Str, ip_addr).
-        {"", Port} ->
+
+%% @doc support the following format:
+%%  - 127.0.0.1:1883
+%%  - ::1:1883
+%%  - [::1]:1883
+%%  - :1883
+%%  - :::1883
+%%  - example.com:80
+to_host_port(Str) ->
+    to_host_port(Str, hostname).
+
+%%  - example.com:80
+to_host_port(Str, IpOrHost) ->
+    case split_host_port(Str) of
+        {"", Port} when IpOrHost =:= ip_addr ->
+            %% this is a local address
             {ok, list_to_integer(Port)};
-        {Ip, Port} ->
+        {"", _Port} ->
+            %% must specify host part when it's a remote endpoint
+            {error, bad_host_port};
+        {MaybeIp, Port} ->
             PortVal = list_to_integer(Port),
-            case inet:parse_address(Ip) of
+            case inet:parse_address(MaybeIp) of
-                {ok, R} ->
+                {ok, IpTuple} ->
-                    {ok, {R, PortVal}};
+                    {ok, {IpTuple, PortVal}};
-                _ ->
+                _ when IpOrHost =:= hostname ->
                     %% check is a rfc1035's hostname
-                    case inet_parse:domain(Ip) of
+                    case inet_parse:domain(MaybeIp) of
                         true ->
-                            {ok, {Ip, PortVal}};
+                            {ok, {MaybeIp, PortVal}};
                         _ ->
-                            {error, Str}
+                            {error, bad_hostname}
-                    end
                     end;
                 _ ->
-            {error, Str}
+                    {error, bad_ip_port}
+            end;
+        _ ->
+            {error, bad_ip_port}
     end.
 
-split_ip_port(Str0) ->
+split_host_port(Str0) ->
     Str = re:replace(Str0, " ", "", [{return, list}, global]),
     case lists:split(string:rchr(Str, $:), Str) of
-        %% no port
+        %% no colon
         {[], Str} ->
-            error;
+            try
+                %% if it's just a port number, then return as-is
+                _ = list_to_integer(Str),
+                {"", Str}
+            catch
+                _:_ ->
+                    error
+            end;
         {IpPlusColon, PortString} ->
             IpStr0 = lists:droplast(IpPlusColon),
             case IpStr0 of

apps/emqx/test/emqx_schema_tests.erl

@@ -175,3 +175,30 @@ ssl_opts_gc_after_handshake_test_not_rancher_listener_test() ->
         Checked
     ),
     ok.
+
+to_ip_port_test_() ->
+    Ip = fun emqx_schema:to_ip_port/1,
+    Host = fun(Str) ->
+        case Ip(Str) of
+            {ok, {_, _} = Res} ->
+                %% assert
+                {ok, Res} = emqx_schema:to_host_port(Str);
+            _ ->
+                emqx_schema:to_host_port(Str)
+        end
+    end,
+    [
+        ?_assertEqual({ok, 80}, Ip("80")),
+        ?_assertEqual({error, bad_host_port}, Host("80")),
+        ?_assertEqual({ok, 80}, Ip(":80")),
+        ?_assertEqual({error, bad_host_port}, Host(":80")),
+        ?_assertEqual({error, bad_ip_port}, Ip("localhost:80")),
+        ?_assertEqual({ok, {"localhost", 80}}, Host("localhost:80")),
+        ?_assertEqual({ok, {"example.com", 80}}, Host("example.com:80")),
+        ?_assertEqual({ok, {{127, 0, 0, 1}, 80}}, Ip("127.0.0.1:80")),
+        ?_assertEqual({error, bad_ip_port}, Ip("$:1900")),
+        ?_assertEqual({error, bad_hostname}, Host("$:1900")),
+        ?_assertMatch({ok, {_, 1883}}, Ip("[::1]:1883")),
+        ?_assertMatch({ok, {_, 1883}}, Ip("::1:1883")),
+        ?_assertMatch({ok, {_, 1883}}, Ip(":::1883"))
+    ].

apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl

@@ -55,7 +55,7 @@ fields("connector") ->
             )},
         {server,
             sc(
-                emqx_schema:ip_port(),
+                emqx_schema:host_port(),
                 #{
                     required => true,
                     desc => ?DESC("server")

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -656,8 +656,8 @@ typename_to_spec("file()", _Mod) ->
     #{type => string, example => <<"/path/to/file">>};
 typename_to_spec("ip_port()", _Mod) ->
     #{type => string, example => <<"127.0.0.1:80">>};
-typename_to_spec("ip_ports()", _Mod) ->
+typename_to_spec("host_port()", _Mod) ->
-    #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>};
+    #{type => string, example => <<"example.host.domain:80">>};
 typename_to_spec("url()", _Mod) ->
     #{type => string, example => <<"http://127.0.0.1">>};
 typename_to_spec("connect_timeout()", Mod) ->

rel/emqx_conf.template.en.md

@@ -143,7 +143,19 @@ to even set complex values from environment variables.
 For example, this environment variable sets an array value.
 
 ```
-export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]"
+export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]'
+```
+
+However this also means a string value should be quoted if it happen to contain special
+characters such as `=` and `:`.
+
+For example, a string value `"localhost:1883"` would be 
+parsed into object (struct): `{"localhost": 1883}`.
+
+To keep it as a string, one should quote the value like below:
+
+```
+EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"'
 ```
 
 ::: tip Tip

rel/emqx_conf.template.zh.md

@@ -127,14 +127,24 @@ authentication.1.enable = true
 
 例如 `node.name` 的重载变量名是 `EMQX_NODE__NAME`。
 
-环境变量的值，是解析成HOCON值的。所以这也使得环境变量可以用来传递复杂数据类型的值。
+环境变量的值，是按 HOCON 值解析的，这也使得环境变量可以用来传递复杂数据类型的值。
 
 例如，下面这个环境变量传入一个数组类型的值。
 
 ```
-export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]"
+export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]'
 ```
 
+这也意味着有些带特殊字符（例如`:` 和 `=`），则需要用双引号对这个值包起来。
+
+例如`localhost:1883` 会被解析成一个结构体 `{"localhost": 1883}`。
+想要把它当字符串使用时，就必需使用引号，如下：
+
+```
+EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"'
+```
+
+
 ::: tip Tip
 未定义的根路径会被EMQX忽略，例如 `EMQX_UNKNOWN_ROOT__FOOBAR` 这个环境变量会被EMQX忽略，
 因为 `UNKNOWN_ROOT` 不是预先定义好的根路径。