From 1db9d54d1733593767856eb4aee7f6c9f3f529c8 Mon Sep 17 00:00:00 2001 From: zmstone Date: Mon, 27 May 2024 16:29:33 +0200 Subject: [PATCH 1/4] fix: handle unicode in data integration template strings The bug was introduced in 5.7.0 (#12893) --- apps/emqx/src/emqx_schema.erl | 26 ++++++++++++++------------ apps/emqx/test/emqx_schema_tests.erl | 10 ++++++++++ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 948849ccb..da8b885c6 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2471,7 +2471,7 @@ converter_ciphers(Ciphers, _Opts) when is_binary(Ciphers) -> default_ciphers(Which) -> lists:map( - fun erlang:iolist_to_binary/1, + fun unicode:characters_to_binary/1, do_default_ciphers(Which) ). @@ -2494,7 +2494,7 @@ bin_str_converter(I, _) when is_integer(I) -> integer_to_binary(I); bin_str_converter(X, _) -> try - iolist_to_binary(X) + unicode:characters_to_binary(X) catch _:_ -> throw("must_quote") @@ -2649,7 +2649,7 @@ to_comma_separated_list(Str) -> {ok, string:tokens(Str, ", ")}. to_comma_separated_binary(Str) -> - {ok, lists:map(fun erlang:list_to_binary/1, string:tokens(Str, ", "))}. + {ok, lists:map(fun unicode:characters_to_binary/1, string:tokens(Str, ", "))}. to_comma_separated_atoms(Str) -> {ok, lists:map(fun to_atom/1, string:tokens(Str, ", "))}. @@ -2658,7 +2658,7 @@ to_url(Str) -> case emqx_http_lib:uri_parse(Str) of {ok, URIMap} -> URIString = emqx_http_lib:normalize(URIMap), - {ok, iolist_to_binary(URIString)}; + {ok, unicode:characters_to_binary(URIString)}; Error -> Error end. @@ -2666,13 +2666,13 @@ to_url(Str) -> to_json_binary(Str) -> case emqx_utils_json:safe_decode(Str) of {ok, _} -> - {ok, iolist_to_binary(Str)}; + {ok, unicode:characters_to_binary(Str)}; Error -> Error end. to_template(Str) -> - {ok, iolist_to_binary(Str)}. + {ok, unicode:characters_to_binary(Str, utf8)}. to_template_str(Str) -> {ok, unicode:characters_to_list(Str, utf8)}. @@ -2768,7 +2768,7 @@ validate_keepalive_multiplier(_Multiplier) -> {error, #{reason => keepalive_multiplier_out_of_range, min => 1, max => 65535}}. validate_tcp_keepalive(Value) -> - case iolist_to_binary(Value) of + case unicode:characters_to_binary(Value) of <<"none">> -> ok; _ -> @@ -2949,7 +2949,7 @@ convert_servers(undefined) -> convert_servers(Map) when is_map(Map) -> try List = convert_hocon_map_host_port(Map), - iolist_to_binary(string:join(List, ",")) + unicode:characters_to_binary(string:join(List, ",")) catch _:_ -> throw("bad_host_port") @@ -2957,13 +2957,13 @@ convert_servers(Map) when is_map(Map) -> convert_servers([H | _] = Array) when is_binary(H) orelse is_list(H) -> %% if the old config was a string array %% we want to make sure it's converted to a comma-separated - iolist_to_binary([[I, ","] || I <- Array]); + unicode:characters_to_binary([[I, ","] || I <- Array]); convert_servers(Str) -> normalize_host_port_str(Str). %% remove spaces around comma (,) normalize_host_port_str(Str) -> - iolist_to_binary(re:replace(Str, "(\s)*,(\s)*", ",")). + unicode:characters_to_binary(re:replace(Str, "(\s)*,(\s)*", ",")). %% @doc Shared validation function for both 'server' and 'servers' string. %% NOTE: Validator is called after converter. @@ -3442,8 +3442,10 @@ ensure_default_listener(Map, ListenerType) -> NewMap = Map#{<<"default">> => default_listener(ListenerType)}, keep_default_tombstone(NewMap, #{}). -cert_file(_File, client) -> undefined; -cert_file(File, server) -> iolist_to_binary(filename:join(["${EMQX_ETC_DIR}", "certs", File])). +cert_file(_File, client) -> + undefined; +cert_file(File, server) -> + unicode:characters_to_binary(filename:join(["${EMQX_ETC_DIR}", "certs", File])). mqtt_converter(#{<<"keepalive_multiplier">> := Multi} = Mqtt, _Opts) -> case round(Multi * 100) =:= round(?DEFAULT_MULTIPLIER * 100) of diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index 2f3f2aa1d..b94815995 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -930,3 +930,13 @@ timeout_types_test_() -> typerefl:from_string(emqx_schema:timeout_duration_s(), <<"4294967001ms">>) ) ]. + +unicode_template_test() -> + Sc = #{ + roots => [root], + fields => #{root => [{template, #{type => emqx_schema:template()}}]} + }, + HoconText = <<"root = {template = \"中文\"}"/utf8>>, + {ok, Hocon} = hocon:binary(HoconText), + _ = hocon_tconf:check_plain(Sc, Hocon), + ok. From 78a6100346ceb837bcab156d0f7bf456c00586e1 Mon Sep 17 00:00:00 2001 From: zmstone Date: Wed, 29 May 2024 21:56:22 +0200 Subject: [PATCH 2/4] chore: fix app vsn bumps only bug fixes so far --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_auth/src/emqx_auth.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index c391bc1c8..97769fe1f 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.4.0"}, + {vsn, "5.3.1"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_auth/src/emqx_auth.app.src b/apps/emqx_auth/src/emqx_auth.app.src index 35f3ff8b0..db4b69ded 100644 --- a/apps/emqx_auth/src/emqx_auth.app.src +++ b/apps/emqx_auth/src/emqx_auth.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth, [ {description, "EMQX Authentication and authorization"}, - {vsn, "0.4.0"}, + {vsn, "0.3.1"}, {modules, []}, {registered, [emqx_auth_sup]}, {applications, [ diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index f01f5c7b9..7c99b70cd 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.3.0"}, + {vsn, "5.2.1"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [ From 9fe29bf5f748252493d2825e97f0731fc6038265 Mon Sep 17 00:00:00 2001 From: zmstone Date: Wed, 29 May 2024 21:57:08 +0200 Subject: [PATCH 3/4] chore: bump kafka bridge app vsn --- apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src index d7af76b8b..ea7bfaba8 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_kafka, [ {description, "EMQX Enterprise Kafka Bridge"}, - {vsn, "0.3.0"}, + {vsn, "0.3.1"}, {registered, [emqx_bridge_kafka_consumer_sup]}, {applications, [ kernel, From 532812f6f6efafbcec81c0ceadc48f216d7a9028 Mon Sep 17 00:00:00 2001 From: zmstone Date: Wed, 29 May 2024 22:10:34 +0200 Subject: [PATCH 4/4] test: assert results --- apps/emqx/test/emqx_schema_tests.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index b94815995..8f72c3d48 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -938,5 +938,7 @@ unicode_template_test() -> }, HoconText = <<"root = {template = \"中文\"}"/utf8>>, {ok, Hocon} = hocon:binary(HoconText), - _ = hocon_tconf:check_plain(Sc, Hocon), - ok. + ?assertEqual( + #{<<"root">> => #{<<"template">> => <<"中文"/utf8>>}}, + hocon_tconf:check_plain(Sc, Hocon) + ).