From 03d8e06ff76e45e32fd8512ced4a25cab4a27021 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 6 Oct 2023 12:18:54 +0200 Subject: [PATCH 01/13] chore(emqx_rule_funcs): regroup export functions --- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index ae8e29593..c353742ae 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.27"}, + {vsn, "5.0.28"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 37797cb4f..aa5e271eb 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -80,14 +80,18 @@ tanh/1 ]). -%% Bits Funcs +%% Bitwise operations -export([ bitnot/1, bitand/2, bitor/2, bitxor/2, bitsl/2, - bitsr/2, + bitsr/2 +]). + +%% binary and bitstring Funcs +-export([ bitsize/1, bytesize/1, subbits/2, From 4654458cb08c4a67b1a287851313c0471d2a9a05 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 6 Oct 2023 12:44:05 +0200 Subject: [PATCH 02/13] build: use env variable to disalbe buildx provenance attestations So it can work with older version docker buildkit --- build | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/build b/build index 874e4088c..5c2bb556b 100755 --- a/build +++ b/build @@ -407,10 +407,10 @@ make_docker() { PRODUCT_DESCRIPTION='Official docker image for EMQX Enterprise, an enterprise MQTT platform at scale. ' DOCUMENTATION_URL='https://docs.emqx.com/en/enterprise/latest/' fi - # shellcheck disable=SC2155 - local ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" - # shellcheck disable=SC2155 - local GIT_REVISION="$(git rev-parse HEAD)" + local ISO_8601_DATE GIT_REVISION + ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + GIT_REVISION="$(git rev-parse HEAD)" + export BUILDX_NO_DEFAULT_ATTESTATIONS=1 local DOCKER_BUILDX_ARGS=( --build-arg BUILD_FROM="${EMQX_BUILDER}" \ --build-arg RUN_FROM="${EMQX_RUNNER}" \ @@ -430,7 +430,6 @@ make_docker() { --label org.opencontainers.image.licenses="${LICENSE}" \ --label org.opencontainers.image.otp.version="${EMQX_BUILDER_OTP}" \ --tag "${EMQX_IMAGE_TAG}" \ - --provenance false \ --pull ) if [ "${DOCKER_BUILD_NOCACHE:-false}" = true ]; then From 497e08448d0d85d0a3f9f493709f7193159fad2f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 2 Oct 2023 20:29:25 +0200 Subject: [PATCH 03/13] feat(cluster): support ipv6 and tls on ipv6 for clustering Made possible to configure inet6_tls for Erlang distribution Also, added support to configure ipv6 listener for gen_rpc --- apps/emqx_conf/src/emqx_conf_schema.erl | 24 +++++++++++++++++++++--- bin/emqx | 18 ++++++++---------- rel/i18n/emqx_conf_schema.hocon | 11 ++++++++++- 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 051e6885d..481eb59f2 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -195,7 +195,7 @@ fields("cluster") -> )}, {"proto_dist", sc( - hoconsc:enum([inet_tcp, inet6_tcp, inet_tls]), + hoconsc:enum([inet_tcp, inet6_tcp, inet_tls, inet6_tls]), #{ mapping => "ekka.proto_dist", default => inet_tcp, @@ -948,7 +948,16 @@ fields("rpc") -> } )}, {"ciphers", emqx_schema:ciphers_schema(tls_all_available)}, - {"tls_versions", emqx_schema:tls_versions_schema(tls_all_available)} + {"tls_versions", emqx_schema:tls_versions_schema(tls_all_available)}, + {"listen_address", + sc( + string(), + #{ + default => "0.0.0.0", + desc => ?DESC(rpc_listen_address), + importance => ?IMPORTANCE_MEDIUM + } + )} ]; fields("log") -> [ @@ -1133,7 +1142,16 @@ translation("gen_rpc") -> [ {"default_client_driver", fun tr_default_config_driver/1}, {"ssl_client_options", fun tr_gen_rpc_ssl_options/1}, - {"ssl_server_options", fun tr_gen_rpc_ssl_options/1} + {"ssl_server_options", fun tr_gen_rpc_ssl_options/1}, + {"socket_ip", fun(Conf) -> + Addr = conf_get("rpc.listen_address", Conf), + case inet:parse_address(Addr) of + {ok, Tuple} -> + Tuple; + {error, _Reason} -> + throw(#{bad_ip_address => Addr}) + end + end} ]; translation("prometheus") -> [ diff --git a/bin/emqx b/bin/emqx index 5484a82e5..f24210cdf 100755 --- a/bin/emqx +++ b/bin/emqx @@ -522,17 +522,13 @@ else ## only one emqx node is running, get running args from 'ps -ef' output tmp_nodename=$(echo -e "$PS_LINE" | $GREP -oE "\s-s?name.*" | awk '{print $2}' || true) tmp_cookie=$(echo -e "$PS_LINE" | $GREP -oE "\s-setcookie.*" | awk '{print $2}' || true) + tmp_proto_dist=$(echo -e "$PS_LINE" | $GREP -oE '\s-ekka_proto_dist.*' | awk '{print $2}' || echo 'inet_tcp') SSL_DIST_OPTFILE="$(echo -e "$PS_LINE" | $GREP -oE '\-ssl_dist_optfile\s.+\s' | awk '{print $2}' || true)" tmp_ticktime="$(echo -e "$PS_LINE" | $GREP -oE '\s-kernel\snet_ticktime\s.+\s' | awk '{print $3}' || true)" # data_dir is actually not needed, but kept anyway tmp_datadir="$(echo -e "$PS_LINE" | $GREP -oE "\-emqx_data_dir.*" | sed -E 's#.+emqx_data_dir[[:blank:]]##g' | sed -E 's#[[:blank:]]--$##g' || true)" - if [ -z "$SSL_DIST_OPTFILE" ]; then - tmp_proto='inet_tcp' - else - tmp_proto='inet_tls' - fi ## Make the format like what call_hocon multi_get prints out, but only need 4 args - EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}" + EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto_dist}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}" else if [ "$RUNNING_NODES_COUNT" -gt 1 ]; then if [ -z "${EMQX_NODE__NAME:-}" ]; then @@ -567,7 +563,7 @@ TICKTIME="$(get_boot_config 'node.dist_net_ticktime' || echo '120')" # this environment variable is required by ekka_dist module # because proto_dist is overriden to ekka, and there is a lack of ekka_tls module export EKKA_PROTO_DIST_MOD="${PROTO_DIST:-inet_tcp}" -if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ]; then +if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ] || [ "$EKKA_PROTO_DIST_MOD" = 'inet6_tls' ]; then if [ "$IS_BOOT_COMMAND" = 'yes' ]; then SSL_DIST_OPTFILE=${EMQX_SSL_DIST_OPTFILE:-"$EMQX_ETC_DIR/ssl_dist.conf"} case "$SSL_DIST_OPTFILE" in @@ -1216,7 +1212,6 @@ case "${COMMAND}" in export PROGNAME # Store passed arguments since they will be erased by `set` - # add emqx_data_dir to boot command so it is visible from 'ps -ef' ARGS="$*" # shellcheck disable=SC2086 @@ -1247,10 +1242,13 @@ case "${COMMAND}" in fi # Log the startup - logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS} -emqx_data_dir ${DATA_DIR}" + logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS} -ekka_proto_dist ${EKKA_PROTO_DIST_MOD} -emqx_data_dir ${DATA_DIR}" # Start the VM - exec "$@" -- ${1+$ARGS} -emqx_data_dir "${DATA_DIR}" + # add ekka_proto_dist emqx_data_dir to boot command so it is visible from 'ps -ef' + # NTOE: order matters! emqx_data_dir has to be positioned at the end of the line to simplify the + # line parsing when file path contains spaces + exec "$@" -- ${1+$ARGS} -ekka_proto_dist "${EKKA_PROTO_DIST_MOD}" -emqx_data_dir "${DATA_DIR}" ;; ctl) diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index e1f2120ca..13f852b3f 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -60,7 +60,9 @@ node_etc_dir.label: cluster_proto_dist.desc: """The Erlang distribution protocol for the cluster.
- inet_tcp: IPv4 TCP
-- inet_tls: IPv4 TLS, works together with etc/ssl_dist.conf""" +- inet_tls: IPv4 TLS, works together with etc/ssl_dist.conf
+- inet6_tcp: IPv6 TCP
+- inet6_tls: IPv6 TLS, works together with etc/ssl_dist.conf""" cluster_proto_dist.label: """Cluster Protocol Distribution""" @@ -192,6 +194,13 @@ rpc_insecure_fallback.desc: rpc_insecure_fallback.label: """RPC insecure fallback""" +rpc_listen_address.desc: +"""Specify which IP address the RPC server should listen on. +For example "0.0.0.0" (default) for IPv4 and "::" for IPv6""" + +rpc_listen_address.label: +"""RPC Listen Address""" + cluster_mcast_buffer.desc: """Size of the user-level buffer.""" From 1e93d2f1fc7028f4c9f9d6eb0a638c2fb087e1c1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 9 Oct 2023 16:53:19 +0200 Subject: [PATCH 04/13] chore: upgrade to ekka 0.15.16 gen_rpc 3.2.0 * ekka 0.15.16 supports 'inet6_tls' as ekka.proto_dist * gen_rpc 3.2.0 supports true | false as gen_rpc.ipv6_only --- apps/emqx/rebar.config | 4 ++-- mix.exs | 4 ++-- rebar.config | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index a69060b9d..544848889 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -28,8 +28,8 @@ {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.15"}}}, - {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.1"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}}, + {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.0"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.16"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/mix.exs b/mix.exs index 6f727e911..07f6b0209 100644 --- a/mix.exs +++ b/mix.exs @@ -55,8 +55,8 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.7", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.15.15", override: true}, - {:gen_rpc, github: "emqx/gen_rpc", tag: "3.1.1", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true}, + {:gen_rpc, github: "emqx/gen_rpc", tag: "3.2.0", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.13", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true}, diff --git a/rebar.config b/rebar.config index 70341ef31..e2e1a7cf0 100644 --- a/rebar.config +++ b/rebar.config @@ -62,8 +62,8 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.15"}}} - , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.1"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}} + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.0"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.8"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.13"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}} From 2a291dfd275551e79432a5e0e8728ec1c6fb785e Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 10 Oct 2023 12:31:19 +0700 Subject: [PATCH 05/13] fix(audit): handle abstract forms when logging `eval_erl` --- apps/emqx_ctl/src/emqx_ctl.erl | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index ed6aa7bb1..8f6b3309b 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -331,14 +331,13 @@ safe_to_existing_atom(Str) -> is_initialized() -> ets:info(?CMD_TAB) =/= undefined. -audit_log(Level, From, Log = #{args := Args}) -> +audit_log(Level, From, Log) -> case lookup_command(audit) of {error, _} -> ignore; {ok, {Mod, Fun}} -> try - Log1 = Log#{args => [unicode:characters_to_binary(A) || A <- Args]}, - apply(Mod, Fun, [Level, From, Log1]) + apply(Mod, Fun, [Level, From, normalize_audit_log_args(Log)]) catch _:Reason:Stacktrace -> ?LOG_ERROR(#{ @@ -349,12 +348,18 @@ audit_log(Level, From, Log = #{args := Args}) -> end end. --define(TOO_SLOW, 3000). - audit_level(ok, _Duration) -> info; audit_level({ok, _}, _Duration) -> info; audit_level(_, _) -> error. +normalize_audit_log_args(Log = #{args := [Parsed | _] = Exprs, cmd := eval_erl}) when + is_tuple(Parsed) +-> + String = erl_pp:exprs(Exprs, [{linewidth, 10000}]), + Log#{args => [unicode:characters_to_binary(String)]}; +normalize_audit_log_args(Log = #{args := Args}) -> + Log#{args => [unicode:characters_to_binary(A) || A <- Args]}. + eval_erl([Parsed | _] = Expr) when is_tuple(Parsed) -> eval_expr(Expr); eval_erl([String]) -> From e07937a3efee646e1f0f0c3e5ec9498b39d840d2 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 10 Oct 2023 13:49:36 +0800 Subject: [PATCH 06/13] fix(ldap): escape the escape character (\) --- .../test/emqx_authn_ldap_SUITE.erl | 7 +++++- .../test/emqx_authn_ldap_bind_SUITE.erl | 2 +- apps/emqx_ldap/src/emqx_ldap.erl | 8 +++---- apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl | 23 +++++++++++++++---- apps/emqx_ldap/test/data/emqx.io.ldif | 9 ++++++++ .../emqx_ldap/test/emqx_ldap_filter_SUITE.erl | 4 ++++ 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index a4a85b49b..e75a9a617 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -241,7 +241,12 @@ user_seeds() -> New(<<"mqttuser0006">>, <<"mqttuser0006">>, {error, user_disabled}), %% IsSuperuser New(<<"mqttuser0007">>, <<"mqttuser0007">>, {ok, #{is_superuser => true}}), - New(<<"mqttuser0008 (test)">>, <<"mqttuser0008 (test)">>, {ok, #{is_superuser => true}}) + New(<<"mqttuser0008 (test)">>, <<"mqttuser0008 (test)">>, {ok, #{is_superuser => true}}), + New( + <<"mqttuser0009 \\\\test\\\\">>, + <<"mqttuser0009 \\\\test\\\\">>, + {ok, #{is_superuser => true}} + ) | Valid ]. diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl index 5773b4f49..a796b8e01 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl @@ -235,7 +235,7 @@ user_seeds() -> lists:seq(1, 5) ), - Specials = [<<"mqttuser0008 (test)">>], + Specials = [<<"mqttuser0008 (test)">>, <<"mqttuser0009 \\\\test\\\\">>], Valid = lists:map( diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 2db801b9b..94b8992e0 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -319,10 +319,8 @@ do_prepare_template([], State) -> filter_escape(Binary) when is_binary(Binary) -> filter_escape(erlang:binary_to_list(Binary)); -filter_escape([$\\ | T]) -> - [$\\, $\\ | filter_escape(T)]; filter_escape([Char | T]) -> - case lists:member(Char, filter_control_chars()) of + case lists:member(Char, filter_special_chars()) of true -> [$\\, Char | filter_escape(T)]; _ -> @@ -331,5 +329,5 @@ filter_escape([Char | T]) -> filter_escape([]) -> []. -filter_control_chars() -> - [$(, $), $&, $|, $=, $!, $~, $>, $<, $:, $*, $\t, $\n, $\r]. +filter_special_chars() -> + [$(, $), $&, $|, $=, $!, $~, $>, $<, $:, $*, $\t, $\n, $\r, $\\]. diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl index 7c4d62bc9..3b4851fc4 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl @@ -2,8 +2,8 @@ Definitions. Control = [()&|!=~><:*] White = [\s\t\n\r]+ -StringChars = [^()&|!=~><:*\t\n\r] -Escape = \\{Control}|\\{White} +StringChars = [^()&|!=~><:*\t\n\r\\] +Escape = \\\\|\\{Control}|\\{White} String = ({Escape}|{StringChars})+ Rules. @@ -23,7 +23,7 @@ Rules. {White} : skip_token. {String} : {token, {string, TokenLine, to_string(TokenChars)}}. %% Leex will hang if a composite operation is missing a character -{Control} : {error, lists:flatten(io_lib:format("Unexpected Tokens:~ts", [TokenChars]))}. +{Control} : {error, format("Unexpected Tokens:~ts", [TokenChars])}. Erlang code. @@ -34,4 +34,19 @@ Erlang code. %% so after the tokenization we should remove all escape character to_string(TokenChars) -> String = string:trim(TokenChars), - lists:flatten(string:replace(String, "\\", "", all)). + trim_escape(String). + +%% because of the below situation, we can't directly use the `replace` to trim the escape character +%%trim_escape([$\\, $\\ | T]) -> +%% [$\\ | trim_escape(T)]; +trim_escape([$\\, Char | T]) -> + [Char | trim_escape(T)]; +%% the underneath is impossible to occur because it is not valid in the lexer +%% trim_escape([$\\]) +trim_escape([Char | T]) -> + [Char | trim_escape(T)]; +trim_escape([]) -> + []. + +format(Fmt, Args) -> + lists:flatten(io_lib:format(Fmt, Args)). diff --git a/apps/emqx_ldap/test/data/emqx.io.ldif b/apps/emqx_ldap/test/data/emqx.io.ldif index 62ed4c3d5..bc9012b00 100644 --- a/apps/emqx_ldap/test/data/emqx.io.ldif +++ b/apps/emqx_ldap/test/data/emqx.io.ldif @@ -166,6 +166,15 @@ uid: mqttuser0008 (test) isSuperuser: TRUE userPassword: {SHA}FCzJLOp66OwsZ9DQzXSxdTd9c0U= +objectClass: top +dn:uid=mqttuser0009 \\test\\,ou=testdevice,dc=emqx,dc=io +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0009 \\test\\ +isSuperuser: TRUE +userPassword: {SHA}awxXARLqWYx+xy0677D/TLjlyHA= + ## Try to test with base DN 'ou=dashboard,dc=emqx,dc=io' ## with a filter ugroup=group1 ## this should return 2 users in the query and fail the test diff --git a/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl index eb2b6161c..e1aacef88 100644 --- a/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl @@ -235,6 +235,10 @@ t_escape(_Config) -> ?assertEqual( 'or'([equalityMatch("a", "name (1) *")]), parse("(|(a=name\\ \\(1\\) \\*))") + ), + ?assertEqual( + 'and'([equalityMatch("a", "\\value\\")]), + parse("(&(a=\\\\value\\\\))") ). t_value_eql_dn(_Config) -> From e7e696cd66d8eb6abd9477ca42a78f39d684ae92 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 9 Oct 2023 17:08:31 +0200 Subject: [PATCH 07/13] feat(rpc): add ipv6_only config --- apps/emqx_conf/src/emqx_conf_schema.erl | 10 ++++++++++ rel/i18n/emqx_conf_schema.hocon | 13 ++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 481eb59f2..7ec5348fc 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -957,6 +957,16 @@ fields("rpc") -> desc => ?DESC(rpc_listen_address), importance => ?IMPORTANCE_MEDIUM } + )}, + {"ipv6_only", + sc( + boolean(), + #{ + default => false, + mapping => "gen_rpc.ipv6_only", + desc => ?DESC(rpc_ipv6_only), + importance => ?IMPORTANCE_LOW + } )} ]; fields("log") -> diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index 13f852b3f..fde0f7ff3 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -195,11 +195,18 @@ rpc_insecure_fallback.label: """RPC insecure fallback""" rpc_listen_address.desc: -"""Specify which IP address the RPC server should listen on. -For example "0.0.0.0" (default) for IPv4 and "::" for IPv6""" +"""Indicates the IP address for the RPC server to listen on. For example, use "0.0.0.0" for IPv4 or "::" for IPv6.""" rpc_listen_address.label: -"""RPC Listen Address""" +"""RPC Listen IP Address""" + +rpc_ipv6_only.desc: +"""This setting is effective only when rpc.listen_address is assigned an IPv6 address. +If set to true, the RPC client will exclusively use IPv6 for connections. +Otherwise, the client might opt for IPv4, even if the server is on IPv6.""" + +rpc_ipv6_only.label: +"""Use IPv6 Only""" cluster_mcast_buffer.desc: """Size of the user-level buffer.""" From 2c13a26cac47420326433c54509c1400732e9bc6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 9 Oct 2023 17:41:10 +0200 Subject: [PATCH 08/13] test: add support of ipv6 in start-two-nodes-in-docker.sh --- scripts/test/start-two-nodes-in-docker.sh | 43 ++++++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/scripts/test/start-two-nodes-in-docker.sh b/scripts/test/start-two-nodes-in-docker.sh index 53689b207..a6fe4d522 100755 --- a/scripts/test/start-two-nodes-in-docker.sh +++ b/scripts/test/start-two-nodes-in-docker.sh @@ -16,6 +16,7 @@ NET='emqx.io' NODE1="node1.$NET" NODE2="node2.$NET" COOKIE='this-is-a-secret' +IPV6=0 cleanup() { docker rm -f haproxy >/dev/null 2>&1 || true @@ -24,31 +25,61 @@ cleanup() { docker network rm "$NET" >/dev/null 2>&1 || true } -while getopts ":Pc" opt +show_help() { + echo "Usage: $0 [options] EMQX_IMAGE1 [EMQX_IAMGE2]" + echo "" + echo "Specifiy which docker image to run with EMQX_IMAGE1" + echo "EMQX_IMAGE2 is the same as EMQX_IMAGE1 if not set" + echo "" + echo "Options:" + echo " -h, --help Show this help message and exit." + echo " -P Add -p options for docker run to expose more HAProxy container ports." + echo " -6 Test with IPv6" + echo " -c Cleanup: delete docker network, force delete the containers." +} + +while getopts "hc6P:" opt do case $opt in # -P option is treated similarly to docker run -P: # publish ports to random available host ports P) HAPROXY_PORTS=(-p 18083 -p 8883 -p 8084);; c) cleanup; exit 0;; + h) show_help; exit 0;; + 6) IPV6=1;; *) ;; esac done shift $((OPTIND - 1)) -IMAGE1="${1}" +IMAGE1="${1:-}" IMAGE2="${2:-${IMAGE1}}" +if [ -z "${IMAGE1:-}" ] || [ -z "${IMAGE2:-}" ]; then + show_help + exit 1 +fi + cleanup -docker network create "$NET" +if [ ${IPV6} = 1 ]; then + docker network create --ipv6 --subnet 2001:0DB8::/112 "$NET" + RPC_ADDRESS="::" + PROTO_DIST='inet6_tls' +else + docker network create "$NET" + RPC_ADDRESS="0.0.0.0" + PROTO_DIST='inet_tls' +fi docker run -d -t --restart=always --name "$NODE1" \ --net "$NET" \ -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \ -e EMQX_NODE_NAME="emqx@$NODE1" \ -e EMQX_NODE_COOKIE="$COOKIE" \ - -e EMQX_CLUSTER__PROTO_DIST='inet_tls' \ + -e EMQX_CLUSTER__PROTO_DIST="${PROTO_DIST}" \ + -e EMQX_RPC__LISTEN_ADDRESS="${RPC_ADDRESS}" \ + -e EMQX_RPC__IPV6_ONLY="true" \ -e EMQX_listeners__ssl__default__enable=false \ -e EMQX_listeners__wss__default__enable=false \ -e EMQX_listeners__tcp__default__proxy_protocol=true \ @@ -60,7 +91,9 @@ docker run -d -t --restart=always --name "$NODE2" \ -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \ -e EMQX_NODE_NAME="emqx@$NODE2" \ -e EMQX_NODE_COOKIE="$COOKIE" \ - -e EMQX_CLUSTER__PROTO_DIST='inet_tls' \ + -e EMQX_CLUSTER__PROTO_DIST="${PROTO_DIST}" \ + -e EMQX_RPC__LISTEN_ADDRESS="${RPC_ADDRESS}" \ + -e EMQX_RPC__IPV6_ONLY="true" \ -e EMQX_listeners__ssl__default__enable=false \ -e EMQX_listeners__wss__default__enable=false \ -e EMQX_listeners__tcp__default__proxy_protocol=true \ From d651764ff491e394d5fbbc4edf6ea4e8a41e92c2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 9 Oct 2023 20:06:12 +0200 Subject: [PATCH 09/13] docs: add changelog for pull request 11734 --- changes/ce/fix-11734.en.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ce/fix-11734.en.md diff --git a/changes/ce/fix-11734.en.md b/changes/ce/fix-11734.en.md new file mode 100644 index 000000000..576799ecf --- /dev/null +++ b/changes/ce/fix-11734.en.md @@ -0,0 +1,3 @@ +Fix clustering in IPv6 network. + +Added new configurations `rpc.listen_address` and `rpc.ipv6_only` to allow EMQX cluster RPC server and client to use IPv6. From 143673b703f8b4fddfd3afea2565a29e8ef851c9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 10 Oct 2023 14:35:46 +0200 Subject: [PATCH 10/13] chore: bump release version to 5.3.1-alpha.1 --- apps/emqx/include/emqx_release.hrl | 4 ++-- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- deploy/charts/emqx/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index e77ff8656..904c432b4 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.3.0"). +-define(EMQX_RELEASE_CE, "5.3.1.alpha.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.3.0"). +-define(EMQX_RELEASE_EE, "5.3.1.alpha.1"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index cd004f0a5..c881481d5 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.3.0 +version: 5.3.1-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.3.0 +appVersion: 5.3.-alpha.1 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index f1c26cc8d..54d36eee3 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.3.0 +version: 5.3.1-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.3.0 +appVersion: 5.3.1-alpha.1 From bfb2218392bcd18a6bdcb15dd05b6043180dcb1c Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 10 Oct 2023 14:50:49 +0200 Subject: [PATCH 11/13] chore: fix release version --- apps/emqx/include/emqx_release.hrl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 904c432b4..4a9340d11 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.3.1.alpha.1"). +-define(EMQX_RELEASE_CE, "5.3.1-alpha.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.3.1.alpha.1"). +-define(EMQX_RELEASE_EE, "5.3.1-alpha.1"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). From f5b0f965219a16132e608088f8e279a4c27020d1 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 10 Oct 2023 14:51:07 +0200 Subject: [PATCH 12/13] chore(docker): use public.ecr.aws registry for base image --- Makefile | 2 +- deploy/docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 42ae88e95..3d11491f9 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-debian11 -export EMQX_DEFAULT_RUNNER = debian:11-slim +export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:11-slim export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 76ded75eb..ec227cb61 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,5 +1,5 @@ ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-debian11 -ARG RUN_FROM=debian:11-slim +ARG RUN_FROM=public.ecr.aws/debian/debian:11-slim FROM ${BUILD_FROM} AS builder COPY . /emqx From 5502d342254b0e0d3091b3d532f5bd3bac60f05e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 10 Oct 2023 15:28:02 +0200 Subject: [PATCH 13/13] chore: fix enterprise chart version typo --- deploy/charts/emqx-enterprise/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index c881481d5..5537d4b89 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -18,4 +18,4 @@ version: 5.3.1-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.3.-alpha.1 +appVersion: 5.3.1-alpha.1