From 423b8b4274e47a5618bb8eac1b410b7a40b0c398 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Wed, 27 Dec 2023 13:55:53 +0200 Subject: [PATCH 01/19] fix(emqx_opentelemetry): use converter to convert legacy metrics config HOCON converter is cleaner and safer option, that will also work for configurations defined in emqx.conf. Previously used `upgrade_raw_conf/1` is currently not supported in hocon_cli (used in bin/emqx). Fixes: EMQX-11643 --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 3 +- .../src/emqx_opentelemetry.app.src | 2 +- .../src/emqx_otel_schema.erl | 63 +++++++++---------- .../test/emqx_otel_schema_SUITE.erl | 3 +- changes/ce/fix-12234.en.md | 1 + 6 files changed, 36 insertions(+), 38 deletions(-) create mode 100644 changes/ce/fix-12234.en.md diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 7f495a3cd..761f36a11 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.32"}, + {vsn, "0.1.33"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index eeb0d6de8..8cfa9a8ea 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -77,8 +77,7 @@ %% Callback to upgrade config after loaded from config file but before validation. upgrade_raw_conf(RawConf) -> - RawConf1 = emqx_connector_schema:transform_bridges_v1_to_connectors_and_bridges_v2(RawConf), - emqx_otel_schema:upgrade_legacy_metrics(RawConf1). + emqx_connector_schema:transform_bridges_v1_to_connectors_and_bridges_v2(RawConf). namespace() -> emqx. diff --git a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src index 95249ff85..4cde26326 100644 --- a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src +++ b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src @@ -1,6 +1,6 @@ {application, emqx_opentelemetry, [ {description, "OpenTelemetry for EMQX Broker"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {mod, {emqx_otel_app, []}}, {applications, [ diff --git a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl index be14a2b29..04d12c8c7 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl @@ -24,40 +24,15 @@ desc/1 ]). --export([upgrade_legacy_metrics/1]). - -%% Compatibility with the previous schema that defined only metric fields -upgrade_legacy_metrics(RawConf) -> - case RawConf of - #{<<"opentelemetry">> := Otel} -> - Otel1 = - case maps:take(<<"enable">>, Otel) of - {MetricsEnable, OtelConf} -> - emqx_utils_maps:deep_put( - [<<"metrics">>, <<"enable">>], OtelConf, MetricsEnable - ); - error -> - Otel - end, - Otel2 = - case Otel1 of - #{<<"exporter">> := #{<<"interval">> := Interval} = Exporter} -> - emqx_utils_maps:deep_put( - [<<"metrics">>, <<"interval">>], - Otel1#{<<"exporter">> => maps:remove(<<"interval">>, Exporter)}, - Interval - ); - _ -> - Otel1 - end, - RawConf#{<<"opentelemetry">> => Otel2}; - _ -> - RawConf - end. - namespace() -> opentelemetry. -roots() -> ["opentelemetry"]. +roots() -> + [ + {"opentelemetry", + ?HOCON(?R_REF("opentelemetry"), #{ + converter => fun legacy_metrics_converter/2 + })} + ]. fields("opentelemetry") -> [ @@ -259,3 +234,27 @@ desc("otel_metrics") -> ?DESC(otel_metrics); desc("otel_traces") -> ?DESC(otel_traces); desc("trace_filter") -> ?DESC(trace_filter); desc(_) -> undefined. + +%% Compatibility with the previous schema that defined only metrics fields +legacy_metrics_converter(OtelConf, _Opts) when is_map(OtelConf) -> + Otel1 = + case maps:take(<<"enable">>, OtelConf) of + {MetricsEnable, OtelConf1} -> + emqx_utils_maps:deep_put( + [<<"metrics">>, <<"enable">>], OtelConf1, MetricsEnable + ); + error -> + OtelConf + end, + case Otel1 of + #{<<"exporter">> := #{<<"interval">> := Interval} = Exporter} -> + emqx_utils_maps:deep_put( + [<<"metrics">>, <<"interval">>], + Otel1#{<<"exporter">> => maps:remove(<<"interval">>, Exporter)}, + Interval + ); + _ -> + Otel1 + end; +legacy_metrics_converter(Conf, _Opts) -> + Conf. diff --git a/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl b/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl index abc6548f9..22c93e307 100644 --- a/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl +++ b/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl @@ -22,8 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -%% Backward compatibility suite for `upgrade_raw_conf/1`, -%% expected callback is `emqx_otel_schema:upgrade_legacy_metrics/1` +%% Backward compatibility suite for legacy metrics converter -define(OLD_CONF_ENABLED, << "\n" diff --git a/changes/ce/fix-12234.en.md b/changes/ce/fix-12234.en.md new file mode 100644 index 000000000..ac6f105ef --- /dev/null +++ b/changes/ce/fix-12234.en.md @@ -0,0 +1 @@ +Fix old (prior to EMQX 5.4.0) Open Telemetry configuration incompatibility when the config is defined in emqx.conf. From b6391cf18c7caf78ffa9b2212105f0ba1efe491f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 28 Dec 2023 11:36:48 +0800 Subject: [PATCH 02/19] fix(bridge-http): compatible conf with the error format introduced in 5.3.2 --- .../src/emqx_bridge_http.app.src | 2 +- .../src/emqx_bridge_http_schema.erl | 12 +++- .../emqx_connector/src/emqx_connector.app.src | 2 +- .../src/schema/emqx_connector_schema.erl | 17 ++++- scripts/conf-test/old-confs/e5.3.2.conf | 63 +++++++++++++++++++ 5 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 scripts/conf-test/old-confs/e5.3.2.conf diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src index a7f8688d4..c03a15b01 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src +++ b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_http, [ {description, "EMQX HTTP Bridge and Connector Application"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {applications, [kernel, stdlib, emqx_connector, emqx_resource, ehttpc]}, {env, [{emqx_action_info_modules, [emqx_bridge_http_action_info]}]}, diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl index 91ad25d31..d3702bcfd 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl @@ -97,7 +97,10 @@ fields("http_action") -> required => true, desc => ?DESC("config_parameters_opts") })} - ] ++ emqx_connector_schema:resource_opts_ref(?MODULE, action_resource_opts); + ] ++ + emqx_connector_schema:resource_opts_ref( + ?MODULE, action_resource_opts, fun legacy_action_resource_opts_converter/2 + ); fields(action_resource_opts) -> UnsupportedOpts = [batch_size, batch_time], lists:filter( @@ -342,6 +345,13 @@ mark_request_field_deperecated(Fields) -> Fields ). +legacy_action_resource_opts_converter(Conf, _Opts) when is_map(Conf) -> + %% In e5.3.0, we accidentally added `start_after_created` and `start_timeout` to the action resource opts. + %% Since e5.4.0, we have removed them. This function is used to convert the old config to the new one. + maps:without([<<"start_after_created">>, <<"start_timeout">>], Conf); +legacy_action_resource_opts_converter(Conf, _Opts) -> + Conf. + %%-------------------------------------------------------------------- %% Examples diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 3e781dae0..261c70d34 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.36"}, + {vsn, "0.1.37"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index f67cd6991..f203d0d1b 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -51,7 +51,8 @@ common_resource_opts_subfields_bin/0, resource_opts_fields/0, resource_opts_fields/1, - resource_opts_ref/2 + resource_opts_ref/2, + resource_opts_ref/3 ]). -export([examples/1]). @@ -528,13 +529,23 @@ status_and_actions_fields() -> } )} ]. - resource_opts_ref(Module, RefName) -> + resource_opts_ref(Module, RefName, undefined). + +resource_opts_ref(Module, RefName, ConverterFun) -> + Meta = + case ConverterFun of + undefined -> + emqx_resource_schema:resource_opts_meta(); + _ -> + M = emqx_resource_schema:resource_opts_meta(), + M#{converter => ConverterFun} + end, [ {resource_opts, mk( ref(Module, RefName), - emqx_resource_schema:resource_opts_meta() + Meta )} ]. diff --git a/scripts/conf-test/old-confs/e5.3.2.conf b/scripts/conf-test/old-confs/e5.3.2.conf new file mode 100644 index 000000000..64badf7ce --- /dev/null +++ b/scripts/conf-test/old-confs/e5.3.2.conf @@ -0,0 +1,63 @@ +actions { + http { + x_WH_D { + connector = connector_x_WH_D + enable = true + parameters { + body = "${clientid}" + headers {} + max_retries = 2 + method = post + path = "" + } + resource_opts { + health_check_interval = 15s + inflight_window = 100 + max_buffer_bytes = 1GB + query_mode = async + request_ttl = 45s + start_after_created = true + start_timeout = 5s + worker_pool_size = 4 + } + } + } +} + +connectors { + http { + connector_x_WH_D { + connect_timeout = 15s + enable = true + enable_pipelining = 100 + headers {content-type = "application/json"} + pool_size = 8 + pool_type = hash + ssl { + ciphers = [] + depth = 10 + enable = false + hibernate_after = 5s + log_level = notice + reuse_sessions = true + secure_renegotiate = true + verify = verify_peer + versions = [tlsv1.3, tlsv1.2] + } + url = "http://127.0.0.1:18083" + } + } +} + +rule_engine { + rules { + x_WH_D { + actions = ["webhook:x_WH_D"] + description = x + enable = true + metadata {created_at = 1699341635802} + name = "" + sql = "SELECT\n *\nFROM\n \"#\",\n \"$events/message_delivered\",\n \"$events/message_acked\"" + } + } +} From c0aba14d2729bb58a2493bbc71b036d9adcaca8b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 28 Dec 2023 11:41:20 +0800 Subject: [PATCH 03/19] chore: update changes --- changes/ce/fix-12238.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-12238.en.md diff --git a/changes/ce/fix-12238.en.md b/changes/ce/fix-12238.en.md new file mode 100644 index 000000000..56d9931aa --- /dev/null +++ b/changes/ce/fix-12238.en.md @@ -0,0 +1 @@ +Compatible with the configuration of error formats introduced by HTTP Action in 5.3.2. From 3ce2f225d7dd1c1709bf177a0ac93ad76e115cf1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 28 Dec 2023 13:48:51 +0800 Subject: [PATCH 04/19] test: update the test config --- scripts/conf-test/old-confs/e5.3.2.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/conf-test/old-confs/e5.3.2.conf b/scripts/conf-test/old-confs/e5.3.2.conf index 64badf7ce..17bf029fd 100644 --- a/scripts/conf-test/old-confs/e5.3.2.conf +++ b/scripts/conf-test/old-confs/e5.3.2.conf @@ -1,3 +1,14 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + actions { http { x_WH_D { From 1fd67564ce5f3b0f85dbc0f55b64588d186ba5e2 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 28 Dec 2023 14:43:11 +0800 Subject: [PATCH 05/19] fix: return raw config when update ft's config --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_ft/src/emqx_ft.app.src | 2 +- apps/emqx_ft/src/emqx_ft_api.erl | 18 ++++++++---------- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 3 +++ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index f837161be..4dd2fca24 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.1.17"}, + {vsn, "5.1.18"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index 8f78bae35..9edc2fd89 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -1,6 +1,6 @@ {application, emqx_ft, [ {description, "EMQX file transfer over MQTT"}, - {vsn, "0.1.11"}, + {vsn, "0.1.12"}, {registered, []}, {mod, {emqx_ft_app, []}}, {applications, [ diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index ace8c7b83..9f43cd1fe 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -176,13 +176,13 @@ check_ft_enabled(Params, _Meta) -> end. '/file_transfer'(get, _Meta) -> - {200, format_config(emqx_ft_conf:get())}; + {200, format_config(emqx_ft_conf:get_raw())}; '/file_transfer'(put, #{body := ConfigIn}) -> OldConf = emqx_ft_conf:get_raw(), UpdateConf = emqx_utils:deobfuscate(ConfigIn, OldConf), case emqx_ft_conf:update(UpdateConf) of - {ok, #{config := Config}} -> - {200, format_config(Config)}; + {ok, #{raw_config := RawConfig}} -> + {200, format_config(RawConfig)}; {error, Error = #{kind := validation_error}} -> {400, error_msg('INVALID_CONFIG', format_validation_error(Error))}; {error, Error} -> @@ -199,13 +199,11 @@ format_page(#{items := Files}) -> <<"files">> => lists:map(fun format_file_info/1, Files) }. -format_config(Config) -> - Schema = emqx_hocon:make_schema(emqx_ft_schema:fields(file_transfer)), - hocon_tconf:make_serializable( - Schema, - emqx_utils_maps:binary_key_map(Config), - #{obfuscate_sensitive_values => true} - ). +format_config(RawConf) -> + RootKey = <<"file_transfer">>, + Opts = #{obfuscate_sensitive_values => true, make_serializable => true}, + Conf = emqx_config:fill_defaults(emqx_ft_schema, #{RootKey => RawConf}, Opts), + maps:get(RootKey, Conf). format_validation_error(Error) -> emqx_logger_jsonfmt:best_effort_json(Error). diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 66f770a22..8fbd2eeeb 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -345,6 +345,7 @@ test_configure(Uri, Config) -> <<"host">> => <<"localhost">>, <<"port">> => 9000, <<"bucket">> => <<"emqx">>, + <<"url_expire_time">> => <<"2h">>, <<"secret_access_key">> => ?SECRET_ACCESS_KEY, <<"transport_options">> => #{ <<"ssl">> => #{ @@ -384,6 +385,8 @@ test_configure(Uri, Config) -> <<"keyfile">> := <<"/", _KeyFilepath/bytes>> } }, + %% ensure 2h is unchanged + <<"url_expire_time">> := <<"2h">>, <<"secret_access_key">> := <<"******">> } } From 4669c4d552a43f90434f81053ecde23534ea83ed Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 27 Dec 2023 17:47:51 +0100 Subject: [PATCH 06/19] fix(bridge/mqtt): ensure short clientid Some mqtt brokers do not allow long client IDs. To make it compatible with this limitation, this commit tries to limit the number of bytes under 23 with a best-effort attempt to derive it from the bridge name. --- apps/emqx/include/emqx_release.hrl | 4 +- apps/emqx/src/emqx.app.src | 2 +- apps/emqx/src/emqx_passwd.erl | 3 +- apps/emqx_bridge_mqtt/rebar.config | 3 +- .../src/emqx_bridge_mqtt.app.src | 2 +- .../src/emqx_bridge_mqtt_connector.erl | 28 ++++++--- .../src/emqx_bridge_mqtt_egress.erl | 2 +- .../src/emqx_bridge_mqtt_ingress.erl | 2 +- .../src/emqx_bridge_mqtt_lib.erl | 57 +++++++++++++++++++ .../test/emqx_bridge_mqtt_SUITE.erl | 38 +++++++------ changes/ce/fix-12236.en.md | 1 + 11 files changed, 109 insertions(+), 33 deletions(-) create mode 100644 apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_lib.erl create mode 100644 changes/ce/fix-12236.en.md diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index c96f93d8e..0115edc91 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.4.0"). +-define(EMQX_RELEASE_CE, "5.4.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.4.0"). +-define(EMQX_RELEASE_EE, "5.4.1"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index f837161be..4dd2fca24 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.1.17"}, + {vsn, "5.1.18"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx/src/emqx_passwd.erl b/apps/emqx/src/emqx_passwd.erl index 1232dfcb4..29067244f 100644 --- a/apps/emqx/src/emqx_passwd.erl +++ b/apps/emqx/src/emqx_passwd.erl @@ -141,4 +141,5 @@ pbkdf2(MacFun, Password, Salt, Iterations, DKLength) -> pbkdf2:pbkdf2(MacFun, Password, Salt, Iterations, DKLength). hex(X) when is_binary(X) -> - pbkdf2:to_hex(X). + %% TODO: change to binary:encode_hex(X, lowercase) when OTP version is always > 25 + string:lowercase(binary:encode_hex(X)). diff --git a/apps/emqx_bridge_mqtt/rebar.config b/apps/emqx_bridge_mqtt/rebar.config index 35ccc1a37..189ba970d 100644 --- a/apps/emqx_bridge_mqtt/rebar.config +++ b/apps/emqx_bridge_mqtt/rebar.config @@ -1,3 +1,4 @@ {deps, [ - {emqx, {path, "../../apps/emqx"}} + {emqx, {path, "../../apps/emqx"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}} ]}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index 716626bdf..e6fe78ab8 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 61e9353ce..cc2296d3c 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -17,6 +17,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). -behaviour(emqx_resource). @@ -35,6 +36,8 @@ -export([on_async_result/2]). -define(HEALTH_CHECK_TIMEOUT, 1000). +-define(INGRESS, "I"). +-define(EGRESS, "E"). %% =================================================================== %% When use this bridge as a data source, ?MODULE:on_message_received will be called @@ -66,7 +69,7 @@ on_start(ResourceId, Conf) -> end. start_ingress(ResourceId, Conf) -> - ClientOpts = mk_client_opts(ResourceId, "ingress", Conf), + ClientOpts = mk_client_opts(ResourceId, ?INGRESS, Conf), case mk_ingress_config(ResourceId, Conf) of Ingress = #{} -> start_ingress(ResourceId, Ingress, ClientOpts); @@ -91,6 +94,8 @@ start_ingress(ResourceId, Ingress, ClientOpts) -> {error, Reason} end. +choose_ingress_pool_size(<>, _) -> + 1; choose_ingress_pool_size( ResourceId, #{remote := #{topic := RemoteTopic}, pool_size := PoolSize} @@ -119,7 +124,7 @@ start_egress(ResourceId, Conf) -> % NOTE % We are ignoring the user configuration here because there's currently no reliable way % to ensure proper session recovery according to the MQTT spec. - ClientOpts = maps:put(clean_start, true, mk_client_opts(ResourceId, "egress", Conf)), + ClientOpts = maps:put(clean_start, true, mk_client_opts(ResourceId, ?EGRESS, Conf)), case mk_egress_config(Conf) of Egress = #{} -> start_egress(ResourceId, Egress, ClientOpts); @@ -326,9 +331,10 @@ mk_client_opts( ], Config ), + Name = parse_id_to_name(ResourceId), mk_client_opt_password(Options#{ hosts => [HostPort], - clientid => clientid(ResourceId, ClientScope, Config), + clientid => clientid(Name, ClientScope, Config), connect_timeout => 30, keepalive => ms_to_s(KeepAlive), force_ping => true, @@ -336,6 +342,12 @@ mk_client_opts( ssl_opts => maps:to_list(maps:remove(enable, Ssl)) }). +parse_id_to_name(<>) -> + Name; +parse_id_to_name(Id) -> + {_Type, Name} = emqx_bridge_resource:parse_bridge_id(Id, #{atom_name => false}), + Name. + mk_client_opt_password(Options = #{password := Secret}) -> %% TODO: Teach `emqtt` to accept 0-arity closures as passwords. Options#{password := emqx_secret:unwrap(Secret)}; @@ -345,7 +357,9 @@ mk_client_opt_password(Options) -> ms_to_s(Ms) -> erlang:ceil(Ms / 1000). -clientid(Id, ClientScope, _Conf = #{clientid_prefix := Prefix}) when is_binary(Prefix) -> - iolist_to_binary([Prefix, ":", Id, ":", ClientScope, ":", atom_to_list(node())]); -clientid(Id, ClientScope, _Conf) -> - iolist_to_binary([Id, ":", ClientScope, ":", atom_to_list(node())]). +clientid(Name, ClientScope, _Conf = #{clientid_prefix := Prefix}) when + is_binary(Prefix) andalso Prefix =/= <<>> +-> + emqx_bridge_mqtt_lib:clientid_base([Prefix, $:, Name, ClientScope]); +clientid(Name, ClientScope, _Conf) -> + emqx_bridge_mqtt_lib:clientid_base([Name, ClientScope]). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl index a9415294c..2573cad8b 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl @@ -81,7 +81,7 @@ mk_client_opts(WorkerId, ClientOpts = #{clientid := ClientId}) -> ClientOpts#{clientid := mk_clientid(WorkerId, ClientId)}. mk_clientid(WorkerId, ClientId) -> - iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). + emqx_bridge_mqtt_lib:bytes23(ClientId, WorkerId). connect(Pid, Name) -> case emqtt:connect(Pid) of diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl index 3174cdb6f..a051ffbd8 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl @@ -81,7 +81,7 @@ mk_client_opts(Name, WorkerId, Ingress, ClientOpts = #{clientid := ClientId}) -> }. mk_clientid(WorkerId, ClientId) -> - iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). + emqx_bridge_mqtt_lib:bytes23(ClientId, WorkerId). mk_client_event_handler(Name, Ingress = #{}) -> IngressVars = maps:with([server], Ingress), diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_lib.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_lib.erl new file mode 100644 index 000000000..c5f763e6c --- /dev/null +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_lib.erl @@ -0,0 +1,57 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_mqtt_lib). + +-export([clientid_base/1, bytes23/2]). + +%% @doc Make the base ID of client IDs. +%% A base ID is used to concatenate with pool worker ID to build a +%% full ID. +%% In order to avoid client ID clashing when EMQX is clustered, +%% the base ID is the resource name concatenated with +%% broker node name SHA-hash and truncated to 8 hex characters. +clientid_base(Name) -> + bin([Name, shortener(atom_to_list(node()), 8)]). + +%% @doc Limit the number of bytes for client ID under 23 bytes. +%% If Prefix and suffix concatenated is longer than 23 bytes +%% it hashes the concatenation and replace the non-random suffix. +bytes23(Prefix, SeqNo) -> + Suffix = integer_to_binary(SeqNo), + Concat = bin([Prefix, $:, Suffix]), + case size(Concat) =< 23 of + true -> + Concat; + false -> + shortener(Concat, 23) + end. + +%% @private SHA hash a string and return the prefix of +%% the given length as hex string in binary format. +shortener(Str, Length) when is_list(Str) -> + shortener(bin(Str), Length); +shortener(Str, Length) when is_binary(Str) -> + true = size(Str) > 0, + true = (Length > 0 andalso Length =< 40), + Sha = crypto:hash(sha, Str), + %% TODO: change to binary:encode_hex(X, lowercase) when OTP version is always > 25 + Hex = string:lowercase(binary:encode_hex(Sha)), + <> = Hex, + UniqueEnough. + +bin(IoList) -> + iolist_to_binary(IoList). diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index bde546bd0..e887a98f4 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -495,7 +495,6 @@ t_mqtt_conn_bridge_egress(_) -> <<"egress">> => ?EGRESS_CONF } ), - ResourceID = emqx_bridge_resource:resource_id(?TYPE_MQTT, ?BRIDGE_NAME_EGRESS), %% we now test if the bridge works as expected LocalTopic = <>, @@ -510,11 +509,6 @@ t_mqtt_conn_bridge_egress(_) -> #{?snk_kind := buffer_worker_flush_ack} ), - %% we should receive a message on the "remote" broker, with specified topic - Msg = assert_mqtt_msg_received(RemoteTopic, Payload), - Size = byte_size(ResourceID), - ?assertMatch(<>, Msg#message.from), - %% verify the metrics of the bridge ?retry( _Interval = 200, @@ -538,7 +532,6 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> <<"egress">> => ?EGRESS_CONF_NO_PAYLOAD_TEMPLATE } ), - ResourceID = emqx_bridge_resource:resource_id(?TYPE_MQTT, ?BRIDGE_NAME_EGRESS), %% we now test if the bridge works as expected LocalTopic = <>, @@ -555,8 +548,6 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> %% we should receive a message on the "remote" broker, with specified topic Msg = assert_mqtt_msg_received(RemoteTopic), - %% the MapMsg is all fields outputed by Rule-Engine. it's a binary coded json here. - ?assertMatch(<>, Msg#message.from), ?assertMatch(#{<<"payload">> := Payload}, emqx_utils_json:decode(Msg#message.payload)), %% verify the metrics of the bridge @@ -575,15 +566,29 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> ok. -t_egress_custom_clientid_prefix(_Config) -> +t_egress_short_clientid(_Config) -> + %% Name is short, expect the actual client ID in use is hashed from + %% E: + Name = "abc01234", + BaseId = emqx_bridge_mqtt_lib:clientid_base([Name, "E"]), + ExpectedClientId = iolist_to_binary([BaseId, $:, "1"]), + test_egress_clientid(Name, ExpectedClientId). + +t_egress_long_clientid(_Config) -> + %% Expect the actual client ID in use is hashed from + %% E: + Name = "abc01234567890123456789", + BaseId = emqx_bridge_mqtt_lib:clientid_base([Name, "E"]), + ExpectedClientId = emqx_bridge_mqtt_lib:bytes23(BaseId, 1), + test_egress_clientid(Name, ExpectedClientId). + +test_egress_clientid(Name, ExpectedClientId) -> BridgeIDEgress = create_bridge( ?SERVER_CONF#{ - <<"clientid_prefix">> => <<"my-custom-prefix">>, - <<"name">> => ?BRIDGE_NAME_EGRESS, - <<"egress">> => ?EGRESS_CONF + <<"name">> => Name, + <<"egress">> => (?EGRESS_CONF)#{<<"pool_size">> => 1} } ), - ResourceID = emqx_bridge_resource:resource_id(?TYPE_MQTT, ?BRIDGE_NAME_EGRESS), LocalTopic = <>, RemoteTopic = <>, Payload = <<"hello">>, @@ -592,8 +597,7 @@ t_egress_custom_clientid_prefix(_Config) -> emqx:publish(emqx_message:make(LocalTopic, Payload)), Msg = assert_mqtt_msg_received(RemoteTopic, Payload), - Size = byte_size(ResourceID), - ?assertMatch(<<"my-custom-prefix:", _ResouceID:Size/binary, _/binary>>, Msg#message.from), + ?assertEqual(ExpectedClientId, Msg#message.from), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), ok. @@ -843,7 +847,6 @@ t_mqtt_conn_bridge_egress_reconnect(_) -> <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF, <<"resource_opts">> => #{ - <<"worker_pool_size">> => 2, <<"query_mode">> => <<"sync">>, %% using a long time so we can test recovery <<"request_ttl">> => <<"15s">>, @@ -949,7 +952,6 @@ t_mqtt_conn_bridge_egress_async_reconnect(_) -> <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF, <<"resource_opts">> => #{ - <<"worker_pool_size">> => 2, <<"query_mode">> => <<"async">>, %% using a long time so we can test recovery <<"request_ttl">> => <<"15s">>, diff --git a/changes/ce/fix-12236.en.md b/changes/ce/fix-12236.en.md new file mode 100644 index 000000000..1cb763488 --- /dev/null +++ b/changes/ce/fix-12236.en.md @@ -0,0 +1 @@ +Ensure short client ID for MQTT bridges. From 33ef964582c62b2cb135ac84c37b525008ff7fb2 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 28 Dec 2023 10:20:22 +0100 Subject: [PATCH 07/19] fix(s3): handle extra configured headers correctly * Header names in config become atoms. (Probably because of blanket `unsafe_atom_key_map/1` in `emqx_config`) * Header names should be normalized to lowercase. (Otherwise signature will be incorrect) --- apps/emqx_s3/src/emqx_s3_client.erl | 4 +- apps/emqx_s3/test/emqx_s3_client_SUITE.erl | 43 ++++++++++++++++------ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index 73cf667a1..fe0058433 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -388,7 +388,7 @@ with_path_and_query_only(Url, Fun) -> %% Users provide headers as a map, but erlcloud expects a list of tuples with string keys and values. headers_user_to_erlcloud_request(UserHeaders) -> - [{to_list_string(K), V} || {K, V} <- maps:to_list(UserHeaders)]. + [{string:to_lower(to_list_string(K)), V} || {K, V} <- maps:to_list(UserHeaders)]. %% Ehttpc returns operates on headers as a list of tuples with binary keys. %% Erlcloud expects a list of tuples with string values and lowcase string keys @@ -409,6 +409,8 @@ to_binary(Val) when is_binary(Val) -> Val. to_list_string(Val) when is_binary(Val) -> binary_to_list(Val); +to_list_string(Val) when is_atom(Val) -> + atom_to_list(Val); to_list_string(Val) when is_list(Val) -> Val. diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl index 434510867..2522af9b7 100644 --- a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -142,6 +142,22 @@ t_no_acl(Config) -> ok = emqx_s3_client:put_object(Client, Key, <<"data">>). +t_extra_headers(Config0) -> + Config = [{extra_headers, #{'Content-Type' => <<"application/json">>}} | Config0], + Key = ?config(key, Config), + + Client = client(Config), + Data = #{foo => bar}, + ok = emqx_s3_client:put_object(Client, Key, emqx_utils_json:encode(Data)), + + Url = emqx_s3_client:uri(Client, Key), + + {ok, {{_StatusLine, 200, "OK"}, _Headers, Content}} = httpc:request(Url), + ?_assertEqual( + Data, + emqx_utils_json:decode(Content) + ). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -154,17 +170,22 @@ client(Config) -> profile_config(Config) -> ProfileConfig0 = emqx_s3_test_helpers:base_config(?config(conn_type, Config)), - ProfileConfig1 = maps:put( - bucket, - ?config(bucket, Config), - ProfileConfig0 - ), - ProfileConfig2 = emqx_utils_maps:deep_put( - [transport_options, pool_type], - ProfileConfig1, - ?config(pool_type, Config) - ), - ProfileConfig2. + maps:fold( + fun inject_config/3, + ProfileConfig0, + #{ + bucket => ?config(bucket, Config), + [transport_options, pool_type] => ?config(pool_type, Config), + [transport_options, headers] => ?config(extra_headers, Config) + } + ). + +inject_config(_Key, undefined, ProfileConfig) -> + ProfileConfig; +inject_config(KeyPath, Value, ProfileConfig) when is_list(KeyPath) -> + emqx_utils_maps:deep_put(KeyPath, ProfileConfig, Value); +inject_config(Key, Value, ProfileConfig) -> + maps:put(Key, Value, ProfileConfig). data(Size) -> iolist_to_binary([$a || _ <- lists:seq(1, Size)]). From 1d9374125c8d2df24772b786957dffb5dcdc37cb Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 28 Dec 2023 14:12:33 +0100 Subject: [PATCH 08/19] chore: bump `emqx_s3` to 5.0.13 --- apps/emqx_s3/src/emqx_s3.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src index 2658dfe69..3da9ea7ca 100644 --- a/apps/emqx_s3/src/emqx_s3.app.src +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -1,6 +1,6 @@ {application, emqx_s3, [ {description, "EMQX S3"}, - {vsn, "5.0.12"}, + {vsn, "5.0.13"}, {modules, []}, {registered, [emqx_s3_sup]}, {applications, [ From 18dc9b251652d496859c8f40a835e00de4b681d2 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 28 Dec 2023 14:13:22 +0100 Subject: [PATCH 09/19] fix(s3): remove `emqx_bridge_http` from runtime dependencies Otherwise it's impossible to test `emqx_s3` in isolation. --- apps/emqx_s3/src/emqx_s3.app.src | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src index 3da9ea7ca..ef59859ce 100644 --- a/apps/emqx_s3/src/emqx_s3.app.src +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -8,8 +8,7 @@ stdlib, gproc, erlcloud, - ehttpc, - emqx_bridge_http + ehttpc ]}, {mod, {emqx_s3_app, []}} ]}. From 72677f69d40dcad1d6af565e8d21b736c90219cc Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 28 Dec 2023 14:49:05 +0100 Subject: [PATCH 10/19] chore: add changelog entry --- changes/ee/fix-12241.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/fix-12241.en.md diff --git a/changes/ee/fix-12241.en.md b/changes/ee/fix-12241.en.md new file mode 100644 index 000000000..e9f5be6aa --- /dev/null +++ b/changes/ee/fix-12241.en.md @@ -0,0 +1 @@ +Fix an issue where setting up extra HTTP headers for communication with S3 API would break File Transfers using S3 storage backend. From 1649f679968c791f98abb082301543ec01245a90 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 2 Jan 2024 09:03:43 +0100 Subject: [PATCH 11/19] chore: do not expose 11883 by default in docker as we do not listen on it --- Dockerfile.ubuntu20.04.runner | 3 +-- .../emqx-enterprise/templates/StatefulSet.yaml | 4 ---- .../emqx-enterprise/templates/service.yaml | 17 ----------------- deploy/charts/emqx/templates/StatefulSet.yaml | 4 ---- deploy/charts/emqx/templates/service.yaml | 17 ----------------- deploy/docker/Dockerfile | 3 +-- 6 files changed, 2 insertions(+), 46 deletions(-) diff --git a/Dockerfile.ubuntu20.04.runner b/Dockerfile.ubuntu20.04.runner index 1bb44a6e9..1b340aaf8 100644 --- a/Dockerfile.ubuntu20.04.runner +++ b/Dockerfile.ubuntu20.04.runner @@ -30,11 +30,10 @@ VOLUME ["/opt/emqx/log", "/opt/emqx/data"] # - 8083 for WebSocket/HTTP # - 8084 for WSS/HTTPS # - 8883 port for MQTT(SSL) -# - 11883 port for internal MQTT/TCP # - 18083 for dashboard and API # - 4370 default Erlang distrbution port # - 5369 for backplain gen_rpc -EXPOSE 1883 8083 8084 8883 11883 18083 4370 5369 +EXPOSE 1883 8083 8084 8883 18083 4370 5369 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] diff --git a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml index a70b6d168..96702ebd7 100644 --- a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml +++ b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml @@ -111,10 +111,6 @@ spec: containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT__BIND | default 8084 }} - name: dashboard containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - - name: internalmqtt - containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND }} - {{- end }} {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND) }} - name: dashboardtls containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND }} diff --git a/deploy/charts/emqx-enterprise/templates/service.yaml b/deploy/charts/emqx-enterprise/templates/service.yaml index 525390a90..bb45adb50 100644 --- a/deploy/charts/emqx-enterprise/templates/service.yaml +++ b/deploy/charts/emqx-enterprise/templates/service.yaml @@ -41,17 +41,6 @@ spec: {{- else if eq .Values.service.type "ClusterIP" }} nodePort: null {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - - name: internalmqtt - port: {{ .Values.service.internalmqtt | default 11883 }} - protocol: TCP - targetPort: internalmqtt - {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.internalmqtt)) }} - nodePort: {{ .Values.service.nodePorts.internalmqtt }} - {{- else if eq .Values.service.type "ClusterIP" }} - nodePort: null - {{- end }} - {{ end }} - name: mqttssl port: {{ .Values.service.mqttssl | default 8883 }} protocol: TCP @@ -124,12 +113,6 @@ spec: port: {{ .Values.service.mqtt | default 1883 }} protocol: TCP targetPort: mqtt - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - - name: internalmqtt - port: {{ .Values.service.internalmqtt | default 11883 }} - protocol: TCP - targetPort: internalmqtt - {{ end }} - name: mqttssl port: {{ .Values.service.mqttssl | default 8883 }} protocol: TCP diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 624f0f2ab..1eba3d1ba 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -111,10 +111,6 @@ spec: containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT__BIND | default 8084 }} - name: dashboard containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTP__BIND | default 18083 }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - - name: internalmqtt - containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND }} - {{- end }} {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTPS__BIND) }} - name: dashboardtls containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTPS__BIND }} diff --git a/deploy/charts/emqx/templates/service.yaml b/deploy/charts/emqx/templates/service.yaml index 525390a90..bb45adb50 100644 --- a/deploy/charts/emqx/templates/service.yaml +++ b/deploy/charts/emqx/templates/service.yaml @@ -41,17 +41,6 @@ spec: {{- else if eq .Values.service.type "ClusterIP" }} nodePort: null {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - - name: internalmqtt - port: {{ .Values.service.internalmqtt | default 11883 }} - protocol: TCP - targetPort: internalmqtt - {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.internalmqtt)) }} - nodePort: {{ .Values.service.nodePorts.internalmqtt }} - {{- else if eq .Values.service.type "ClusterIP" }} - nodePort: null - {{- end }} - {{ end }} - name: mqttssl port: {{ .Values.service.mqttssl | default 8883 }} protocol: TCP @@ -124,12 +113,6 @@ spec: port: {{ .Values.service.mqtt | default 1883 }} protocol: TCP targetPort: mqtt - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - - name: internalmqtt - port: {{ .Values.service.internalmqtt | default 11883 }} - protocol: TCP - targetPort: internalmqtt - {{ end }} - name: mqttssl port: {{ .Values.service.mqttssl | default 8883 }} protocol: TCP diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index b2dfbb1f6..6aaf1bf2e 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -64,11 +64,10 @@ VOLUME ["/opt/emqx/log", "/opt/emqx/data"] # - 8083 for WebSocket/HTTP # - 8084 for WSS/HTTPS # - 8883 port for MQTT(SSL) -# - 11883 port for internal MQTT/TCP # - 18083 for dashboard and API # - 4370 default Erlang distribution port # - 5369 for backplane gen_rpc -EXPOSE 1883 8083 8084 8883 11883 18083 4370 5369 +EXPOSE 1883 8083 8084 8883 18083 4370 5369 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] From 4c761b0ff9f25e9d4f87ad1513fdec7ac0188430 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 2 Jan 2024 10:50:15 +0100 Subject: [PATCH 12/19] docs: add changelog entry --- changes/ee/fix-12246.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/fix-12246.en.md diff --git a/changes/ee/fix-12246.en.md b/changes/ee/fix-12246.en.md new file mode 100644 index 000000000..29ea2419e --- /dev/null +++ b/changes/ee/fix-12246.en.md @@ -0,0 +1 @@ +Do not expose 11883 port by default in docker and remove it from helm chart since this port is no longer in use. From d4c83cf463618cdfbf674fb4772ee8346d1fef4d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 2 Jan 2024 14:37:45 +0100 Subject: [PATCH 13/19] chore: tag 5.4.1-alpha.1 --- apps/emqx/include/emqx_release.hrl | 4 ++-- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- deploy/charts/emqx/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 0115edc91..cb97b5853 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.4.1"). +-define(EMQX_RELEASE_CE, "5.4.1-alpha.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.4.1"). +-define(EMQX_RELEASE_EE, "5.4.1-alpha.1"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index acbcaffb2..d774147ca 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.4.0 +version: 5.4.1-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.4.0 +appVersion: 5.4.1-alpha.1 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 41ee9a4cb..f88333572 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.4.0 +version: 5.4.1-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.4.0 +appVersion: 5.4.1-alpha.1 From 00e8a36dbea9c20c65ad4ec07f153d84915a13d2 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 3 Jan 2024 11:13:57 +0800 Subject: [PATCH 14/19] fix: the API prompt garbled characters for updating the configuration --- apps/emqx_conf/src/emqx_conf_cli.erl | 2 +- apps/emqx_conf/test/emqx_conf_cli_SUITE.erl | 2 +- .../test/emqx_mgmt_api_configs_SUITE.erl | 17 ++++++++++++++--- changes/ee/fix-12240.en.md | 1 + changes/ee/fix-12249.en.md | 1 + 5 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 changes/ee/fix-12240.en.md create mode 100644 changes/ee/fix-12249.en.md diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index f2ced3327..b6ba0dfc9 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -35,7 +35,7 @@ -define(CLUSTER_CALL, cluster_call). -define(CONF, conf). -define(AUDIT_MOD, audit). --define(UPDATE_READONLY_KEYS_PROHIBITED, "update_readonly_keys_prohibited"). +-define(UPDATE_READONLY_KEYS_PROHIBITED, <<"update_readonly_keys_prohibited">>). -dialyzer({no_match, [load/0]}). diff --git a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl index b3ecdd0cf..a3630e91d 100644 --- a/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_cli_SUITE.erl @@ -165,7 +165,7 @@ t_load_readonly(Config) -> ConfBin0 = hocon_pp:do(Base1#{KeyBin => Conf}, #{}), ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config), ?assertEqual( - {error, "update_readonly_keys_prohibited"}, + {error, <<"update_readonly_keys_prohibited">>}, emqx_conf_cli:conf(["load", ConfFile0]) ), %% reload etc/emqx.conf changed readonly keys diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 2bfff17a6..bb3ca87b7 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -331,7 +331,7 @@ t_configs_key(_Config) -> Log ), Log1 = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"error">>), - ?assertEqual([], update_configs_with_binary(iolist_to_binary(hocon_pp:do(Log1, #{})))), + ?assertEqual(<<>>, update_configs_with_binary(iolist_to_binary(hocon_pp:do(Log1, #{})))), ?assertEqual(<<"error">>, read_conf([<<"log">>, <<"console">>, <<"level">>])), BadLog = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"erro1r">>), {error, Error} = update_configs_with_binary(iolist_to_binary(hocon_pp:do(BadLog, #{}))), @@ -345,6 +345,17 @@ t_configs_key(_Config) -> } }, ?assertEqual(ExpectError, emqx_utils_json:decode(Error, [return_maps])), + ReadOnlyConf = #{ + <<"cluster">> => + #{ + <<"autoclean">> => <<"23h">>, + <<"autoheal">> => true, + <<"discovery_strategy">> => <<"manual">> + } + }, + ReadOnlyBin = iolist_to_binary(hocon_pp:do(ReadOnlyConf, #{})), + {error, ReadOnlyError} = update_configs_with_binary(ReadOnlyBin), + ?assertEqual(<<"update_readonly_keys_prohibited">>, ReadOnlyError), ok. t_get_configs_in_different_accept(_Config) -> @@ -394,7 +405,7 @@ t_create_webhook_v1_bridges_api(Config) -> WebHookFile = filename:join(?config(data_dir, Config), "webhook_v1.conf"), ?assertMatch({ok, _}, hocon:files([WebHookFile])), {ok, WebHookBin} = file:read_file(WebHookFile), - ?assertEqual([], update_configs_with_binary(WebHookBin)), + ?assertEqual(<<>>, update_configs_with_binary(WebHookBin)), Actions = #{ <<"http">> => @@ -531,7 +542,7 @@ update_configs_with_binary(Bin) -> Path = emqx_mgmt_api_test_util:api_path(["configs"]), Auth = emqx_mgmt_api_test_util:auth_header_(), Headers = [{"accept", "text/plain"}, Auth], - case httpc:request(put, {Path, Headers, "text/plain", Bin}, [], []) of + case httpc:request(put, {Path, Headers, "text/plain", Bin}, [], [{body_format, binary}]) of {ok, {{"HTTP/1.1", Code, _}, _Headers, Body}} when Code >= 200 andalso Code =< 299 -> diff --git a/changes/ee/fix-12240.en.md b/changes/ee/fix-12240.en.md new file mode 100644 index 000000000..064eaa7c7 --- /dev/null +++ b/changes/ee/fix-12240.en.md @@ -0,0 +1 @@ +Modified the /file_transfer API to return the file transfer configuration in raw format rather than converting time units like "1h" to seconds, providing callers with the original configured values for consistency with other getter APIs diff --git a/changes/ee/fix-12249.en.md b/changes/ee/fix-12249.en.md new file mode 100644 index 000000000..9fe98cba3 --- /dev/null +++ b/changes/ee/fix-12249.en.md @@ -0,0 +1 @@ +Fixed issue where the response message from the /configs API would be garbled when attempting to update a read-only configuration value From 90404b62a49039ee5ac4068dea3beb1e1d5117da Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 3 Jan 2024 16:06:25 +0800 Subject: [PATCH 15/19] fix: incorrectly attempt to update the field value when the previous field don't have value --- apps/emqx_utils/src/emqx_utils.app.src | 2 +- apps/emqx_utils/src/emqx_utils.erl | 25 ++++++++++++++++++++++++- changes/ee/fix-12250.en.md | 1 + 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 changes/ee/fix-12250.en.md diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index 893ad707e..76d503331 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.13"}, + {vsn, "5.0.14"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index fd9f5a311..cadeabf70 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -760,7 +760,11 @@ deobfuscate(NewConf, OldConf) -> fun(K, V, Acc) -> case maps:find(K, OldConf) of error -> - Acc#{K => V}; + case is_redacted(K, V) of + %% don't put redacted value into new config + true -> Acc; + false -> Acc#{K => V} + end; {ok, OldV} when is_map(V), is_map(OldV) -> Acc#{K => deobfuscate(V, OldV)}; {ok, OldV} -> @@ -879,6 +883,25 @@ redact2_test_() -> Keys = [secret, passcode], [{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys]. +deobfuscate_test() -> + NewConf0 = #{foo => <<"bar0">>, password => <<"123456">>}, + ?assertEqual(NewConf0, deobfuscate(NewConf0, #{foo => <<"bar">>, password => <<"654321">>})), + + NewConf1 = #{foo => <<"bar1">>, password => <>}, + ?assertEqual( + #{foo => <<"bar1">>, password => <<"654321">>}, + deobfuscate(NewConf1, #{foo => <<"bar">>, password => <<"654321">>}) + ), + + %% Don't have password before and ignore to put redact_val into new config + NewConf2 = #{foo => <<"bar2">>, password => ?REDACT_VAL}, + ?assertEqual(#{foo => <<"bar2">>}, deobfuscate(NewConf2, #{foo => <<"bar">>})), + + %% Don't have password before and should allow put non-redact-val into new config + NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>}, + ?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})), + ok. + redact_is_authorization_test_() -> Types = [string, binary], Keys = ["auThorization", "Authorization", "authorizaTion"], diff --git a/changes/ee/fix-12250.en.md b/changes/ee/fix-12250.en.md new file mode 100644 index 000000000..4d81ff5c9 --- /dev/null +++ b/changes/ee/fix-12250.en.md @@ -0,0 +1 @@ +Fixed incorrect attempt to update the file_transfer configuration's secret_access_key value to masked stars ('*****') From 304fe84434e5be3e3027566e1bbc81216c6abbee Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 4 Jan 2024 13:06:54 +0100 Subject: [PATCH 16/19] fix(mysql): expect password may be missing from config --- .../test/emqx_bridge_mysql_SUITE.erl | 2 - apps/emqx_mysql/src/emqx_mysql.app.src | 2 +- apps/emqx_mysql/src/emqx_mysql.erl | 11 +- apps/emqx_mysql/test/emqx_mysql_SUITE.erl | 124 +++++++++++------- 4 files changed, 85 insertions(+), 54 deletions(-) diff --git a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl index 96fcf6d24..61847a6b0 100644 --- a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl +++ b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl @@ -112,8 +112,6 @@ end_per_group(_Group, _Config) -> ok. init_per_suite(Config) -> - emqx_common_test_helpers:clear_screen(), - Config. end_per_suite(_Config) -> diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index e9f7f6f98..9ae3234cc 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index a9b132570..d33f313bc 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -289,10 +289,17 @@ do_check_prepares(_NoTemplates) -> connect(Options) -> %% TODO: teach `tdengine` to accept 0-arity closures as passwords. - {value, {password, Secret}, Rest} = lists:keytake(password, 1, Options), - NOptions = [{password, emqx_secret:unwrap(Secret)} | Rest], + NOptions = init_connect_opts(Options), mysql:start_link(NOptions). +init_connect_opts(Options) -> + case lists:keytake(password, 1, Options) of + {value, {password, Secret}, Rest} -> + [{password, emqx_secret:unwrap(Secret)} | Rest]; + false -> + Options + end. + init_prepare(State = #{query_templates := Templates}) -> case maps:size(Templates) of 0 -> diff --git a/apps/emqx_mysql/test/emqx_mysql_SUITE.erl b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl index b8886f1cd..25e9cce5a 100644 --- a/apps/emqx_mysql/test/emqx_mysql_SUITE.erl +++ b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl @@ -1,17 +1,17 @@ -% %%-------------------------------------------------------------------- -% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -% %% -% %% Licensed under the Apache License, Version 2.0 (the "License"); -% %% you may not use this file except in compliance with the License. -% %% You may obtain a copy of the License at -% %% http://www.apache.org/licenses/LICENSE-2.0 -% %% -% %% Unless required by applicable law or agreed to in writing, software -% %% distributed under the License is distributed on an "AS IS" BASIS, -% %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% %% See the License for the specific language governing permissions and -% %% limitations under the License. -% %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_mysql_SUITE). @@ -19,40 +19,30 @@ -compile(export_all). -include_lib("emqx_connector/include/emqx_connector.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). -define(MYSQL_HOST, "mysql"). +-define(MYSQL_USER, "root"). +-define(MYSQL_PASSWORD, "public"). -define(MYSQL_RESOURCE_MOD, emqx_mysql). all() -> emqx_common_test_helpers:all(?MODULE). -groups() -> - []. - init_per_suite(Config) -> case emqx_common_test_helpers:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_DEFAULT_PORT) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource]), - {ok, _} = application:ensure_all_started(emqx_connector), - Config; + Apps = emqx_cth_suite:start( + [emqx_conf, emqx_connector], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{apps, Apps} | Config]; false -> {skip, no_mysql} end. -end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector). - -init_per_testcase(_, Config) -> - Config. - -end_per_testcase(_, _Config) -> - ok. +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(proplists:get_value(apps, Config)). % %%------------------------------------------------------------------------------ % %% Testcases @@ -64,6 +54,12 @@ t_lifecycle(_Config) -> mysql_config() ). +t_lifecycle_passwordless(_Config) -> + perform_lifecycle_check( + <<"emqx_mysql_SUITE:passwordless">>, + mysql_config(passwordless) + ). + perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?MYSQL_RESOURCE_MOD, InitialConfig), @@ -136,25 +132,55 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> % %%------------------------------------------------------------------------------ mysql_config() -> - RawConfig = list_to_binary( - io_lib:format( - "" - "\n" - " auto_reconnect = true\n" - " database = mqtt\n" - " username= root\n" - " password = public\n" - " pool_size = 8\n" - " server = \"~s:~b\"\n" - " " - "", - [?MYSQL_HOST, ?MYSQL_DEFAULT_PORT] - ) - ), + mysql_config(default). - {ok, Config} = hocon:binary(RawConfig), +mysql_config(default) -> + parse_mysql_config( + "\n auto_reconnect = true" + "\n database = mqtt" + "\n username = ~p" + "\n password = ~p" + "\n pool_size = 8" + "\n server = \"~s:~b\"" + "\n", + [?MYSQL_USER, ?MYSQL_PASSWORD, ?MYSQL_HOST, ?MYSQL_DEFAULT_PORT] + ); +mysql_config(passwordless) -> + ok = run_admin_query("CREATE USER IF NOT EXISTS 'nopwd'@'%'"), + ok = run_admin_query("GRANT ALL ON mqtt.* TO 'nopwd'@'%'"), + parse_mysql_config( + "\n auto_reconnect = true" + "\n database = mqtt" + "\n username = nopwd" + "\n pool_size = 8" + "\n server = \"~s:~b\"" + "\n", + [?MYSQL_HOST, ?MYSQL_DEFAULT_PORT] + ). + +parse_mysql_config(FormatString, Args) -> + {ok, Config} = hocon:binary(io_lib:format(FormatString, Args)), #{<<"config">> => Config}. +run_admin_query(Query) -> + Pid = connect_mysql(), + try + mysql:query(Pid, Query) + after + mysql:stop(Pid) + end. + +connect_mysql() -> + Opts = [ + {host, ?MYSQL_HOST}, + {port, ?MYSQL_DEFAULT_PORT}, + {user, ?MYSQL_USER}, + {password, ?MYSQL_PASSWORD}, + {database, "mysql"} + ], + {ok, Pid} = mysql:start_link(Opts), + Pid. + test_query_no_params() -> {sql, <<"SELECT 1">>}. From ace60817928742203f4922341c9c7fec9ad91ef4 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 4 Jan 2024 13:10:06 +0100 Subject: [PATCH 17/19] chore: add changelog entry --- changes/ee/fix-12256.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/fix-12256.en.md diff --git a/changes/ee/fix-12256.en.md b/changes/ee/fix-12256.en.md new file mode 100644 index 000000000..5db28dd68 --- /dev/null +++ b/changes/ee/fix-12256.en.md @@ -0,0 +1 @@ +Fix an issue where connections to passwordless MySQL resources could not be established. From 4edd4b4db80057eccd44342f9e5cb89ecd41515c Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 5 Jan 2024 11:38:29 +0800 Subject: [PATCH 18/19] chore: upgrade dashboard to e1.4.1 for ee --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6897d7f5a..8afe7bd81 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ endif # Dashboard version # from https://github.com/emqx/emqx-dashboard5 export EMQX_DASHBOARD_VERSION ?= v1.6.0 -export EMQX_EE_DASHBOARD_VERSION ?= e1.4.0 +export EMQX_EE_DASHBOARD_VERSION ?= e1.4.1 PROFILE ?= emqx REL_PROFILES := emqx emqx-enterprise From d04ac08d1dfa6889bd46deb8da149db602cee826 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 5 Jan 2024 10:37:03 +0100 Subject: [PATCH 19/19] chore: release 5.4.1 --- apps/emqx/include/emqx_release.hrl | 4 ++-- changes/e5.4.1.en.md | 21 +++++++++++++++++++++ changes/v5.4.1.en.md | 13 +++++++++++++ deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- deploy/charts/emqx/Chart.yaml | 4 ++-- 5 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 changes/e5.4.1.en.md create mode 100644 changes/v5.4.1.en.md diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index cb97b5853..0115edc91 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.4.1-alpha.1"). +-define(EMQX_RELEASE_CE, "5.4.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.4.1-alpha.1"). +-define(EMQX_RELEASE_EE, "5.4.1"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/changes/e5.4.1.en.md b/changes/e5.4.1.en.md new file mode 100644 index 000000000..c41ff0a55 --- /dev/null +++ b/changes/e5.4.1.en.md @@ -0,0 +1,21 @@ +# e5.4.1 + +## Bug Fixes + +- [#12234](https://github.com/emqx/emqx/pull/12234) Resolved compatibility issues with Open Telemetry configurations defined in `emqx.conf` from versions before EMQX 5.4.0, ensuring smooth integration of legacy configurations with the latest EMQX release. + +- [#12236](https://github.com/emqx/emqx/pull/12236) Fixed client ID generation in MQTT broker data integration to comply with MQTT 3.1 specification of 23-byte limit. Client ID is now prefixed with user-assigned connector name, followed by the first 8 bytes of node name's SHA hash and pool member ID. If the resulting ID exceeds 23 bytes, additional SHA hash and truncation are applied to ensure compliance. + +- [#12238](https://github.com/emqx/emqx/pull/12238) Resolved compatibility issue with the error format configurations introduced in the HTTP Action feature of EMQX version 5.3.2. + +- [#12240](https://github.com/emqx/emqx/pull/12240) Modified the `/file_transfer` API to return file transfer configurations in their original raw format. This change prevents the conversion of time units, such as "1h", to seconds, ensuring that callers receive the initially configured values. This modification aligns with other getter APIs, maintaining consistency in data representation. + +- [#12241](https://github.com/emqx/emqx/pull/12241) Fixed a bug where configuring additional HTTP headers for S3 API interactions disrupted file transfers using the S3 storage backend, ensuring stable and uninterrupted file transfer operations. + +- [#12246](https://github.com/emqx/emqx/pull/12246) Stopped exposing port 11883 by default in Docker and removed it from Helm charts, as this port is no longer in use. + +- [#12249](https://github.com/emqx/emqx/pull/12249) Fixed an issue in the `/configs` API where attempting to modify a read-only configuration value resulted in a garbled response message. + +- [#12250](https://github.com/emqx/emqx/pull/12250) Resolved an issue where the `file_transfer` configuration's `secret_access_key` value was erroneously being updated to masked stars (`*****`), ensuring that the original key value remains unaltered and secure. + +- [#12256](https://github.com/emqx/emqx/pull/12256) Fixed an issue that prevented establishing connections to MySQL resources without a password. diff --git a/changes/v5.4.1.en.md b/changes/v5.4.1.en.md new file mode 100644 index 000000000..beb4c822a --- /dev/null +++ b/changes/v5.4.1.en.md @@ -0,0 +1,13 @@ +# v5.4.1 + +## Bug Fixes + +- [#12234](https://github.com/emqx/emqx/pull/12234) Resolved compatibility issues with Open Telemetry configurations defined in `emqx.conf` from versions before EMQX 5.4.0, ensuring smooth integration of legacy configurations with the latest EMQX release. + +- [#12236](https://github.com/emqx/emqx/pull/12236) Fixed client ID generation in MQTT broker data integration to comply with MQTT 3.1 specification of 23-byte limit. Client ID is now prefixed with user-assigned connector name, followed by the first 8 bytes of node name's SHA hash and pool member ID. If the resulting ID exceeds 23 bytes, additional SHA hash and truncation are applied to ensure compliance. + +- [#12238](https://github.com/emqx/emqx/pull/12238) Resolved compatibility issue with the error format configurations introduced in the HTTP Action feature of EMQX version 5.3.2. + +- [#12246](https://github.com/emqx/emqx/pull/12246) Stopped exposing port 11883 by default in Docker and removed it from Helm charts, as this port is no longer in use. + +- [#12249](https://github.com/emqx/emqx/pull/12249) Fixed an issue in the `/configs` API where attempting to modify a read-only configuration value resulted in a garbled response message. diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index d774147ca..5f4dc4cdc 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.4.1-alpha.1 +version: 5.4.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.4.1-alpha.1 +appVersion: 5.4.1 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index f88333572..c95573c0a 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.4.1-alpha.1 +version: 5.4.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.4.1-alpha.1 +appVersion: 5.4.1